diff --git a/dagshub/data_engine/annotation/metadata.py b/dagshub/data_engine/annotation/metadata.py index 8b5d632c..60e9e645 100644 --- a/dagshub/data_engine/annotation/metadata.py +++ b/dagshub/data_engine/annotation/metadata.py @@ -1,25 +1,27 @@ -from typing import TYPE_CHECKING, Optional, Sequence, Tuple, Union, Literal, Dict +from typing import TYPE_CHECKING, Dict, Literal, Optional, Sequence, Tuple, Union -from dagshub_annotation_converter.formats.label_studio.task import parse_ls_task, LabelStudioTask -from dagshub_annotation_converter.formats.yolo import import_lookup, import_yolo_result, YoloContext +from dagshub_annotation_converter.formats.label_studio.task import LabelStudioTask, parse_ls_task +from dagshub_annotation_converter.formats.yolo import YoloContext, import_lookup, import_yolo_result from dagshub_annotation_converter.formats.yolo.categories import Categories from dagshub_annotation_converter.ir.image import ( - IRBBoxImageAnnotation, CoordinateStyle, - IRSegmentationImageAnnotation, - IRSegmentationPoint, + IRBBoxImageAnnotation, IRPoseImageAnnotation, IRPosePoint, + IRSegmentationImageAnnotation, + IRSegmentationPoint, ) from dagshub_annotation_converter.ir.image.annotations.base import IRAnnotationBase, IRImageAnnotationBase from dagshub.common.api import UserAPI from dagshub.common.helpers import log_message +from dagshub.data_engine.util.not_implemented import NotImplementedMeta if TYPE_CHECKING: - from dagshub.data_engine.model.datapoint import Datapoint import ultralytics.engine.results + from dagshub.data_engine.model.datapoint import Datapoint + class AnnotationMetaDict(dict): def __init__(self, annotation: "MetadataAnnotations", *args, **kwargs): @@ -315,3 +317,23 @@ def _generate_yolo_context(annotation_type, categories: Dict[int, str]) -> YoloC for cat_id, cat_name in categories.items(): cats.add(cat_name, cat_id) return YoloContext(annotation_type=annotation_type, categories=cats) + + +class UnsupportedMetadataAnnotations(MetadataAnnotations, metaclass=NotImplementedMeta): + def __init__( + self, + datapoint: "Datapoint", + field: str, + original_value: bytes, + ): + super().__init__(datapoint, field, None, None, original_value) + + @property + def value(self) -> Optional[bytes]: + return self._original_value + + def to_ls_task(self) -> Optional[bytes]: + return self._original_value + + def __repr__(self): + return "Label Studio annotations of unrecognized type" diff --git a/dagshub/data_engine/model/query_result.py b/dagshub/data_engine/model/query_result.py index 6c326eab..2a0e706e 100644 --- a/dagshub/data_engine/model/query_result.py +++ b/dagshub/data_engine/model/query_result.py @@ -30,6 +30,7 @@ from dagshub.common.rich_util import get_rich_progress from dagshub.common.util import lazy_load, multi_urljoin from dagshub.data_engine.annotation import MetadataAnnotations +from dagshub.data_engine.annotation.metadata import UnsupportedMetadataAnnotations from dagshub.data_engine.annotation.voxel_conversion import ( add_ls_annotations, add_voxel_annotations, @@ -425,8 +426,11 @@ def _get_blob_fn(dp: Datapoint, field: str, url: str, blob_path: Path): for dp in self: for fld in document_fields: if fld in dp.metadata: - # Override the load_into_memory flag, because we need the contents - if not load_into_memory: + # Defensive check to not mangle annotation fields by accident + if isinstance(dp.metadata[fld], MetadataAnnotations): + continue + # Force load the content into memory, even if load_into_memory was set to False + if not load_into_memory or isinstance(dp.metadata[fld], Path): dp.metadata[fld] = Path(dp.metadata[fld]).read_bytes() dp.metadata[fld] = dp.metadata[fld].decode("utf-8") @@ -435,38 +439,49 @@ def _get_blob_fn(dp: Datapoint, field: str, url: str, blob_path: Path): def _convert_annotation_fields(self, *fields, load_into_memory): # Convert any downloaded annotation column annotation_fields = [f for f in fields if f in self.annotation_fields] + if not annotation_fields: + return + # List of datapoints with annotations that couldn't be parsed bad_annotations = defaultdict(list) - if annotation_fields: - # Convert them - for dp in self: - for fld in annotation_fields: - if fld in dp.metadata: - # Already loaded - skip - if isinstance(dp.metadata[fld], MetadataAnnotations): - continue - # Override the load_into_memory flag, because we need the contents - if not load_into_memory: - dp.metadata[fld] = Path(dp.metadata[fld]).read_bytes() - try: - dp.metadata[fld] = MetadataAnnotations.from_ls_task( - datapoint=dp, field=fld, ls_task=dp.metadata[fld] - ) - except ValidationError: - bad_annotations[fld].append(dp.path) - else: - dp.metadata[fld] = MetadataAnnotations(datapoint=dp, field=fld) + for dp in self: + for fld in annotation_fields: + metadata_value = dp.metadata.get(fld) + # No value - create ampty annotation container + if metadata_value is None: + dp.metadata[fld] = MetadataAnnotations(datapoint=dp, field=fld) + continue + # Already loaded - skip + elif isinstance(metadata_value, MetadataAnnotations): + continue + # Parse annotation from the content of the field + else: + # Force load the content into memory, even if load_into_memory was set to False + if not load_into_memory or isinstance(dp.metadata[fld], Path): + metadata_value = Path(metadata_value).read_bytes() + try: + dp.metadata[fld] = MetadataAnnotations.from_ls_task( + datapoint=dp, field=fld, ls_task=metadata_value + ) + except ValidationError: + dp.metadata[fld] = UnsupportedMetadataAnnotations( + datapoint=dp, field=fld, original_value=metadata_value + ) + bad_annotations[fld].append(dp.path) if bad_annotations: log_message( - "Warning: The following datapoints had invalid annotations, " - "any annotation-related operations will not work on these:" + "Warning: The following datapoints had unsupported or invalid annotations, " + "convenience functions like `add_bounding_box` won't work on these:" ) err_msg = "" for fld, dps in bad_annotations.items(): - err_msg += f'Field "{fld}" in datapoints:\n\t' - err_msg += "\n\t".join(dps) + err_msg += f'\nField "{fld}" in datapoints:\n\t' + if len(dps) > 10: + err_msg += "\n\t".join(dps[:10]) + f"\n\t... and {len(dps) - 10} more" + else: + err_msg += "\n\t".join(dps) log_message(err_msg) def download_binary_columns( diff --git a/dagshub/data_engine/util/__init__.py b/dagshub/data_engine/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dagshub/data_engine/util/not_implemented.py b/dagshub/data_engine/util/not_implemented.py new file mode 100644 index 00000000..d9a81285 --- /dev/null +++ b/dagshub/data_engine/util/not_implemented.py @@ -0,0 +1,48 @@ +class NotImplementedMeta(type): + """ + A metaclass that replaces all parent class methods and properties that aren't overridden in the subclass + with NotImplementedError. + """ + + def __new__(mcs, name, bases, namespace): + # Get all attributes from base classes + for base in bases: + for attr_name in dir(base): + if attr_name.startswith("_"): + continue + + # Skip if already defined in subclass + if attr_name in namespace: + continue + + base_attr = getattr(base, attr_name) + + # Handle properties + if isinstance(base_attr, property): + # Create a property that raises NotImplementedError + def make_not_implemented_property(prop_name): + def getter(self): + raise NotImplementedError(f"Property '{prop_name}' not implemented") + + def setter(self, value): + raise NotImplementedError(f"Property '{prop_name}' not implemented") + + def deleter(self): + raise NotImplementedError(f"Property '{prop_name}' not implemented") + + return property(getter, setter, deleter) + + namespace[attr_name] = make_not_implemented_property(attr_name) + + # Handle regular methods + elif callable(base_attr): + + def make_not_implemented(method_name): + def not_impl(self, *args, **kwargs): + raise NotImplementedError(f"Method '{method_name}' not implemented") + + return not_impl + + namespace[attr_name] = make_not_implemented(attr_name) + + return super().__new__(mcs, name, bases, namespace) diff --git a/tests/data_engine/annotation_import/res/audio_annotation.json b/tests/data_engine/annotation_import/res/audio_annotation.json new file mode 100644 index 00000000..adc356e2 --- /dev/null +++ b/tests/data_engine/annotation_import/res/audio_annotation.json @@ -0,0 +1,82 @@ +{ + "id": 41, + "data": { + "audio": "https://example.com/some-non-existent-file.mp3", + "media type": "audio/mpeg", + "size": 111699 + }, + "meta": { + "datapoint_id": 12345678, + "datasource_id": 6565 + }, + "created_at": "2025-12-20T13:44:02.316027Z", + "updated_at": "2026-01-26T15:00:13.046967Z", + "is_labeled": true, + "project": 1, + "annotations": [ + { + "completed_by": 1, + "result": [ + { + "type": "choices", + "value": { + "choices": [ + "true" + ] + }, + "origin": "manual", + "to_name": "audio", + "from_name": "further_utterance", + "id": "deadbeef1" + }, + { + "type": "rating", + "value": { + "rating": 1 + }, + "origin": "manual", + "to_name": "audio", + "from_name": "difficulty", + "id": "deadbeef1" + }, + { + "type": "textarea", + "value": { + "text": [ + "kirill@dagshub.com" + ] + }, + "origin": "manual", + "to_name": "audio", + "from_name": "email_address", + "id": "deadbeef1" + }, + { + "to_name": "audio", + "from_name": "first_name", + "id": "Qzu1dR2RQ8", + "type": "textarea", + "value": { + "text": [ + "Kirill" + ] + }, + "origin": "manual" + }, + { + "type": "textarea", + "value": { + "text": [ + "Bolashev" + ] + }, + "origin": "manual", + "to_name": "audio", + "from_name": "last_name", + "id": "deadbeef1" + } + ], + "ground_truth": false + } + ] +} diff --git a/tests/data_engine/annotation_import/test_annotation_parsing.py b/tests/data_engine/annotation_import/test_annotation_parsing.py index 66840ecb..6fce8d78 100644 --- a/tests/data_engine/annotation_import/test_annotation_parsing.py +++ b/tests/data_engine/annotation_import/test_annotation_parsing.py @@ -1,19 +1,23 @@ import json +from os import PathLike from pathlib import Path +from typing import Union from unittest.mock import MagicMock import pytest from dagshub_annotation_converter.ir.image import IRSegmentationImageAnnotation +from pytest import MonkeyPatch from dagshub.data_engine.annotation import MetadataAnnotations +from dagshub.data_engine.annotation.metadata import UnsupportedMetadataAnnotations from dagshub.data_engine.dtypes import MetadataFieldType, ReservedTags from dagshub.data_engine.model import query_result from dagshub.data_engine.model.datasource import Datasource +from dagshub.data_engine.model.query_result import QueryResult from tests.data_engine.util import add_metadata_field _annotation_field_name = "annotation" _dp_path = "data/sample_datapoint.jpg" -_annotation_hash = "annotation1" # Corresponds to a resource JSON _res_folder = Path(__file__).parent / "res" @@ -51,17 +55,21 @@ def mock_annotation_query_result( return query_result.QueryResult.from_gql_query(data_dict, ds) -def mock_get_blob(*args, **kwargs) -> bytes: +def mock_get_blob(*args, **kwargs) -> Union[bytes, PathLike]: download_url: str = args[0] blob_hash = download_url.split("/")[-1] + load_into_memory = args[4] blob_path = _res_folder / f"{blob_hash}.json" + if not blob_path.exists(): raise FileNotFoundError(f"Mock blob file not found: {blob_path}") - return blob_path.read_bytes() + if load_into_memory: + return blob_path.read_bytes() + else: + return blob_path -@pytest.fixture -def ds_with_document_annotation(ds, monkeypatch): +def _ds_with_annotation(ds: "Datasource", monkeypatch: MonkeyPatch, annotation_hash: str): add_metadata_field( ds, _annotation_field_name, @@ -70,18 +78,57 @@ def ds_with_document_annotation(ds, monkeypatch): ) ds.source.client.get_datapoints = MagicMock( - return_value=mock_annotation_query_result(ds, _annotation_field_name, _dp_path, _annotation_hash) + return_value=mock_annotation_query_result(ds, _annotation_field_name, _dp_path, annotation_hash) ) monkeypatch.setattr(query_result, "_get_blob", mock_get_blob) - yield ds + return ds + + +@pytest.fixture +def ds_with_document_annotation(ds, monkeypatch): + yield _ds_with_annotation(ds, monkeypatch, "annotation1") + + +@pytest.fixture +def ds_with_unsupported_annotation(ds, monkeypatch): + yield _ds_with_annotation(ds, monkeypatch, "audio_annotation") def test_annotation_with_document_are_parsed_as_annotation(ds_with_document_annotation): qr = ds_with_document_annotation.all() + _test_annotation(qr) + + +def test_double_loading_annotation_works(ds_with_document_annotation): + qr = ds_with_document_annotation.all() + qr.get_blob_fields(_annotation_field_name) + _test_annotation(qr) + + +def _test_annotation(qr: QueryResult): annotation: MetadataAnnotations = qr[0].metadata[_annotation_field_name] assert isinstance(annotation, MetadataAnnotations) # Check that the annotation got parsed correctly, the JSON should have one segmentation annotation in it assert len(annotation.annotations) == 1 assert isinstance(annotation.annotations[0], IRSegmentationImageAnnotation) + + +def test_handling_unsupported_annotation(ds_with_unsupported_annotation): + qr = ds_with_unsupported_annotation.all() + + annotation: MetadataAnnotations = qr[0].metadata[_annotation_field_name] + + assert isinstance(annotation, UnsupportedMetadataAnnotations) + # Unsupported annotation is still a subclass of regular annotation + # This is crucial for logic that checks if annotation metadata was parsed already, + # so if this starts failing, that logic will need to be changed too + assert isinstance(annotation, MetadataAnnotations) + + with pytest.raises(NotImplementedError): + annotation.add_image_bbox("cat", 0, 0, 10, 10, 1920, 1080) + + expected_content = (_res_folder / "audio_annotation.json").read_bytes() + assert annotation.value == expected_content + assert annotation.to_ls_task() == expected_content