Bug: cannot query metadata with unsupported annotations

dagshub/data_engine/annotation/metadata.py

@@ -1,25 +1,27 @@
-from typing import TYPE_CHECKING, Optional, Sequence, Tuple, Union, Literal, Dict
+from typing import TYPE_CHECKING, Dict, Literal, Optional, Sequence, Tuple, Union
 
-from dagshub_annotation_converter.formats.label_studio.task import parse_ls_task, LabelStudioTask
-from dagshub_annotation_converter.formats.yolo import import_lookup, import_yolo_result, YoloContext
+from dagshub_annotation_converter.formats.label_studio.task import LabelStudioTask, parse_ls_task
+from dagshub_annotation_converter.formats.yolo import YoloContext, import_lookup, import_yolo_result
 from dagshub_annotation_converter.formats.yolo.categories import Categories
 from dagshub_annotation_converter.ir.image import (
-    IRBBoxImageAnnotation,
     CoordinateStyle,
-    IRSegmentationImageAnnotation,
-    IRSegmentationPoint,
+    IRBBoxImageAnnotation,
     IRPoseImageAnnotation,
     IRPosePoint,
+    IRSegmentationImageAnnotation,
+    IRSegmentationPoint,
 )
 from dagshub_annotation_converter.ir.image.annotations.base import IRAnnotationBase, IRImageAnnotationBase
 
 from dagshub.common.api import UserAPI
 from dagshub.common.helpers import log_message
+from dagshub.data_engine.util.not_implemented import NotImplementedMeta
 
 if TYPE_CHECKING:
-    from dagshub.data_engine.model.datapoint import Datapoint
     import ultralytics.engine.results
 
+    from dagshub.data_engine.model.datapoint import Datapoint
+
 
 class AnnotationMetaDict(dict):
     def __init__(self, annotation: "MetadataAnnotations", *args, **kwargs):
@@ -315,3 +317,23 @@ def _generate_yolo_context(annotation_type, categories: Dict[int, str]) -> YoloC
         for cat_id, cat_name in categories.items():
             cats.add(cat_name, cat_id)
         return YoloContext(annotation_type=annotation_type, categories=cats)
+
+
+class UnsupportedMetadataAnnotations(MetadataAnnotations, metaclass=NotImplementedMeta):
+    def __init__(
+        self,
+        datapoint: "Datapoint",
+        field: str,
+        original_value: bytes,
+    ):
+        super().__init__(datapoint, field, None, None, original_value)
+
+    @property
+    def value(self) -> Optional[bytes]:
+        return self._original_value
+
+    def to_ls_task(self) -> Optional[bytes]:
+        return self._original_value
+
+    def __repr__(self):
+        return "Label Studio annotations of unrecognized type"

dagshub/data_engine/model/query_result.py

@@ -30,6 +30,7 @@
 from dagshub.common.rich_util import get_rich_progress
 from dagshub.common.util import lazy_load, multi_urljoin
 from dagshub.data_engine.annotation import MetadataAnnotations
+from dagshub.data_engine.annotation.metadata import UnsupportedMetadataAnnotations
 from dagshub.data_engine.annotation.voxel_conversion import (
     add_ls_annotations,
     add_voxel_annotations,
@@ -425,8 +426,11 @@ def _get_blob_fn(dp: Datapoint, field: str, url: str, blob_path: Path):
             for dp in self:
                 for fld in document_fields:
                     if fld in dp.metadata:
-                        # Override the load_into_memory flag, because we need the contents
-                        if not load_into_memory:
+                        # Defensive check to not mangle annotation fields by accident
+                        if isinstance(dp.metadata[fld], MetadataAnnotations):
+                            continue
+                        # Force load the content into memory, even if load_into_memory was set to False
+                        if not load_into_memory or isinstance(dp.metadata[fld], Path):
                             dp.metadata[fld] = Path(dp.metadata[fld]).read_bytes()
                         dp.metadata[fld] = dp.metadata[fld].decode("utf-8")
 
@@ -435,38 +439,49 @@ def _get_blob_fn(dp: Datapoint, field: str, url: str, blob_path: Path):
     def _convert_annotation_fields(self, *fields, load_into_memory):
         # Convert any downloaded annotation column
         annotation_fields = [f for f in fields if f in self.annotation_fields]
+        if not annotation_fields:
+            return
 
+        # List of datapoints with annotations that couldn't be parsed
         bad_annotations = defaultdict(list)
 
-        if annotation_fields:
-            # Convert them
-            for dp in self:
-                for fld in annotation_fields:
-                    if fld in dp.metadata:
-                        # Already loaded - skip
-                        if isinstance(dp.metadata[fld], MetadataAnnotations):
-                            continue
-                        # Override the load_into_memory flag, because we need the contents
-                        if not load_into_memory:
-                            dp.metadata[fld] = Path(dp.metadata[fld]).read_bytes()
-                        try:
-                            dp.metadata[fld] = MetadataAnnotations.from_ls_task(
-                                datapoint=dp, field=fld, ls_task=dp.metadata[fld]
-                            )
-                        except ValidationError:
-                            bad_annotations[fld].append(dp.path)
-                    else:
-                        dp.metadata[fld] = MetadataAnnotations(datapoint=dp, field=fld)
+        for dp in self:
+            for fld in annotation_fields:
+                metadata_value = dp.metadata.get(fld)
+                # No value - create ampty annotation container
+                if metadata_value is None:
+                    dp.metadata[fld] = MetadataAnnotations(datapoint=dp, field=fld)
+                    continue
+                # Already loaded - skip
+                elif isinstance(metadata_value, MetadataAnnotations):
+                    continue
+                # Parse annotation from the content of the field
+                else:
+                    # Force load the content into memory, even if load_into_memory was set to False
+                    if not load_into_memory or isinstance(dp.metadata[fld], Path):
+                        metadata_value = Path(metadata_value).read_bytes()
+                    try:
+                        dp.metadata[fld] = MetadataAnnotations.from_ls_task(
+                            datapoint=dp, field=fld, ls_task=metadata_value
+                        )
+                    except ValidationError:
+                        dp.metadata[fld] = UnsupportedMetadataAnnotations(
+                            datapoint=dp, field=fld, original_value=metadata_value
+                        )
+                        bad_annotations[fld].append(dp.path)
 
         if bad_annotations:
             log_message(
-                "Warning: The following datapoints had invalid annotations, "
-                "any annotation-related operations will not work on these:"
+                "Warning: The following datapoints had unsupported or invalid annotations, "
+                "convenience functions like `add_bounding_box` won't work on these:"
             )
             err_msg = ""
             for fld, dps in bad_annotations.items():
-                err_msg += f'Field "{fld}" in datapoints:\n\t'
-                err_msg += "\n\t".join(dps)
+                err_msg += f'\nField "{fld}" in datapoints:\n\t'
+                if len(dps) > 10:
+                    err_msg += "\n\t".join(dps[:10]) + f"\n\t... and {len(dps) - 10} more"
+                else:
+                    err_msg += "\n\t".join(dps)
             log_message(err_msg)
 
     def download_binary_columns(

dagshub/data_engine/util/not_implemented.py

@@ -0,0 +1,48 @@
+class NotImplementedMeta(type):
+    """
+    A metaclass that replaces all parent class methods and properties that aren't overridden in the subclass
+    with NotImplementedError.
+    """
+
+    def __new__(mcs, name, bases, namespace):
+        # Get all attributes from base classes
+        for base in bases:
+            for attr_name in dir(base):
+                if attr_name.startswith("_"):
+                    continue
+
+                # Skip if already defined in subclass
+                if attr_name in namespace:
+                    continue
+
+                base_attr = getattr(base, attr_name)
+
+                # Handle properties
+                if isinstance(base_attr, property):
+                    # Create a property that raises NotImplementedError
+                    def make_not_implemented_property(prop_name):
+                        def getter(self):
+                            raise NotImplementedError(f"Property '{prop_name}' not implemented")
+
+                        def setter(self, value):
+                            raise NotImplementedError(f"Property '{prop_name}' not implemented")
+
+                        def deleter(self):
+                            raise NotImplementedError(f"Property '{prop_name}' not implemented")
+
+                        return property(getter, setter, deleter)
+
+                    namespace[attr_name] = make_not_implemented_property(attr_name)
+
+                # Handle regular methods
+                elif callable(base_attr):
+
+                    def make_not_implemented(method_name):
+                        def not_impl(self, *args, **kwargs):
+                            raise NotImplementedError(f"Method '{method_name}' not implemented")
+
+                        return not_impl
+
+                    namespace[attr_name] = make_not_implemented(attr_name)
+
+        return super().__new__(mcs, name, bases, namespace)

tests/data_engine/annotation_import/res/audio_annotation.json

@@ -0,0 +1,82 @@
+{
+  "id": 41,
+  "data": {
+    "audio": "https://example.com/some-non-existent-file.mp3",
+    "media type": "audio/mpeg",
+    "size": 111699
+  },
+  "meta": {
+    "datapoint_id": 12345678,
+    "datasource_id": 6565
+  },
+  "created_at": "2025-12-20T13:44:02.316027Z",
+  "updated_at": "2026-01-26T15:00:13.046967Z",
+  "is_labeled": true,
+  "project": 1,
+  "annotations": [
+    {
+      "completed_by": 1,
+      "result": [
+        {
+          "type": "choices",
+          "value": {
+            "choices": [
+              "true"
+            ]
+          },
+          "origin": "manual",
+          "to_name": "audio",
+          "from_name": "further_utterance",
+          "id": "deadbeef1"
+        },
+        {
+          "type": "rating",
+          "value": {
+            "rating": 1
+          },
+          "origin": "manual",
+          "to_name": "audio",
+          "from_name": "difficulty",
+          "id": "deadbeef1"
+        },
+        {
+          "type": "textarea",
+          "value": {
+            "text": [
+              "kirill@dagshub.com"
+            ]
+          },
+          "origin": "manual",
+          "to_name": "audio",
+          "from_name": "email_address",
+          "id": "deadbeef1"
+        },
+        {
+          "to_name": "audio",
+          "from_name": "first_name",
+          "id": "Qzu1dR2RQ8",
+          "type": "textarea",
+          "value": {
+            "text": [
+              "Kirill"
+            ]
+          },
+          "origin": "manual"
+        },
+        {
+          "type": "textarea",
+          "value": {
+            "text": [
+              "Bolashev"
+            ]
+          },
+          "origin": "manual",
+          "to_name": "audio",
+          "from_name": "last_name",
+          "id": "deadbeef1"
+        }
+      ],
+      "ground_truth": false
+    }
+  ]
+}

tests/data_engine/annotation_import/test_annotation_parsing.py

@@ -1,19 +1,23 @@
 import json
+from os import PathLike
 from pathlib import Path
+from typing import Union
 from unittest.mock import MagicMock
 
 import pytest
 from dagshub_annotation_converter.ir.image import IRSegmentationImageAnnotation
+from pytest import MonkeyPatch
 
 from dagshub.data_engine.annotation import MetadataAnnotations
+from dagshub.data_engine.annotation.metadata import UnsupportedMetadataAnnotations
 from dagshub.data_engine.dtypes import MetadataFieldType, ReservedTags
 from dagshub.data_engine.model import query_result
 from dagshub.data_engine.model.datasource import Datasource
+from dagshub.data_engine.model.query_result import QueryResult
 from tests.data_engine.util import add_metadata_field
 
 _annotation_field_name = "annotation"
 _dp_path = "data/sample_datapoint.jpg"
-_annotation_hash = "annotation1"  # Corresponds to a resource JSON
 _res_folder = Path(__file__).parent / "res"
 
 
@@ -51,17 +55,21 @@ def mock_annotation_query_result(
     return query_result.QueryResult.from_gql_query(data_dict, ds)
 
 
-def mock_get_blob(*args, **kwargs) -> bytes:
+def mock_get_blob(*args, **kwargs) -> Union[bytes, PathLike]:
     download_url: str = args[0]
     blob_hash = download_url.split("/")[-1]
+    load_into_memory = args[4]
     blob_path = _res_folder / f"{blob_hash}.json"
+
     if not blob_path.exists():
         raise FileNotFoundError(f"Mock blob file not found: {blob_path}")
-    return blob_path.read_bytes()
+    if load_into_memory:
+        return blob_path.read_bytes()
+    else:
+        return blob_path
 
 
-@pytest.fixture
-def ds_with_document_annotation(ds, monkeypatch):
+def _ds_with_annotation(ds: "Datasource", monkeypatch: MonkeyPatch, annotation_hash: str):
     add_metadata_field(
         ds,
         _annotation_field_name,
@@ -70,18 +78,57 @@ def ds_with_document_annotation(ds, monkeypatch):
     )
 
     ds.source.client.get_datapoints = MagicMock(
-        return_value=mock_annotation_query_result(ds, _annotation_field_name, _dp_path, _annotation_hash)
+        return_value=mock_annotation_query_result(ds, _annotation_field_name, _dp_path, annotation_hash)
     )
 
     monkeypatch.setattr(query_result, "_get_blob", mock_get_blob)
 
-    yield ds
+    return ds
+
+
+@pytest.fixture
+def ds_with_document_annotation(ds, monkeypatch):
+    yield _ds_with_annotation(ds, monkeypatch, "annotation1")
+
+
+@pytest.fixture
+def ds_with_unsupported_annotation(ds, monkeypatch):
+    yield _ds_with_annotation(ds, monkeypatch, "audio_annotation")
 
 
 def test_annotation_with_document_are_parsed_as_annotation(ds_with_document_annotation):
     qr = ds_with_document_annotation.all()
+    _test_annotation(qr)
+
+
+def test_double_loading_annotation_works(ds_with_document_annotation):
+    qr = ds_with_document_annotation.all()
+    qr.get_blob_fields(_annotation_field_name)
+    _test_annotation(qr)
+
+
+def _test_annotation(qr: QueryResult):
     annotation: MetadataAnnotations = qr[0].metadata[_annotation_field_name]
     assert isinstance(annotation, MetadataAnnotations)
     # Check that the annotation got parsed correctly, the JSON should have one segmentation annotation in it
     assert len(annotation.annotations) == 1
     assert isinstance(annotation.annotations[0], IRSegmentationImageAnnotation)
+
+
+def test_handling_unsupported_annotation(ds_with_unsupported_annotation):
+    qr = ds_with_unsupported_annotation.all()
+
+    annotation: MetadataAnnotations = qr[0].metadata[_annotation_field_name]
+
+    assert isinstance(annotation, UnsupportedMetadataAnnotations)
+    # Unsupported annotation is still a subclass of regular annotation
+    # This is crucial for logic that checks if annotation metadata was parsed already,
+    # so if this starts failing, that logic will need to be changed too
+    assert isinstance(annotation, MetadataAnnotations)
+
+    with pytest.raises(NotImplementedError):
+        annotation.add_image_bbox("cat", 0, 0, 10, 10, 1920, 1080)
+
+    expected_content = (_res_folder / "audio_annotation.json").read_bytes()
+    assert annotation.value == expected_content
+    assert annotation.to_ls_task() == expected_content