fix(zarr-metadata)!: make entity types assignable to metadata fields; validators return tuples

packages/zarr-metadata/changes/317.bugfix.md

@@ -0,0 +1,25 @@
+Concrete v3 entity types are now assignable to the fields they describe.
+Previously, none of the package's canonical codec / chunk-grid /
+chunk-key-encoding / data-type types (e.g. `BloscCodecMetadata`,
+`RegularChunkGridMetadata`) satisfied `ZarrV3MetadataFieldJSON`, so a
+type checker rejected putting them into the very fields they document
+(`codecs`, `chunk_grid`, `data_type`, ...). Three changes fix this:
+
+- `ZarrV3NamedConfigJSON.name` and `.configuration` are now `ReadOnly`
+  (PEP 705), making them covariant so concrete `name: Literal[...]` and
+  required-`configuration` shapes are accepted.
+- `ZarrV3NamedConfigJSON` is now `closed` (PEP 728): the spec's
+  named-configuration envelope has exactly `name` / `configuration` /
+  `must_understand`, and closing the type also makes it usable as a
+  `JSONValue` (needed for e.g. the `sharding_indexed` inner `codecs`).
+- Every concrete `*Object` / `*Configuration` TypedDict is now `closed`,
+  and object forms declare `must_understand: NotRequired[bool]` (any v3
+  metadata field may carry the extension member).
+
+**Soft-breaking** for type-checking consumers: dicts with keys beyond the
+declared shape no longer satisfy the closed types, and `name` /
+`configuration` can no longer be mutated through `ZarrV3NamedConfigJSON`.
+Both were previously accepted by type checkers but produced documents
+outside the spec's shapes. `zarr_metadata.pydantic` serializers now
+declare their return schema via the pydantic-facing shadow types, so
+pydantic schema generation stays warning-free.

packages/zarr-metadata/changes/317.misc.1.md

@@ -0,0 +1,5 @@
+`ZarrV2ConsolidatedMetadataJSON.zarr_consolidated_format` is typed
+`Literal[1]` rather than `int`. Format 1 is the only defined `.zmetadata`
+format and the runtime validator already rejected anything else, so the
+type now carries the constraint the validator enforces instead of
+contradicting it.

packages/zarr-metadata/changes/317.misc.md

@@ -0,0 +1,6 @@
+**Breaking:** every `validate_*` function in `zarr_metadata.model` now
+returns `tuple[ValidationProblem, ...]` instead of `list[ValidationProblem]`,
+and `MetadataValidationError.problems` is a tuple. Iteration and indexing are
+unchanged; callers that mutate reports must first copy them with
+`list(problems)`. `MetadataValidationError` still accepts any problem
+sequence.

packages/zarr-metadata/pyproject.toml

@@ -95,6 +95,14 @@ include = [
 extend = "../../pyproject.toml"
 target-version = "py311"
 
+[tool.ruff.lint]
+# `Any` defeats the point of a package whose product is precise types, so it
+# is banned in annotations here rather than merely discouraged. Use `object`
+# for "any value" (the caller must narrow) and the document TypedDicts where
+# the shape is known; a genuinely dynamic annotation needs an explicit noqa
+# saying why.
+extend-select = ["ANN401"]
+
 [tool.pytest.ini_options]
 minversion = "7"
 testpaths = ["tests"]

packages/zarr-metadata/src/zarr_metadata/_common.py

@@ -6,45 +6,59 @@
 `zarr_metadata.v3.data_type`.
 """
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from typing import NotRequired
 
-from typing_extensions import TypeAliasType, TypedDict
+from typing_extensions import ReadOnly, TypeAliasType, TypedDict
 
 JSONValue = TypeAliasType(
     "JSONValue",
-    int | float | bool | str | Sequence["JSONValue"] | Mapping[str, "JSONValue"] | None,
+    int
+    | float
+    | bool
+    | str
+    | list["JSONValue"]
+    | tuple["JSONValue", ...]
+    | Mapping[str, "JSONValue"]
+    | None,
 )
 """A recursive type alias for JSON-encodable values.
 
 Defined via `TypeAliasType` (rather than a plain `TypeAlias`) so the
 self-reference is a named recursion point that pydantic can resolve when
 building a `TypeAdapter`; a bare recursive `TypeAlias` raises
 `PydanticUserError`/`RecursionError` at validation time.
-
-The array arm is the covariant `Sequence` rather than the invariant
-`list["JSONValue"] | tuple["JSONValue", ...]`, so values typed with a
-*narrower* element type still count as JSON values: a `list[str]` field on a
-TypedDict is assignable to `JSONValue` under `Sequence` but not under
-`list[JSONValue]` (`list` is invariant in its element type, and pyright's
-diagnostic for that failure suggests exactly this change). This is what lets
-downstream TypedDicts give their fields precise types (`Sequence[str]`,
-`list[int]`, ...) while remaining assignable to `Mapping[str, JSONValue]`.
-The type-level cost, accepted deliberately: `Sequence` says nothing about the
-concrete container, and it admits `str`/`bytes` (`str` was already a union
-arm); runtime code narrowing a JSON array must exclude `str`/`bytes`/
-`bytearray` regardless of how this alias is spelled.
 """
 
 
-class ZarrV3NamedConfigJSON(TypedDict):
+class ZarrV3NamedConfigJSON(TypedDict, closed=True):
     """
     Externally-tagged union member for a metadata field.
 
     The optional `configuration` mapping holds arbitrary JSON-encodable
     values. `must_understand` is implicitly true when absent.
+
+    `name` and `configuration` are `ReadOnly` (PEP 705) so that concrete
+    entity types — `BloscCodecObject`, `RegularChunkGridObject`, and the
+    rest — are assignable to this type, and therefore to
+    `ZarrV3MetadataFieldJSON`. Without `ReadOnly` both items are invariant,
+    so a concrete `name: Literal["blosc"]` does not satisfy `name: str`, and
+    a required `configuration` does not satisfy a `NotRequired` one. That
+    made the package's own codec types unusable in the very fields they
+    describe (`codecs`, `data_type`, `chunk_grid`, ...), and made
+    `TypeIs`-based codec classification impossible to declare, since `TypeIs`
+    requires the narrowed type to be assignable to the input type.
+
+    `must_understand` stays writable: nothing needs to narrow it, and
+    keeping it mutable lets writers set it on an already-constructed field.
+
+    The type is `closed` (PEP 728): the spec's named-configuration envelope
+    has exactly these three members, and closing it is also what makes this
+    type — and every concrete entity type embedding it, e.g. the
+    `sharding_indexed` configuration's inner `codecs` list — assignable to
+    `Mapping[str, JSONValue]` (i.e. usable as a `JSONValue`).
     """
 
-    name: str
-    configuration: NotRequired[Mapping[str, JSONValue]]
+    name: ReadOnly[str]
+    configuration: NotRequired[ReadOnly[Mapping[str, JSONValue]]]
     must_understand: NotRequired[bool]

packages/zarr-metadata/src/zarr_metadata/model/_array.py

@@ -61,10 +61,14 @@ class ZarrV3NamedConfig:
     def to_json(self) -> ZarrV3MetadataFieldJSON:
         if not self.configuration and self.must_understand:
             return self.name
-        out: ZarrV3NamedConfigJSON = {"name": self.name}
-        if self.configuration:
-            # to_json output shares no mutable state with the model.
-            out["configuration"] = copy.deepcopy(self.configuration)
+        # `configuration` is ReadOnly, so it is set in the literal rather than
+        # assigned afterwards. to_json output shares no mutable state with the
+        # model.
+        out: ZarrV3NamedConfigJSON = (
+            {"name": self.name, "configuration": copy.deepcopy(self.configuration)}
+            if self.configuration
+            else {"name": self.name}
+        )
         if not self.must_understand:
             out["must_understand"] = False
         return out
@@ -464,7 +468,7 @@ def from_json(cls, data: object) -> ZarrV2ArrayMetadata:
 
     @classmethod
     def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata:
-        zarray_raw = cast("object", load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY))
+        zarray_raw = load_store_json(mapping, ZARR_V2_ARRAY_METADATA_STORE_KEY)
         if not isinstance(zarray_raw, Mapping):
             return cls.from_json(zarray_raw)
         zarray = cast("Mapping[str, object]", zarray_raw)
@@ -479,7 +483,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2ArrayMetadata:
                 ]
             )
         if ZARR_V2_ATTRIBUTES_STORE_KEY in mapping:
-            zattrs = cast("object", load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY))
+            zattrs = load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY)
             return cls.from_json({**zarray, "attributes": zattrs})
         return cls.from_json(zarray)
 

packages/zarr-metadata/src/zarr_metadata/model/_group.py

@@ -226,7 +226,7 @@ def to_json(self) -> ZarrV3ConsolidatedMetadataJSON:
     def from_json(cls, data: object) -> ZarrV3ConsolidatedMetadata:
         normalized = arrays_to_tuples(data)
         problems = validate_consolidated_metadata_v3(normalized)
-        if problems:
+        if len(problems) != 0:
             raise MetadataValidationError(problems)
         env = cast("Mapping[str, object]", normalized)
         entries: dict[str, ZarrV3ArrayMetadata | ZarrV3GroupMetadata] = {}
@@ -314,7 +314,7 @@ def from_json(cls, data: object) -> ZarrV2GroupMetadata:
 
     @classmethod
     def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata:
-        zgroup_raw = cast("object", load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY))
+        zgroup_raw = load_store_json(mapping, ZARR_V2_GROUP_METADATA_STORE_KEY)
         if not isinstance(zgroup_raw, Mapping):
             return cls.from_json(zgroup_raw)
         zgroup = cast("Mapping[str, object]", zgroup_raw)
@@ -329,7 +329,7 @@ def from_key_value(cls, mapping: Mapping[str, bytes]) -> ZarrV2GroupMetadata:
                 ]
             )
         if ZARR_V2_ATTRIBUTES_STORE_KEY in mapping:
-            zattrs = cast("object", load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY))
+            zattrs = load_store_json(mapping, ZARR_V2_ATTRIBUTES_STORE_KEY)
             return cls.from_json({**zgroup, "attributes": zattrs})
         return cls.from_json(zgroup)
 
@@ -416,7 +416,7 @@ def from_json(cls, data: object) -> ZarrV2ConsolidatedMetadata:
                         )
                         for problem in validate_json(value)
                     )
-        if problems:
+        if len(problems) != 0:
             raise MetadataValidationError(problems)
         entries_tupled = cast(
             "dict[str, JSONValue]",