diff --git a/packages/zarr-metadata/changes/319.feature.1.md b/packages/zarr-metadata/changes/319.feature.1.md new file mode 100644 index 0000000000..92aaaea379 --- /dev/null +++ b/packages/zarr-metadata/changes/319.feature.1.md @@ -0,0 +1,18 @@ +Added `zarr_metadata.v3._extension_points`: a table of the v3 extension +points — `data_type`, `chunk_grid`, `chunk_key_encoding`, `codecs`, +`storage_transformers` — recording, per identifier, where it was +standardized and where its definition lives, and per point, whether +`must_understand: false` is permitted there and whether the field holds +one entity or a sequence. + +Keyed by `(field, name)`, because names are unique only *within* an +extension point: `bytes` is both a core codec and a registered extension +data type. + +Provenance is `CORE`, `REGISTERED`, or `PROPOSED`; `zstd` is marked +proposed because its cited specification is still an open pull request. + +`canonical_name(field, name)` is identity except for raw-byte data types: +all `r` spellings, including invalid ones such as `r12`, map to +`RAW_BYTES_FAMILY`. This lets validation report malformed family members +instead of treating them as unknown extensions. diff --git a/packages/zarr-metadata/changes/319.feature.2.md b/packages/zarr-metadata/changes/319.feature.2.md new file mode 100644 index 0000000000..8b9e3bf2fb --- /dev/null +++ b/packages/zarr-metadata/changes/319.feature.2.md @@ -0,0 +1,8 @@ +Added `zarr_metadata.v3.codec.kind`: codec kind classification. One +branded union per spec pipeline kind over the concrete codec types +(`ArrayArrayCodecMetadata`, `ArrayBytesCodecMetadata`, +`BytesBytesCodecMetadata`, plus `KnownCodecMetadata` and the paired +`*_CODEC_NAMES` constants). Shape-exact `TypeIs` guards narrow canonical +codec metadata to those unions. `codec_kind_of_name` classifies known +names without validating object shape and returns `None` for unknown +names. All names are re-exported from `zarr_metadata.v3.codec`. diff --git a/packages/zarr-metadata/changes/319.feature.4.md b/packages/zarr-metadata/changes/319.feature.4.md new file mode 100644 index 0000000000..199f1bbc4a --- /dev/null +++ b/packages/zarr-metadata/changes/319.feature.4.md @@ -0,0 +1,16 @@ +Added `zarr_metadata.builder`: incremental, validated construction of v3 +array metadata documents over the plain JSON TypedDict shapes. +`ZarrV3ArrayMetadataBuilder` accumulates a +`ZarrV3ArrayMetadataJSONPartial` and returns updated copies. +`with_fields(**kwargs)` types standard fields; PEP 728 checkers also accept +extension fields, while `with_extension` works across checkers. +`without_fields` removes keys, properties return `T | UNSET`, `build()` returns +a complete validated document, and `to_partial_json()` returns the +current fragment. Inputs are copied and JSON arrays normalize to tuples. + +Composition rules fire after each change once their dependencies are +present. They check fill values, known entity shapes, codec pipelines, +dimension counts, chunk grids, transpose codecs, and sharding. Unknown +entity names are left unjudged; known names must use their canonical +shape. A failing update raises one `MetadataValidationError` containing +all problems found. diff --git a/packages/zarr-metadata/docs/api/builder.md b/packages/zarr-metadata/docs/api/builder.md index 50bf9ac278..77cdb3a291 100644 --- a/packages/zarr-metadata/docs/api/builder.md +++ b/packages/zarr-metadata/docs/api/builder.md @@ -3,3 +3,8 @@ title: builder --- ::: zarr_metadata.builder + +See `examples/build_v3_array.py` in the package for incremental +construction, rejected invalid updates, JSON round-tripping, and +consolidated metadata. It ships in the sdist and is executed by the test +suite, so it cannot drift from the API it demonstrates. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 1a73ec3842..9b5ce91c21 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -13,7 +13,8 @@ The package is organized to mirror the structure of the Zarr specifications: ordering, chunk geometry), plus whole-document `validate`/`is`/`parse` trios combining structure and composition - [`zarr_metadata.builder`](builder.md) — validated construction: - one-shot `create_*` factories, one per document type + one-shot `create_*` factories per document type and an incremental + builder for v3 arrays - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents diff --git a/packages/zarr-metadata/examples/build_v3_array.py b/packages/zarr-metadata/examples/build_v3_array.py new file mode 100644 index 0000000000..955ff8b969 --- /dev/null +++ b/packages/zarr-metadata/examples/build_v3_array.py @@ -0,0 +1,92 @@ +"""Construct, serialize, and validate a Zarr v3 metadata hierarchy. + +Run from ``packages/zarr-metadata`` with:: + + uv run python examples/build_v3_array.py + +The example uses only public APIs. Assertions make it useful as an executable +smoke test as well as a starting point for applications. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from zarr_metadata.builder import ( + ZarrV3ArrayMetadataBuilder, + create_zarr_v3_consolidated_metadata_json, + create_zarr_v3_group_metadata_json, +) +from zarr_metadata.model import MetadataValidationError +from zarr_metadata.rules import Valid, check_group_metadata_v3 + +if TYPE_CHECKING: + from collections.abc import Callable + + +def expect_rejected(label: str, operation: Callable[[], object]) -> None: + """Run one deliberately invalid operation and show why it was rejected.""" + try: + operation() + except MetadataValidationError as error: + print(f"Rejected {label}: {error.problems[0]}") + else: # pragma: no cover - this script is also an executable assertion + raise AssertionError(f"Expected {label} to be rejected") + + +def main() -> None: + # Composition errors are rejected as soon as all fields involved are known. + expect_rejected( + "an out-of-range uint8 fill value", + lambda: ZarrV3ArrayMetadataBuilder().with_fields(data_type="uint8", fill_value=300), + ) + + # Known extension names are checked against their canonical configuration. + expect_rejected( + "an invalid chunk-key separator", + lambda: ZarrV3ArrayMetadataBuilder().with_fields( + chunk_key_encoding={"name": "default", "configuration": {"separator": "!"}} + ), + ) + + array = ( + ZarrV3ArrayMetadataBuilder() + .with_fields(zarr_format=3, node_type="array", shape=(100, 200)) + .with_fields(data_type="uint16", fill_value=0) + .with_fields( + chunk_grid={"name": "regular", "configuration": {"chunk_shape": (10, 20)}}, + chunk_key_encoding={"name": "default", "configuration": {"separator": "/"}}, + codecs=( + {"name": "transpose", "configuration": {"order": (1, 0)}}, + {"name": "bytes", "configuration": {"endian": "little"}}, + "crc32c", + ), + dimension_names=("y", "x"), + attributes={"units": "counts"}, + ) + .build() + ) + + # JSON round-tripping changes tuples into lists; the read-side API normalizes + # them back before returning a typed, composition-valid document. + loaded_array = json.loads(json.dumps(array)) + consolidated = create_zarr_v3_consolidated_metadata_json( + kind="inline", must_understand=False, metadata={"measurements": loaded_array} + ) + group = create_zarr_v3_group_metadata_json( + zarr_format=3, + node_type="group", + attributes={"title": "Example hierarchy"}, + extensions={"consolidated_metadata": consolidated}, + ) + + checked = check_group_metadata_v3(json.loads(json.dumps(group))) + assert isinstance(checked, Valid) + assert checked.document == group + + print(json.dumps(checked.document, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/packages/zarr-metadata/justfile b/packages/zarr-metadata/justfile index 0f1861ed7d..6bcb9d2043 100644 --- a/packages/zarr-metadata/justfile +++ b/packages/zarr-metadata/justfile @@ -21,7 +21,7 @@ pyright_version := "1.1.404" # stdlib it cannot parse, so pin the interpreter to match CI. # Type-check the package sources typecheck: - uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright src + uv run --python 3.11 --group test --with 'pyright=={{ pyright_version }}' pyright src examples # Run everything CI runs for this package check: lint typecheck test docs-check diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index 081f2b3c48..11cd33af5d 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -74,8 +74,9 @@ packages = ["src/zarr_metadata"] # An allowlist, so nothing that merely happens to sit in the package directory # — a scratch script, a stray notebook — can ride along in a release. The list # keeps an sdist self-testing and self-documenting: every fixture this suite -# reads is a JSON file sitting next to the test module that loads it, so -# `/tests` is the whole test dependency, and `/docs` plus `/mkdocs.yml` are a +# reads is a JSON file sitting next to the test module that loads it; the +# one exception is `/examples`, which `tests/test_examples.py` executes, so +# both ship, and `/docs` plus `/mkdocs.yml` are a # self-contained site (mkdocstrings reads `src`, nothing reaches outside the # package) so `just docs-check` runs from an unpacked sdist too. `changes/` # and `.readthedocs.yaml` are deliberately absent: towncrier fragments are @@ -85,6 +86,7 @@ packages = ["src/zarr_metadata"] include = [ "/src", "/tests", + "/examples", "/docs", "/mkdocs.yml", "/justfile", @@ -129,7 +131,7 @@ checks = [ # class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel # relies on. Use the same pin locally; unpin when the fix lands. [tool.pyright] -include = ["src"] +include = ["src", "examples"] enableExperimentalFeatures = true typeCheckingMode = "strict" pythonVersion = "3.11" diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py b/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py index 10dcb0dfe9..1b38b08ba1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/builder/__init__.py @@ -5,10 +5,14 @@ at literal-keyword call sites, and the runtime pass normalizes the input and applies structural and composition validation, raising one `MetadataValidationError` carrying every problem. +`ZarrV3ArrayMetadataBuilder` supports incremental v3 array construction, +running applicable composition rules after each update and checking +completeness at `build()`. Use `zarr_metadata.rules` to validate documents read from storage. """ +from zarr_metadata.builder._array_v3 import ZarrV3ArrayMetadataBuilder from zarr_metadata.builder._create import ( create_zarr_v2_array_metadata_json, create_zarr_v2_consolidated_metadata_json, @@ -21,6 +25,7 @@ ) __all__ = [ + "ZarrV3ArrayMetadataBuilder", "create_zarr_v2_array_metadata_json", "create_zarr_v2_consolidated_metadata_json", "create_zarr_v2_group_metadata_json", diff --git a/packages/zarr-metadata/src/zarr_metadata/builder/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/builder/_array_v3.py new file mode 100644 index 0000000000..6a32162a2a --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/builder/_array_v3.py @@ -0,0 +1,251 @@ +"""Incremental, validated construction of v3 array metadata documents.""" + +from __future__ import annotations + +import copy +from typing import TYPE_CHECKING, Literal, Self, cast + +from typing_extensions import Unpack + +from zarr_metadata.model._sentinel import UNSET +from zarr_metadata.model._validation import ( + ARRAY_METADATA_STANDARD_KEYS_V3, + MetadataValidationError, + ValidationProblem, + arrays_to_tuples, + parse_array_metadata_v3, +) +from zarr_metadata.rules import ZARR_V3_ARRAY_RULES, applicable, run_rules + +if TYPE_CHECKING: + from collections.abc import Mapping + from collections.abc import Set as AbstractSet + + from zarr_metadata._common import JSONValue + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + from zarr_metadata.v3.array import ( + ZarrV3ArrayMetadataJSON, + ZarrV3ArrayMetadataJSONPartial, + ZarrV3ExtensionField, + ) + + +def _normalized(inner: ZarrV3ArrayMetadataJSONPartial) -> ZarrV3ArrayMetadataJSONPartial: + """Copy `inner` and convert JSON arrays to tuples.""" + return cast("ZarrV3ArrayMetadataJSONPartial", arrays_to_tuples(copy.deepcopy(inner))) + + +class ZarrV3ArrayMetadataBuilder: + """Immutable accumulator for a v3 array metadata document. + + Holds a possibly incomplete `ZarrV3ArrayMetadataJSONPartial` and + returns updated copies: + + doc = ( + ZarrV3ArrayMetadataBuilder() + .with_fields(zarr_format=3, node_type="array") + .with_fields(shape=(4, 4), data_type="uint8", fill_value=0) + .with_fields( + chunk_grid={"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + chunk_key_encoding="default", + codecs=("bytes",), + ) + .build() + ) + + `with_fields` types standard fields; PEP 728 checkers also accept extension + fields. `with_extension` works across checkers and rejects standard + names. `without_fields` removes keys. Properties return `UNSET` for absent + fields, distinct from JSON `null`. + + Applicable composition rules run after every change. `build` adds + structural validation and returns a complete document. Inputs and + outputs are copied, and JSON arrays normalize to tuples. + """ + + __slots__ = ("_inner",) + _inner: ZarrV3ArrayMetadataJSONPartial + + def __init__(self, inner: ZarrV3ArrayMetadataJSONPartial | None = None) -> None: + self._inner = _normalized(inner) if inner is not None else {} + self._check(changed=frozenset(self._inner.keys())) + + # -- updates ------------------------------------------------------------ + + def with_fields(self, **kwargs: Unpack[ZarrV3ArrayMetadataJSONPartial]) -> Self: + """A new builder with the given fields replaced. + + Each given field fully replaces its previous value. Raises + `MetadataValidationError` if the merged state violates any + dependency-complete semantic rule. + """ + return self._evolved(cast("ZarrV3ArrayMetadataJSONPartial", dict(kwargs))) + + def with_extension(self, name: str, value: ZarrV3ExtensionField) -> Self: + """A new builder with extension field `name` set to `value`. + + Extension fields are the document keys outside the standard v3 + array metadata keys; `name` must not collide with a standard key. + This method supports extension names on checkers without PEP 728. + """ + if name in ARRAY_METADATA_STANDARD_KEYS_V3: + raise MetadataValidationError( + [ + ValidationProblem( + (name,), + f"{name!r} is a standard v3 array metadata key; set it via with_fields()", + "invalid_value", + ) + ] + ) + return self._evolved(cast("ZarrV3ArrayMetadataJSONPartial", {name: value})) + + def without_fields(self, *keys: str) -> Self: + """A new builder with the given keys absent. + + Removing an already-absent key is a no-op. This is the only way to + unset a field: `with_fields` never stores an "unset" sentinel, so a key + is UNSET exactly when it is not in the document. + """ + inner = cast( + "ZarrV3ArrayMetadataJSONPartial", + {key: value for key, value in self._inner.items() if key not in keys}, + ) + return type(self)(inner) + + def _evolved(self, changes: ZarrV3ArrayMetadataJSONPartial) -> Self: + new = type(self).__new__(type(self)) + new._inner = _normalized(cast("ZarrV3ArrayMetadataJSONPartial", {**self._inner, **changes})) + new._check(changed=frozenset(changes.keys())) + return new + + def _check(self, changed: AbstractSet[str]) -> None: + """Run every dependency-complete rule; raise with all problems found. + + `changed` names the keys set by the triggering call, used to + attribute a conflict between a just-set field and one set earlier. + """ + problems: list[ValidationProblem] = [] + for rule in applicable(ZARR_V3_ARRAY_RULES, self._inner.keys()): + found = rule.check(self._inner) + if len(found) == 0: + continue + earlier = rule.requires - changed + just_set = rule.requires & changed + if len(earlier) != 0 and len(just_set) != 0: + hint = ( + f" [{', '.join(sorted(just_set))} set in this call conflicts with " + f"{', '.join(sorted(earlier))} set earlier; with_fields() can change " + "both at once]" + ) + found = tuple( + ValidationProblem(problem.loc, problem.message + hint, problem.kind) + for problem in found + ) + problems.extend(found) + if len(problems) != 0: + raise MetadataValidationError(problems) + + # -- output ------------------------------------------------------------- + + def build(self) -> ZarrV3ArrayMetadataJSON: + """The complete, validated metadata document. + + Validates structurally (required keys derived from the TypedDict, + field shapes) via the model layer's parser, then semantically via + the full rule set, and raises `MetadataValidationError` carrying + every problem from both passes. The returned dict shares no + mutable state with the builder. + """ + structural: tuple[ValidationProblem, ...] = () + parsed: ZarrV3ArrayMetadataJSON | None = None + try: + parsed = parse_array_metadata_v3(copy.deepcopy(dict(self._inner))) + except MetadataValidationError as error: + structural = error.problems + semantic = run_rules(ZARR_V3_ARRAY_RULES, self._inner) + if len(structural) != 0 or len(semantic) != 0: + raise MetadataValidationError(structural + semantic) + assert parsed is not None + return parsed + + def to_partial_json(self) -> dict[str, JSONValue]: + """The accumulated document fragment, under an honest partial type. + + Unlike `build`, this always succeeds. The name says "partial" + because the output is not necessarily a valid metadata document — + do not persist it to a store as one. Key absence is preserved + exactly: a key missing from the builder is missing here, and a + stored `None` (JSON `null`) is emitted as `null`, at any depth. + """ + return cast("dict[str, JSONValue]", copy.deepcopy(dict(self._inner))) + + # -- introspection ------------------------------------------------------ + + @property + def zarr_format(self) -> Literal[3] | UNSET: + return copy.deepcopy(self._inner.get("zarr_format", UNSET)) + + @property + def node_type(self) -> Literal["array"] | UNSET: + return copy.deepcopy(self._inner.get("node_type", UNSET)) + + @property + def shape(self) -> tuple[int, ...] | UNSET: + return copy.deepcopy(self._inner.get("shape", UNSET)) + + @property + def data_type(self) -> ZarrV3MetadataFieldJSON | UNSET: + return copy.deepcopy(self._inner.get("data_type", UNSET)) + + @property + def chunk_grid(self) -> ZarrV3MetadataFieldJSON | UNSET: + return copy.deepcopy(self._inner.get("chunk_grid", UNSET)) + + @property + def chunk_key_encoding(self) -> ZarrV3MetadataFieldJSON | UNSET: + return copy.deepcopy(self._inner.get("chunk_key_encoding", UNSET)) + + @property + def fill_value(self) -> JSONValue | UNSET: + return copy.deepcopy(self._inner.get("fill_value", UNSET)) + + @property + def codecs(self) -> tuple[ZarrV3MetadataFieldJSON, ...] | UNSET: + return copy.deepcopy(self._inner.get("codecs", UNSET)) + + @property + def attributes(self) -> Mapping[str, JSONValue] | UNSET: + return copy.deepcopy(self._inner.get("attributes", UNSET)) + + @property + def storage_transformers(self) -> tuple[ZarrV3MetadataFieldJSON, ...] | UNSET: + return copy.deepcopy(self._inner.get("storage_transformers", UNSET)) + + @property + def dimension_names(self) -> tuple[str | None, ...] | UNSET: + return copy.deepcopy(self._inner.get("dimension_names", UNSET)) + + @property + def extension_fields(self) -> dict[str, ZarrV3ExtensionField]: + """The accumulated extension fields (keys outside the standard set).""" + return { + key: copy.deepcopy(cast("ZarrV3ExtensionField", value)) + for key, value in self._inner.items() + if key not in ARRAY_METADATA_STANDARD_KEYS_V3 + } + + # -- value semantics ---------------------------------------------------- + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ZarrV3ArrayMetadataBuilder): + return NotImplemented + return self._inner == other._inner + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.to_partial_json()!r})" + + +__all__ = [ + "ZarrV3ArrayMetadataBuilder", +] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py index 03a8472a2b..0c7e524803 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_extension_points.py @@ -1,20 +1,58 @@ -"""The Zarr v3 extension points, and how names are keyed under them. +"""Known identifiers and policies for Zarr v3 extension points. -Names are unique only within an extension point (`bytes` is both a core -codec and a registered data type), so every table in this package is -keyed by `(field, canonical name)`. +Entries are keyed by `(field, name)` because names may occur at multiple +extension points. Each entry records provenance and a specification URL; +each point records cardinality and `must_understand: false` policy. -`canonical_name` is identity except for raw-byte data types: every `r` -spelling, valid or not, maps to `RAW_BYTES_FAMILY`, so a malformed member -of that family is reported as a misspelling rather than passing as an -unknown extension. Canonical names are lookup keys and are never emitted. +`canonical_name` is identity except for raw-byte data types: every +`r` spelling, valid or not, maps to `RAW_BYTES_FAMILY`. Canonical names +are lookup keys and are never emitted. Registry-allocated names are +authoritative; private entities that reuse them are judged as the +registered entity. """ from __future__ import annotations -from typing import Final, Literal +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Final, Literal +from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID_NAME +from zarr_metadata.v3.chunk_grid.regular import REGULAR_CHUNK_GRID_NAME +from zarr_metadata.v3.chunk_key_encoding.default import DEFAULT_CHUNK_KEY_ENCODING_NAME +from zarr_metadata.v3.chunk_key_encoding.v2 import V2_CHUNK_KEY_ENCODING_NAME +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME +from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME +from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME +from zarr_metadata.v3.data_type.bool import BOOL_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.bytes import BYTES_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex64 import COMPLEX64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.complex128 import COMPLEX128_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float16 import FLOAT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float32 import FLOAT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.float64 import FLOAT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int8 import INT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int16 import INT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int32 import INT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.int64 import INT64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_datetime64 import NUMPY_DATETIME64_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.numpy_timedelta64 import NUMPY_TIMEDELTA64_DATA_TYPE_NAME from zarr_metadata.v3.data_type.raw import RAW_BYTES_NAME_PATTERN +from zarr_metadata.v3.data_type.string import STRING_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.struct import STRUCT_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint8 import UINT8_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint16 import UINT16_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint32 import UINT32_DATA_TYPE_NAME +from zarr_metadata.v3.data_type.uint64 import UINT64_DATA_TYPE_NAME + +if TYPE_CHECKING: + from collections.abc import Mapping ExtensionPointField = Literal[ "data_type", "chunk_grid", "chunk_key_encoding", "codecs", "storage_transformers" @@ -30,25 +68,258 @@ RAW_BYTES_FAMILY: Final = "r" """Canonical key for the parameterized raw-bytes data type family. -Spelled as the spec writes the family; the angle brackets keep it -unforgeable by a real name. +Spelled as the spec writes the family so that a table dump reads as +documentation. The angle brackets keep it unforgeable by a real name. """ +class Provenance(Enum): + """Where an identifier was standardized, and how settled that is.""" + + CORE = "core" + """Defined normatively by the Zarr v3 core specification.""" + + REGISTERED = "registered" + """Defined by a registered extension in the zarr-extensions repository.""" + + PROPOSED = "proposed" + """Defined only by an open proposal; the shape may still change.""" + + +@dataclass(frozen=True, slots=True) +class ExtensionIdentifier: + """One name an extension point accepts, and where it comes from.""" + + name: str + """The canonical name (see the module docstring on canonicalization).""" + + provenance: Provenance + reference: str + """URL of the definition this package models.""" + + +@dataclass(frozen=True, slots=True) +class ExtensionPoint: + """One v3 extension point and the identifiers this package models for it.""" + + field: ExtensionPointField + identifiers: Mapping[str, ExtensionIdentifier] + must_understand_false_permitted: bool + """Whether the spec allows `must_understand: false` at this point. + + False for `data_type`, `chunk_grid`, and `chunk_key_encoding`: an + implementation that does not recognize one of those cannot proceed + by ignoring it, so opting out of understanding is meaningless there. + """ + + holds_sequence: bool + """Whether the field holds a list of entities rather than a single one.""" + + +def _identifiers(*entries: ExtensionIdentifier) -> Mapping[str, ExtensionIdentifier]: + return {entry.name: entry for entry in entries} + + +_CORE_DATA_TYPES = "https://zarr-specs.readthedocs.io/en/latest/v3/data-types/index.html" +_CORE_SPEC = "https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html" +_EXTENSIONS = "https://github.com/zarr-developers/zarr-extensions/tree/main" + + +def _core_dtype(name: str) -> ExtensionIdentifier: + return ExtensionIdentifier(name, Provenance.CORE, _CORE_DATA_TYPES) + + +EXTENSION_POINTS: Final[Mapping[ExtensionPointField, ExtensionPoint]] = { + DATA_TYPE: ExtensionPoint( + field=DATA_TYPE, + identifiers=_identifiers( + _core_dtype(BOOL_DATA_TYPE_NAME), + _core_dtype(INT8_DATA_TYPE_NAME), + _core_dtype(INT16_DATA_TYPE_NAME), + _core_dtype(INT32_DATA_TYPE_NAME), + _core_dtype(INT64_DATA_TYPE_NAME), + _core_dtype(UINT8_DATA_TYPE_NAME), + _core_dtype(UINT16_DATA_TYPE_NAME), + _core_dtype(UINT32_DATA_TYPE_NAME), + _core_dtype(UINT64_DATA_TYPE_NAME), + _core_dtype(FLOAT16_DATA_TYPE_NAME), + _core_dtype(FLOAT32_DATA_TYPE_NAME), + _core_dtype(FLOAT64_DATA_TYPE_NAME), + _core_dtype(COMPLEX64_DATA_TYPE_NAME), + _core_dtype(COMPLEX128_DATA_TYPE_NAME), + ExtensionIdentifier(RAW_BYTES_FAMILY, Provenance.CORE, _CORE_SPEC), + ExtensionIdentifier( + BYTES_DATA_TYPE_NAME, Provenance.REGISTERED, f"{_EXTENSIONS}/data-types/bytes" + ), + ExtensionIdentifier( + STRING_DATA_TYPE_NAME, Provenance.REGISTERED, f"{_EXTENSIONS}/data-types/string" + ), + ExtensionIdentifier( + NUMPY_DATETIME64_DATA_TYPE_NAME, + Provenance.REGISTERED, + f"{_EXTENSIONS}/data-types/numpy.datetime64", + ), + ExtensionIdentifier( + NUMPY_TIMEDELTA64_DATA_TYPE_NAME, + Provenance.REGISTERED, + f"{_EXTENSIONS}/data-types/numpy.timedelta64", + ), + ExtensionIdentifier( + STRUCT_DATA_TYPE_NAME, Provenance.REGISTERED, f"{_EXTENSIONS}/data-types/struct" + ), + ), + must_understand_false_permitted=False, + holds_sequence=False, + ), + CHUNK_GRID: ExtensionPoint( + field=CHUNK_GRID, + identifiers=_identifiers( + ExtensionIdentifier( + REGULAR_CHUNK_GRID_NAME, Provenance.CORE, f"{_CORE_SPEC}#regular-grids" + ), + ExtensionIdentifier( + RECTILINEAR_CHUNK_GRID_NAME, + Provenance.REGISTERED, + f"{_EXTENSIONS}/chunk-grids/rectilinear", + ), + ), + must_understand_false_permitted=False, + holds_sequence=False, + ), + CHUNK_KEY_ENCODING: ExtensionPoint( + field=CHUNK_KEY_ENCODING, + identifiers=_identifiers( + ExtensionIdentifier( + DEFAULT_CHUNK_KEY_ENCODING_NAME, Provenance.CORE, f"{_CORE_SPEC}#chunk-key-encoding" + ), + ExtensionIdentifier( + V2_CHUNK_KEY_ENCODING_NAME, Provenance.CORE, f"{_CORE_SPEC}#chunk-key-encoding" + ), + ), + must_understand_false_permitted=False, + holds_sequence=False, + ), + CODECS: ExtensionPoint( + field=CODECS, + identifiers=_identifiers( + ExtensionIdentifier( + BLOSC_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/blosc/index.html", + ), + ExtensionIdentifier( + BYTES_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/bytes/index.html", + ), + ExtensionIdentifier( + CRC32C_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/crc32c/index.html", + ), + ExtensionIdentifier( + GZIP_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/gzip/index.html", + ), + ExtensionIdentifier( + SHARDING_INDEXED_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/index.html", + ), + ExtensionIdentifier( + TRANSPOSE_CODEC_NAME, + Provenance.CORE, + "https://zarr-specs.readthedocs.io/en/latest/v3/codecs/transpose/index.html", + ), + ExtensionIdentifier( + CAST_VALUE_CODEC_NAME, Provenance.REGISTERED, f"{_EXTENSIONS}/codecs/cast_value" + ), + ExtensionIdentifier( + SCALE_OFFSET_CODEC_NAME, Provenance.REGISTERED, f"{_EXTENSIONS}/codecs/scale_offset" + ), + # The zstd codec's specification is an open pull request, not + # merged text: anything typed against it is typed against a draft. + ExtensionIdentifier( + ZSTD_CODEC_NAME, + Provenance.PROPOSED, + "https://github.com/zarr-developers/zarr-specs/pull/256", + ), + ), + must_understand_false_permitted=True, + holds_sequence=True, + ), + STORAGE_TRANSFORMERS: ExtensionPoint( + field=STORAGE_TRANSFORMERS, + # A real extension point that this package models no identifiers + # for. Recorded explicitly so its emptiness is a stated fact rather + # than an oversight. + identifiers=_identifiers(), + must_understand_false_permitted=True, + holds_sequence=True, + ), +} +"""Every v3 extension point, keyed by the document field that carries it.""" + + def canonical_name(field: ExtensionPointField, name: str) -> str: - """`name` reduced to the key this package tables it under.""" + """`name` reduced to the key this package tables it under. + + Identity except for the raw-bytes data type family, which reduces to + `RAW_BYTES_FAMILY`. By grammar shape, not validity: `r12` and `r0` + canonicalize too, so a malformed member of a family we model is + reported as a misspelling rather than mistaken for an unknown + third-party extension. + """ if field == DATA_TYPE and RAW_BYTES_NAME_PATTERN.fullmatch(name) is not None: return RAW_BYTES_FAMILY return name +def identifier_of(field: ExtensionPointField, name: str) -> ExtensionIdentifier | None: + """What this package knows about `name` at `field`, or None if nothing. + + None means the name is not one this package models — an unregistered + or newer extension, which is not an error in itself. + """ + point = EXTENSION_POINTS.get(field) + if point is None: + return None + return point.identifiers.get(canonical_name(field, name)) + + +def provenance_of(field: ExtensionPointField, name: str) -> Provenance | None: + """Where `name` at `field` was standardized, or None if not modelled.""" + identifier = identifier_of(field, name) + return None if identifier is None else identifier.provenance + + +def identifiers_with(field: ExtensionPointField, provenance: Provenance) -> frozenset[str]: + """The canonical names at `field` with the given provenance.""" + point = EXTENSION_POINTS.get(field) + if point is None: + return frozenset() + return frozenset( + name + for name, identifier in point.identifiers.items() + if identifier.provenance is provenance + ) + + __all__ = [ "CHUNK_GRID", "CHUNK_KEY_ENCODING", "CODECS", "DATA_TYPE", + "EXTENSION_POINTS", "RAW_BYTES_FAMILY", "STORAGE_TRANSFORMERS", + "ExtensionIdentifier", + "ExtensionPoint", "ExtensionPointField", + "Provenance", "canonical_name", + "identifier_of", + "identifiers_with", + "provenance_of", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py index a31cc04d52..7d586f2437 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_shape.py @@ -656,9 +656,27 @@ def blocking_problems( return tuple(problem for problem in problems if problem.kind != "unknown_key") +def is_valid_known_codec_name(value: object) -> str | None: + """The codec name of `value` if it is a valid known codec, else None. + + The single primitive behind the `TypeIs` guards in + `zarr_metadata.v3.codec.kind`: a non-None answer certifies that + `value` is an instance of the named codec's canonical metadata type. + `TypeIs` narrowing is two-sided, so the shape verdict must be exact: + a value carrying an extra member is not an instance of a closed + TypedDict, and is refused here even though the rules layer reports it + as a non-blocking `unknown_key`. + """ + problems = validate_known_codec_metadata(value) + if problems is None or len(problems) != 0: + return None + return entity_name(value) + + __all__ = [ "blocking_problems", "entity_name", + "is_valid_known_codec_name", "modelled_entities", "validate_known_chunk_grid_metadata", "validate_known_codec_metadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py index f22a2280f9..11278f9bb5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/__init__.py @@ -14,8 +14,9 @@ `codecs` list and in sharding's inner pipelines), import `ZarrV3MetadataFieldJSON` from `zarr_metadata.v3`. -The `kind` submodule sorts the known codec names into the spec's three -pipeline kinds (`array -> array`, `array -> bytes`, `bytes -> bytes`). +The `kind` submodule sorts the known codecs into the spec's three pipeline +kinds (`array -> array`, `array -> bytes`, `bytes -> bytes`); its kind +unions and `TypeIs` classification guards are re-exported here. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ @@ -29,8 +30,14 @@ ARRAY_ARRAY_CODEC_NAMES, ARRAY_BYTES_CODEC_NAMES, BYTES_BYTES_CODEC_NAMES, - CodecKind, - codec_kind_of_name, + ArrayArrayCodecMetadata, + ArrayBytesCodecMetadata, + BytesBytesCodecMetadata, + KnownCodecMetadata, + is_array_array_codec, + is_array_bytes_codec, + is_bytes_bytes_codec, + is_known_codec, ) from zarr_metadata.v3.codec.scale_offset import ScaleOffsetCodecMetadata from zarr_metadata.v3.codec.sharding_indexed import ShardingIndexedCodecMetadata @@ -41,15 +48,21 @@ "ARRAY_ARRAY_CODEC_NAMES", "ARRAY_BYTES_CODEC_NAMES", "BYTES_BYTES_CODEC_NAMES", + "ArrayArrayCodecMetadata", + "ArrayBytesCodecMetadata", "BloscCodecMetadata", + "BytesBytesCodecMetadata", "BytesCodecMetadata", "CastValueCodecMetadata", - "CodecKind", "Crc32cCodecMetadata", "GzipCodecMetadata", + "KnownCodecMetadata", "ScaleOffsetCodecMetadata", "ShardingIndexedCodecMetadata", "TransposeCodecMetadata", "ZstdCodecMetadata", - "codec_kind_of_name", + "is_array_array_codec", + "is_array_bytes_codec", + "is_bytes_bytes_codec", + "is_known_codec", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py index a5853c0f97..23c24745ed 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/kind.py @@ -1,24 +1,34 @@ """Classify Zarr v3 codecs by pipeline kind. -The v3 spec sorts codecs into three kinds — `array -> array`, -`array -> bytes`, `bytes -> bytes` — and a pipeline is -`array->array* array->bytes bytes->bytes*`. `codec_kind_of_name` -classifies a known name; unknown names have no kind. +The `TypeIs` guards are shape-exact against canonical codec TypedDicts; +normalize JSON arrays to tuples before using them. `codec_kind_of_name` +classifies a known name without validating its object shape, which is +useful for pipeline ordering. Unknown names return no kind. See https://zarr-specs.readthedocs.io/en/latest/v3/codecs/index.html """ from typing import Final, Literal -from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME -from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME -from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME -from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME -from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME -from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME -from zarr_metadata.v3.codec.sharding_indexed import SHARDING_INDEXED_CODEC_NAME -from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME -from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME +from typing_extensions import TypeIs + +from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON +from zarr_metadata.v3._shape import is_valid_known_codec_name +from zarr_metadata.v3.codec.blosc import BLOSC_CODEC_NAME, BloscCodecMetadata +from zarr_metadata.v3.codec.bytes import BYTES_CODEC_NAME, BytesCodecMetadata +from zarr_metadata.v3.codec.cast_value import CAST_VALUE_CODEC_NAME, CastValueCodecMetadata +from zarr_metadata.v3.codec.crc32c import CRC32C_CODEC_NAME, Crc32cCodecMetadata +from zarr_metadata.v3.codec.gzip import GZIP_CODEC_NAME, GzipCodecMetadata +from zarr_metadata.v3.codec.scale_offset import SCALE_OFFSET_CODEC_NAME, ScaleOffsetCodecMetadata +from zarr_metadata.v3.codec.sharding_indexed import ( + SHARDING_INDEXED_CODEC_NAME, + ShardingIndexedCodecMetadata, +) +from zarr_metadata.v3.codec.transpose import TRANSPOSE_CODEC_NAME, TransposeCodecMetadata +from zarr_metadata.v3.codec.zstd import ZSTD_CODEC_NAME, ZstdCodecMetadata + +ArrayArrayCodecMetadata = TransposeCodecMetadata | CastValueCodecMetadata | ScaleOffsetCodecMetadata +"""Permitted JSON shapes of the `array -> array` codecs this package defines.""" ARRAY_ARRAY_CODEC_NAMES: Final = ( TRANSPOSE_CODEC_NAME, @@ -27,9 +37,17 @@ ) """Tuple of the `name` field values of the known `array -> array` codecs.""" +ArrayBytesCodecMetadata = BytesCodecMetadata | ShardingIndexedCodecMetadata +"""Permitted JSON shapes of the `array -> bytes` codecs this package defines.""" + ARRAY_BYTES_CODEC_NAMES: Final = (BYTES_CODEC_NAME, SHARDING_INDEXED_CODEC_NAME) """Tuple of the `name` field values of the known `array -> bytes` codecs.""" +BytesBytesCodecMetadata = ( + BloscCodecMetadata | Crc32cCodecMetadata | GzipCodecMetadata | ZstdCodecMetadata +) +"""Permitted JSON shapes of the `bytes -> bytes` codecs this package defines.""" + BYTES_BYTES_CODEC_NAMES: Final = ( BLOSC_CODEC_NAME, CRC32C_CODEC_NAME, @@ -38,6 +56,30 @@ ) """Tuple of the `name` field values of the known `bytes -> bytes` codecs.""" +KnownCodecMetadata = ArrayArrayCodecMetadata | ArrayBytesCodecMetadata | BytesBytesCodecMetadata +"""Permitted JSON shapes of every codec this package defines.""" + + +def is_array_array_codec(codec: ZarrV3MetadataFieldJSON) -> TypeIs[ArrayArrayCodecMetadata]: + """Whether `codec` is an instance of a known `array -> array` codec type.""" + return is_valid_known_codec_name(codec) in ARRAY_ARRAY_CODEC_NAMES + + +def is_array_bytes_codec(codec: ZarrV3MetadataFieldJSON) -> TypeIs[ArrayBytesCodecMetadata]: + """Whether `codec` is an instance of a known `array -> bytes` codec type.""" + return is_valid_known_codec_name(codec) in ARRAY_BYTES_CODEC_NAMES + + +def is_bytes_bytes_codec(codec: ZarrV3MetadataFieldJSON) -> TypeIs[BytesBytesCodecMetadata]: + """Whether `codec` is an instance of a known `bytes -> bytes` codec type.""" + return is_valid_known_codec_name(codec) in BYTES_BYTES_CODEC_NAMES + + +def is_known_codec(codec: ZarrV3MetadataFieldJSON) -> TypeIs[KnownCodecMetadata]: + """Whether `codec` is an instance of any codec type this package defines.""" + return is_valid_known_codec_name(codec) is not None + + CodecKind = Literal["array_array", "array_bytes", "bytes_bytes"] """The three pipeline positions the v3 spec sorts codecs into.""" @@ -45,8 +87,10 @@ def codec_kind_of_name(name: str) -> CodecKind | None: """The pipeline kind of the codec named `name`, or None if unknown. - Classifies by name alone, with no judgment of the entry's spelling or - configuration; the rules layer judges those separately. + Classifies by name alone, with no spelling judgment: `"transpose"` + answers `"array_array"` here even though the bare-string spelling is + not valid transpose metadata (the `TypeIs` guards answer False for + it). See the module docstring for when to use which surface. """ if name in ARRAY_ARRAY_CODEC_NAMES: return "array_array" @@ -61,6 +105,14 @@ def codec_kind_of_name(name: str) -> CodecKind | None: "ARRAY_ARRAY_CODEC_NAMES", "ARRAY_BYTES_CODEC_NAMES", "BYTES_BYTES_CODEC_NAMES", + "ArrayArrayCodecMetadata", + "ArrayBytesCodecMetadata", + "BytesBytesCodecMetadata", "CodecKind", + "KnownCodecMetadata", "codec_kind_of_name", + "is_array_array_codec", + "is_array_bytes_codec", + "is_bytes_bytes_codec", + "is_known_codec", ] diff --git a/packages/zarr-metadata/tests/builder/test_array_v3.py b/packages/zarr-metadata/tests/builder/test_array_v3.py new file mode 100644 index 0000000000..98f7970857 --- /dev/null +++ b/packages/zarr-metadata/tests/builder/test_array_v3.py @@ -0,0 +1,413 @@ +"""Tests for `ZarrV3ArrayMetadataBuilder`.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from zarr_metadata.builder import ZarrV3ArrayMetadataBuilder +from zarr_metadata.model import UNSET, MetadataValidationError + +if TYPE_CHECKING: + from collections.abc import Sequence + +# A structurally- and semantically-valid document, assembled below in +# different with_fields() orders. +COMPLETE: dict[str, object] = { + "zarr_format": 3, + "node_type": "array", + "shape": (4, 4), + "data_type": "uint8", + "fill_value": 0, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (2, 2)}}, + "chunk_key_encoding": "default", + "codecs": ("bytes",), +} + +LITTLE_ENDIAN_BYTES: dict[str, object] = { + "name": "bytes", + "configuration": {"endian": "little"}, +} + + +def _steps(*chunks: dict[str, object]) -> tuple[dict[str, object], ...]: + return chunks + + +# (with_fields-call payloads applied in order, expected build() output). Every +# entry must build successfully; error paths get their own tests below. +CASES: dict[str, tuple[Sequence[dict[str, object]], dict[str, object]]] = { + "one-call": (_steps(COMPLETE), COMPLETE), + "field-at-a-time": (_steps(*({k: v} for k, v in COMPLETE.items())), COMPLETE), + "fill-before-dtype": ( + _steps( + {"fill_value": "NaN"}, + {"data_type": "float32"}, + { + **{ + k: v + for k, v in COMPLETE.items() + if k not in ("fill_value", "data_type", "codecs") + }, + "codecs": (LITTLE_ENDIAN_BYTES,), + }, + ), + { + **COMPLETE, + "fill_value": "NaN", + "data_type": "float32", + "codecs": (LITTLE_ENDIAN_BYTES,), + }, + ), + "conflict-escape-by-pair": ( + # uint8/0 established, then both members of the couple change at once. + _steps( + COMPLETE, + { + "data_type": "float64", + "fill_value": "Infinity", + "codecs": (LITTLE_ENDIAN_BYTES,), + }, + ), + { + **COMPLETE, + "data_type": "float64", + "fill_value": "Infinity", + "codecs": (LITTLE_ENDIAN_BYTES,), + }, + ), + "field-replacement": ( + _steps(COMPLETE, {"shape": (8, 8)}, {"chunk_grid": COMPLETE["chunk_grid"]}), + {**COMPLETE, "shape": (8, 8)}, + ), + "with-optionals": ( + _steps( + COMPLETE, + { + "attributes": {"unit": "kelvin", "nothing": None}, + "dimension_names": ("y", None), + "storage_transformers": (), + }, + ), + { + **COMPLETE, + "attributes": {"unit": "kelvin", "nothing": None}, + "dimension_names": ("y", None), + "storage_transformers": (), + }, + ), + "full-pipeline": ( + _steps( + COMPLETE, + { + "codecs": ( + {"name": "transpose", "configuration": {"order": (1, 0)}}, + "bytes", + {"name": "gzip", "configuration": {"level": 5}}, + "crc32c", + ) + }, + ), + { + **COMPLETE, + "codecs": ( + {"name": "transpose", "configuration": {"order": (1, 0)}}, + "bytes", + {"name": "gzip", "configuration": {"level": 5}}, + "crc32c", + ), + }, + ), + "unknown-codec-passes": ( + # An unclassifiable codec imposes no ordering constraint and may be + # the pipeline's array->bytes stage. + _steps(COMPLETE, {"codecs": ({"name": "lightspeed"},)}), + {**COMPLETE, "codecs": ({"name": "lightspeed"},)}, + ), + "unknown-dtype-accepts-any-fill": ( + _steps(COMPLETE, {"data_type": {"name": "bfloat16"}, "fill_value": "whatever"}), + {**COMPLETE, "data_type": {"name": "bfloat16"}, "fill_value": "whatever"}, + ), + "null-fill-value-is-a-value": ( + # JSON null is a stored value, not absence; builds only for a dtype + # whose fill values this package does not judge. + _steps(COMPLETE, {"data_type": {"name": "unknowable"}, "fill_value": None}), + {**COMPLETE, "data_type": {"name": "unknowable"}, "fill_value": None}, + ), + "json-loads-input-normalizes-to-tuples": ( + # Arrays arriving as lists (straight from json.loads) are + # materialized as tuples at ingestion, so the built document is + # spelling-identical to its tuple-spelled twin. + _steps( + { + **COMPLETE, + "shape": [4, 4], + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [2, 2]}}, + "codecs": ["bytes"], + } + ), + COMPLETE, + ), + "bare-spellings-where-permitted": ( + # scale_offset/bytes/crc32c have no required configuration, so + # their bare short-hand spellings are canonical and pass the + # spelling rule. + _steps(COMPLETE, {"codecs": ("scale_offset", "bytes", "crc32c")}), + {**COMPLETE, "codecs": ("scale_offset", "bytes", "crc32c")}, + ), +} + + +@pytest.mark.parametrize(("steps", "expected"), CASES.values(), ids=list(CASES)) +def test_build(steps: Sequence[dict[str, object]], expected: dict[str, object]) -> None: + builder = ZarrV3ArrayMetadataBuilder() + for step in steps: + builder = builder.with_fields(**step) + assert builder.build() == expected + + +def test_extension_fields() -> None: + builder = ZarrV3ArrayMetadataBuilder(COMPLETE).with_extension( + "my_ext", {"must_understand": False, "level": 3} + ) + assert builder.extension_fields == {"my_ext": {"must_understand": False, "level": 3}} + built = builder.build() + assert built["my_ext"] == {"must_understand": False, "level": 3} + + +def test_properties_unset_vs_value() -> None: + empty = ZarrV3ArrayMetadataBuilder() + assert empty.shape is UNSET + assert empty.fill_value is UNSET + assert empty.dimension_names is UNSET + full = ZarrV3ArrayMetadataBuilder(COMPLETE) + assert full.shape == (4, 4) + assert full.zarr_format == 3 + assert full.codecs == ("bytes",) + # optional-but-absent stays UNSET even on a buildable document + assert full.attributes is UNSET + + +def test_without_unsets() -> None: + builder = ZarrV3ArrayMetadataBuilder(COMPLETE).with_fields(dimension_names=("y", "x")) + assert builder.without_fields("dimension_names").dimension_names is UNSET + # removing an absent key is a no-op + assert builder.without_fields("attributes") == builder + # a removed required key is UNSET, not null + assert builder.without_fields("fill_value").fill_value is UNSET + + +def test_immutability() -> None: + source: dict[str, object] = dict(COMPLETE) + builder = ZarrV3ArrayMetadataBuilder(source) + source["shape"] = (9,) # the builder copied on ingest + evolved = builder.with_fields(shape=(8, 8)) + assert builder.shape == (4, 4) # with_fields did not mutate its receiver + assert evolved.shape == (8, 8) + built = evolved.build() + built["attributes"] = {"corrupted": True} # outputs are isolated copies + assert evolved.attributes is UNSET + partial = builder.to_partial_json() + assert partial == builder.to_partial_json() + partial["shape"] = (1,) + assert builder.shape == (4, 4) + + +def test_runtime_list_input_cannot_corrupt_builder() -> None: + # Regression: `shape`/`dimension_names` used to hand out the internal + # object, so a list smuggled past the type checker could be mutated in + # place, corrupting the builder behind the eager rules' back. + builder = ZarrV3ArrayMetadataBuilder().with_fields(shape=[2, 2], dimension_names=["a", "b"]) # type: ignore[arg-type] + assert builder.shape == (2, 2) # normalized to a tuple: no .append to abuse + assert builder.dimension_names == ("a", "b") + # and equality is spelling-insensitive as a consequence + assert builder == ZarrV3ArrayMetadataBuilder().with_fields( + shape=(2, 2), dimension_names=("a", "b") + ) + + +def test_to_partial_json_always_succeeds() -> None: + fragment = ZarrV3ArrayMetadataBuilder().with_fields(shape=(2,), fill_value=None) + # incomplete and unbuildable, but honestly serializable — and the stored + # null survives (key omission is never decided by value inspection) + assert fragment.to_partial_json() == {"shape": (2,), "fill_value": None} + with pytest.raises(MetadataValidationError): + fragment.build() + + +# -- error cases, one test per failure mode --------------------------------- + + +def test_error_fill_dtype_conflict_names_both_fields() -> None: + builder = ZarrV3ArrayMetadataBuilder().with_fields(fill_value="NaN") + with pytest.raises(MetadataValidationError) as info: + builder.with_fields(data_type="uint8") + (problem,) = info.value.problems + assert problem.loc == ("fill_value",) + assert problem.kind == "invalid_value" + # the conflict names the field just set AND the one set earlier, and + # points at the batch-with_fields escape hatch + assert "data_type set in this call" in problem.message + assert "fill_value set earlier" in problem.message + assert "with_fields()" in problem.message + + +def test_error_fill_out_of_range() -> None: + with pytest.raises(MetadataValidationError, match=r"\[0, 255\]"): + ZarrV3ArrayMetadataBuilder().with_fields(data_type="uint8", fill_value=300) + + +def test_error_codec_order() -> None: + with pytest.raises(MetadataValidationError, match="may not follow"): + ZarrV3ArrayMetadataBuilder().with_fields( + codecs=("bytes", {"name": "transpose", "configuration": {"order": (0, 1)}}) + ) + + +def test_error_two_array_bytes_codecs() -> None: + with pytest.raises(MetadataValidationError, match="exactly one"): + ZarrV3ArrayMetadataBuilder().with_fields(codecs=("bytes", "bytes")) + + +def test_error_no_array_bytes_codec() -> None: + with pytest.raises(MetadataValidationError, match="no array->bytes codec"): + ZarrV3ArrayMetadataBuilder().with_fields(codecs=("crc32c",)) + + +def test_error_dimension_names_length() -> None: + with pytest.raises(MetadataValidationError, match="2 entries.*3 dimensions"): + ZarrV3ArrayMetadataBuilder().with_fields(shape=(1, 2, 3), dimension_names=("a", "b")) + + +def test_error_regular_grid_dimensions() -> None: + with pytest.raises(MetadataValidationError, match="chunk_shape has 1"): + ZarrV3ArrayMetadataBuilder().with_fields( + shape=(4, 4), + chunk_grid={"name": "regular", "configuration": {"chunk_shape": (2,)}}, + ) + + +def test_error_bare_spelling_of_config_required_codec() -> None: + # Regression: bare "transpose" used to pass as an unknown extension, + # suppressing both the spelling check and the exactly-one-array->bytes + # count — build() would emit a pipeline with no array->bytes codec. + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields(codecs=("transpose",)) + messages = [p.message for p in info.value.problems] + assert any("no bare short-hand form" in m for m in messages) + assert any("no array->bytes codec" in m for m in messages) + + +def test_error_known_codec_missing_configuration_key() -> None: + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields( + codecs=("bytes", {"name": "gzip", "configuration": {}}) + ) + (problem,) = info.value.problems + assert problem.loc == ("codecs", 1, "configuration", "level") + assert problem.kind == "missing_key" + + +def test_error_known_codec_missing_configuration_object() -> None: + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields(codecs=({"name": "transpose"}, "bytes")) + (problem,) = info.value.problems + assert problem.loc == ("codecs", 0, "configuration") + assert problem.kind == "missing_key" + + +def test_error_known_codec_bad_configuration_literal() -> None: + # Known-name configurations are held to their full canonical shapes, + # value types included — not just key presence. + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields( + codecs=({"name": "bytes", "configuration": {"endian": "middle"}},) + ) + (problem,) = info.value.problems + assert problem.loc == ("codecs", 0, "configuration", "endian") + assert problem.kind == "invalid_value" + + +def test_error_known_codec_bad_configuration_value_type() -> None: + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields( + codecs=("bytes", {"name": "gzip", "configuration": {"level": "high"}}) + ) + (problem,) = info.value.problems + assert problem.loc == ("codecs", 1, "configuration", "level") + assert problem.kind == "invalid_type" + + +def test_error_known_codec_unexpected_configuration_key() -> None: + with pytest.raises(MetadataValidationError, match="unexpected key"): + ZarrV3ArrayMetadataBuilder().with_fields( + codecs=("bytes", {"name": "gzip", "configuration": {"level": 1, "speed": "max"}}) + ) + + +def test_error_bare_chunk_grid_spelling() -> None: + with pytest.raises(MetadataValidationError, match="no bare short-hand form"): + ZarrV3ArrayMetadataBuilder().with_fields(chunk_grid="regular") + + +def test_error_chunk_grid_missing_configuration_object() -> None: + with pytest.raises(MetadataValidationError, match="requires a 'configuration' object"): + ZarrV3ArrayMetadataBuilder().with_fields(chunk_grid={"name": "regular"}) + + +def test_error_chunk_grid_missing_configuration_key() -> None: + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields( + chunk_grid={"name": "regular", "configuration": {}} + ) + (problem,) = info.value.problems + assert problem.loc == ("chunk_grid", "configuration", "chunk_shape") + assert problem.kind == "missing_key" + + +def test_spelling_verdicts_agree_across_model_normalization() -> None: + # Regression: the model layer collapses empty-config codecs to bare + # names, and bare "gzip" used to classify as an unknown extension — + # so a document the builder rejected round-tripped through the model + # into one the rules accepted. Both spellings must now be rejected. + from zarr_metadata.model import ZarrV3ArrayMetadata + from zarr_metadata.rules import ZARR_V3_ARRAY_RULES, run_rules + + doc = { + **COMPLETE, + "codecs": ({"name": "gzip", "configuration": {}}, {"name": "bytes", "configuration": {}}), + } + with pytest.raises(MetadataValidationError): + ZarrV3ArrayMetadataBuilder(doc) + normalized = ZarrV3ArrayMetadata.from_json(doc).to_json() + assert normalized["codecs"] == ("gzip", "bytes") # the collapsed spelling + assert run_rules(ZARR_V3_ARRAY_RULES, normalized) # still rejected + + +def test_error_build_incomplete_reports_every_missing_key() -> None: + with pytest.raises(MetadataValidationError) as info: + ZarrV3ArrayMetadataBuilder().with_fields(shape=(2, 2)).build() + missing = {p.loc[0] for p in info.value.problems if p.kind == "missing_key"} + # every absent required key is reported at once, not one per attempt + assert {"zarr_format", "node_type", "data_type", "fill_value"} <= missing + + +def test_error_build_reports_structural_problems() -> None: + # The eager with_fields/constructor pass runs semantic rules only; the + # structural pass belongs to build(). An untyped caller smuggling in a + # structurally-invalid value is caught there. + builder = ZarrV3ArrayMetadataBuilder({**COMPLETE, "zarr_format": 2}) + with pytest.raises(MetadataValidationError) as info: + builder.build() + assert any(p.loc == ("zarr_format",) for p in info.value.problems) + + +def test_error_constructor_validates() -> None: + with pytest.raises(MetadataValidationError, match="fill_value invalid"): + ZarrV3ArrayMetadataBuilder({"data_type": "uint8", "fill_value": "NaN"}) + + +def test_error_evolve_extension_rejects_standard_key() -> None: + with pytest.raises(MetadataValidationError, match="standard v3 array metadata key"): + ZarrV3ArrayMetadataBuilder().with_extension("shape", (1, 2)) diff --git a/packages/zarr-metadata/tests/test_examples.py b/packages/zarr-metadata/tests/test_examples.py new file mode 100644 index 0000000000..dd2508075f --- /dev/null +++ b/packages/zarr-metadata/tests/test_examples.py @@ -0,0 +1,11 @@ +"""Executable examples remain complete, current, and self-contained.""" + +from __future__ import annotations + +import runpy +from pathlib import Path + + +def test_build_v3_array_example() -> None: + example = Path(__file__).parents[1] / "examples" / "build_v3_array.py" + runpy.run_path(str(example), run_name="__main__") diff --git a/packages/zarr-metadata/tests/v3/codec/test_kind.py b/packages/zarr-metadata/tests/v3/codec/test_kind.py index 996d469035..c023106c64 100644 --- a/packages/zarr-metadata/tests/v3/codec/test_kind.py +++ b/packages/zarr-metadata/tests/v3/codec/test_kind.py @@ -2,12 +2,125 @@ from __future__ import annotations +from typing import TYPE_CHECKING, cast + import pytest -from zarr_metadata.v3.codec.kind import codec_kind_of_name +from zarr_metadata.v3.codec.kind import ( + codec_kind_of_name, + is_array_array_codec, + is_array_bytes_codec, + is_bytes_bytes_codec, + is_known_codec, +) + +if TYPE_CHECKING: + from zarr_metadata.v3._common import ZarrV3MetadataFieldJSON + +# (codec entry, expected kind) — kind is one of "aa", "ab", "bb", or None +# for entries no guard should claim. Object forms use minimal spec-shaped +# configurations; bare strings appear only for codecs whose canonical type +# permits the short-hand form. +CASES: dict[str, tuple[ZarrV3MetadataFieldJSON, str | None]] = { + "transpose": ({"name": "transpose", "configuration": {"order": (1, 0)}}, "aa"), + "cast_value": ( + {"name": "cast_value", "configuration": {"data_type": "uint8"}}, + "aa", + ), + "scale_offset": ( + {"name": "scale_offset", "configuration": {"scale": 2, "offset": 1}}, + "aa", + ), + "scale_offset-bare": ("scale_offset", "aa"), + "bytes": ({"name": "bytes", "configuration": {"endian": "little"}}, "ab"), + "bytes-bare": ("bytes", "ab"), + "sharding_indexed": ( + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (2, 2), + "codecs": ("bytes",), + "index_codecs": ("bytes", "crc32c"), + }, + }, + "ab", + ), + "blosc": ( + { + "name": "blosc", + "configuration": { + "cname": "zstd", + "clevel": 5, + "shuffle": "shuffle", + "blocksize": 0, + }, + }, + "bb", + ), + "crc32c": ({"name": "crc32c"}, "bb"), + "crc32c-bare": ("crc32c", "bb"), + "gzip": ({"name": "gzip", "configuration": {"level": 5}}, "bb"), + "zstd": ({"name": "zstd", "configuration": {"level": 3, "checksum": False}}, "bb"), + # Unknown codecs have unknown kind. + "unknown-object": ({"name": "lightspeed"}, None), + "unknown-bare": ("lightspeed", None), + # A bare name is only classified when the codec's spec permits the + # short-hand form; "transpose" as a bare string is not valid transpose + # metadata, so no guard claims it. + "transpose-bare-invalid": ("transpose", None), + "blosc-bare-invalid": ("blosc", None), + # Object forms that are not instances of their codec's canonical type: + # `TypeIs` narrowing is two-sided, so a guard claiming any of these + # would lie to the type checker. The guards deep-check shape. + "transpose-missing-config": ({"name": "transpose"}, None), + "gzip-missing-config": ({"name": "gzip"}, None), + "gzip-empty-config": ({"name": "gzip", "configuration": {}}, None), + "gzip-bool-level": ({"name": "gzip", "configuration": {"level": True}}, None), + "zstd-missing-checksum": ({"name": "zstd", "configuration": {"level": 1}}, None), + "bytes-bad-endian": ( + cast("ZarrV3MetadataFieldJSON", {"name": "bytes", "configuration": {"endian": "middle"}}), + None, + ), + "crc32c-nonempty-config": ( + cast("ZarrV3MetadataFieldJSON", {"name": "crc32c", "configuration": {"x": 1}}), + None, + ), + "crc32c-extra-key": ( + cast("ZarrV3MetadataFieldJSON", {"name": "crc32c", "bogus": 1}), + None, + ), + "crc32c-nonbool-must-understand": ( + cast("ZarrV3MetadataFieldJSON", {"name": "crc32c", "must_understand": "yes"}), + None, + ), + # Judgments are at the canonical data level: JSON arrays are tuples. + "transpose-list-order": ( + cast("ZarrV3MetadataFieldJSON", {"name": "transpose", "configuration": {"order": [0, 1]}}), + None, + ), + "no-name": (cast("ZarrV3MetadataFieldJSON", {}), None), + "non-string-name": (cast("ZarrV3MetadataFieldJSON", {"name": 3}), None), + # ...and valid instances of the optional-configuration codecs in every + # spelling their types permit. + "bytes-no-config": ({"name": "bytes"}, "ab"), + "scale_offset-no-config": ({"name": "scale_offset"}, "aa"), + "crc32c-empty-config": ({"name": "crc32c", "configuration": {}}, "bb"), + "crc32c-must-understand": ({"name": "crc32c", "must_understand": False}, "bb"), +} + + +@pytest.mark.parametrize(("codec", "kind"), CASES.values(), ids=list(CASES)) +def test_classification(codec: ZarrV3MetadataFieldJSON, kind: str | None) -> None: + assert is_array_array_codec(codec) is (kind == "aa") + assert is_array_bytes_codec(codec) is (kind == "ab") + assert is_bytes_bytes_codec(codec) is (kind == "bb") + assert is_known_codec(codec) is (kind is not None) + -# (codec name, expected kind). Classification is by name alone. -CASES: dict[str, str | None] = { +# (codec name, expected kind) — `codec_kind_of_name` classifies by name +# alone, so a name answers its kind even where the bare spelling is not +# valid metadata for that codec (unlike the spelling-strict guards above). +NAME_CASES: dict[str, str | None] = { "transpose": "array_array", "cast_value": "array_array", "scale_offset": "array_array", @@ -21,6 +134,6 @@ } -@pytest.mark.parametrize(("name", "kind"), CASES.items(), ids=list(CASES)) +@pytest.mark.parametrize(("name", "kind"), NAME_CASES.items(), ids=list(NAME_CASES)) def test_kind_of_name(name: str, kind: str | None) -> None: assert codec_kind_of_name(name) == kind diff --git a/packages/zarr-metadata/tests/v3/test_extension_points.py b/packages/zarr-metadata/tests/v3/test_extension_points.py index 7a539fe28d..0eda703f5a 100644 --- a/packages/zarr-metadata/tests/v3/test_extension_points.py +++ b/packages/zarr-metadata/tests/v3/test_extension_points.py @@ -1,15 +1,29 @@ -"""Tests for extension-point name canonicalization.""" +"""Tests for the extension-point table and name canonicalization.""" from __future__ import annotations +import importlib +import pkgutil + import pytest -from zarr_metadata.rules import validate_array_metadata_v3 +import zarr_metadata.v3.chunk_grid +import zarr_metadata.v3.chunk_key_encoding +import zarr_metadata.v3.codec +import zarr_metadata.v3.data_type from zarr_metadata.v3._extension_points import ( + CHUNK_GRID, + CHUNK_KEY_ENCODING, CODECS, DATA_TYPE, + EXTENSION_POINTS, RAW_BYTES_FAMILY, + STORAGE_TRANSFORMERS, + Provenance, canonical_name, + identifier_of, + identifiers_with, + provenance_of, ) # (field, name, expected canonical key) — identity everywhere except the @@ -38,11 +52,119 @@ def test_canonical_name(field: str, name: str, expected: str) -> None: assert canonical_name(field, name) == expected # type: ignore[arg-type] +def test_name_collision_resolves_per_extension_point() -> None: + # `bytes` is a core codec and, separately, a registered extension data + # type. This is the case a name-keyed table could not represent. + assert provenance_of(CODECS, "bytes") is Provenance.CORE + assert provenance_of(DATA_TYPE, "bytes") is Provenance.REGISTERED + assert identifier_of(CODECS, "bytes").reference != identifier_of(DATA_TYPE, "bytes").reference + + +def test_raw_family_members_share_one_entry() -> None: + entry = identifier_of(DATA_TYPE, "r8") + assert entry is identifier_of(DATA_TYPE, "r4096") + assert entry.provenance is Provenance.CORE + assert entry.name == RAW_BYTES_FAMILY + + +def test_unmodelled_names_are_not_errors() -> None: + # An unregistered or newer extension is simply unknown, not invalid. + assert provenance_of(CODECS, "zfpy") is None + assert identifier_of(DATA_TYPE, "float8_e4m3") is None + + +def test_zstd_is_marked_proposed() -> None: + # Its specification is an open pull request, not merged text. + entry = identifier_of(CODECS, "zstd") + assert entry.provenance is Provenance.PROPOSED + assert "pull" in entry.reference + + +def test_must_understand_policy_matches_the_spec() -> None: + # The spec excludes must_understand: false at exactly these three + # points; an implementation cannot proceed without understanding them. + for field in (DATA_TYPE, CHUNK_GRID, CHUNK_KEY_ENCODING): + assert EXTENSION_POINTS[field].must_understand_false_permitted is False + for field in (CODECS, STORAGE_TRANSFORMERS): + assert EXTENSION_POINTS[field].must_understand_false_permitted is True + + +def test_sequence_valued_points() -> None: + assert EXTENSION_POINTS[CODECS].holds_sequence is True + assert EXTENSION_POINTS[STORAGE_TRANSFORMERS].holds_sequence is True + assert EXTENSION_POINTS[DATA_TYPE].holds_sequence is False + + +def test_storage_transformers_is_recorded_as_empty() -> None: + # A real extension point this package models no identifiers for; its + # emptiness is a stated fact, not an oversight. + assert EXTENSION_POINTS[STORAGE_TRANSFORMERS].identifiers == {} + + +def _module_names(package: object, suffix: str) -> set[str]: + found: set[str] = set() + for info in pkgutil.iter_modules(package.__path__): # type: ignore[attr-defined] + if info.name.startswith("_"): + continue + module = importlib.import_module(f"{package.__name__}.{info.name}") # type: ignore[attr-defined] + found.update( + value + for attribute, value in vars(module).items() + if attribute.endswith(suffix) and isinstance(value, str) + ) + return found + + +# The table is hand-written because provenance is irreducible knowledge — +# nothing in the type modules records where a name was standardized. These +# drift tests tie it to the names those modules actually define, so a new +# codec or data type cannot ship without a provenance entry. +DRIFT_CASES: dict[str, tuple[str, object, str, set[str]]] = { + "codecs": (CODECS, zarr_metadata.v3.codec, "_CODEC_NAME", set()), + "chunk_grid": (CHUNK_GRID, zarr_metadata.v3.chunk_grid, "_CHUNK_GRID_NAME", set()), + "chunk_key_encoding": ( + CHUNK_KEY_ENCODING, + zarr_metadata.v3.chunk_key_encoding, + "_CHUNK_KEY_ENCODING_NAME", + set(), + ), + # `raw` defines a grammar, not a name constant, so the family key has + # no counterpart to scan for. + "data_type": (DATA_TYPE, zarr_metadata.v3.data_type, "_DATA_TYPE_NAME", {RAW_BYTES_FAMILY}), +} + + +@pytest.mark.parametrize( + ("field", "package", "suffix", "extra"), DRIFT_CASES.values(), ids=list(DRIFT_CASES) +) +def test_table_matches_the_modules( + field: str, package: object, suffix: str, extra: set[str] +) -> None: + tabled = set(EXTENSION_POINTS[field].identifiers) # type: ignore[index] + assert tabled == _module_names(package, suffix) | extra + + +def test_every_identifier_cites_a_reference() -> None: + for point in EXTENSION_POINTS.values(): + for identifier in point.identifiers.values(): + assert identifier.reference.startswith("https://"), identifier + + +def test_identifiers_with_partitions_a_point() -> None: + point = EXTENSION_POINTS[CODECS] + by_provenance = { + name for provenance in Provenance for name in identifiers_with(CODECS, provenance) + } + assert by_provenance == set(point.identifiers) + + def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: # Zarr identifiers are registry-allocated. A private codec named # `bytes` has left the compatibility contract, and saying so is the # correct answer rather than a limitation, so nothing here defends # against collisions. + from zarr_metadata.rules import validate_array_metadata_v3 + document = { "zarr_format": 3, "node_type": "array", @@ -58,9 +180,12 @@ def test_squatted_names_are_judged_against_the_definition_they_squat() -> None: def test_forging_the_family_sentinel_cannot_change_a_verdict() -> None: - # A literal "r" data type mislabels nothing: the rules layer matches - # the family through the name pattern, not through the table key, so - # no validation verdict depends on the sentinel being unforgeable. + # A literal "r" data type mislabels its provenance and nothing + # else: the rules layer matches the family through the name pattern, + # not through the table key, so no validation verdict depends on the + # sentinel being unforgeable. + from zarr_metadata.rules import validate_array_metadata_v3 + assert canonical_name(DATA_TYPE, RAW_BYTES_FAMILY) == RAW_BYTES_FAMILY document = { "zarr_format": 3,