fix(chunk-grids): keep explicit per-chunk lists rectilinear

Drop the regular-grid collapse for explicit list input in
normalize_chunks_1d: a per-chunk size list now always produces
VaryingDimension, even when the sizes are uniform or uniform plus a
short tail. Scalar specs (ints, numpy integers, and the -1 sentinel)
still produce FixedDimension, which is the path that makes array
creation O(1) in per-dimension chunk count; explicit lists are already
O(n) in the input, so nothing is lost.

The grid kind now follows the input syntax, matching 3.2.x behavior:
previously an explicitly rectilinear spec whose edges happened to look
regular was silently stored as RegularChunkGridMetadata, which changes
resize semantics — a regular grid grows by extending the uniform
pattern while a rectilinear grid appends an edge chunk, breaking
append-oriented layouts like (168,) * 13 + (24,). This is resolution
option 1 from gh-4272.

Uniform dimensions of mixed scalar/list specs still serialize as the
spec's bare-int step-size shorthand; explicit lists serialize as edge
lists.

The touched-chunk-keys assertions in the resize regression test follow
the approach from gh-4290, which fixes the same issue on main via a
requested_rectilinear flag.

Fixes #4272

Co-authored-by: Shurong Cao <CAOShurong@users.noreply.github.com>
Assisted-by: ClaudeCode:claude-fable-5

Author: d-v-b

changes/4272.bugfix.md

@@ -0,0 +1,11 @@
+Explicit per-chunk size lists now always produce a rectilinear chunk grid,
+even when the sizes happen to describe a regular grid (all equal, or all equal
+with a smaller trailing chunk). Previously such input was silently collapsed to
+a regular grid, which changed resize semantics: a regular grid grows by
+extending the uniform pattern, while a rectilinear grid appends a new edge
+chunk — the behavior an append-oriented layout like `(168,) * 13 + (24,)`
+relies on. The grid kind now follows the input syntax, matching 3.2.x:
+scalar chunk sizes (including numpy integers and the `-1` sentinel) produce a
+regular grid, nested sequences produce a rectilinear grid. Rectilinear grids
+remain gated behind `zarr.config.set({"array.rectilinear_chunks": True})`.
+See #4174 for the accompanying O(1) chunk normalization change.

src/zarr/core/chunk_grids.py

@@ -703,16 +703,18 @@ def _guess_regular_chunks(
 def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionGrid:
     """
     Normalize a one-dimensional chunk specification into a dimension grid:
-    `FixedDimension` for uniform chunk sizes, `VaryingDimension` for explicit
-    per-chunk sizes that genuinely vary. Both variants bind the chunk sizes to
+    `FixedDimension` for scalar chunk sizes, `VaryingDimension` for explicit
+    per-chunk size lists. Both variants bind the chunk sizes to
     the span, and the uniform form is O(1) in the number of chunks — a
     dimension with `2**62` chunks must not materialize one entry per chunk.
 
     `-1` means "one chunk covering the entire span."
-    Explicit chunk size lists must sum to the span exactly; lists that describe
-    a regular grid (all sizes equal, or equal with a smaller boundary chunk)
-    collapse to `FixedDimension`. For uniform sizes the last chunk may overhang
-    the span.
+    Explicit chunk size lists must sum to the span exactly and always produce
+    `VaryingDimension`, even when the sizes happen to be uniform: the input
+    syntax declares the grid kind, so a per-chunk list is preserved as a
+    rectilinear dimension rather than silently collapsed to a regular one,
+    which would change how the dimension grows on resize. For scalar sizes
+    the last chunk may overhang the span.
     """
     # `numbers.Integral` rather than `int` so that numpy integer scalars (which are not
     # `int` subclasses) take the uniform-chunk path instead of being treated as a sequence.
@@ -750,8 +752,6 @@ def normalize_chunks_1d(chunks: int | Iterable[object], span: int) -> DimensionG
             raise ValueError(f"All chunk sizes must be positive, got {ints}")
         if sum(ints) != span:
             raise ValueError(f"Chunk sizes {ints} do not sum to span {span}")
-        if is_regular_1d(ints):
-            return FixedDimension(size=ints[0], extent=span)
         return VaryingDimension(ints, extent=span)
 
 

tests/test_chunk_grids.py

@@ -73,12 +73,12 @@ def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None:
         ((30, -1, -1), (100, 20, 10), (30, 20, 10)),
         ((30, 20, -1), (100, 20, 10), (30, 20, 10)),
         ((30, 20, 10), (100, 20, 10), (30, 20, 10)),
-        # dask-style chunks describing a regular grid collapse to the uniform form
-        (((100, 100, 100), (50, 50)), (300, 100), (100, 50)),
-        (((100, 100, 50),), (250,), (100,)),
-        (((100,),), (100,), (100,)),
-        # genuinely irregular explicit chunks keep the per-chunk form
-        (((10, 20, 70), (50, 50)), (100, 100), ((10, 20, 70), 50)),
+        # dask-style explicit lists always keep the per-chunk rectilinear form,
+        # even when the sizes describe a regular grid (gh-4272)
+        (((100, 100, 100), (50, 50)), (300, 100), ((100, 100, 100), (50, 50))),
+        (((100, 100, 50),), (250,), ((100, 100, 50),)),
+        (((100,),), (100,), ((100,),)),
+        (((10, 20, 70), (50, 50)), (100, 100), ((10, 20, 70), (50, 50))),
         # no chunking (False means each dimension is one chunk spanning the full extent)
         (False, (100,), (100,)),
         (False, (100, 50), (100, 50)),
@@ -96,7 +96,7 @@ def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None:
         ((np.int32(30), np.int64(-1)), (100, 20), (30, 20)),
         (np.array([10, 10]), (100, 100), (10, 10)),
         # rectilinear chunks given as numpy arrays
-        ((np.array([60, 40]), np.array([50, 50])), (100, 100), (60, 50)),
+        ((np.array([60, 40]), np.array([50, 50])), (100, 100), ((60, 40), (50, 50))),
     ],
 )
 def test_normalize_chunks(
@@ -112,9 +112,9 @@ def test_normalize_chunks(
         ((100,), (10,), None, (10,), None),
         # explicit regular shards
         ((100,), (10,), (50,), (50,), (10,)),
-        # rectilinear shards describing a regular-with-boundary grid collapse
-        ((100,), (10,), ((60, 40),), (60,), (10,)),
-        # genuinely irregular rectilinear shards keep the per-chunk form
+        # rectilinear shards keep the per-chunk form even when the sizes
+        # describe a regular-with-boundary grid (gh-4272)
+        ((100,), (10,), ((60, 40),), ((60, 40),), (10,)),
         ((100,), (10,), ((30, 60, 10),), ((30, 60, 10),), (10,)),
         # dict-style shards
         ((100, 100), (10, 10), {"shape": (50, 50)}, (50, 50), (10, 10)),
@@ -291,18 +291,18 @@ def test_normalize_chunks_nd_errors(case: ExpectFail[tuple[Any, tuple[int, ...]]
         Expect(input=(-1, 100), output=FixedDimension(size=100, extent=100), id="full-span"),
         # zero-length span preserves the declared chunk size.
         Expect(input=(10, 0), output=FixedDimension(size=10, extent=0), id="uniform-zero-span"),
-        # explicit lists that describe a regular grid collapse to the uniform form.
+        # explicit lists always keep the per-chunk form, even when the sizes
+        # describe a regular grid (gh-4272).
         Expect(
             input=([10, 10, 10], 30),
-            output=FixedDimension(size=10, extent=30),
+            output=VaryingDimension([10, 10, 10], extent=30),
             id="explicit-regular",
         ),
         Expect(
             input=([10, 10, 5], 25),
-            output=FixedDimension(size=10, extent=25),
+            output=VaryingDimension([10, 10, 5], extent=25),
             id="explicit-boundary",
         ),
-        # genuinely irregular edges keep the explicit per-chunk form.
         Expect(
             input=([10, 20, 70], 100),
             output=VaryingDimension([10, 20, 70], extent=100),
@@ -314,8 +314,8 @@ def test_normalize_chunks_nd_errors(case: ExpectFail[tuple[Any, tuple[int, ...]]
 def test_normalize_chunks_1d(
     case: Expect[tuple[Any, int], FixedDimension | VaryingDimension],
 ) -> None:
-    """Both output variants bind chunk sizes to the span: uniform specs become
-    `FixedDimension` (O(1) regardless of chunk count), irregular explicit
+    """Both output variants bind chunk sizes to the span: scalar specs become
+    `FixedDimension` (O(1) regardless of chunk count), explicit per-chunk
     lists become `VaryingDimension`."""
     chunks, span = case.input
     assert normalize_chunks_1d(chunks, span) == case.output

tests/test_properties.py

@@ -452,7 +452,7 @@ def test_array_metadata_meets_spec(meta: ArrayV2Metadata | ArrayV3Metadata) -> N
 
 def test_chunks_param_from_rectilinear_bare_int_roundtrip() -> None:
     """Bare-int dims in rectilinear metadata (the spec's step-size shorthand,
-    produced when a uniform dimension of a mixed grid is collapsed) must pass
+    produced by a scalar dimension of a mixed chunk spec) must pass
     through the `chunks=` conversion unchanged. Wrapping one in a
     single-element list turns "repeat to cover the axis" into "exactly one
     chunk" and re-creation fails the sum-to-span check."""
@@ -461,9 +461,7 @@ def test_chunks_param_from_rectilinear_bare_int_roundtrip() -> None:
     from zarr.testing.strategies import chunks_param_from_rectilinear
 
     with zarr.config.set({"array.rectilinear_chunks": True}):
-        src = zarr.create_array(
-            MemoryStore(), shape=(3, 3), chunks=[[1, 2], [1, 1, 1]], dtype="uint8"
-        )
+        src = zarr.create_array(MemoryStore(), shape=(3, 3), chunks=([1, 2], 1), dtype="uint8")
         grid = src.metadata.chunk_grid  # type: ignore[union-attr]
         assert isinstance(grid, RectilinearChunkGridMetadata)
         assert grid.chunk_shapes == ((1, 2), 1)

tests/test_unified_chunk_grid.py

@@ -1027,6 +1027,96 @@ def test_e2e_chunk_grid_name_regular_from_dict(tmp_path: Path) -> None:
     assert chunk_grid_dict["name"] == "regular"
 
 
+# ---------------------------------------------------------------------------
+# Input syntax determines grid kind (gh-4272)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("shape", "chunks", "expected_type"),
+    [
+        # scalar specs (including numpy integers and the -1 sentinel) produce a regular grid
+        ((30,), (10,), RegularChunkGridMetadata),
+        ((30,), 10, RegularChunkGridMetadata),
+        ((30,), (np.int64(10),), RegularChunkGridMetadata),
+        ((30,), -1, RegularChunkGridMetadata),
+        # explicit per-chunk lists stay rectilinear even when the sizes are uniform
+        ((30,), [[10, 10, 10]], RectilinearChunkGridMetadata),
+        # ... or uniform with a short tail (the append-oriented time-series layout of gh-4272)
+        ((168 * 13 + 24,), [[168] * 13 + [24]], RectilinearChunkGridMetadata),
+        # genuinely varying edges are rectilinear too
+        ((60,), [[10, 20, 30]], RectilinearChunkGridMetadata),
+    ],
+    ids=[
+        "scalar-tuple",
+        "scalar-int",
+        "scalar-numpy-int",
+        "full-span-sentinel",
+        "explicit-uniform",
+        "explicit-uniform-short-tail",
+        "explicit-varying",
+    ],
+)
+def test_chunk_grid_kind_follows_input_syntax(
+    tmp_path: Path, shape: tuple[int, ...], chunks: Any, expected_type: type
+) -> None:
+    """The stored grid kind is decided by the input syntax, not the chunk sizes:
+    scalar specs produce a regular grid, while explicit per-chunk lists produce
+    a rectilinear grid even when the sizes happen to describe a regular one
+    (gh-4272)."""
+    from zarr.core.metadata.v3 import ArrayV3Metadata
+
+    arr = zarr.create_array(store=tmp_path / "arr.zarr", shape=shape, chunks=chunks, dtype="int32")
+    assert isinstance(arr.metadata, ArrayV3Metadata)
+    assert isinstance(arr.metadata.chunk_grid, expected_type)
+
+
+def test_mixed_scalar_and_list_dims_keep_shorthand(tmp_path: Path) -> None:
+    """In a mixed spec, a scalar dimension serializes as the bare-int step-size
+    shorthand while an explicit list keeps its per-chunk edges."""
+    from zarr.core.metadata.v3 import ArrayV3Metadata
+
+    arr = zarr.create_array(
+        store=tmp_path / "arr.zarr", shape=(30, 100), chunks=(5, [10, 20, 70]), dtype="int32"
+    )
+    assert isinstance(arr.metadata, ArrayV3Metadata)
+    grid = arr.metadata.chunk_grid
+    assert isinstance(grid, RectilinearChunkGridMetadata)
+    assert grid.chunk_shapes == (5, (10, 20, 70))
+
+
+def test_resize_uniform_rectilinear_appends_edge() -> None:
+    """Growing an explicitly rectilinear array whose edges look regular appends
+    a new edge chunk, while the same sizes declared as a scalar extend the
+    uniform pattern instead (gh-4272), so an append-only workload writing the
+    grown region touches exactly one new chunk."""
+    from zarr.core.metadata.v3 import ArrayV3Metadata
+
+    rect_store: dict[str, Any] = {}
+    rect = zarr.create_array(store=rect_store, shape=(30,), chunks=[[10, 10, 10]], dtype="int32")
+    rect.resize((45,))
+    assert isinstance(rect.metadata, ArrayV3Metadata)
+    rect_grid = rect.metadata.chunk_grid
+    assert isinstance(rect_grid, RectilinearChunkGridMetadata)
+    assert rect_grid.chunk_shapes == ((10, 10, 10, 15),)
+    assert rect.nchunks == 4
+    # the appended region is exactly the new edge chunk
+    rect[30:45] = 1
+    assert sorted(k for k in rect_store if k.startswith("c/")) == ["c/3"]
+
+    reg_store: dict[str, Any] = {}
+    reg = zarr.create_array(store=reg_store, shape=(30,), chunks=(10,), dtype="int32")
+    reg.resize((45,))
+    assert isinstance(reg.metadata, ArrayV3Metadata)
+    reg_grid = reg.metadata.chunk_grid
+    assert isinstance(reg_grid, RegularChunkGridMetadata)
+    assert reg_grid.chunk_shape == (10,)
+    assert reg.nchunks == 5
+    # the same write straddles two chunks of the extended uniform pattern
+    reg[30:45] = 1
+    assert sorted(k for k in reg_store if k.startswith("c/")) == ["c/3", "c/4"]
+
+
 # ---------------------------------------------------------------------------
 # Sharding compatibility tests
 # ---------------------------------------------------------------------------