Skip to content

Commit 7afdc8d

Browse files
d-v-bd-v-b
authored
feat(zarr-metadata): partial metadata types (#3982)
* docs(plan): add Partial* TypedDicts implementation plan Plan for adding ArrayMetadataV3Partial, GroupMetadataV3Partial, ArrayMetadataV2Partial, and GroupMetadataV2Partial to zarr-metadata as siblings of the existing full TypedDicts, plus an equivalence test that prevents drift. * feat(zarr-metadata): add ArrayMetadataV3Partial Sibling TypedDict to ArrayMetadataV3 with total=False, intended for typing dicts that intentionally hold a subset of a complete v3 array metadata document (test fixtures, fragment templates). Drift between the two is prevented by a new equivalence test. * feat(zarr-metadata): add GroupMetadataV3Partial Sibling TypedDict to GroupMetadataV3 with total=False. * feat(zarr-metadata): add ArrayMetadataV2Partial Sibling TypedDict to ArrayMetadataV2 with total=False. * feat(zarr-metadata): add GroupMetadataV2Partial Sibling TypedDict to GroupMetadataV2 with total=False. Added for symmetry with the other v2/v3 *Partial types; the only practical difference from GroupMetadataV2 is that zarr_format becomes optional. * docs(zarr-metadata): align Partial* docstring shapes across all four types After landing all four *Partial classes, the docstrings had drifted into two shapes: ArrayMetadataV2Partial / GroupMetadataV2Partial used the self-contained form (summary → use-case → NotRequired rationale → drift), while GroupMetadataV3Partial deferred its use-case paragraph to ArrayMetadataV3Partial, and ArrayMetadataV3Partial put its drift sentence before the NotRequired rationale. Standardize on the self-contained form with consistent paragraph order so each Partial reads on its own and any future prose edit only needs to touch the one class it concerns. * chore: remove superpowers docs
1 parent 8bfcc66 commit 7afdc8d

6 files changed

Lines changed: 189 additions & 3 deletions

File tree

‎packages/zarr-metadata/src/zarr_metadata/__init__.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,34 +4,39 @@
44
from zarr_metadata.v2.array import (
55
ArrayDimensionSeparatorV2,
66
ArrayMetadataV2,
7+
ArrayMetadataV2Partial,
78
ArrayOrderV2,
89
DataTypeMetadataV2,
910
ZArrayMetadata,
1011
)
1112
from zarr_metadata.v2.attributes import ZAttrsMetadata
1213
from zarr_metadata.v2.codec import CodecMetadataV2
1314
from zarr_metadata.v2.consolidated import ConsolidatedMetadataV2
14-
from zarr_metadata.v2.group import GroupMetadataV2, ZGroupMetadata
15+
from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial, ZGroupMetadata
1516
from zarr_metadata.v3._common import MetadataFieldV3
16-
from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3
17+
from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial, ExtensionFieldV3
1718
from zarr_metadata.v3.consolidated import ConsolidatedMetadataV3
18-
from zarr_metadata.v3.group import GroupMetadataV3
19+
from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial
1920

2021
__version__ = version("zarr-metadata")
2122

2223

2324
__all__ = [
2425
"ArrayDimensionSeparatorV2",
2526
"ArrayMetadataV2",
27+
"ArrayMetadataV2Partial",
2628
"ArrayMetadataV3",
29+
"ArrayMetadataV3Partial",
2730
"ArrayOrderV2",
2831
"CodecMetadataV2",
2932
"ConsolidatedMetadataV2",
3033
"ConsolidatedMetadataV3",
3134
"DataTypeMetadataV2",
3235
"ExtensionFieldV3",
3336
"GroupMetadataV2",
37+
"GroupMetadataV2Partial",
3438
"GroupMetadataV3",
39+
"GroupMetadataV3Partial",
3540
"MetadataFieldV3",
3641
"NamedConfig",
3742
"ZArrayMetadata",

‎packages/zarr-metadata/src/zarr_metadata/v2/array.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,53 @@ class ArrayMetadataV2(TypedDict):
9898
"""
9999

100100

101+
class ArrayMetadataV2Partial(TypedDict, total=False):
102+
"""
103+
Partial form of `ArrayMetadataV2`: every field is `NotRequired`.
104+
105+
Field annotations mirror `ArrayMetadataV2` exactly. The only difference is
106+
`total=False`, which makes every key optional at the type level.
107+
108+
Use this when typing dicts that intentionally hold a subset of a complete
109+
v2 array metadata document — e.g. test fixtures that override only a few
110+
fields of a base template, or callers that build a fragment to be merged
111+
into a complete document elsewhere.
112+
113+
The `NotRequired[...]` wrappers on `dimension_separator` and `attributes`
114+
are intentional: keeping them preserves byte-identical `__annotations__`
115+
with `ArrayMetadataV2` so the `==` check in
116+
`tests/test_partial_equivalence.py` passes without special-casing those
117+
fields (PEP 655 explicitly permits `NotRequired` inside `total=False`).
118+
119+
Note: v2 array metadata has no `extra_items` setting (the v2 spec has no
120+
extension-field concept), so this partial inherits the same closed shape.
121+
122+
Drift between this type and `ArrayMetadataV2` is prevented by
123+
`tests/test_partial_equivalence.py`.
124+
"""
125+
126+
zarr_format: Literal[2]
127+
shape: tuple[int, ...]
128+
chunks: tuple[int, ...]
129+
dtype: DataTypeMetadataV2
130+
compressor: CodecMetadataV2 | None
131+
fill_value: object
132+
order: ArrayOrderV2
133+
filters: tuple[CodecMetadataV2, ...] | None
134+
dimension_separator: NotRequired[ArrayDimensionSeparatorV2]
135+
attributes: NotRequired[Mapping[str, object]]
136+
"""User attributes from the sibling `.zattrs` file (not part of `.zarray`).
137+
138+
See the class docstring for the rationale behind the merged representation.
139+
"""
140+
141+
101142
__all__ = [
102143
"ARRAY_DIMENSION_SEPARATOR_V2",
103144
"ARRAY_ORDER_V2",
104145
"ArrayDimensionSeparatorV2",
105146
"ArrayMetadataV2",
147+
"ArrayMetadataV2Partial",
106148
"ArrayOrderV2",
107149
"DataTypeMetadataV2",
108150
"ZArrayMetadata",

‎packages/zarr-metadata/src/zarr_metadata/v2/group.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,38 @@ class GroupMetadataV2(TypedDict):
4242
attributes: NotRequired[Mapping[str, object]]
4343

4444

45+
class GroupMetadataV2Partial(TypedDict, total=False):
46+
"""
47+
Partial form of `GroupMetadataV2`: every field is `NotRequired`.
48+
49+
Field annotations mirror `GroupMetadataV2` exactly. The only difference is
50+
`total=False`, which makes every key optional at the type level.
51+
52+
Use this when typing dicts that intentionally hold a subset of a complete
53+
v2 group metadata document — e.g. test fixtures that override only a few
54+
fields of a base template, or callers that build a fragment to be merged
55+
into a complete document elsewhere. Provided for symmetry with the other
56+
`*Partial` types; the practical effect is that `zarr_format` becomes optional.
57+
58+
The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it
59+
preserves byte-identical `__annotations__` with `GroupMetadataV2` so the
60+
`==` check in `tests/test_partial_equivalence.py` passes without
61+
special-casing that field (PEP 655 explicitly permits `NotRequired` inside
62+
`total=False`).
63+
64+
Note: v2 group metadata has no `extra_items` setting (the v2 spec has no
65+
extension-field concept), so this partial inherits the same closed shape.
66+
67+
Drift between this type and `GroupMetadataV2` is prevented by
68+
`tests/test_partial_equivalence.py`.
69+
"""
70+
71+
zarr_format: Literal[2]
72+
attributes: NotRequired[Mapping[str, object]]
73+
74+
4575
__all__ = [
4676
"GroupMetadataV2",
77+
"GroupMetadataV2Partial",
4778
"ZGroupMetadata",
4879
]

‎packages/zarr-metadata/src/zarr_metadata/v3/array.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,44 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[
6262
dimension_names: NotRequired[tuple[str | None, ...]]
6363

6464

65+
class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg]
66+
"""
67+
Partial form of `ArrayMetadataV3`: every field is `NotRequired`.
68+
69+
Field annotations and `extra_items=` mirror `ArrayMetadataV3` exactly.
70+
The only difference is `total=False`, which makes every key optional
71+
at the type level.
72+
73+
Use this when typing dicts that intentionally hold a subset of a complete
74+
v3 array metadata document — e.g. test fixtures that override only a few
75+
fields of a base template, or callers that build a fragment to be merged
76+
into a complete document elsewhere.
77+
78+
The `NotRequired[...]` wrappers on `attributes`, `storage_transformers`,
79+
and `dimension_names` are intentional: keeping them preserves byte-identical
80+
`__annotations__` with `ArrayMetadataV3` so the `==` check in
81+
`tests/test_partial_equivalence.py` passes without special-casing those
82+
fields (PEP 655 explicitly permits `NotRequired` inside `total=False`).
83+
84+
Drift between this type and `ArrayMetadataV3` is prevented by
85+
`tests/test_partial_equivalence.py`.
86+
"""
87+
88+
zarr_format: Literal[3]
89+
node_type: Literal["array"]
90+
data_type: MetadataFieldV3
91+
shape: tuple[int, ...]
92+
chunk_grid: MetadataFieldV3
93+
chunk_key_encoding: MetadataFieldV3
94+
fill_value: object
95+
codecs: tuple[MetadataFieldV3, ...]
96+
attributes: NotRequired[Mapping[str, object]]
97+
storage_transformers: NotRequired[tuple[MetadataFieldV3, ...]]
98+
dimension_names: NotRequired[tuple[str | None, ...]]
99+
100+
65101
__all__ = [
66102
"ArrayMetadataV3",
103+
"ArrayMetadataV3Partial",
67104
"ExtensionFieldV3",
68105
]

‎packages/zarr-metadata/src/zarr_metadata/v3/group.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,35 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[
2525
attributes: NotRequired[Mapping[str, object]]
2626

2727

28+
class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg]
29+
"""
30+
Partial form of `GroupMetadataV3`: every field is `NotRequired`.
31+
32+
Field annotations and `extra_items=` mirror `GroupMetadataV3` exactly.
33+
The only difference is `total=False`, which makes every key optional
34+
at the type level.
35+
36+
Use this when typing dicts that intentionally hold a subset of a complete
37+
v3 group metadata document — e.g. test fixtures that override only a few
38+
fields of a base template, or callers that build a fragment to be merged
39+
into a complete document elsewhere.
40+
41+
The `NotRequired[...]` wrapper on `attributes` is intentional: keeping it
42+
preserves byte-identical `__annotations__` with `GroupMetadataV3` so the
43+
`==` check in `tests/test_partial_equivalence.py` passes without
44+
special-casing that field (PEP 655 explicitly permits `NotRequired` inside
45+
`total=False`).
46+
47+
Drift between this type and `GroupMetadataV3` is prevented by
48+
`tests/test_partial_equivalence.py`.
49+
"""
50+
51+
zarr_format: Literal[3]
52+
node_type: Literal["group"]
53+
attributes: NotRequired[Mapping[str, object]]
54+
55+
2856
__all__ = [
2957
"GroupMetadataV3",
58+
"GroupMetadataV3Partial",
3059
]

‎packages/zarr-metadata/tests/test_partial_equivalence.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""Drift-prevention tests for Partial* TypedDict variants.
2+
3+
Each *Partial TypedDict in the package must declare the same fields
4+
(with the same annotations) and the same extra_items setting as its
5+
full counterpart. The only intentional difference is total=False
6+
(i.e. every field becomes NotRequired). This test enforces that
7+
invariant so adding a field to the full type without mirroring it
8+
on the partial fails CI.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
from typing import Any
14+
15+
import pytest
16+
17+
from zarr_metadata.v2.array import ArrayMetadataV2, ArrayMetadataV2Partial
18+
from zarr_metadata.v2.group import GroupMetadataV2, GroupMetadataV2Partial
19+
from zarr_metadata.v3.array import ArrayMetadataV3, ArrayMetadataV3Partial
20+
from zarr_metadata.v3.group import GroupMetadataV3, GroupMetadataV3Partial
21+
22+
# (full, partial) pairs to check. Add new pairs here as more are introduced.
23+
PAIRS: list[tuple[type, type]] = [
24+
(ArrayMetadataV3, ArrayMetadataV3Partial),
25+
(GroupMetadataV3, GroupMetadataV3Partial),
26+
(ArrayMetadataV2, ArrayMetadataV2Partial),
27+
(GroupMetadataV2, GroupMetadataV2Partial),
28+
]
29+
30+
31+
@pytest.mark.parametrize(("full", "partial"), PAIRS, ids=lambda p: p.__name__)
32+
def test_partial_matches_full(full: Any, partial: Any) -> None:
33+
"""Partial TypedDict has identical fields and extra_items, only total differs."""
34+
assert full.__annotations__ == partial.__annotations__, (
35+
f"{partial.__name__} fields drifted from {full.__name__}: "
36+
f"full={set(full.__annotations__)}, partial={set(partial.__annotations__)}"
37+
)
38+
assert getattr(full, "__extra_items__", None) == getattr(partial, "__extra_items__", None), (
39+
f"{partial.__name__} extra_items differs from {full.__name__}"
40+
)
41+
assert full.__total__ is True, f"{full.__name__} must be declared with total=True (default)"
42+
assert partial.__total__ is False, f"{partial.__name__} must be declared with total=False"

0 commit comments

Comments
 (0)