Merge branch 'v3' into batched-codec-pipeline

Author: normanrz

.flake8

@@ -1,2 +1,4 @@
 # lint codebase with black and ruff
 4e348d6b80c96da461fd866576c971b8a659ba15
+# migrate from black to ruff format
+22cea005629913208a85799372e045f353744add

.git-blame-ignore-revs

@@ -10,15 +10,22 @@ on:
     branches: [ v3 ]
 
 jobs:
-  run-tests:
+  test:
+    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
 
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11']
+        numpy-version: ['1.24', '1.26']
+        dependency-set: ["minimal", "optional"]
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: ${{ matrix.python-version }}
         cache: 'pip'
     - name: Install Hatch
       run: |
@@ -29,8 +36,8 @@ jobs:
         hatch env create
     - name: Run Tests
       run: |
-        hatch run test:run
+        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run
     - name: Run mypy
       continue-on-error: true
       run: |
-        hatch run test:run-mypy
+        hatch run test:run-mypy

.github/workflows/test-v3.yml

@@ -7,31 +7,25 @@ default_language_version:
   python: python3
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    # Ruff version.
-    rev: 'v0.0.224'
+    rev: 'v0.2.1'
     hooks:
     - id: ruff
-      # Respect `exclude` and `extend-exclude` settings.
-      args: ["--force-exclude"]
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
-    hooks:
-    - id: black
+      args: ["--fix", "--show-fixes"]
+    - id: ruff-format
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
         args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-yaml
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
       - id: mypy
         files: src
-        exclude: ^src/zarr/v3
         args: []
         additional_dependencies:
           - types-redis

.pre-commit-config.yaml

@@ -8,7 +8,6 @@
 from zarr import blosc
 
 if __name__ == "__main__":
-
     sys.path.insert(0, "..")
 
     # setup

bench/compress_normal.py

@@ -19,13 +19,4 @@ latest GitHub main::
 
     $ pip install git+https://github.com/zarr-developers/zarr-python.git
 
-To work with Zarr source code in development, install from GitHub::
-
-    $ git clone --recursive https://github.com/zarr-developers/zarr-python.git
-    $ cd zarr-python
-    $ python -m pip install -e .
-
-To verify that Zarr has been fully installed, run the test suite::
-
-    $ pip install pytest
-    $ python -m pytest -v --pyargs zarr
+To work with Zarr source code in development, see `Contributing <contributing.html>`_.

docs/installation.rst

@@ -127,7 +127,8 @@ serve = "sphinx-autobuild docs docs/_build --ignore 'docs/_autoapi/**/*' --host
 
 [tool.ruff]
 line-length = 100
-exclude = [
+force-exclude = true
+extend-exclude = [
     ".bzr",
     ".direnv",
     ".eggs",
@@ -146,25 +147,35 @@ exclude = [
     "docs"
 ]
 
-[tool.black]
-line-length = 100
-exclude = '''
-/(
-    \.git
-  | \.mypy_cache
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-  | docs
-)/
-'''
-
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.10"
 ignore_missing_imports = true
-follow_imports = "silent"
+namespace_packages = false
+
+warn_unused_configs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+strict_equality = true
+strict_concatenate = true
+
+check_untyped_defs = true
+disallow_untyped_decorators = true
+
+[[tool.mypy.overrides]]
+module = [
+    "zarr._storage.store",
+    "zarr._storage.v3_storage_transformers",
+    "zarr.v3.group",
+    "zarr.core",
+    "zarr.hierarchy",
+    "zarr.indexing",
+    "zarr.storage",
+    "zarr.sync",
+    "zarr.util",
+    "tests.*",
+]
+check_untyped_defs = false
+
 
 [tool.pytest.ini_options]
 doctest_optionflags = [

pyproject.toml

@@ -87,7 +87,7 @@ def __init__(
                 "https://{}.blob.core.windows.net/".format(account_name),
                 container,
                 credential=account_key,
-                **blob_service_kwargs
+                **blob_service_kwargs,
             )
 
         self.client = client
@@ -240,7 +240,6 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
     def rmdir(self, path=None):
-
         if not path:
             # Currently allowing clear to delete everything as in v2
 

src/zarr/_storage/absstore.py

@@ -629,7 +629,6 @@ def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None:
 
 
 def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
-
     meta_dir = meta_root + path
     meta_dir = meta_dir.rstrip("/")
     _rmdir_from_keys(store, meta_dir)
@@ -643,10 +642,10 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None:
     sfx = _get_metadata_suffix(store)
     array_meta_file = meta_dir + ".array" + sfx
     if array_meta_file in store:
-        store.erase(array_meta_file)  # type: ignore
+        store.erase(array_meta_file)
     group_meta_file = meta_dir + ".group" + sfx
     if group_meta_file in store:
-        store.erase(group_meta_file)  # type: ignore
+        store.erase(group_meta_file)
 
 
 def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]:

src/zarr/_storage/store.py

@@ -118,7 +118,6 @@ def _get_files_and_dirs_from_path(store, path):
 
 
 class FSStoreV3(FSStore, StoreV3):
-
     # FSStoreV3 doesn't use this (FSStore uses it within _normalize_key)
     _META_KEYS = ()
 

src/zarr/_storage/v3.py

@@ -367,7 +367,7 @@ def erase_prefix(self, prefix):
 
     def rmdir(self, path=None):
         path = normalize_storage_path(path)
-        _rmdir_from_keys_v3(self, path)  # type: ignore
+        _rmdir_from_keys_v3(self, path)
 
     def __contains__(self, key):
         if self._is_data_key(key):

src/zarr/_storage/v3_storage_transformers.py

@@ -1,3 +1,4 @@
+from typing import Any
 import warnings
 from collections.abc import MutableMapping
 
@@ -26,7 +27,6 @@ class Attributes(MutableMapping):
     """
 
     def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None):
-
         self._version = getattr(store, "_store_version", 2)
         _Store = Store if self._version == 2 else StoreV3
         self.store = _Store._ensure_store(store)
@@ -40,7 +40,7 @@ def _get_nosync(self):
         try:
             data = self.store[self.key]
         except KeyError:
-            d = dict()
+            d: dict[str, Any] = dict()
             if self._version > 2:
                 d["attributes"] = {}
         else:
@@ -73,7 +73,6 @@ def __getitem__(self, item):
         return self.asdict()[item]
 
     def _write_op(self, f, *args, **kwargs):
-
         # guard condition
         if self.read_only:
             raise PermissionError("attributes are read-only")
@@ -89,7 +88,6 @@ def __setitem__(self, item, value):
         self._write_op(self._setitem_nosync, item, value)
 
     def _setitem_nosync(self, item, value):
-
         # load existing data
         d = self._get_nosync()
 
@@ -106,7 +104,6 @@ def __delitem__(self, item):
         self._write_op(self._delitem_nosync, item)
 
     def _delitem_nosync(self, key):
-
         # load existing data
         d = self._get_nosync()
 
@@ -128,7 +125,6 @@ def put(self, d):
             self._write_op(self._put_nosync, dict(attributes=d))
 
     def _put_nosync(self, d):
-
         d_to_check = d if self._version == 2 else d["attributes"]
         if not all(isinstance(item, str) for item in d_to_check):
             # TODO: Raise an error for non-string keys
@@ -178,7 +174,6 @@ def update(self, *args, **kwargs):
         self._write_op(self._update_nosync, *args, **kwargs)
 
     def _update_nosync(self, *args, **kwargs):
-
         # load existing data
         d = self._get_nosync()