diff --git a/.github/workflows/ci-gpu.yml b/.github/workflows/ci-gpu.yml index 3c3e193..554b6dd 100644 --- a/.github/workflows/ci-gpu.yml +++ b/.github/workflows/ci-gpu.yml @@ -47,7 +47,7 @@ jobs: with: enable-cache: true # "auto" is `false` on non-GitHub runners - name: Install package - run: uv pip install --system -e .[test,full] cupy-cuda12x --extra-index-url=https://pypi.nvidia.com --index-strategy=unsafe-best-match + run: uv pip install --system --group=test -e .[full] cupy-cuda12x --extra-index-url=https://pypi.nvidia.com --index-strategy=unsafe-best-match - name: List installed packages run: uv pip list - name: Run tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4e24fe..a902689 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: envs: ${{ steps.get-envs.outputs.envs }} pythons: ${{ steps.get-pythons.outputs.pythons }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: { fetch-depth: 0, filter: "blob:none" } - uses: astral-sh/setup-uv@v7 - name: Get test environments @@ -50,7 +50,7 @@ jobs: python: "3.13" os: macos-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: { fetch-depth: 0, filter: "blob:none" } - uses: astral-sh/setup-uv@v7 with: @@ -73,13 +73,13 @@ jobs: name: CPU Benchmarks runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: { fetch-depth: 0, filter: "blob:none" } - uses: actions/setup-python@v6 with: python-version: '3.13' - uses: astral-sh/setup-uv@v7 - - run: uv pip install --system -e .[test,full] + - run: uv pip install --system --group=test -e .[full] - uses: CodSpeedHQ/action@v3 with: run: pytest -m benchmark --codspeed -n auto @@ -88,7 +88,7 @@ jobs: name: Import Tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: { fetch-depth: 0, filter: "blob:none" } - uses: actions/setup-python@v6 with: @@ -100,20 +100,15 @@ jobs: - run: python -c 'import testing.fast_array_utils as tfau; print(tfau.ArrayType("numpy", "ndarray"))' check: name: Static Checks - needs: get-environments runs-on: ubuntu-latest - strategy: - matrix: - python-version: ${{ fromJSON(needs.get-environments.outputs.pythons) }} env: SKIP: no-commit-to-branch # this CI runs on the main branch steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: { fetch-depth: 0, filter: "blob:none" } - - uses: actions/setup-python@v6 + - uses: j178/prek-action@v1 with: - python-version: ${{ matrix.python-version }} - - uses: pre-commit/action@v3.0.1 + working-directory: ${{ github.workspace }} pass: name: All Checks if: always() diff --git a/.gitignore b/.gitignore index c7ff898..09ee092 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ _version.py /build/ /dist/ /.python-version +/*.lock # Testing /test-data/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2f0718f..8d59284 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,43 +1,108 @@ repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - id: no-commit-to-branch - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.5 - hooks: - - id: ruff-check - args: [--fix, --exit-non-zero-on-fix] - - id: ruff-check - args: [--preview, --select=CPY] - - id: ruff-format - - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.11.1 - hooks: - - id: pyproject-fmt - - repo: https://github.com/biomejs/pre-commit - rev: v2.3.5 - hooks: - - id: biome-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.18.2 - hooks: - - id: mypy - args: [--config-file=pyproject.toml, .] - pass_filenames: false - additional_dependencies: - - pytest - - pytest-codspeed!=4.0.0 # https://github.com/CodSpeedHQ/pytest-codspeed/pull/84 - - numba - - numpy - - scipy-stubs - - dask - - zarr - - h5py - - anndata - - types-docutils - - sphinx +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: no-commit-to-branch +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.14.5 + hooks: + - id: ruff-check + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-check + args: [--preview, --select=CPY] + - id: ruff-format +- repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.11.1 + hooks: + - id: pyproject-fmt +- repo: https://github.com/biomejs/pre-commit + rev: v2.3.5 + hooks: + - id: biome-format +- repo: https://github.com/H4rryK4ne/update-mypy-hook + rev: a8b56c4055ff0c7c589794c02813ef8e9d5704fc + hooks: + - id: update-mypy-hook +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.18.2 + hooks: + - id: mypy + args: [--config-file=pyproject.toml, .] + pass_filenames: false + language_version: '3.13' + additional_dependencies: + - alabaster==1.0.0 + - anndata==0.12.6 + - array-api-compat==1.12.0 + - babel==2.17.0 + - certifi==2025.11.12 + - cffi==2.0.0 + - charset-normalizer==3.4.4 + - click==8.3.1 + - cloudpickle==3.1.2 + - colorama==0.4.6 ; sys_platform == 'win32' + - coverage==7.12.0 + - dask==2025.11.0 + - docutils==0.22.3 + - donfig==0.8.1.post1 + - execnet==2.1.2 + - fsspec==2025.10.0 + - google-crc32c==1.7.1 + - h5py==3.15.1 + - idna==3.11 + - imagesize==1.4.1 + - iniconfig==2.3.0 + - jinja2==3.1.6 + - joblib==1.5.2 + - legacy-api-wrap==1.5 + - llvmlite==0.45.1 + - locket==1.0.0 + - markdown-it-py==4.0.0 + - markupsafe==3.0.3 + - mdurl==0.1.2 + - natsort==8.4.0 + - numba==0.62.1 + - numcodecs==0.16.5 + - numpy==2.3.5 + - numpy-typing-compat==20250818.2.3 + - optype==0.14.0 + - packaging==25.0 + - pandas==2.3.3 + - partd==1.4.2 + - pluggy==1.6.0 + - pycparser==2.23 ; implementation_name != 'PyPy' + - pygments==2.19.2 + - pytest==9.0.1 + - pytest-codspeed==4.2.0 + - pytest-doctestplus==1.6.0 + - pytest-xdist==3.8.0 + - python-dateutil==2.9.0.post0 + - pytz==2025.2 + - pyyaml==6.0.3 + - requests==2.32.5 + - rich==14.2.0 + - roman-numerals==3.1.0 + - scikit-learn==1.7.2 + - scipy==1.16.3 + - scipy-stubs==1.16.3.2 + - six==1.17.0 + - snowballstemmer==3.0.1 + - sphinx==9.0.1 + - sphinxcontrib-applehelp==2.0.0 + - sphinxcontrib-devhelp==2.0.0 + - sphinxcontrib-htmlhelp==2.1.0 + - sphinxcontrib-jsmath==1.0.1 + - sphinxcontrib-qthelp==2.0.0 + - sphinxcontrib-serializinghtml==2.0.0 + - threadpoolctl==3.6.0 + - toolz==1.1.0 + - types-docutils==0.22.3.20251115 + - tzdata==2025.2 + - urllib3==2.5.0 + - zarr==3.1.5 ci: - skip: [mypy] # too big + skip: + - mypy # too big + - update-mypy-hook # offline? diff --git a/.readthedocs.yml b/.readthedocs.yml index f32ea6c..1b8283c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,12 +3,12 @@ build: os: ubuntu-24.04 tools: python: "3.13" -python: - install: - - method: pip - path: . - extra_requirements: - - doc -sphinx: - configuration: docs/conf.py - fail_on_warning: true + jobs: + create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - uvx hatch run docs:build + - mv docs/_build $READTHEDOCS_OUTPUT diff --git a/docs/conf.py b/docs/conf.py index 657de4e..77cd5cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,8 +43,6 @@ "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.napoleon", - # "scanpydoc.definition_list_typed_field", - "scanpydoc.elegant_typehints", "sphinx_autofixture", ] diff --git a/pyproject.toml b/pyproject.toml index ae26151..f7d5a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ build-backend = "hatchling.build" requires = [ "hatch-docstring-description>=1.1.1", "hatch-fancy-pypi-readme", - "hatch-min-requirements", "hatch-vcs", "hatchling", ] @@ -27,23 +26,23 @@ dynamic = [ "description", "readme", "version" ] dependencies = [ "numpy>=2" ] optional-dependencies.accel = [ "numba>=0.57" ] optional-dependencies.dask = [ "dask>=2023.6.1" ] -optional-dependencies.doc = [ - "furo>=2024.8.6", - "pytest>=8.4", - "scanpydoc>=0.15.4", - "sphinx>=8.2.3", - "sphinx-autodoc-typehints>=3.2", - "sphinx-autofixture>=0.4.1", -] optional-dependencies.full = [ "fast-array-utils[accel,dask,sparse]", "h5py", "zarr" ] optional-dependencies.sparse = [ "scipy>=1.13" ] -optional-dependencies.test = [ +optional-dependencies.testing = [ "packaging" ] +urls.'Documentation' = "https://icb-fast-array-utils.readthedocs-hosted.com/" +urls.'Issue Tracker' = "https://github.com/scverse/fast-array-utils/issues" +urls.'Source Code' = "https://github.com/scverse/fast-array-utils" +entry-points.pytest11.fast_array_utils = "testing.fast_array_utils.pytest" + +[dependency-groups] +test = [ "anndata", - "fast-array-utils[accel,test-min]", - "numcodecs<0.16", # zarr 2 needs this - "zarr<3", # anndata needs this + "fast-array-utils[accel]", + "scikit-learn", + "zarr", + { include-group = "test-min" }, ] -optional-dependencies.test-min = [ +test-min = [ "coverage[toml]", "fast-array-utils[sparse,testing]", # include sparse for testing numba-less to_dense "pytest", @@ -51,12 +50,21 @@ optional-dependencies.test-min = [ "pytest-doctestplus", "pytest-xdist", ] -optional-dependencies.testing = [ "packaging" ] -urls.'Documentation' = "https://icb-fast-array-utils.readthedocs-hosted.com/" -urls.'Issue Tracker' = "https://github.com/scverse/fast-array-utils/issues" -urls.'Source Code' = "https://github.com/scverse/fast-array-utils" - -entry-points.pytest11.fast_array_utils = "testing.fast_array_utils.pytest" +doc = [ + "furo>=2024.8.6", + "pytest>=8.4", + "sphinx>=9.0.1", + "sphinx-autofixture>=0.4.1", +] +# for update-mypy-hook +mypy = [ + "fast-array-utils[full]", + "scipy-stubs", + # TODO: replace sphinx with this: { include-group = "doc" }, + "sphinx", + "types-docutils", + { include-group = "test" }, +] [tool.hatch.version] source = "vcs" @@ -69,7 +77,6 @@ path = "README.rst" start-after = ".. begin" [tool.hatch.metadata.hooks.docstring-description] -[tool.hatch.metadata.hooks.min_requirements] [tool.hatch.build.targets.wheel] packages = [ "src/testing", "src/fast_array_utils" ] @@ -78,29 +85,26 @@ packages = [ "src/testing", "src/fast_array_utils" ] installer = "uv" [tool.hatch.envs.docs] -features = [ "doc" ] +dependency-groups = [ "doc" ] scripts.build = "sphinx-build -M html docs docs/_build" scripts.clean = "git clean -fdX docs" scripts.open = "python -m webbrowser -t docs/_build/html/index.html" [tool.hatch.envs.hatch-test] default-args = [ ] -features = [ "test-min" ] -extra-dependencies = [ "ipykernel", "ipycytoscape" ] +dependency-groups = [ "test-min" ] +# TODO: remove scipy once https://github.com/pypa/hatch/pull/2127 is released +extra-dependencies = [ "ipykernel", "ipycytoscape", "scipy" ] env-vars.CODSPEED_PROFILE_FOLDER = "test-data/codspeed" overrides.matrix.extras.features = [ { if = [ "full" ], value = "full" }, - { if = [ "full" ], value = "test" }, -] -overrides.matrix.extras.dependencies = [ - { if = [ "full" ], value = "scipy-stubs" }, - { if = [ "full" ], value = "scikit-learn" }, ] -overrides.matrix.resolution.features = [ - { if = [ "lowest" ], value = "min-reqs" }, # feature added by hatch-min-requirements +overrides.matrix.extras.dependency-groups = [ + { if = [ "full" ], value = "test" }, ] overrides.matrix.resolution.dependencies = [ - # TODO: move to min dep once this is fixed: https://github.com/tlambert03/hatch-min-requirements/issues/11 + # TODO: move to `min-reqs` feature once this is fixed: https://github.com/tlambert03/hatch-min-requirements/issues/11 + { if = [ "lowest" ], value = "numpy==2" }, { if = [ "lowest" ], value = "dask==2023.6.1" }, { if = [ "lowest" ], value = "scipy==1.13.0" }, ] @@ -114,6 +118,9 @@ python = [ "3.12" ] extras = [ "full" ] resolution = [ "lowest" ] +[tool.uv] +override-dependencies = [ "sphinx>=9.0.1" ] + [tool.ruff] line-length = 160 namespace-packages = [ "src/testing" ] @@ -154,10 +161,10 @@ lint.isort.lines-after-imports = 2 lint.pydocstyle.convention = "numpy" lint.future-annotations = true -[tool.pytest.ini_options] +[tool.pytest] +strict = true addopts = [ "--import-mode=importlib", - "--strict-markers", "--doctest-modules", "--doctest-plus", "--pyargs", @@ -179,7 +186,6 @@ filterwarnings = [ markers = [ "benchmark: marks tests as benchmark (to run with `--codspeed`)", ] -xfail_strict = true [tool.coverage] run.data_file = "test-data/.coverage" diff --git a/src/fast_array_utils/types.py b/src/fast_array_utils/types.py index 41e6c6f..c1fbf26 100644 --- a/src/fast_array_utils/types.py +++ b/src/fast_array_utils/types.py @@ -109,7 +109,7 @@ from anndata.abc import CSCDataset, CSRDataset # type: ignore[import-untyped] else: # pragma: no cover try: # only exists in anndata 0.11+ - from anndata.abc import CSCDataset, CSRDataset # type: ignore[import-untyped] + from anndata.abc import CSCDataset, CSRDataset except ImportError: CSRDataset = type("CSRDataset", (), {}) CSCDataset = type("CSCDataset", (), {}) diff --git a/src/fast_array_utils/typing.py b/src/fast_array_utils/typing.py index aebcdbb..aee112c 100644 --- a/src/fast_array_utils/typing.py +++ b/src/fast_array_utils/typing.py @@ -3,27 +3,22 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from numpy.typing import NDArray from . import types -if TYPE_CHECKING: - from typing import TypeAlias - - __all__ = ["CpuArray", "DiskArray", "GpuArray"] -# change to `type` syntax once this is released: https://github.com/sphinx-doc/sphinx/pull/13508 -CpuArray: TypeAlias = NDArray[Any] | types.CSBase # noqa: UP040 +type CpuArray = NDArray[Any] | types.CSBase """Arrays and matrices stored in CPU memory.""" -GpuArray: TypeAlias = types.CupyArray | types.CupyCSMatrix # noqa: UP040 +type GpuArray = types.CupyArray | types.CupyCSMatrix """Arrays and matrices stored in GPU memory.""" # TODO(flying-sheep): types.CSDataset # noqa: TD003 -DiskArray: TypeAlias = types.H5Dataset | types.ZarrArray # noqa: UP040 +type DiskArray = types.H5Dataset | types.ZarrArray # type: ignore[type-arg] """Arrays and matrices stored on disk.""" diff --git a/src/testing/fast_array_utils/_array_type.py b/src/testing/fast_array_utils/_array_type.py index b5649a3..9db7335 100644 --- a/src/testing/fast_array_utils/_array_type.py +++ b/src/testing/fast_array_utils/_array_type.py @@ -264,7 +264,7 @@ def _to_h5py_dataset(self, x: ArrayLike | Array, /, *, dtype: DTypeLike | None = return ctx.hdf5_file.create_dataset("data", arr.shape, arr.dtype, data=arr) @classmethod - def _to_zarr_array(cls, x: ArrayLike | Array, /, *, dtype: DTypeLike | None = None) -> types.ZarrArray: + def _to_zarr_array(cls, x: ArrayLike | Array, /, *, dtype: DTypeLike | None = None) -> types.ZarrArray[Any]: """Convert to a zarr array.""" import zarr @@ -298,8 +298,8 @@ def _to_cs_dataset(self, x: ArrayLike | Array, /, *, dtype: DTypeLike | None = N cls = cast("type[types.csr_array[Any, tuple[int, int]] | types.csc_array]", csr_array if self.cls is types.CSRDataset else csc_array) x_sparse = self._to_scipy_sparse(x, dtype=dtype, cls=cls) - anndata.io.write_elem(grp, "/", x_sparse) - return anndata.io.sparse_dataset(grp) + anndata.io.write_elem(grp, "/mtx", x_sparse) + return anndata.io.sparse_dataset(grp["mtx"]) def _to_scipy_sparse( self, diff --git a/tests/test_stats.py b/tests/test_stats.py index 8d0a777..334d1fd 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -292,7 +292,7 @@ def test_mean_var_sparse_64(array_type: ArrayType[types.CSArray], axis: Literal[ @pytest.mark.skipif(not find_spec("sklearn"), reason="sklearn not installed") @pytest.mark.array_type(Flags.Sparse, skip=Flags.Matrix | Flags.Dask | Flags.Disk | Flags.Gpu) -def test_mean_var_sparse_32(array_type: ArrayType[types.CSArray]) -> None: +def test_mean_var_sparse_32(array_type: ArrayType[types.CSArray], subtests: pytest.Subtests) -> None: """Test whether we are more accurate for 32 bit.""" from sklearn.utils.sparsefuncs import mean_variance_axis @@ -304,10 +304,11 @@ def test_mean_var_sparse_32(array_type: ArrayType[types.CSArray]) -> None: fau[n_bit] = stats.mean_var(mtx, axis=0) skl[n_bit] = mean_variance_axis(mtx, 0) - for stat, _ in enumerate(["mean", "var"]): - resid_fau = np.mean(np.abs(fau[64][stat] - fau[32][stat])) - resid_skl = np.mean(np.abs(skl[64][stat] - skl[32][stat])) - assert resid_fau < resid_skl + for stat, name in enumerate(["mean", "var"]): + with subtests.test(stat=name): + resid_fau = np.mean(np.abs(fau[64][stat] - fau[32][stat])) + resid_skl = np.mean(np.abs(skl[64][stat] - skl[32][stat])) + assert resid_fau < resid_skl @pytest.mark.array_type({at for at in SUPPORTED_TYPES if at.flags & Flags.Sparse and at.flags & Flags.Dask}) diff --git a/typings/cupy/_creation/from_data.pyi b/typings/cupy/_creation/from_data.pyi index da6f030..b1ba1ab 100644 --- a/typings/cupy/_creation/from_data.pyi +++ b/typings/cupy/_creation/from_data.pyi @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -from typing import Literal +from typing import Any, Literal import h5py import zarr @@ -8,7 +8,7 @@ from numpy.typing import ArrayLike, DTypeLike from .._core import ndarray def asarray( - a: ArrayLike | h5py.Dataset | zarr.Array, + a: ArrayLike | h5py.Dataset | zarr.Array[Any], dtype: DTypeLike | None = None, order: Literal["C", "F", "A", "K"] | None = None, *, diff --git a/typings/h5py.pyi b/typings/h5py.pyi index 7e4809f..bfb92bc 100644 --- a/typings/h5py.pyi +++ b/typings/h5py.pyi @@ -14,7 +14,8 @@ class Dataset(HLObject): shape: tuple[int, ...] ndim: int -class Group(HLObject): ... +class Group(HLObject): + def __getitem__(self, name: str) -> Group | Dataset: ... class File(Group, closing[File]): # not actually a subclass of closing filename: str diff --git a/typings/sklearn/utils/sparsefuncs.pyi b/typings/sklearn/utils/sparsefuncs.pyi index 2a73b01..51eb3cc 100644 --- a/typings/sklearn/utils/sparsefuncs.pyi +++ b/typings/sklearn/utils/sparsefuncs.pyi @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MPL-2.0 -from typing import Literal +from typing import Any, Literal import numpy as np from numpy.typing import NDArray @@ -8,6 +8,6 @@ from scipy.sparse import csc_array, csc_matrix, csr_array, csr_matrix def mean_variance_axis( X: csc_array | csc_matrix | csr_array | csr_matrix, # noqa: N803 axis: Literal[0, 1], - weights: NDArray[np.floating] | None = None, + weights: NDArray[np.floating[Any]] | None = None, return_sum_weights: bool = False, ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: ...