Merge branch 'main' into feature-cluster-decision-tree

Author: joe-jhou2

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.6
+    rev: v0.11.8
     hooks:
     -   id: ruff
         args: ["--fix"]

benchmarks/benchmarks/preprocessing_log.py

@@ -8,7 +8,6 @@
 from typing import TYPE_CHECKING
 
 import scanpy as sc
-from scanpy.preprocessing._utils import _get_mean_var
 
 from ._utils import get_dataset, param_skipper
 
@@ -75,19 +74,3 @@ def time_scale(*_):
 
 def peakmem_scale(*_):
     sc.pp.scale(adata, max_value=10)
-
-
-class FastSuite:
-    """Suite for fast preprocessing operations."""
-
-    params: tuple[list[Dataset], list[KeyX]] = (
-        ["pbmc3k", "pbmc68k_reduced", "bmmc", "lung93k"],
-        [None, "off-axis"],
-    )
-    param_names = ("dataset", "layer")
-
-    def time_mean_var(self, *_):
-        _get_mean_var(adata.X)
-
-    def peakmem_mean_var(self, *_):
-        _get_mean_var(adata.X)

docs/conf.py

@@ -128,6 +128,10 @@
     cycler=("https://matplotlib.org/cycler/", None),
     dask=("https://docs.dask.org/en/stable/", None),
     dask_ml=("https://ml.dask.org/", None),
+    fast_array_utils=(
+        "https://icb-fast-array-utils.readthedocs-hosted.com/en/stable/",
+        None,
+    ),
     h5py=("https://docs.h5py.org/en/stable/", None),
     zarr=("https://zarr.readthedocs.io/en/stable/", None),
     ipython=("https://ipython.readthedocs.io/en/stable/", None),

docs/release-notes/3598.dev.md

@@ -0,0 +1 @@
+Replaced several internal utilities with their {mod}`fast_array_utils` counterparts {smaller}`P Angerer`

docs/tutorials/index.md

@@ -1,7 +1,7 @@
 # Tutorials
 
 :::{seealso}
-For more tutorials featureing scanpy and other [scverse](https://scverse.org) ecosystem tools, check out the curated set of tutorials at [scverse.org/learn](https://scverse.org/learn)
+For more tutorials featuring scanpy and other [scverse](https://scverse.org) ecosystem tools, check out the curated set of tutorials at [scverse.org/learn](https://scverse.org/learn)
 :::
 
 ## Basic workflows

pyproject.toml

@@ -48,6 +48,7 @@ classifiers = [
 dependencies = [
     "anndata>=0.9",
     "numpy>=1.25",
+    "fast-array-utils[accel,sparse]>=1.2.1",
     "matplotlib>=3.7",
     "pandas >=2.0",
     "scipy>=1.11",
@@ -213,7 +214,7 @@ exclude_also = [
     "if __name__ == .__main__.:",
     "if TYPE_CHECKING:",
     # https://github.com/numba/numba/issues/4268
-    '@(numba\.|nb\.)njit.*',
+    '@(numba\.|nb\.)?njit.*',
 ]
 
 [tool.ruff]

src/scanpy/_utils/__init__.py

@@ -34,9 +34,8 @@
 from packaging.version import Version
 
 from .. import logging as logg
-from .._compat import CSBase, DaskArray, _CSArray, _CSMatrix, pkg_version
+from .._compat import CSBase, DaskArray, _CSArray, pkg_version
 from .._settings import settings
-from .compute.is_constant import is_constant  # noqa: F401
 
 if Version(anndata_version) >= Version("0.10.0"):
     from anndata._core.sparse_dataset import (
@@ -53,7 +52,7 @@
 
     from anndata import AnnData
     from igraph import Graph
-    from numpy.typing import ArrayLike, DTypeLike, NDArray
+    from numpy.typing import ArrayLike, NDArray
 
     from .._compat import CSRBase
     from ..neighbors import NeighborsParams, RPForestDict
@@ -546,27 +545,6 @@ def get_literal_vals(typ: UnionType | Any) -> KeysView[Any]:
 # --------------------------------------------------------------------------------
 
 
-@singledispatch
-def elem_mul(x: _SupportedArray, y: _SupportedArray) -> _SupportedArray:
-    raise NotImplementedError
-
-
-@elem_mul.register(np.ndarray)
-@elem_mul.register(CSBase)
-def _elem_mul_in_mem(x: _MemoryArray, y: _MemoryArray) -> _MemoryArray:
-    if isinstance(x, CSBase):
-        # returns coo_matrix, so cast back to input type
-        return type(x)(x.multiply(y))
-    return x * y
-
-
-@elem_mul.register(DaskArray)
-def _elem_mul_dask(x: DaskArray, y: DaskArray) -> DaskArray:
-    import dask.array as da
-
-    return da.map_blocks(elem_mul, x, y)
-
-
 if TYPE_CHECKING:
     Scaling_T = TypeVar("Scaling_T", DaskArray, np.ndarray)
 
@@ -606,7 +584,7 @@ def axis_mul_or_truediv(
 @axis_mul_or_truediv.register(CSBase)
 def _(
     X: CSBase,
-    scaling_array,
+    scaling_array: np.ndarray,
     axis: Literal[0, 1],
     op: Callable[[Any, Any], Any],
     *,
@@ -746,78 +724,6 @@ def _(X: DaskArray, axis: Literal[0, 1]) -> DaskArray:
     )
 
 
-@overload
-def axis_sum(
-    X: _CSMatrix,
-    *,
-    axis: tuple[Literal[0, 1], ...] | Literal[0, 1] | None = None,
-    dtype: DTypeLike | None = None,
-) -> np.matrix: ...
-
-
-@overload
-def axis_sum(
-    X: np.ndarray,  # TODO: or sparray
-    *,
-    axis: tuple[Literal[0, 1], ...] | Literal[0, 1] | None = None,
-    dtype: DTypeLike | None = None,
-) -> np.ndarray: ...
-
-
-@singledispatch
-def axis_sum(
-    X: np.ndarray | CSBase,
-    *,
-    axis: tuple[Literal[0, 1], ...] | Literal[0, 1] | None = None,
-    dtype: DTypeLike | None = None,
-) -> np.ndarray | np.matrix:
-    return np.sum(X, axis=axis, dtype=dtype)
-
-
-@axis_sum.register(DaskArray)
-def _(
-    X: DaskArray,
-    *,
-    axis: tuple[Literal[0, 1], ...] | Literal[0, 1] | None = None,
-    dtype: DTypeLike | None = None,
-) -> DaskArray:
-    import dask.array as da
-
-    if dtype is None:
-        dtype = getattr(np.zeros(1, dtype=X.dtype).sum(), "dtype", object)
-
-    if isinstance(X._meta, np.ndarray) and not isinstance(X._meta, np.matrix):
-        return X.sum(axis=axis, dtype=dtype)
-
-    def sum_drop_keepdims(*args, **kwargs):
-        kwargs.pop("computing_meta", None)
-        # masked operations on sparse produce which numpy matrices gives the same API issues handled here
-        if isinstance(X._meta, _CSMatrix | np.matrix) or isinstance(
-            args[0], _CSMatrix | np.matrix
-        ):
-            kwargs.pop("keepdims", None)
-            axis = kwargs["axis"]
-            if isinstance(axis, tuple):
-                if len(axis) != 1:
-                    msg = f"`axis_sum` can only sum over one axis when `axis` arg is provided but got {axis} instead"
-                    raise ValueError(msg)
-                kwargs["axis"] = axis[0]
-        # returns a np.matrix normally, which is undesireable
-        return np.array(np.sum(*args, dtype=dtype, **kwargs))
-
-    def aggregate_sum(*args, **kwargs):
-        return np.sum(args[0], dtype=dtype, **kwargs)
-
-    return da.reduction(
-        X,
-        sum_drop_keepdims,
-        aggregate_sum,
-        axis=axis,
-        dtype=dtype,
-        meta=np.array([], dtype=dtype),
-    )
-
-
 @singledispatch
 def check_nonnegative_integers(X: _SupportedArray) -> bool | DaskArray:
     """Check values of X to ensure it is count data."""