From e066a6c559e9d7f31c359ea95da42d0e45c585ce Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 19 Mar 2024 11:32:32 +0100 Subject: [PATCH 01/65] replace the use of `numpy.array_api` with `array_api_strict` This would make it a dependency of `namedarray`, and not allow behavior that is allowed but not required by the array API standard. Otherwise we can: - use the main `numpy` namespace - use `array_api_compat` (would also be a new dependency) to allow optional behavior --- xarray/namedarray/_array_api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/xarray/namedarray/_array_api.py b/xarray/namedarray/_array_api.py index 977d011c685..4bf81603fa7 100644 --- a/xarray/namedarray/_array_api.py +++ b/xarray/namedarray/_array_api.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from types import ModuleType from typing import Any @@ -21,14 +20,6 @@ ) from xarray.namedarray.core import NamedArray -with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - r"The numpy.array_api submodule is still experimental", - category=UserWarning, - ) - import numpy.array_api as nxp # noqa: F401 - def _get_data_namespace(x: NamedArray[Any, Any]) -> ModuleType: if isinstance(x._data, _arrayapi): From 0c14425c62c00fd6069a9acea947ce83b71cf498 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 19 Mar 2024 12:09:45 +0100 Subject: [PATCH 02/65] replace `numpy.array_api` with `array_api_strict` in the tests --- xarray/tests/__init__.py | 1 - xarray/tests/test_array_api.py | 15 ++------------- xarray/tests/test_namedarray.py | 9 +-------- xarray/tests/test_strategies.py | 12 +++++++----- 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 5007db9eeb2..f4d3da7164a 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -143,7 +143,6 @@ def _importorskip( requires_pandas_version_two = pytest.mark.skipif( not has_pandas_version_two, reason="requires pandas 2.0.0" ) -has_numpy_array_api, requires_numpy_array_api = _importorskip("numpy", "1.26.0") has_h5netcdf_ros3, requires_h5netcdf_ros3 = _importorskip("h5netcdf", "1.3.0") has_netCDF4_1_6_2_or_above, requires_netCDF4_1_6_2_or_above = _importorskip( diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index a5ffb37a109..5e9954fcac9 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -6,20 +6,9 @@ from xarray.testing import assert_equal np = pytest.importorskip("numpy", minversion="1.22") +xp = pytest.importorskip("array_api_strict") -try: - import warnings - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - - import numpy.array_api as xp - from numpy.array_api._array_object import Array -except ImportError: - # for `numpy>=2.0` - xp = pytest.importorskip("array_api_strict") - - from array_api_strict._array_object import Array # type: ignore[no-redef] +from array_api_strict._array_object import Array # isort:skip # type: ignore[no-redef] @pytest.fixture diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 2a3faf32b85..13ddd45d9c1 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -1,7 +1,6 @@ from __future__ import annotations import copy -import warnings from abc import abstractmethod from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Generic, cast, overload @@ -358,13 +357,7 @@ def test_duck_array_typevar( test_duck_array_typevar(custom_a) # Test numpy's array api: - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - r"The numpy.array_api submodule is still experimental", - category=UserWarning, - ) - import numpy.array_api as nxp + import array_api_strict as nxp # TODO: nxp doesn't use dtype typevars, so can only use Any for the moment: arrayapi_a: duckarray[Any, Any] # duckarray[Any, np.dtype[np.int64]] diff --git a/xarray/tests/test_strategies.py b/xarray/tests/test_strategies.py index 44f0d56cde8..26c5906ad19 100644 --- a/xarray/tests/test_strategies.py +++ b/xarray/tests/test_strategies.py @@ -1,6 +1,7 @@ import numpy as np import numpy.testing as npt import pytest +from packaging.version import Version pytest.importorskip("hypothesis") # isort: split @@ -19,7 +20,6 @@ unique_subset_of, variables, ) -from xarray.tests import requires_numpy_array_api ALLOWED_ATTRS_VALUES_TYPES = (int, bool, str, np.ndarray) @@ -199,7 +199,6 @@ def dodgy_array_strategy_fn(*, shape=None, dtype=None): ) ) - @requires_numpy_array_api @given(st.data()) def test_make_strategies_namespace(self, data): """ @@ -208,9 +207,12 @@ def test_make_strategies_namespace(self, data): We still want to generate dtypes not in the array API by default, but this checks we don't accidentally override the user's choice of dtypes with non-API-compliant ones. """ - from numpy import ( - array_api as np_array_api, # requires numpy>=1.26.0, and we expect a UserWarning to be raised - ) + if Version(np.__version__) >= Version("2.0.0.dev0"): + np_array_api = np + else: + from numpy import ( + array_api as np_array_api, # requires numpy>=1.26.0, and we expect a UserWarning to be raised + ) np_array_api_st = make_strategies_namespace(np_array_api) From c76adc919278b2a05ab718f86f886ec0c9dac561 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Mar 2024 10:26:32 +0100 Subject: [PATCH 03/65] replace the use of the removed `nxp` with just plain `numpy` --- xarray/namedarray/_array_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/_array_api.py b/xarray/namedarray/_array_api.py index 4bf81603fa7..405b71c1efa 100644 --- a/xarray/namedarray/_array_api.py +++ b/xarray/namedarray/_array_api.py @@ -59,7 +59,7 @@ def astype( Examples -------- - >>> narr = NamedArray(("x",), nxp.asarray([1.5, 2.5])) + >>> narr = NamedArray(("x",), np.asarray([1.5, 2.5])) >>> narr Size: 16B Array([1.5, 2.5], dtype=float64) @@ -100,7 +100,7 @@ def imag( Examples -------- - >>> narr = NamedArray(("x",), np.asarray([1.0 + 2j, 2 + 4j])) # TODO: Use nxp + >>> narr = NamedArray(("x",), np.asarray([1.0 + 2j, 2 + 4j])) >>> imag(narr) Size: 16B array([2., 4.]) @@ -132,7 +132,7 @@ def real( Examples -------- - >>> narr = NamedArray(("x",), np.asarray([1.0 + 2j, 2 + 4j])) # TODO: Use nxp + >>> narr = NamedArray(("x",), np.asarray([1.0 + 2j, 2 + 4j])) >>> real(narr) Size: 16B array([1., 2.]) @@ -170,7 +170,7 @@ def expand_dims( Examples -------- - >>> x = NamedArray(("x", "y"), nxp.asarray([[1.0, 2.0], [3.0, 4.0]])) + >>> x = NamedArray(("x", "y"), np.asarray([[1.0, 2.0], [3.0, 4.0]])) >>> expand_dims(x) Size: 32B Array([[[1., 2.], From 57cd907e03febefc82c8093475c16e980436b78b Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Mar 2024 10:32:27 +0100 Subject: [PATCH 04/65] directly pass the `numpy` dtype --- xarray/tests/test_namedarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 13ddd45d9c1..3317c46da7f 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -361,7 +361,7 @@ def test_duck_array_typevar( # TODO: nxp doesn't use dtype typevars, so can only use Any for the moment: arrayapi_a: duckarray[Any, Any] # duckarray[Any, np.dtype[np.int64]] - arrayapi_a = nxp.asarray([2.1, 4], dtype=np.dtype(np.int64)) + arrayapi_a = nxp.asarray([2.1, 4], dtype=np.int64) test_duck_array_typevar(arrayapi_a) def test_new_namedarray(self) -> None: From 3a7552fabf41374da0b3d79832a65da984085d9f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Mar 2024 10:59:42 +0100 Subject: [PATCH 05/65] replace `dtype.type` with `type(dtype)` for `isnull` --- xarray/core/duck_array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index ef497e78ebf..3571dd8944d 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -141,7 +141,7 @@ def fail_on_dask_array_input(values, msg=None, func_name=None): def isnull(data): data = asarray(data) - scalar_type = data.dtype.type + scalar_type = type(data.dtype) if issubclass(scalar_type, (np.datetime64, np.timedelta64)): # datetime types use NaT for null # note: must check timedelta64 before integers, because currently From 4fa8767319277d33f7a4562990467e8578768c45 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 22 Mar 2024 11:48:07 +0100 Subject: [PATCH 06/65] use a new function to compare dtypes --- xarray/core/duck_array_ops.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 3571dd8944d..948f750f44d 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -139,19 +139,33 @@ def fail_on_dask_array_input(values, msg=None, func_name=None): ) +def extract_dtype(dtype): + return getattr(dtype, "_np_dtype", dtype) + + +def issubdtype(dtype, dtype_classes): + if not isinstance(dtype_classes, tuple): + return np.issubdtype(extract_dtype(dtype), dtype_classes) + + return any( + np.issubdtype(extract_dtype(dtype), dtype_class) + for dtype_class in dtype_classes + ) + + def isnull(data): data = asarray(data) - scalar_type = type(data.dtype) - if issubclass(scalar_type, (np.datetime64, np.timedelta64)): + scalar_type = data.dtype + if issubdtype(scalar_type, (np.datetime64, np.timedelta64)): # datetime types use NaT for null # note: must check timedelta64 before integers, because currently # timedelta64 inherits from np.integer return isnat(data) - elif issubclass(scalar_type, np.inexact): + elif issubdtype(scalar_type, np.inexact): # float types use NaN for null xp = get_array_namespace(data) return xp.isnan(data) - elif issubclass(scalar_type, (np.bool_, np.integer, np.character, np.void)): + elif issubdtype(scalar_type, (np.bool_, np.integer, np.character, np.void)): # these types cannot represent missing values return full_like(data, dtype=bool, fill_value=False) else: From 4063e117af66c84c077f2eab3364c6ce270c6b92 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 10 Apr 2024 23:04:28 +0200 Subject: [PATCH 07/65] use `array_api_strict`'s version of `int64` --- ci/install-upstream-wheels.sh | 10 +++++----- xarray/tests/test_namedarray.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index d9c797e27cd..9a5a7abd7b6 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash # install cython for building cftime without build isolation -micromamba install "cython>=0.29.20" py-cpuinfo +mamba install "cython>=0.29.20" py-cpuinfo # temporarily (?) remove numbagg and numba -micromamba remove -y numba numbagg sparse +mamba remove -y numba numbagg sparse # temporarily remove numexpr -micromamba remove -y numexpr +mamba remove -y numexpr # temporarily remove backends -micromamba remove -y cf_units hdf5 h5py netcdf4 +mamba remove -y cf_units hdf5 h5py netcdf4 # forcibly remove packages to avoid artifacts -micromamba remove -y --force \ +mamba remove -y --force \ numpy \ scipy \ pandas \ diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index 3317c46da7f..ff236ef2486 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -361,7 +361,7 @@ def test_duck_array_typevar( # TODO: nxp doesn't use dtype typevars, so can only use Any for the moment: arrayapi_a: duckarray[Any, Any] # duckarray[Any, np.dtype[np.int64]] - arrayapi_a = nxp.asarray([2.1, 4], dtype=np.int64) + arrayapi_a = nxp.asarray([2.1, 4], dtype=nxp.int64) test_duck_array_typevar(arrayapi_a) def test_new_namedarray(self) -> None: From fdd6c801c8f2abcea45222131f48719e45607eae Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 10 Apr 2024 23:28:10 +0200 Subject: [PATCH 08/65] use `array_api_strict`'s dtypes when interacting with its `Array` class --- xarray/tests/test_array_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index 5e9954fcac9..1958bbc4681 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -54,8 +54,8 @@ def test_aggregation_skipna(arrays) -> None: def test_astype(arrays) -> None: np_arr, xp_arr = arrays expected = np_arr.astype(np.int64) - actual = xp_arr.astype(np.int64) - assert actual.dtype == np.int64 + actual = xp_arr.astype(xp.int64) + assert actual.dtype == xp.int64 assert isinstance(actual.data, Array) assert_equal(actual, expected) From 3ef997a8267541ae2f08b87cdfee15ae6256121c Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 10 Apr 2024 23:58:24 +0200 Subject: [PATCH 09/65] Revert the (unintentional) switch to `mamba` [skip-ci] --- ci/install-upstream-wheels.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 9a5a7abd7b6..d9c797e27cd 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash # install cython for building cftime without build isolation -mamba install "cython>=0.29.20" py-cpuinfo +micromamba install "cython>=0.29.20" py-cpuinfo # temporarily (?) remove numbagg and numba -mamba remove -y numba numbagg sparse +micromamba remove -y numba numbagg sparse # temporarily remove numexpr -mamba remove -y numexpr +micromamba remove -y numexpr # temporarily remove backends -mamba remove -y cf_units hdf5 h5py netcdf4 +micromamba remove -y cf_units hdf5 h5py netcdf4 # forcibly remove packages to avoid artifacts -mamba remove -y --force \ +micromamba remove -y --force \ numpy \ scipy \ pandas \ From 2e0211c133189b637f6d9fd1da2ec38eccea1580 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 13:05:59 +0200 Subject: [PATCH 10/65] use the array API in `result_type` --- xarray/core/dtypes.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index ccf84146819..1dc02d34884 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -166,6 +166,14 @@ def is_datetime_like(dtype): return np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64) +def isdtype(dtype, compare, xp=None): + if xp is None or xp is np: + # need to take this path to allow checking for datetime/timedelta/strings + return np.issubdtype(dtype, compare) + else: + return xp.isdtype(dtype, compare) + + def result_type( *arrays_and_dtypes: np.typing.ArrayLike | np.typing.DTypeLike, ) -> np.dtype: @@ -184,12 +192,17 @@ def result_type( ------- numpy.dtype for the result. """ - types = {np.result_type(t).type for t in arrays_and_dtypes} + from xarray.core.duck_array_ops import get_array_namespace + + namespaces = {get_array_namespace(t) for t in arrays_and_dtypes} + [xp] = namespaces + + types = {xp.result_type(t) for t in arrays_and_dtypes} for left, right in PROMOTE_TO_OBJECT: - if any(issubclass(t, left) for t in types) and any( - issubclass(t, right) for t in types + if any(isdtype(t, left, xp=xp) for t in types) and any( + isdtype(t, right, xp=xp) for t in types ): - return np.dtype(object) + return xp.dtype(object) - return np.result_type(*arrays_and_dtypes) + return xp.result_type(*arrays_and_dtypes) From 73da2ba33bd1187b491f311854a6e6850ed1fb4d Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 13:06:25 +0200 Subject: [PATCH 11/65] skip modifying the casting rules if no numpy dtype is involved --- xarray/core/dtypes.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 1dc02d34884..37959dcffcc 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -199,10 +199,13 @@ def result_type( types = {xp.result_type(t) for t in arrays_and_dtypes} - for left, right in PROMOTE_TO_OBJECT: - if any(isdtype(t, left, xp=xp) for t in types) and any( - isdtype(t, right, xp=xp) for t in types - ): - return xp.dtype(object) + if any(isinstance(t, np.dtype) for t in types): + # only check if there's numpy dtypes – the array API does not + # define the types we're checking for + for left, right in PROMOTE_TO_OBJECT: + if any(isdtype(t, left, xp=xp) for t in types) and any( + isdtype(t, right, xp=xp) for t in types + ): + return xp.dtype(object) return xp.result_type(*arrays_and_dtypes) From 63c0f6c152480405a15ab57c8672b0fd1192b301 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 16:14:12 +0200 Subject: [PATCH 12/65] don't use isdtype for modules that don't have it --- xarray/core/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 37959dcffcc..11f17544993 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -167,7 +167,7 @@ def is_datetime_like(dtype): def isdtype(dtype, compare, xp=None): - if xp is None or xp is np: + if xp in (None, np) or not hasattr(xp, "isdtype"): # need to take this path to allow checking for datetime/timedelta/strings return np.issubdtype(dtype, compare) else: From 0326ae311f451a28a03b21beda50d717d57b6356 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 16:14:33 +0200 Subject: [PATCH 13/65] allow mixing numpy arrays with others This is not explicitly allowed by the array API specification (it was declared out of scope), so I'm not sure if this is the right way to do this. --- xarray/core/dtypes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 11f17544993..c3768bace97 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -195,7 +195,11 @@ def result_type( from xarray.core.duck_array_ops import get_array_namespace namespaces = {get_array_namespace(t) for t in arrays_and_dtypes} - [xp] = namespaces + non_numpy = namespaces - {np} + if non_numpy: + [xp] = non_numpy + else: + xp = np types = {xp.result_type(t) for t in arrays_and_dtypes} From b1e259d1f300f9bd8501fb38909dcd557c54974a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 19:08:59 +0200 Subject: [PATCH 14/65] use the array api to implement `nbytes` --- xarray/namedarray/core.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 135dabc0656..b813818c0cb 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -472,8 +472,18 @@ def nbytes(self) -> _IntOrUnknown: """ if hasattr(self._data, "nbytes"): return self._data.nbytes # type: ignore[no-any-return] + + if isinstance(self._data, _arrayapi): + xp = self._data.__array_namespace__() + + if xp.isdtype(self.dtype, "integral"): + itemsize = xp.iinfo(self.dtype).bits // 8 + else: + itemsize = xp.finfo(self.dtype).bits // 8 else: - return self.size * self.dtype.itemsize + itemsize = self.dtype.itemsize + + return self.size * itemsize @property def dims(self) -> _Dims: From 9d94dc18a0f2405d7186ec0e036d8317c8a94b59 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 19:44:49 +0200 Subject: [PATCH 15/65] refactor `isdtype` --- xarray/core/dtypes.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index c3768bace97..294d7f57b54 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -166,12 +166,34 @@ def is_datetime_like(dtype): return np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64) -def isdtype(dtype, compare, xp=None): +def isdtype(dtype, kind, xp=None): if xp in (None, np) or not hasattr(xp, "isdtype"): # need to take this path to allow checking for datetime/timedelta/strings - return np.issubdtype(dtype, compare) + long_names = { + "bool": "b", + "signed integer": "i", + "unsigned integer": "u", + "integral": "ui", + "real floating": "f", + "complex floating": "c", + "numeric": "uifc", + "object": "O", + "character": "U", + } + + if isinstance(kind, str): + return dtype.kind in long_names.get(kind, kind) + elif isinstance(kind, np.dtype) or issubclass(kind, np.generic): + return np.issubdtype(dtype, kind) + elif not isinstance(kind, tuple): + raise TypeError(f"unknown dtype kind: {kind}") + + if all(isinstance(k, str) for k in kind): + return dtype.kind in "".join(long_names.get(k, k) for k in kind) + else: + return any(np.issubdtype(dtype, k) for k in kind) else: - return xp.isdtype(dtype, compare) + return xp.isdtype(dtype, kind) def result_type( From 16e2403282679ce646b44de8e2272a419b8260fc Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 20:40:00 +0200 Subject: [PATCH 16/65] refactor `isdtype` to be a more general dtype checking mechanism --- xarray/core/dtypes.py | 66 ++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 294d7f57b54..c88457debe7 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -167,33 +167,55 @@ def is_datetime_like(dtype): def isdtype(dtype, kind, xp=None): - if xp in (None, np) or not hasattr(xp, "isdtype"): - # need to take this path to allow checking for datetime/timedelta/strings - long_names = { - "bool": "b", - "signed integer": "i", - "unsigned integer": "u", - "integral": "ui", - "real floating": "f", - "complex floating": "c", - "numeric": "uifc", - "object": "O", - "character": "U", - } - - if isinstance(kind, str): + array_api_names = { + "bool": "b", + "signed integer": "i", + "unsigned integer": "u", + "integral": "ui", + "real floating": "f", + "complex floating": "c", + "numeric": "uifc", + } + numpy_names = { + "object": "O", + "character": "U", + "string": "S", + } + long_names = array_api_names | numpy_names + + def compare_dtype(dtype, kind): + if isinstance(kind, np.dtype): + return dtype == kind + elif isinstance(kind, str): return dtype.kind in long_names.get(kind, kind) - elif isinstance(kind, np.dtype) or issubclass(kind, np.generic): + elif isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)): return np.issubdtype(dtype, kind) - elif not isinstance(kind, tuple): + else: raise TypeError(f"unknown dtype kind: {kind}") - if all(isinstance(k, str) for k in kind): - return dtype.kind in "".join(long_names.get(k, k) for k in kind) - else: - return any(np.issubdtype(dtype, k) for k in kind) + def is_numpy_kind(kind): + return (isinstance(kind, str) and kind in numpy_names) or ( + isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)) + ) + + def split_numpy_kinds(kinds): + if not isinstance(kinds, tuple): + kinds = (kinds,) + + numpy_kinds = tuple(kind for kind in kinds if is_numpy_kind(kind)) + non_numpy_kinds = tuple(kind for kind in kinds if not is_numpy_kind(kind)) + + return numpy_kinds, non_numpy_kinds + + numpy_kinds, non_numpy_kinds = split_numpy_kinds(kind) + if xp in (None, np) or not hasattr(xp, "isdtype"): + # need to take this path to allow checking for datetime/timedelta/strings + return any(compare_dtype(dtype, k) for k in (numpy_kinds + non_numpy_kinds)) + elif non_numpy_kinds: + return xp.isdtype(dtype, non_numpy_kinds) else: - return xp.isdtype(dtype, kind) + # can't compare numpy kinds with non-numpy dtypes + return False def result_type( From 79bc7f5d2b01a5a62e4efafdd9b72d9bd0910a9c Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 20:45:50 +0200 Subject: [PATCH 17/65] replace all `dtype.kind` calls with `dtypes.isdtype` --- xarray/core/duck_array_ops.py | 51 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 948f750f44d..d7f024b52b3 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -143,29 +143,21 @@ def extract_dtype(dtype): return getattr(dtype, "_np_dtype", dtype) -def issubdtype(dtype, dtype_classes): - if not isinstance(dtype_classes, tuple): - return np.issubdtype(extract_dtype(dtype), dtype_classes) - - return any( - np.issubdtype(extract_dtype(dtype), dtype_class) - for dtype_class in dtype_classes - ) - - def isnull(data): data = asarray(data) + + xp = get_array_namespace(data) scalar_type = data.dtype - if issubdtype(scalar_type, (np.datetime64, np.timedelta64)): + if dtypes.isdtype(scalar_type, (np.datetime64, np.timedelta64), xp=xp): # datetime types use NaT for null # note: must check timedelta64 before integers, because currently # timedelta64 inherits from np.integer return isnat(data) - elif issubdtype(scalar_type, np.inexact): + elif dtypes.isdtype(scalar_type, ("real floating", "complex floating"), xp=xp): # float types use NaN for null xp = get_array_namespace(data) return xp.isnan(data) - elif issubdtype(scalar_type, (np.bool_, np.integer, np.character, np.void)): + elif dtypes.isdtype(scalar_type, ("bool", "integral", "character", np.void), xp=xp): # these types cannot represent missing values return full_like(data, dtype=bool, fill_value=False) else: @@ -408,13 +400,19 @@ def f(values, axis=None, skipna=None, **kwargs): if invariant_0d and axis == (): return values - values = asarray(values) + xp = get_array_namespace(values) + values = asarray(values, xp=xp) - if coerce_strings and values.dtype.kind in "SU": + if coerce_strings and dtypes.isdtype(values.dtype, ("string", "character")): values = astype(values, object) func = None - if skipna or (skipna is None and values.dtype.kind in "cfO"): + if skipna or ( + skipna is None + and dtypes.isdtype( + values.dtype, ("complex floating", "real floating", "object"), xp=xp + ) + ): nanname = "nan" + name func = getattr(nanops, nanname) else: @@ -479,7 +477,8 @@ def _datetime_nanmin(array): - numpy nanmin() don't work on datetime64 (all versions at the moment of writing) - dask min() does not work on datetime64 (all versions at the moment of writing) """ - assert array.dtype.kind in "mM" + # no need for `xp` since this is only datetime dtypes + assert dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)) dtype = array.dtype # (NaT).astype(float) does not produce NaN... array = where(pandas_isnull(array), np.nan, array.astype(float)) @@ -517,7 +516,7 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): """ # Set offset to minimum if not given if offset is None: - if array.dtype.kind in "Mm": + if dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): offset = _datetime_nanmin(array) else: offset = min(array) @@ -529,7 +528,7 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): # This map_blocks call is for backwards compatibility. # dask == 2021.04.1 does not support subtracting object arrays # which is required for cftime - if is_duck_dask_array(array) and np.issubdtype(array.dtype, object): + if is_duck_dask_array(array) and dtypes.isdtype(array.dtype, object): array = array.map_blocks(lambda a, b: a - b, offset, meta=array._meta) else: array = array - offset @@ -539,11 +538,11 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): array = np.array(array) # Convert timedelta objects to float by first converting to microseconds. - if array.dtype.kind in "O": + if dtypes.isdtype(array.dtype, "object"): return py_timedelta_to_float(array, datetime_unit or "ns").astype(dtype) # Convert np.NaT to np.nan - elif array.dtype.kind in "mM": + elif dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): # Convert to specified timedelta units. if datetime_unit: array = array / np.timedelta64(1, datetime_unit) @@ -643,7 +642,7 @@ def mean(array, axis=None, skipna=None, **kwargs): from xarray.core.common import _contains_cftime_datetimes array = asarray(array) - if array.dtype.kind in "Mm": + if dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): offset = _datetime_nanmin(array) # xarray always uses np.datetime64[ns] for np.datetime64 data @@ -691,7 +690,9 @@ def cumsum(array, axis=None, **kwargs): def first(values, axis, skipna=None): """Return the first non-NA elements in this array along the given axis""" - if (skipna or skipna is None) and values.dtype.kind not in "iSU": + if (skipna or skipna is None) and not dtypes.isdtype( + values.dtype, ("signed integer", "string", "character") + ): # only bother for dtypes that can hold NaN if is_chunked_array(values): return chunked_nanfirst(values, axis) @@ -702,7 +703,9 @@ def first(values, axis, skipna=None): def last(values, axis, skipna=None): """Return the last non-NA elements in this array along the given axis""" - if (skipna or skipna is None) and values.dtype.kind not in "iSU": + if (skipna or skipna is None) and not dtypes.isdtype( + values.dtype, ("signed integer", "string", "character") + ): # only bother for dtypes that can hold NaN if is_chunked_array(values): return chunked_nanlast(values, axis) From 3cb20ceda7e473b875552b088779795bc41cf009 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 20:52:01 +0200 Subject: [PATCH 18/65] use the proper dtype kind --- xarray/core/duck_array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index d7f024b52b3..92de7a346b7 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -528,7 +528,7 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): # This map_blocks call is for backwards compatibility. # dask == 2021.04.1 does not support subtracting object arrays # which is required for cftime - if is_duck_dask_array(array) and dtypes.isdtype(array.dtype, object): + if is_duck_dask_array(array) and dtypes.isdtype(array.dtype, "object"): array = array.map_blocks(lambda a, b: a - b, offset, meta=array._meta) else: array = array - offset From 1f0d0f31dc9ad9561eb53c805a3fa3d199475f4a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 12 Apr 2024 20:58:37 +0200 Subject: [PATCH 19/65] use `_get_data_namespace` to get the array api namespace --- xarray/namedarray/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index b813818c0cb..59805762b18 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -470,11 +470,13 @@ def nbytes(self) -> _IntOrUnknown: If the underlying data array does not include ``nbytes``, estimates the bytes consumed based on the ``size`` and ``dtype``. """ + from xarray.namedarray._array_api import _get_data_namespace + if hasattr(self._data, "nbytes"): return self._data.nbytes # type: ignore[no-any-return] if isinstance(self._data, _arrayapi): - xp = self._data.__array_namespace__() + xp = _get_data_namespace(self) if xp.isdtype(self.dtype, "integral"): itemsize = xp.iinfo(self.dtype).bits // 8 From 7e952fd4dd07474d7a513bc881b1fdca5feb42f7 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 00:21:17 +0200 Subject: [PATCH 20/65] explicitly handle `bool` when determining the item size --- xarray/namedarray/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 59805762b18..837bf013167 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -478,7 +478,9 @@ def nbytes(self) -> _IntOrUnknown: if isinstance(self._data, _arrayapi): xp = _get_data_namespace(self) - if xp.isdtype(self.dtype, "integral"): + if xp.isdtype(self.dtype, "bool"): + itemsize = 1 + elif xp.isdtype(self.dtype, "integral"): itemsize = xp.iinfo(self.dtype).bits // 8 else: itemsize = xp.finfo(self.dtype).bits // 8 From d0ab11deb5cc0a950e7d42f7f9c15664a145fc11 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 00:54:25 +0200 Subject: [PATCH 21/65] prefer `itemsize` over the array API's version --- xarray/namedarray/core.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xarray/namedarray/core.py b/xarray/namedarray/core.py index 837bf013167..960ab9d4d1d 100644 --- a/xarray/namedarray/core.py +++ b/xarray/namedarray/core.py @@ -475,7 +475,9 @@ def nbytes(self) -> _IntOrUnknown: if hasattr(self._data, "nbytes"): return self._data.nbytes # type: ignore[no-any-return] - if isinstance(self._data, _arrayapi): + if hasattr(self.dtype, "itemsize"): + itemsize = self.dtype.itemsize + elif isinstance(self._data, _arrayapi): xp = _get_data_namespace(self) if xp.isdtype(self.dtype, "bool"): @@ -485,7 +487,9 @@ def nbytes(self) -> _IntOrUnknown: else: itemsize = xp.finfo(self.dtype).bits // 8 else: - itemsize = self.dtype.itemsize + raise TypeError( + "cannot compute the number of bytes (no array API nor nbytes / itemsize)" + ) return self.size * itemsize From 108d40fba39cec59694491dbc6949c04c3a327d7 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 11:15:05 +0200 Subject: [PATCH 22/65] add `array-api-strict` as a test dep to the bare-minimum environment --- ci/requirements/bare-minimum.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements/bare-minimum.yml b/ci/requirements/bare-minimum.yml index 56af319f0bb..2004898eaae 100644 --- a/ci/requirements/bare-minimum.yml +++ b/ci/requirements/bare-minimum.yml @@ -11,6 +11,7 @@ dependencies: - pytest-env - pytest-xdist - pytest-timeout + - array-api-strict - numpy=1.23 - packaging=22.0 - pandas=1.5 From da6fff67ebdb4c3e27a6d8230684c253742b41a4 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 11:30:56 +0200 Subject: [PATCH 23/65] ignore the redefinition of `nxp` --- xarray/tests/test_strategies.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_strategies.py b/xarray/tests/test_strategies.py index 26c5906ad19..b1ce947aa7d 100644 --- a/xarray/tests/test_strategies.py +++ b/xarray/tests/test_strategies.py @@ -208,18 +208,17 @@ def test_make_strategies_namespace(self, data): the user's choice of dtypes with non-API-compliant ones. """ if Version(np.__version__) >= Version("2.0.0.dev0"): - np_array_api = np + nxp = np else: - from numpy import ( - array_api as np_array_api, # requires numpy>=1.26.0, and we expect a UserWarning to be raised - ) + # requires numpy>=1.26.0, and we expect a UserWarning to be raised + from numpy import array_api as nxp # type: ignore[no-redef,unused-ignore] - np_array_api_st = make_strategies_namespace(np_array_api) + nxp_st = make_strategies_namespace(nxp) data.draw( variables( - array_strategy_fn=np_array_api_st.arrays, - dtype=np_array_api_st.scalar_dtypes(), + array_strategy_fn=nxp_st.arrays, + dtype=nxp_st.scalar_dtypes(), ) ) From 423b7ead004c8049bad0b51e98e33478bb0f2266 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 11:54:22 +0200 Subject: [PATCH 24/65] move the array api duck array check into a separate test This allows skipping it if the import fails (and we don't have to add it to the `bare-minimum` ci). --- ci/requirements/bare-minimum.yml | 1 - xarray/tests/test_namedarray.py | 40 ++++++++++++++------------------ 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/ci/requirements/bare-minimum.yml b/ci/requirements/bare-minimum.yml index 2004898eaae..56af319f0bb 100644 --- a/ci/requirements/bare-minimum.yml +++ b/ci/requirements/bare-minimum.yml @@ -11,7 +11,6 @@ dependencies: - pytest-env - pytest-xdist - pytest-timeout - - array-api-strict - numpy=1.23 - packaging=22.0 - pandas=1.5 diff --git a/xarray/tests/test_namedarray.py b/xarray/tests/test_namedarray.py index ff236ef2486..3d3584448de 100644 --- a/xarray/tests/test_namedarray.py +++ b/xarray/tests/test_namedarray.py @@ -78,6 +78,17 @@ def __array_namespace__(self) -> ModuleType: return np +def check_duck_array_typevar(a: duckarray[Any, _DType]) -> duckarray[Any, _DType]: + # Mypy checks a is valid: + b: duckarray[Any, _DType] = a + + # Runtime check if valid: + if isinstance(b, _arrayfunction_or_api): + return b + else: + raise TypeError(f"a ({type(a)}) is not a valid _arrayfunction or _arrayapi") + + class NamedArraySubclassobjects: @pytest.fixture def target(self, data: np.ndarray[Any, Any]) -> Any: @@ -327,42 +338,27 @@ def test_dims_setter( named_array.dims = new_dims assert named_array.dims == tuple(new_dims) - def test_duck_array_class( - self, - ) -> None: - def test_duck_array_typevar( - a: duckarray[Any, _DType], - ) -> duckarray[Any, _DType]: - # Mypy checks a is valid: - b: duckarray[Any, _DType] = a - - # Runtime check if valid: - if isinstance(b, _arrayfunction_or_api): - return b - else: - raise TypeError( - f"a ({type(a)}) is not a valid _arrayfunction or _arrayapi" - ) - + def test_duck_array_class(self) -> None: numpy_a: NDArray[np.int64] numpy_a = np.array([2.1, 4], dtype=np.dtype(np.int64)) - test_duck_array_typevar(numpy_a) + check_duck_array_typevar(numpy_a) masked_a: np.ma.MaskedArray[Any, np.dtype[np.int64]] masked_a = np.ma.asarray([2.1, 4], dtype=np.dtype(np.int64)) # type: ignore[no-untyped-call] - test_duck_array_typevar(masked_a) + check_duck_array_typevar(masked_a) custom_a: CustomArrayIndexable[Any, np.dtype[np.int64]] custom_a = CustomArrayIndexable(numpy_a) - test_duck_array_typevar(custom_a) + check_duck_array_typevar(custom_a) + def test_duck_array_class_array_api(self) -> None: # Test numpy's array api: - import array_api_strict as nxp + nxp = pytest.importorskip("array_api_strict", minversion="1.0") # TODO: nxp doesn't use dtype typevars, so can only use Any for the moment: arrayapi_a: duckarray[Any, Any] # duckarray[Any, np.dtype[np.int64]] arrayapi_a = nxp.asarray([2.1, 4], dtype=nxp.int64) - test_duck_array_typevar(arrayapi_a) + check_duck_array_typevar(arrayapi_a) def test_new_namedarray(self) -> None: dtype_float = np.dtype(np.float32) From 84f0c9575692c38d3ae701ed00cc5d3bb4fc93e1 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 13 Apr 2024 14:47:33 +0200 Subject: [PATCH 25/65] remove `extract_dtype` --- xarray/core/duck_array_ops.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 92de7a346b7..7975bf385fe 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -139,10 +139,6 @@ def fail_on_dask_array_input(values, msg=None, func_name=None): ) -def extract_dtype(dtype): - return getattr(dtype, "_np_dtype", dtype) - - def isnull(data): data = asarray(data) From 7a929c1b72c71c28430c27278cac1df7b281e15f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sun, 21 Apr 2024 23:06:29 +0200 Subject: [PATCH 26/65] try comparing working around extension dtypes --- xarray/core/dtypes.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index c88457debe7..789752dc371 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -183,13 +183,20 @@ def isdtype(dtype, kind, xp=None): } long_names = array_api_names | numpy_names + def issubdtype(dtype, kind): + if isinstance(dtype, np.dtype): + return np.issubdtype(dtype, kind) + else: + # TODO (keewis): find a better way to compare dtypes (like pandas extension dtypes) + return dtype == kind + def compare_dtype(dtype, kind): if isinstance(kind, np.dtype): return dtype == kind elif isinstance(kind, str): return dtype.kind in long_names.get(kind, kind) elif isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)): - return np.issubdtype(dtype, kind) + return issubdtype(dtype, kind) else: raise TypeError(f"unknown dtype kind: {kind}") From 45de4eb21d13635db70455ccc32d09c18b43fe6b Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Mon, 22 Apr 2024 15:27:47 +0200 Subject: [PATCH 27/65] change the `nbytes` test to more clearly communicate the intention --- xarray/tests/test_array_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_array_api.py b/xarray/tests/test_array_api.py index 1958bbc4681..03c77e2365e 100644 --- a/xarray/tests/test_array_api.py +++ b/xarray/tests/test_array_api.py @@ -107,8 +107,10 @@ def test_indexing(arrays: tuple[xr.DataArray, xr.DataArray]) -> None: def test_properties(arrays: tuple[xr.DataArray, xr.DataArray]) -> None: np_arr, xp_arr = arrays - assert np_arr.nbytes == np_arr.data.nbytes - assert xp_arr.nbytes == np_arr.data.nbytes + + expected = np_arr.data.nbytes + assert np_arr.nbytes == expected + assert xp_arr.nbytes == expected def test_reorganizing_operation(arrays: tuple[xr.DataArray, xr.DataArray]) -> None: From a82ec8ba5160605f4d3d5fbda23fc575c62a4288 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 20:05:11 +0200 Subject: [PATCH 28/65] remove the deprecated dtype alias `"a"` --- xarray/tests/test_dtypes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 3c2ee5e8f6f..68665171d12 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -58,7 +58,6 @@ def test_inf(obj) -> None: @pytest.mark.parametrize( "kind, expected", [ - ("a", (np.dtype("O"), "nan")), # dtype('S') ("b", (np.float32, "nan")), # dtype('int8') ("B", (np.float32, "nan")), # dtype('uint8') ("c", (np.dtype("O"), "nan")), # dtype('S1') From 152b983e239986bfdb0ec5b4d8ca100afa11d710 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 20:15:39 +0200 Subject: [PATCH 29/65] refactor to have different code paths for numpy dtypes and others --- xarray/core/dtypes.py | 71 ++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 789752dc371..b4492a246cb 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -4,6 +4,7 @@ from typing import Any import numpy as np +from pandas.api.types import is_extension_array_dtype from xarray.core import utils @@ -168,38 +169,21 @@ def is_datetime_like(dtype): def isdtype(dtype, kind, xp=None): array_api_names = { - "bool": "b", - "signed integer": "i", - "unsigned integer": "u", - "integral": "ui", - "real floating": "f", - "complex floating": "c", - "numeric": "uifc", + "bool": np.bool_, + "signed integer": np.signedinteger, + "unsigned integer": np.unsignedinteger, + "integral": np.integer, + "real floating": np.floating, + "complex floating": np.complexfloating, + "numeric": np.number, } numpy_names = { - "object": "O", - "character": "U", - "string": "S", + "object": np.object_, + "character": np.character, + "string": np.str_, } long_names = array_api_names | numpy_names - def issubdtype(dtype, kind): - if isinstance(dtype, np.dtype): - return np.issubdtype(dtype, kind) - else: - # TODO (keewis): find a better way to compare dtypes (like pandas extension dtypes) - return dtype == kind - - def compare_dtype(dtype, kind): - if isinstance(kind, np.dtype): - return dtype == kind - elif isinstance(kind, str): - return dtype.kind in long_names.get(kind, kind) - elif isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)): - return issubdtype(dtype, kind) - else: - raise TypeError(f"unknown dtype kind: {kind}") - def is_numpy_kind(kind): return (isinstance(kind, str) and kind in numpy_names) or ( isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)) @@ -214,15 +198,32 @@ def split_numpy_kinds(kinds): return numpy_kinds, non_numpy_kinds - numpy_kinds, non_numpy_kinds = split_numpy_kinds(kind) - if xp in (None, np) or not hasattr(xp, "isdtype"): - # need to take this path to allow checking for datetime/timedelta/strings - return any(compare_dtype(dtype, k) for k in (numpy_kinds + non_numpy_kinds)) - elif non_numpy_kinds: - return xp.isdtype(dtype, non_numpy_kinds) + def numpy_isdtype(dtype, kinds): + translated_kinds = [long_names.get(kind, kind) for kind in kinds] + if isinstance(dtype, np.generic): + return any(isinstance(dtype, kind) for kind in translated_kinds) + else: + return any(np.issubdtype(dtype, kind) for kind in translated_kinds) + + if xp is None: + xp = np + + if not isinstance(kind, tuple): + kinds = (kind,) + else: + kinds = kind + + if isinstance(dtype, np.dtype): + return numpy_isdtype(dtype, kinds) + elif is_extension_array_dtype(dtype): + return any(dtype == kind for kind in kinds) else: - # can't compare numpy kinds with non-numpy dtypes - return False + numpy_kinds, non_numpy_kinds = split_numpy_kinds(kind) + + if not non_numpy_kinds: + return False + + return xp.isdtype(dtype, non_numpy_kinds) def result_type( From d9426ec61ff537d0b017b1a8af9b92003bf28709 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 20:38:12 +0200 Subject: [PATCH 30/65] use `isdtype` for all other dtype checks in `xarray.core.dtypes` --- xarray/core/dtypes.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index b4492a246cb..cbdbbdbe3c9 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -61,22 +61,22 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # N.B. these casting rules should match pandas dtype_: np.typing.DTypeLike fill_value: Any - if np.issubdtype(dtype, np.floating): + if isdtype(dtype, "floating"): dtype_ = dtype fill_value = np.nan - elif np.issubdtype(dtype, np.timedelta64): + elif isdtype(dtype, np.timedelta64): # See https://github.com/numpy/numpy/issues/10685 # np.timedelta64 is a subclass of np.integer # Check np.timedelta64 before np.integer fill_value = np.timedelta64("NaT") dtype_ = dtype - elif np.issubdtype(dtype, np.integer): + elif isdtype(dtype, "integer"): dtype_ = np.float32 if dtype.itemsize <= 2 else np.float64 fill_value = np.nan - elif np.issubdtype(dtype, np.complexfloating): + elif isdtype(dtype, "complex floating"): dtype_ = dtype fill_value = np.nan + np.nan * 1j - elif np.issubdtype(dtype, np.datetime64): + elif isdtype(dtype, np.datetime64): dtype_ = dtype fill_value = np.datetime64("NaT") else: @@ -119,16 +119,16 @@ def get_pos_infinity(dtype, max_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if issubclass(dtype.type, np.floating): + if isdtype(dtype, "floating"): return np.inf - if issubclass(dtype.type, np.integer): + if isdtype(dtype, "integer"): if max_for_int: return np.iinfo(dtype).max else: return np.inf - if issubclass(dtype.type, np.complexfloating): + if isdtype(dtype, "complex floating"): return np.inf + 1j * np.inf return INF @@ -147,16 +147,16 @@ def get_neg_infinity(dtype, min_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if issubclass(dtype.type, np.floating): + if isdtype(dtype, "floating"): return -np.inf - if issubclass(dtype.type, np.integer): + if isdtype(dtype, "integer"): if min_for_int: return np.iinfo(dtype).min else: return -np.inf - if issubclass(dtype.type, np.complexfloating): + if isdtype(dtype, "complex floating"): return -np.inf - 1j * np.inf return NINF @@ -164,7 +164,7 @@ def get_neg_infinity(dtype, min_for_int=False): def is_datetime_like(dtype): """Check if a dtype is a subclass of the numpy datetime types""" - return np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64) + return isdtype(dtype, (np.datetime64, np.timedelta64)) def isdtype(dtype, kind, xp=None): From a43c1bff3f7b0818b9679f325246b95cd11d7548 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 20:51:02 +0200 Subject: [PATCH 31/65] use the proper kinds --- xarray/core/dtypes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index cbdbbdbe3c9..9f9dc68c340 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -61,7 +61,7 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # N.B. these casting rules should match pandas dtype_: np.typing.DTypeLike fill_value: Any - if isdtype(dtype, "floating"): + if isdtype(dtype, "real floating"): dtype_ = dtype fill_value = np.nan elif isdtype(dtype, np.timedelta64): @@ -70,7 +70,7 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # Check np.timedelta64 before np.integer fill_value = np.timedelta64("NaT") dtype_ = dtype - elif isdtype(dtype, "integer"): + elif isdtype(dtype, "integral"): dtype_ = np.float32 if dtype.itemsize <= 2 else np.float64 fill_value = np.nan elif isdtype(dtype, "complex floating"): @@ -119,10 +119,10 @@ def get_pos_infinity(dtype, max_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, "floating"): + if isdtype(dtype, "real floating"): return np.inf - if isdtype(dtype, "integer"): + if isdtype(dtype, "integral"): if max_for_int: return np.iinfo(dtype).max else: @@ -147,10 +147,10 @@ def get_neg_infinity(dtype, min_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, "floating"): + if isdtype(dtype, "real floating"): return -np.inf - if isdtype(dtype, "integer"): + if isdtype(dtype, "integral"): if min_for_int: return np.iinfo(dtype).min else: From 0f4d7beb976920f677c5138c7712922e7bd1ab08 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 20:55:36 +0200 Subject: [PATCH 32/65] remove the now unused "always tuple" branch in `split_numpy_kinds` --- xarray/core/dtypes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 9f9dc68c340..8e01962ad2f 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -190,9 +190,6 @@ def is_numpy_kind(kind): ) def split_numpy_kinds(kinds): - if not isinstance(kinds, tuple): - kinds = (kinds,) - numpy_kinds = tuple(kind for kind in kinds if is_numpy_kind(kind)) non_numpy_kinds = tuple(kind for kind in kinds if not is_numpy_kind(kind)) From 7e9562226055f32850a9e43d12106c7810e819af Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Fri, 3 May 2024 21:45:16 +0200 Subject: [PATCH 33/65] raise an error on invalid / unknown kinds --- xarray/core/dtypes.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 8e01962ad2f..3ed40c7afbf 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -195,8 +195,20 @@ def split_numpy_kinds(kinds): return numpy_kinds, non_numpy_kinds + def translate_kind(kind): + if isinstance(kind, str): + translated = long_names.get(kind) + if translated is None: + raise ValueError(f"unknown kind: {kind!r}") + + return translated + elif isinstance(kind, type) and issubclass(kind, np.generic): + return kind + else: + raise TypeError(f"invalid type of kind: {kind!r}") + def numpy_isdtype(dtype, kinds): - translated_kinds = [long_names.get(kind, kind) for kind in kinds] + translated_kinds = [translate_kind(kind) for kind in kinds] if isinstance(dtype, np.generic): return any(isinstance(dtype, kind) for kind in translated_kinds) else: From a088e112d54552120634df673447e00f480a60d9 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 4 May 2024 00:43:13 +0200 Subject: [PATCH 34/65] add tests for `isdtype` --- xarray/tests/__init__.py | 2 ++ xarray/tests/test_dtypes.py | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 494f50b591e..c8a8ba884cf 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -149,6 +149,8 @@ def _importorskip( has_numpy_array_api, requires_numpy_array_api = _importorskip("numpy", "1.26.0") has_numpy_2, requires_numpy_2 = _importorskip("numpy", "2.0.0") +has_array_api_strict, requires_array_api_strict = _importorskip("array_api_strict") + def _importorskip_h5netcdf_ros3(): try: diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 68665171d12..2c4ea699e90 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -1,9 +1,21 @@ from __future__ import annotations import numpy as np +import pandas as pd import pytest from xarray.core import dtypes +from xarray.tests import requires_array_api_strict + +try: + import array_api_strict +except ImportError: + + class DummyArrayAPINamespace: + int32 = None + float64 = None + + array_api_strict = DummyArrayAPINamespace @pytest.mark.parametrize( @@ -97,3 +109,51 @@ def test_nat_types_membership() -> None: assert np.datetime64("NaT").dtype in dtypes.NAT_TYPES assert np.timedelta64("NaT").dtype in dtypes.NAT_TYPES assert np.float64 not in dtypes.NAT_TYPES + + +@pytest.mark.parametrize( + ["dtype", "kinds", "xp", "expected"], + ( + (np.dtype("int32"), "integral", np, True), + (np.dtype("float16"), "real floating", np, True), + (np.dtype("complex128"), "complex floating", np, True), + (np.dtype("datetime64[s]"), (np.datetime64, np.timedelta64), np, True), + (np.dtype("U"), "numeric", np, False), + (np.dtype("int32"), "foo", np, ValueError("unknown kind: 'foo'")), + (np.dtype("float64"), object(), np, TypeError("invalid type of kind: .+")), + (pd.CategoricalDtype, pd.CategoricalDtype, None, True), + pytest.param( + array_api_strict.int32, + "integral", + array_api_strict, + True, + marks=requires_array_api_strict, + id="array_api-int", + ), + pytest.param( + array_api_strict.float64, + "real floating", + array_api_strict, + True, + marks=requires_array_api_strict, + id="array_api-float", + ), + pytest.param( + array_api_strict.bool, + (np.datetime64, np.timedelta64), + array_api_strict, + False, + marks=requires_array_api_strict, + id="array_api-bool", + ), + ), +) +def test_isdtype(dtype, kinds, xp, expected) -> None: + if isinstance(expected, Exception): + with pytest.raises(type(expected), match=expected.args[0]): + dtypes.isdtype(dtype, kinds, xp=xp) + + return + + actual = dtypes.isdtype(dtype, kinds, xp=xp) + assert actual == expected From 26bd6a11ae7d9120ca4d094b3f7cd98884837571 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 4 May 2024 00:43:24 +0200 Subject: [PATCH 35/65] pass in the iterable version of `kind` --- xarray/core/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 3ed40c7afbf..ef0cc039793 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -227,7 +227,7 @@ def numpy_isdtype(dtype, kinds): elif is_extension_array_dtype(dtype): return any(dtype == kind for kind in kinds) else: - numpy_kinds, non_numpy_kinds = split_numpy_kinds(kind) + numpy_kinds, non_numpy_kinds = split_numpy_kinds(kinds) if not non_numpy_kinds: return False From 833f54f767c5f6eaf8e05e5c00b2821144cece8e Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 4 May 2024 00:44:33 +0200 Subject: [PATCH 36/65] remove the array api check --- xarray/tests/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index c8a8ba884cf..d59b594243c 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -146,7 +146,6 @@ def _importorskip( requires_pandas_version_two = pytest.mark.skipif( not has_pandas_version_two, reason="requires pandas 2.0.0" ) -has_numpy_array_api, requires_numpy_array_api = _importorskip("numpy", "1.26.0") has_numpy_2, requires_numpy_2 = _importorskip("numpy", "2.0.0") has_array_api_strict, requires_array_api_strict = _importorskip("array_api_strict") From 75b3b6d8d59ad4ab66b7797d11068be6d2f4e902 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Sat, 4 May 2024 00:45:49 +0200 Subject: [PATCH 37/65] remove the unused `requires_pandas_version_two` --- xarray/tests/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index d59b594243c..58ddfcbf1c6 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -141,11 +141,6 @@ def _importorskip( requires_numbagg_or_bottleneck = pytest.mark.skipif( not has_scipy_or_netCDF4, reason="requires scipy or netCDF4" ) -# _importorskip does not work for development versions -has_pandas_version_two = Version(pd.__version__).major >= 2 -requires_pandas_version_two = pytest.mark.skipif( - not has_pandas_version_two, reason="requires pandas 2.0.0" -) has_numpy_2, requires_numpy_2 = _importorskip("numpy", "2.0.0") has_array_api_strict, requires_array_api_strict = _importorskip("array_api_strict") From fb59d88e9230bfd38d1518237f7b22a155704db6 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 19:32:09 +0200 Subject: [PATCH 38/65] add `bool` to the dummy namespace --- xarray/tests/test_dtypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 2c4ea699e90..7e515b5d962 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -12,6 +12,7 @@ except ImportError: class DummyArrayAPINamespace: + bool = None int32 = None float64 = None From 5c341634a06b4851189270f5cfd019dc41a172c4 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 19:38:57 +0200 Subject: [PATCH 39/65] actual make the extension array dtype test check something --- xarray/tests/test_dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 7e515b5d962..ed942db0edb 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -122,7 +122,7 @@ def test_nat_types_membership() -> None: (np.dtype("U"), "numeric", np, False), (np.dtype("int32"), "foo", np, ValueError("unknown kind: 'foo'")), (np.dtype("float64"), object(), np, TypeError("invalid type of kind: .+")), - (pd.CategoricalDtype, pd.CategoricalDtype, None, True), + (pd.CategoricalDtype([1, 2, 3]), pd.CategoricalDtype, None, True), pytest.param( array_api_strict.int32, "integral", From d72a621a26ea649f92b6d4652ebb910f7aa9bd34 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 19:42:54 +0200 Subject: [PATCH 40/65] actually make the extension array dtype check work --- xarray/core/dtypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index ef0cc039793..86c8984471f 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -225,7 +225,10 @@ def numpy_isdtype(dtype, kinds): if isinstance(dtype, np.dtype): return numpy_isdtype(dtype, kinds) elif is_extension_array_dtype(dtype): - return any(dtype == kind for kind in kinds) + return any( + isinstance(dtype, kind) if isinstance(kind, type) else False + for kind in kinds + ) else: numpy_kinds, non_numpy_kinds = split_numpy_kinds(kinds) From d9f2fb5a40018208536f7f01d519426b9af8dbb2 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 20:03:18 +0200 Subject: [PATCH 41/65] adapt the name of the wrapped array --- xarray/namedarray/_array_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/namedarray/_array_api.py b/xarray/namedarray/_array_api.py index 405b71c1efa..e02aebb93db 100644 --- a/xarray/namedarray/_array_api.py +++ b/xarray/namedarray/_array_api.py @@ -62,10 +62,10 @@ def astype( >>> narr = NamedArray(("x",), np.asarray([1.5, 2.5])) >>> narr Size: 16B - Array([1.5, 2.5], dtype=float64) + array([1.5, 2.5], dtype=float64) >>> astype(narr, np.dtype(np.int32)) Size: 8B - Array([1, 2], dtype=int32) + array([1, 2], dtype=int32) """ if isinstance(x._data, _arrayapi): xp = x._data.__array_namespace__() @@ -173,11 +173,11 @@ def expand_dims( >>> x = NamedArray(("x", "y"), np.asarray([[1.0, 2.0], [3.0, 4.0]])) >>> expand_dims(x) Size: 32B - Array([[[1., 2.], + array([[[1., 2.], [3., 4.]]], dtype=float64) >>> expand_dims(x, dim="z") Size: 32B - Array([[[1., 2.], + array([[[1., 2.], [3., 4.]]], dtype=float64) """ xp = _get_data_namespace(x) From 810cf61f77e150f0d765bd5b3a5b5955cb970a44 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 20:48:12 +0200 Subject: [PATCH 42/65] remove the dtype for those examples that use the default dtype --- xarray/namedarray/_array_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/namedarray/_array_api.py b/xarray/namedarray/_array_api.py index e02aebb93db..acbfc8af4f1 100644 --- a/xarray/namedarray/_array_api.py +++ b/xarray/namedarray/_array_api.py @@ -62,7 +62,7 @@ def astype( >>> narr = NamedArray(("x",), np.asarray([1.5, 2.5])) >>> narr Size: 16B - array([1.5, 2.5], dtype=float64) + array([1.5, 2.5]) >>> astype(narr, np.dtype(np.int32)) Size: 8B array([1, 2], dtype=int32) @@ -174,11 +174,11 @@ def expand_dims( >>> expand_dims(x) Size: 32B array([[[1., 2.], - [3., 4.]]], dtype=float64) + [3., 4.]]]) >>> expand_dims(x, dim="z") Size: 32B array([[[1., 2.], - [3., 4.]]], dtype=float64) + [3., 4.]]]) """ xp = _get_data_namespace(x) dims = x.dims From 3e87ea9a61b04af3a16fa1112ad2233233851695 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 7 May 2024 21:05:10 +0200 Subject: [PATCH 43/65] filter out the warning raised by importing `numpy.array_api` --- xarray/tests/test_strategies.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_strategies.py b/xarray/tests/test_strategies.py index b1ce947aa7d..47f54382c03 100644 --- a/xarray/tests/test_strategies.py +++ b/xarray/tests/test_strategies.py @@ -1,3 +1,5 @@ +import warnings + import numpy as np import numpy.testing as npt import pytest @@ -211,7 +213,13 @@ def test_make_strategies_namespace(self, data): nxp = np else: # requires numpy>=1.26.0, and we expect a UserWarning to be raised - from numpy import array_api as nxp # type: ignore[no-redef,unused-ignore] + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=UserWarning, message=".+See NEP 47." + ) + from numpy import ( # type: ignore[no-redef,unused-ignore] + array_api as nxp, + ) nxp_st = make_strategies_namespace(nxp) From 846b1cbde17267c4cd61610d1ee35c6fff4a936f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 8 May 2024 19:06:21 +0200 Subject: [PATCH 44/65] move the `pandas` isdtype check to a different function --- xarray/core/dtypes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 86c8984471f..cc42a66d972 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -214,6 +214,12 @@ def numpy_isdtype(dtype, kinds): else: return any(np.issubdtype(dtype, kind) for kind in translated_kinds) + def pandas_isdtype(dtype, kinds): + return any( + isinstance(dtype, kind) if isinstance(kind, type) else False + for kind in kinds + ) + if xp is None: xp = np @@ -225,10 +231,7 @@ def numpy_isdtype(dtype, kinds): if isinstance(dtype, np.dtype): return numpy_isdtype(dtype, kinds) elif is_extension_array_dtype(dtype): - return any( - isinstance(dtype, kind) if isinstance(kind, type) else False - for kind in kinds - ) + return pandas_isdtype(dtype, kinds) else: numpy_kinds, non_numpy_kinds = split_numpy_kinds(kinds) From a59edd3479b8b01ea791879fd659426773a6be76 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 8 May 2024 19:06:37 +0200 Subject: [PATCH 45/65] mention that we can remove `numpy_isdtype` once we require `numpy>=2.0` --- xarray/core/dtypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index cc42a66d972..c5d86be5ded 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -229,6 +229,7 @@ def pandas_isdtype(dtype, kinds): kinds = kind if isinstance(dtype, np.dtype): + # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` return numpy_isdtype(dtype, kinds) elif is_extension_array_dtype(dtype): return pandas_isdtype(dtype, kinds) From 911206b6fdcae165daaef1516aabc7055f1e9b11 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Wed, 8 May 2024 19:38:24 +0200 Subject: [PATCH 46/65] use an enum instead --- xarray/core/dtypes.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index c5d86be5ded..d015241bbab 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum import functools from typing import Any @@ -61,7 +62,7 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # N.B. these casting rules should match pandas dtype_: np.typing.DTypeLike fill_value: Any - if isdtype(dtype, "real floating"): + if isdtype(dtype, DtypeKind.real_floating): dtype_ = dtype fill_value = np.nan elif isdtype(dtype, np.timedelta64): @@ -70,10 +71,10 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # Check np.timedelta64 before np.integer fill_value = np.timedelta64("NaT") dtype_ = dtype - elif isdtype(dtype, "integral"): + elif isdtype(dtype, DtypeKind.integral): dtype_ = np.float32 if dtype.itemsize <= 2 else np.float64 fill_value = np.nan - elif isdtype(dtype, "complex floating"): + elif isdtype(dtype, DtypeKind.complex_floating): dtype_ = dtype fill_value = np.nan + np.nan * 1j elif isdtype(dtype, np.datetime64): @@ -119,16 +120,16 @@ def get_pos_infinity(dtype, max_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, "real floating"): + if isdtype(dtype, DtypeKind.real_floating): return np.inf - if isdtype(dtype, "integral"): + if isdtype(dtype, DtypeKind.integral): if max_for_int: return np.iinfo(dtype).max else: return np.inf - if isdtype(dtype, "complex floating"): + if isdtype(dtype, DtypeKind.complex_floating): return np.inf + 1j * np.inf return INF @@ -147,16 +148,16 @@ def get_neg_infinity(dtype, min_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, "real floating"): + if isdtype(dtype, DtypeKind.real_floating): return -np.inf - if isdtype(dtype, "integral"): + if isdtype(dtype, DtypeKind.integral): if min_for_int: return np.iinfo(dtype).min else: return -np.inf - if isdtype(dtype, "complex floating"): + if isdtype(dtype, DtypeKind.complex_floating): return -np.inf - 1j * np.inf return NINF @@ -167,6 +168,19 @@ def is_datetime_like(dtype): return isdtype(dtype, (np.datetime64, np.timedelta64)) +class DtypeKind(enum.Enum): + bool = "bool" + signed_integer = "signed_integer" + unsigned_integer = "unsigned integer" + integral = "integral" + real_floating = "real floating" + complex_floating = "complex floating" + numeric = "numeric" + object = "object" + character = "character" + string = "string" + + def isdtype(dtype, kind, xp=None): array_api_names = { "bool": np.bool_, @@ -228,6 +242,10 @@ def pandas_isdtype(dtype, kinds): else: kinds = kind + kinds = tuple( + kind if not isinstance(kind, DtypeKind) else kind.value for kind in kinds + ) + if isinstance(dtype, np.dtype): # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` return numpy_isdtype(dtype, kinds) From 14c5a5606f994b7acef3f9d2a488f67235dbefe9 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 14:46:13 +0200 Subject: [PATCH 47/65] make `isdtype` simpler --- xarray/core/dtypes.py | 144 +++++++++++++--------------------- xarray/core/duck_array_ops.py | 40 ++++++---- xarray/tests/test_dtypes.py | 8 +- 3 files changed, 80 insertions(+), 112 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index d015241bbab..65ae15fa845 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -1,6 +1,5 @@ from __future__ import annotations -import enum import functools from typing import Any @@ -62,22 +61,22 @@ def maybe_promote(dtype: np.dtype) -> tuple[np.dtype, Any]: # N.B. these casting rules should match pandas dtype_: np.typing.DTypeLike fill_value: Any - if isdtype(dtype, DtypeKind.real_floating): + if isdtype(dtype, "real floating"): dtype_ = dtype fill_value = np.nan - elif isdtype(dtype, np.timedelta64): + elif isinstance(dtype, np.dtype) and np.issubdtype(dtype, np.timedelta64): # See https://github.com/numpy/numpy/issues/10685 # np.timedelta64 is a subclass of np.integer # Check np.timedelta64 before np.integer fill_value = np.timedelta64("NaT") dtype_ = dtype - elif isdtype(dtype, DtypeKind.integral): + elif isdtype(dtype, "integral"): dtype_ = np.float32 if dtype.itemsize <= 2 else np.float64 fill_value = np.nan - elif isdtype(dtype, DtypeKind.complex_floating): + elif isdtype(dtype, "complex floating"): dtype_ = dtype fill_value = np.nan + np.nan * 1j - elif isdtype(dtype, np.datetime64): + elif isinstance(dtype, np.dtype) and np.issubdtype(dtype, np.datetime64): dtype_ = dtype fill_value = np.datetime64("NaT") else: @@ -120,16 +119,16 @@ def get_pos_infinity(dtype, max_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, DtypeKind.real_floating): + if isdtype(dtype, "real floating"): return np.inf - if isdtype(dtype, DtypeKind.integral): + if isdtype(dtype, "integral"): if max_for_int: return np.iinfo(dtype).max else: return np.inf - if isdtype(dtype, DtypeKind.complex_floating): + if isdtype(dtype, "complex floating"): return np.inf + 1j * np.inf return INF @@ -148,16 +147,16 @@ def get_neg_infinity(dtype, min_for_int=False): ------- fill_value : positive infinity value corresponding to this dtype. """ - if isdtype(dtype, DtypeKind.real_floating): + if isdtype(dtype, "real floating"): return -np.inf - if isdtype(dtype, DtypeKind.integral): + if isdtype(dtype, "integral"): if min_for_int: return np.iinfo(dtype).min else: return -np.inf - if isdtype(dtype, DtypeKind.complex_floating): + if isdtype(dtype, "complex floating"): return -np.inf - 1j * np.inf return NINF @@ -165,86 +164,56 @@ def get_neg_infinity(dtype, min_for_int=False): def is_datetime_like(dtype): """Check if a dtype is a subclass of the numpy datetime types""" - return isdtype(dtype, (np.datetime64, np.timedelta64)) + return _is_numpy_subdtype(dtype, (np.datetime64, np.timedelta64)) -class DtypeKind(enum.Enum): - bool = "bool" - signed_integer = "signed_integer" - unsigned_integer = "unsigned integer" - integral = "integral" - real_floating = "real floating" - complex_floating = "complex floating" - numeric = "numeric" - object = "object" - character = "character" - string = "string" +def is_object(dtype): + """Check if a dtype is object""" + return _is_numpy_subdtype(dtype, object) -def isdtype(dtype, kind, xp=None): - array_api_names = { - "bool": np.bool_, - "signed integer": np.signedinteger, - "unsigned integer": np.unsignedinteger, - "integral": np.integer, - "real floating": np.floating, - "complex floating": np.complexfloating, - "numeric": np.number, - } - numpy_names = { - "object": np.object_, - "character": np.character, - "string": np.str_, - } - long_names = array_api_names | numpy_names - - def is_numpy_kind(kind): - return (isinstance(kind, str) and kind in numpy_names) or ( - isinstance(kind, type) and issubclass(kind, (np.dtype, np.generic)) - ) - - def split_numpy_kinds(kinds): - numpy_kinds = tuple(kind for kind in kinds if is_numpy_kind(kind)) - non_numpy_kinds = tuple(kind for kind in kinds if not is_numpy_kind(kind)) - - return numpy_kinds, non_numpy_kinds - - def translate_kind(kind): - if isinstance(kind, str): - translated = long_names.get(kind) - if translated is None: - raise ValueError(f"unknown kind: {kind!r}") - - return translated - elif isinstance(kind, type) and issubclass(kind, np.generic): - return kind - else: - raise TypeError(f"invalid type of kind: {kind!r}") +def is_string(dtype): + """Check if a dtype is a string dtype""" + return _is_numpy_subdtype(dtype, (np.str_, np.character)) - def numpy_isdtype(dtype, kinds): - translated_kinds = [translate_kind(kind) for kind in kinds] - if isinstance(dtype, np.generic): - return any(isinstance(dtype, kind) for kind in translated_kinds) - else: - return any(np.issubdtype(dtype, kind) for kind in translated_kinds) - def pandas_isdtype(dtype, kinds): - return any( - isinstance(dtype, kind) if isinstance(kind, type) else False - for kind in kinds - ) +def _is_numpy_subdtype(dtype, kind): + if not isinstance(dtype, np.dtype): + return False + + kinds = kind if isinstance(kind, tuple) else (kind,) + return any(np.issubdtype(dtype, kind) for kind in kinds) + + +dtype_kinds = { + "bool": np.bool_, + "signed integer": np.signedinteger, + "unsigned integer": np.unsignedinteger, + "integral": np.integer, + "real floating": np.floating, + "complex floating": np.complexfloating, + "numeric": np.number, +} - if xp is None: - xp = np - if not isinstance(kind, tuple): - kinds = (kind,) +def numpy_isdtype(dtype, kinds): + # verified the dtypes already, no need to check again + translated_kinds = [dtype_kinds[kind] for kind in kinds] + if isinstance(dtype, np.generic): + return any(isinstance(dtype, kind) for kind in translated_kinds) else: - kinds = kind + return any(np.issubdtype(dtype, kind) for kind in translated_kinds) - kinds = tuple( - kind if not isinstance(kind, DtypeKind) else kind.value for kind in kinds - ) + +def pandas_isdtype(dtype, kinds): + return False + + +def isdtype(dtype, kind, xp=None): + kinds = kind if isinstance(kind, tuple) else (kind,) + unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] + if unknown_dtypes: + raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") if isinstance(dtype, np.dtype): # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` @@ -252,12 +221,7 @@ def pandas_isdtype(dtype, kinds): elif is_extension_array_dtype(dtype): return pandas_isdtype(dtype, kinds) else: - numpy_kinds, non_numpy_kinds = split_numpy_kinds(kinds) - - if not non_numpy_kinds: - return False - - return xp.isdtype(dtype, non_numpy_kinds) + return xp.isdtype(dtype, kinds) def result_type( @@ -293,8 +257,8 @@ def result_type( # only check if there's numpy dtypes – the array API does not # define the types we're checking for for left, right in PROMOTE_TO_OBJECT: - if any(isdtype(t, left, xp=xp) for t in types) and any( - isdtype(t, right, xp=xp) for t in types + if any(np.issubdtype(t, left) for t in types) and any( + np.issubdtype(t, right) for t in types ): return xp.dtype(object) diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index 9d027d5d3ce..5c1ebca6a71 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -145,7 +145,7 @@ def isnull(data): xp = get_array_namespace(data) scalar_type = data.dtype - if dtypes.isdtype(scalar_type, (np.datetime64, np.timedelta64), xp=xp): + if dtypes.is_datetime_like(scalar_type): # datetime types use NaT for null # note: must check timedelta64 before integers, because currently # timedelta64 inherits from np.integer @@ -154,7 +154,13 @@ def isnull(data): # float types use NaN for null xp = get_array_namespace(data) return xp.isnan(data) - elif dtypes.isdtype(scalar_type, ("bool", "integral", "character", np.void), xp=xp): + elif dtypes.isdtype(scalar_type, ("bool", "integral"), xp=xp) or ( + isinstance(scalar_type, np.dtype) + and ( + np.issubdtype(scalar_type, np.character) + or np.issubdtype(scalar_type, np.void) + ) + ): # these types cannot represent missing values return full_like(data, dtype=bool, fill_value=False) else: @@ -411,14 +417,17 @@ def f(values, axis=None, skipna=None, **kwargs): xp = get_array_namespace(values) values = asarray(values, xp=xp) - if coerce_strings and dtypes.isdtype(values.dtype, ("string", "character")): + if coerce_strings and dtypes.is_string(values.dtype): values = astype(values, object) func = None if skipna or ( skipna is None - and dtypes.isdtype( - values.dtype, ("complex floating", "real floating", "object"), xp=xp + and ( + dtypes.isdtype( + values.dtype, ("complex floating", "real floating"), xp=xp + ) + or dtypes.is_object(values.dtype) ) ): nanname = "nan" + name @@ -485,9 +494,8 @@ def _datetime_nanmin(array): - numpy nanmin() don't work on datetime64 (all versions at the moment of writing) - dask min() does not work on datetime64 (all versions at the moment of writing) """ - # no need for `xp` since this is only datetime dtypes - assert dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)) dtype = array.dtype + assert dtypes.is_datetime_like(dtype) # (NaT).astype(float) does not produce NaN... array = where(pandas_isnull(array), np.nan, array.astype(float)) array = min(array, skipna=True) @@ -524,7 +532,7 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): """ # Set offset to minimum if not given if offset is None: - if dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): + if dtypes.is_datetime_like(array.dtype): offset = _datetime_nanmin(array) else: offset = min(array) @@ -536,7 +544,7 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): # This map_blocks call is for backwards compatibility. # dask == 2021.04.1 does not support subtracting object arrays # which is required for cftime - if is_duck_dask_array(array) and dtypes.isdtype(array.dtype, "object"): + if is_duck_dask_array(array) and dtypes.is_object(array.dtype): array = array.map_blocks(lambda a, b: a - b, offset, meta=array._meta) else: array = array - offset @@ -546,11 +554,11 @@ def datetime_to_numeric(array, offset=None, datetime_unit=None, dtype=float): array = np.array(array) # Convert timedelta objects to float by first converting to microseconds. - if dtypes.isdtype(array.dtype, "object"): + if dtypes.is_object(array.dtype): return py_timedelta_to_float(array, datetime_unit or "ns").astype(dtype) # Convert np.NaT to np.nan - elif dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): + elif dtypes.is_datetime_like(array.dtype): # Convert to specified timedelta units. if datetime_unit: array = array / np.timedelta64(1, datetime_unit) @@ -650,7 +658,7 @@ def mean(array, axis=None, skipna=None, **kwargs): from xarray.core.common import _contains_cftime_datetimes array = asarray(array) - if dtypes.isdtype(array.dtype, (np.datetime64, np.timedelta64)): + if dtypes.is_datetime_like(array.dtype): offset = _datetime_nanmin(array) # xarray always uses np.datetime64[ns] for np.datetime64 data @@ -698,8 +706,8 @@ def cumsum(array, axis=None, **kwargs): def first(values, axis, skipna=None): """Return the first non-NA elements in this array along the given axis""" - if (skipna or skipna is None) and not dtypes.isdtype( - values.dtype, ("signed integer", "string", "character") + if (skipna or skipna is None) and not ( + dtypes.isdtype(values.dtype, "signed integer") or dtypes.is_string(values.dtype) ): # only bother for dtypes that can hold NaN if is_chunked_array(values): @@ -711,8 +719,8 @@ def first(values, axis, skipna=None): def last(values, axis, skipna=None): """Return the last non-NA elements in this array along the given axis""" - if (skipna or skipna is None) and not dtypes.isdtype( - values.dtype, ("signed integer", "string", "character") + if (skipna or skipna is None) and not ( + dtypes.isdtype(values.dtype, "signed integer") or dtypes.is_string(values.dtype) ): # only bother for dtypes that can hold NaN if is_chunked_array(values): diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index ed942db0edb..c6e9fd07c1c 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -1,7 +1,6 @@ from __future__ import annotations import numpy as np -import pandas as pd import pytest from xarray.core import dtypes @@ -118,11 +117,8 @@ def test_nat_types_membership() -> None: (np.dtype("int32"), "integral", np, True), (np.dtype("float16"), "real floating", np, True), (np.dtype("complex128"), "complex floating", np, True), - (np.dtype("datetime64[s]"), (np.datetime64, np.timedelta64), np, True), (np.dtype("U"), "numeric", np, False), - (np.dtype("int32"), "foo", np, ValueError("unknown kind: 'foo'")), - (np.dtype("float64"), object(), np, TypeError("invalid type of kind: .+")), - (pd.CategoricalDtype([1, 2, 3]), pd.CategoricalDtype, None, True), + (np.dtype("int32"), "foo", np, ValueError("unknown dtype kinds:.+'foo'")), pytest.param( array_api_strict.int32, "integral", @@ -141,7 +137,7 @@ def test_nat_types_membership() -> None: ), pytest.param( array_api_strict.bool, - (np.datetime64, np.timedelta64), + "numeric", array_api_strict, False, marks=requires_array_api_strict, From 58d6b8bdc094f0bcdf2c4674587a96980c09600d Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 15:47:23 +0200 Subject: [PATCH 48/65] comment on the empty pandas_isdtype --- xarray/core/dtypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 65ae15fa845..fd39f2e3c6d 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -206,6 +206,7 @@ def numpy_isdtype(dtype, kinds): def pandas_isdtype(dtype, kinds): + # according to the comments in `extension_array.issubdtype` we don't want to match pandas dtypes return False From a9c7a217befea5fc2b51d75882c5c9ea25a225ae Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 15:52:43 +0200 Subject: [PATCH 49/65] drop `pandas_isdtype` in favor of a simple `return `False` --- xarray/core/dtypes.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index fd39f2e3c6d..c13bde3f2b0 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -205,11 +205,6 @@ def numpy_isdtype(dtype, kinds): return any(np.issubdtype(dtype, kind) for kind in translated_kinds) -def pandas_isdtype(dtype, kinds): - # according to the comments in `extension_array.issubdtype` we don't want to match pandas dtypes - return False - - def isdtype(dtype, kind, xp=None): kinds = kind if isinstance(kind, tuple) else (kind,) unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] @@ -220,7 +215,8 @@ def isdtype(dtype, kind, xp=None): # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` return numpy_isdtype(dtype, kinds) elif is_extension_array_dtype(dtype): - return pandas_isdtype(dtype, kinds) + # we never want to match pandas extension array dtypes + return False else: return xp.isdtype(dtype, kinds) From 62eec4851dbacbf06e692799418e2ea0d8a1859a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 15:56:54 +0200 Subject: [PATCH 50/65] move the dtype kind verification to `numpy_isdtype` `xp.isdtype` should already check the same thing. --- xarray/core/dtypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index c13bde3f2b0..71fd8330069 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -197,6 +197,10 @@ def _is_numpy_subdtype(dtype, kind): def numpy_isdtype(dtype, kinds): + unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] + if unknown_dtypes: + raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") + # verified the dtypes already, no need to check again translated_kinds = [dtype_kinds[kind] for kind in kinds] if isinstance(dtype, np.generic): @@ -207,10 +211,6 @@ def numpy_isdtype(dtype, kinds): def isdtype(dtype, kind, xp=None): kinds = kind if isinstance(kind, tuple) else (kind,) - unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] - if unknown_dtypes: - raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") - if isinstance(dtype, np.dtype): # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` return numpy_isdtype(dtype, kinds) From 007e6c94795171a5e0f3c47d8fd6af0fd6645378 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 16:01:01 +0200 Subject: [PATCH 51/65] fall back to `numpy.isdtype` if `xp` is not passed --- xarray/core/dtypes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 71fd8330069..87a6afdb481 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -218,6 +218,8 @@ def isdtype(dtype, kind, xp=None): # we never want to match pandas extension array dtypes return False else: + if xp is None: + xp = np return xp.isdtype(dtype, kinds) From c5f4262cf0dc8b6d77b1727d083f16c0b4df4dfc Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 16:07:03 +0200 Subject: [PATCH 52/65] move `numpy_isdtype` to `npcompat` --- xarray/core/dtypes.py | 32 +++----------------------------- xarray/core/npcompat.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 87a6afdb481..a6e9e2cac5c 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -6,7 +6,7 @@ import numpy as np from pandas.api.types import is_extension_array_dtype -from xarray.core import utils +from xarray.core import npcompat, utils # Use as a sentinel value to indicate a dtype appropriate NA value. NA = utils.ReprObject("") @@ -185,42 +185,16 @@ def _is_numpy_subdtype(dtype, kind): return any(np.issubdtype(dtype, kind) for kind in kinds) -dtype_kinds = { - "bool": np.bool_, - "signed integer": np.signedinteger, - "unsigned integer": np.unsignedinteger, - "integral": np.integer, - "real floating": np.floating, - "complex floating": np.complexfloating, - "numeric": np.number, -} - - -def numpy_isdtype(dtype, kinds): - unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] - if unknown_dtypes: - raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") - - # verified the dtypes already, no need to check again - translated_kinds = [dtype_kinds[kind] for kind in kinds] - if isinstance(dtype, np.generic): - return any(isinstance(dtype, kind) for kind in translated_kinds) - else: - return any(np.issubdtype(dtype, kind) for kind in translated_kinds) - - def isdtype(dtype, kind, xp=None): - kinds = kind if isinstance(kind, tuple) else (kind,) if isinstance(dtype, np.dtype): - # TODO (keewis): replace with `numpy.isdtype` once we drop `numpy<2.0` - return numpy_isdtype(dtype, kinds) + return npcompat.isdtype(dtype, kind) elif is_extension_array_dtype(dtype): # we never want to match pandas extension array dtypes return False else: if xp is None: xp = np - return xp.isdtype(dtype, kinds) + return xp.isdtype(dtype, kind) def result_type( diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index d8a6e300fc0..41a3ab985c4 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -28,3 +28,33 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +try: + # requires numpy>=2.0 + from numpy import isdtype +except ImportError: + import numpy as np + + dtype_kinds = { + "bool": np.bool_, + "signed integer": np.signedinteger, + "unsigned integer": np.unsignedinteger, + "integral": np.integer, + "real floating": np.floating, + "complex floating": np.complexfloating, + "numeric": np.number, + } + + def isdtype(dtype, kind): + kinds = kind if isinstance(kind, tuple) else (kind,) + + unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] + if unknown_dtypes: + raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") + + # verified the dtypes already, no need to check again + translated_kinds = [dtype_kinds[kind] for kind in kinds] + if isinstance(dtype, np.generic): + return any(isinstance(dtype, kind) for kind in translated_kinds) + else: + return any(np.issubdtype(dtype, kind) for kind in translated_kinds) From 032995107194b874549d7920ae096a0fabe0d829 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 16:17:23 +0200 Subject: [PATCH 53/65] typing --- xarray/core/dtypes.py | 10 +++++----- xarray/core/npcompat.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index a6e9e2cac5c..e9eb9f6d105 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -162,22 +162,22 @@ def get_neg_infinity(dtype, min_for_int=False): return NINF -def is_datetime_like(dtype): +def is_datetime_like(dtype) -> bool: """Check if a dtype is a subclass of the numpy datetime types""" return _is_numpy_subdtype(dtype, (np.datetime64, np.timedelta64)) -def is_object(dtype): +def is_object(dtype) -> bool: """Check if a dtype is object""" return _is_numpy_subdtype(dtype, object) -def is_string(dtype): +def is_string(dtype) -> bool: """Check if a dtype is a string dtype""" return _is_numpy_subdtype(dtype, (np.str_, np.character)) -def _is_numpy_subdtype(dtype, kind): +def _is_numpy_subdtype(dtype, kind) -> bool: if not isinstance(dtype, np.dtype): return False @@ -185,7 +185,7 @@ def _is_numpy_subdtype(dtype, kind): return any(np.issubdtype(dtype, kind) for kind in kinds) -def isdtype(dtype, kind, xp=None): +def isdtype(dtype, kind: str, xp=None) -> bool: if isinstance(dtype, np.dtype): return npcompat.isdtype(dtype, kind) elif is_extension_array_dtype(dtype): diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 41a3ab985c4..616fd91d30a 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -31,7 +31,7 @@ try: # requires numpy>=2.0 - from numpy import isdtype + from numpy import isdtype # typing: ignore[attr-defined,unused-ignore] except ImportError: import numpy as np From 63046d003d378d6f45479fd2b319c79a9fda2707 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 16:34:40 +0200 Subject: [PATCH 54/65] fix a type comment --- xarray/core/npcompat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 616fd91d30a..2493c08ca8e 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -31,7 +31,7 @@ try: # requires numpy>=2.0 - from numpy import isdtype # typing: ignore[attr-defined,unused-ignore] + from numpy import isdtype # type: ignore[attr-defined,unused-ignore] except ImportError: import numpy as np From 2e88691c21c76dec4ae2df07630c87813da915de Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 17:50:55 +0200 Subject: [PATCH 55/65] additional code comments Co-authored-by: Stephan Hoyer --- xarray/core/dtypes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index e9eb9f6d105..054573a058a 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -186,6 +186,15 @@ def _is_numpy_subdtype(dtype, kind) -> bool: def isdtype(dtype, kind: str, xp=None) -> bool: + """Compatibility wrapper for isdtype() from the array API standard. + + Unlike xp.isdtype(), kind must be a string. + """ + # TODO(shoyer): remove this wrapper when Xarray requires + # numpy>=2 and pandas extensions arrays are implemented in + # Xarray via the array API + if not isinstance(kind, str): + raise TypeError(f'kind must be a string: {kind}') if isinstance(dtype, np.dtype): return npcompat.isdtype(dtype, kind) elif is_extension_array_dtype(dtype): @@ -217,6 +226,8 @@ def result_type( """ from xarray.core.duck_array_ops import get_array_namespace + # TODO(shoyer): consider moving this logic into get_array_namespace() + # or another helper function. namespaces = {get_array_namespace(t) for t in arrays_and_dtypes} non_numpy = namespaces - {np} if non_numpy: From 499e55369d87f8b90efe9700a761d2ef64fcb74b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 15:51:30 +0000 Subject: [PATCH 56/65] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/core/dtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 054573a058a..04671a055d5 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -187,14 +187,14 @@ def _is_numpy_subdtype(dtype, kind) -> bool: def isdtype(dtype, kind: str, xp=None) -> bool: """Compatibility wrapper for isdtype() from the array API standard. - + Unlike xp.isdtype(), kind must be a string. """ # TODO(shoyer): remove this wrapper when Xarray requires # numpy>=2 and pandas extensions arrays are implemented in # Xarray via the array API if not isinstance(kind, str): - raise TypeError(f'kind must be a string: {kind}') + raise TypeError(f"kind must be a string: {kind}") if isinstance(dtype, np.dtype): return npcompat.isdtype(dtype, kind) elif is_extension_array_dtype(dtype): From 63bacb4a841a94e6d328fd675eeabde43d7dcc4a Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 17:51:19 +0200 Subject: [PATCH 57/65] more typing --- xarray/core/dtypes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 04671a055d5..53d9c1ef371 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -185,7 +185,7 @@ def _is_numpy_subdtype(dtype, kind) -> bool: return any(np.issubdtype(dtype, kind) for kind in kinds) -def isdtype(dtype, kind: str, xp=None) -> bool: +def isdtype(dtype, kind: str | tuple[str, ...], xp=None) -> bool: """Compatibility wrapper for isdtype() from the array API standard. Unlike xp.isdtype(), kind must be a string. @@ -195,6 +195,7 @@ def isdtype(dtype, kind: str, xp=None) -> bool: # Xarray via the array API if not isinstance(kind, str): raise TypeError(f"kind must be a string: {kind}") + if isinstance(dtype, np.dtype): return npcompat.isdtype(dtype, kind) elif is_extension_array_dtype(dtype): From fca4b3cb7354889d55ddec306cc73e2c50d1fe55 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 17:54:07 +0200 Subject: [PATCH 58/65] raise a `TypeError` as `numpy.isdtype` does --- xarray/core/npcompat.py | 2 +- xarray/tests/test_dtypes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index 2493c08ca8e..b5e86be3dd1 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -50,7 +50,7 @@ def isdtype(dtype, kind): unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] if unknown_dtypes: - raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") + raise TypeError(f"unknown dtype kinds: {unknown_dtypes}") # verified the dtypes already, no need to check again translated_kinds = [dtype_kinds[kind] for kind in kinds] diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index c6e9fd07c1c..178b4669f36 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -118,7 +118,7 @@ def test_nat_types_membership() -> None: (np.dtype("float16"), "real floating", np, True), (np.dtype("complex128"), "complex floating", np, True), (np.dtype("U"), "numeric", np, False), - (np.dtype("int32"), "foo", np, ValueError("unknown dtype kinds:.+'foo'")), + (np.dtype("int32"), "foo", np, TypeError("kind")), pytest.param( array_api_strict.int32, "integral", From 7979d448102c8bdb935206aa4d44af3e7aa59d2f Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 18:05:43 +0200 Subject: [PATCH 59/65] also allow tuples of strings as kind --- xarray/core/dtypes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 53d9c1ef371..228dbf48d6c 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -193,8 +193,10 @@ def isdtype(dtype, kind: str | tuple[str, ...], xp=None) -> bool: # TODO(shoyer): remove this wrapper when Xarray requires # numpy>=2 and pandas extensions arrays are implemented in # Xarray via the array API - if not isinstance(kind, str): - raise TypeError(f"kind must be a string: {kind}") + if not isinstance(kind, str) or ( + isinstance(kind, tuple) and not all(isinstance(k, str) for k in kind) + ): + raise TypeError(f"kind must be a string or a tuple of strings: {repr(kind)}") if isinstance(dtype, np.dtype): return npcompat.isdtype(dtype, kind) From 513104bffd8b37f6a03fa9e76c647d01da7f8b42 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 18:10:13 +0200 Subject: [PATCH 60/65] invert the condition --- xarray/core/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 228dbf48d6c..349c7de0530 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -193,7 +193,7 @@ def isdtype(dtype, kind: str | tuple[str, ...], xp=None) -> bool: # TODO(shoyer): remove this wrapper when Xarray requires # numpy>=2 and pandas extensions arrays are implemented in # Xarray via the array API - if not isinstance(kind, str) or ( + if not isinstance(kind, str) or not ( isinstance(kind, tuple) and not all(isinstance(k, str) for k in kind) ): raise TypeError(f"kind must be a string or a tuple of strings: {repr(kind)}") From 6dea06e0cb0737ba94c997b02efc29c9dbd6d9c7 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 18:11:22 +0200 Subject: [PATCH 61/65] final fix, hopefully --- xarray/core/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 349c7de0530..1819ab95bfd 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -193,7 +193,7 @@ def isdtype(dtype, kind: str | tuple[str, ...], xp=None) -> bool: # TODO(shoyer): remove this wrapper when Xarray requires # numpy>=2 and pandas extensions arrays are implemented in # Xarray via the array API - if not isinstance(kind, str) or not ( + if not isinstance(kind, str) and not ( isinstance(kind, tuple) and not all(isinstance(k, str) for k in kind) ): raise TypeError(f"kind must be a string or a tuple of strings: {repr(kind)}") From 48b2e2d3992c827d8109f52149ad6b60e3963e81 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Thu, 16 May 2024 18:14:29 +0200 Subject: [PATCH 62/65] next attempt --- xarray/core/dtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 1819ab95bfd..c8fcdaa1a4d 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -194,7 +194,7 @@ def isdtype(dtype, kind: str | tuple[str, ...], xp=None) -> bool: # numpy>=2 and pandas extensions arrays are implemented in # Xarray via the array API if not isinstance(kind, str) and not ( - isinstance(kind, tuple) and not all(isinstance(k, str) for k in kind) + isinstance(kind, tuple) and all(isinstance(k, str) for k in kind) ): raise TypeError(f"kind must be a string or a tuple of strings: {repr(kind)}") From eb9decef0f17c08371d9c41444c40fe52a1514e7 Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 21 May 2024 21:33:48 +0200 Subject: [PATCH 63/65] raise a `ValueError` for unknown dtype kinds --- xarray/core/npcompat.py | 2 +- xarray/tests/test_dtypes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/core/npcompat.py b/xarray/core/npcompat.py index b5e86be3dd1..2493c08ca8e 100644 --- a/xarray/core/npcompat.py +++ b/xarray/core/npcompat.py @@ -50,7 +50,7 @@ def isdtype(dtype, kind): unknown_dtypes = [kind for kind in kinds if kind not in dtype_kinds] if unknown_dtypes: - raise TypeError(f"unknown dtype kinds: {unknown_dtypes}") + raise ValueError(f"unknown dtype kinds: {unknown_dtypes}") # verified the dtypes already, no need to check again translated_kinds = [dtype_kinds[kind] for kind in kinds] diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index 178b4669f36..bd2466dc6a9 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -118,7 +118,7 @@ def test_nat_types_membership() -> None: (np.dtype("float16"), "real floating", np, True), (np.dtype("complex128"), "complex floating", np, True), (np.dtype("U"), "numeric", np, False), - (np.dtype("int32"), "foo", np, TypeError("kind")), + (np.dtype("int32"), "foo", np, ValueError("kind")), pytest.param( array_api_strict.int32, "integral", From 7302060068a6ccd06ab17b07b6112b517b65354c Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 21 May 2024 21:53:10 +0200 Subject: [PATCH 64/65] split out the tests we expect to raise into a separate function --- xarray/tests/test_dtypes.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index bd2466dc6a9..d8bfa15964e 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -118,7 +118,6 @@ def test_nat_types_membership() -> None: (np.dtype("float16"), "real floating", np, True), (np.dtype("complex128"), "complex floating", np, True), (np.dtype("U"), "numeric", np, False), - (np.dtype("int32"), "foo", np, ValueError("kind")), pytest.param( array_api_strict.int32, "integral", @@ -146,11 +145,17 @@ def test_nat_types_membership() -> None: ), ) def test_isdtype(dtype, kinds, xp, expected) -> None: - if isinstance(expected, Exception): - with pytest.raises(type(expected), match=expected.args[0]): - dtypes.isdtype(dtype, kinds, xp=xp) - - return - actual = dtypes.isdtype(dtype, kinds, xp=xp) assert actual == expected + + +@pytest.mark.parametrize( + ["dtype", "kinds", "xp", "error", "pattern"], + ( + (np.dtype("int32"), "foo", np, (TypeError, ValueError), "kind"), + (np.dtype("int32"), np.signedinteger, np, TypeError, "kind"), + ), +) +def test_isdtype_error(dtype, kinds, xp, error, pattern): + with pytest.raises(error, match=pattern): + dtypes.isdtype(dtype, kinds, xp=xp) From c8ebdc71921fde1cfa470dafe93d3b1a5d63f89e Mon Sep 17 00:00:00 2001 From: Justus Magin Date: Tue, 21 May 2024 22:43:01 +0200 Subject: [PATCH 65/65] add another expected failing test --- xarray/tests/test_dtypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_dtypes.py b/xarray/tests/test_dtypes.py index d8bfa15964e..ed14f735e32 100644 --- a/xarray/tests/test_dtypes.py +++ b/xarray/tests/test_dtypes.py @@ -154,6 +154,7 @@ def test_isdtype(dtype, kinds, xp, expected) -> None: ( (np.dtype("int32"), "foo", np, (TypeError, ValueError), "kind"), (np.dtype("int32"), np.signedinteger, np, TypeError, "kind"), + (np.dtype("float16"), 1, np, TypeError, "kind"), ), ) def test_isdtype_error(dtype, kinds, xp, error, pattern):