ENH/TST: xp_assert_ enhancements (data-apis#267)

* WIP: xp_assert enhancements

* ENH: add xp_assert_less

* Rework prepare_for_test (data-apis#2)

* Fix failures in data-apis#267

* Update tests/test_testing.py

[skip ci]

Co-authored-by: Lucas Colley <[email protected]>

* Update _testing.py

---------

Co-authored-by: Guido Imperiale <[email protected]>
Co-authored-by: Lucas Colley <[email protected]>

Author: 3 people

src/array_api_extra/_lib/_testing.py

@@ -5,27 +5,37 @@
 See also ..testing for public testing utilities.
 """
 
+from __future__ import annotations
+
 import math
 from types import ModuleType
-from typing import cast
+from typing import Any, cast
 
+import numpy as np
 import pytest
 
 from ._utils._compat import (
     array_namespace,
     is_array_api_strict_namespace,
     is_cupy_namespace,
     is_dask_namespace,
+    is_jax_namespace,
+    is_numpy_namespace,
     is_pydata_sparse_namespace,
     is_torch_namespace,
+    to_device,
 )
-from ._utils._typing import Array
+from ._utils._typing import Array, Device
 
-__all__ = ["xp_assert_close", "xp_assert_equal"]
+__all__ = ["as_numpy_array", "xp_assert_close", "xp_assert_equal", "xp_assert_less"]
 
 
 def _check_ns_shape_dtype(
-    actual: Array, desired: Array
+    actual: Array,
+    desired: Array,
+    check_dtype: bool,
+    check_shape: bool,
+    check_scalar: bool,
 ) -> ModuleType:  # numpydoc ignore=RT03
     """
     Assert that namespace, shape and dtype of the two arrays match.
@@ -36,6 +46,11 @@ def _check_ns_shape_dtype(
         The array produced by the tested function.
     desired : Array
         The expected array (typically hardcoded).
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
 
     Returns
     -------
@@ -47,43 +62,67 @@ def _check_ns_shape_dtype(
     msg = f"namespaces do not match: {actual_xp} != f{desired_xp}"
     assert actual_xp == desired_xp, msg
 
-    actual_shape = actual.shape
-    desired_shape = desired.shape
-    if is_dask_namespace(desired_xp):
-        # Dask uses nan instead of None for unknown shapes
-        if any(math.isnan(i) for i in cast(tuple[float, ...], actual_shape)):
-            actual_shape = actual.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
-        if any(math.isnan(i) for i in cast(tuple[float, ...], desired_shape)):
-            desired_shape = desired.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
-
-    msg = f"shapes do not match: {actual_shape} != f{desired_shape}"
-    assert actual_shape == desired_shape, msg
-
-    msg = f"dtypes do not match: {actual.dtype} != {desired.dtype}"
-    assert actual.dtype == desired.dtype, msg
+    if check_shape:
+        actual_shape = actual.shape
+        desired_shape = desired.shape
+        if is_dask_namespace(desired_xp):
+            # Dask uses nan instead of None for unknown shapes
+            if any(math.isnan(i) for i in cast(tuple[float, ...], actual_shape)):
+                actual_shape = actual.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+            if any(math.isnan(i) for i in cast(tuple[float, ...], desired_shape)):
+                desired_shape = desired.compute().shape  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+        msg = f"shapes do not match: {actual_shape} != f{desired_shape}"
+        assert actual_shape == desired_shape, msg
+
+    if check_dtype:
+        msg = f"dtypes do not match: {actual.dtype} != {desired.dtype}"
+        assert actual.dtype == desired.dtype, msg
+
+    if is_numpy_namespace(actual_xp) and check_scalar:
+        # only NumPy distinguishes between scalars and arrays; we do if check_scalar.
+        _msg = (
+            "array-ness does not match:\n Actual: "
+            f"{type(actual)}\n Desired: {type(desired)}"
+        )
+        assert np.isscalar(actual) == np.isscalar(desired), _msg
 
     return desired_xp
 
 
-def _prepare_for_test(array: Array, xp: ModuleType) -> Array:
+def as_numpy_array(array: Array, *, xp: ModuleType) -> np.typing.NDArray[Any]:  # type: ignore[explicit-any]
     """
-    Ensure that the array can be compared with xp.testing or np.testing.
-
-    This involves transferring it from GPU to CPU memory, densifying it, etc.
+    Convert array to NumPy, bypassing GPU-CPU transfer guards and densification guards.
     """
-    if is_torch_namespace(xp):
-        return array.cpu()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+    if is_cupy_namespace(xp):
+        return xp.asnumpy(array)
     if is_pydata_sparse_namespace(xp):
         return array.todense()  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
+
+    if is_torch_namespace(xp):
+        array = to_device(array, "cpu")
     if is_array_api_strict_namespace(xp):
-        # Note: we deliberately did not add a `.to_device` method in _typing.pyi
-        # even if it is required by the standard as many backends don't support it
-        return array.to_device(xp.Device("CPU_DEVICE"))  # type: ignore[attr-defined]  # pyright: ignore[reportAttributeAccessIssue]
-    # Note: nothing to do for CuPy, because it uses a bespoke test function
-    return array
+        cpu: Device = xp.Device("CPU_DEVICE")
+        array = to_device(array, cpu)
+    if is_jax_namespace(xp):
+        import jax
 
+        # Note: only needed if the transfer guard is enabled
+        cpu = cast(Device, jax.devices("cpu")[0])
+        array = to_device(array, cpu)
 
-def xp_assert_equal(actual: Array, desired: Array, err_msg: str = "") -> None:
+    return np.asarray(array)
+
+
+def xp_assert_equal(
+    actual: Array,
+    desired: Array,
+    *,
+    err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
+) -> None:
     """
     Array-API compatible version of `np.testing.assert_array_equal`.
 
@@ -95,34 +134,56 @@ def xp_assert_equal(actual: Array, desired: Array, err_msg: str = "") -> None:
         The expected array (typically hardcoded).
     err_msg : str, optional
         Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
 
     See Also
     --------
     xp_assert_close : Similar function for inexact equality checks.
     numpy.testing.assert_array_equal : Similar function for NumPy arrays.
     """
-    xp = _check_ns_shape_dtype(actual, desired)
-    actual = _prepare_for_test(actual, xp)
-    desired = _prepare_for_test(desired, xp)
+    xp = _check_ns_shape_dtype(actual, desired, check_dtype, check_shape, check_scalar)
+    actual_np = as_numpy_array(actual, xp=xp)
+    desired_np = as_numpy_array(desired, xp=xp)
+    np.testing.assert_array_equal(actual_np, desired_np, err_msg=err_msg)
 
-    if is_cupy_namespace(xp):
-        xp.testing.assert_array_equal(actual, desired, err_msg=err_msg)
-    elif is_torch_namespace(xp):
-        # PyTorch recommends using `rtol=0, atol=0` like this
-        # to test for exact equality
-        xp.testing.assert_close(
-            actual,
-            desired,
-            rtol=0,
-            atol=0,
-            equal_nan=True,
-            check_dtype=False,
-            msg=err_msg or None,
-        )
-    else:
-        import numpy as np  # pylint: disable=import-outside-toplevel
 
-        np.testing.assert_array_equal(actual, desired, err_msg=err_msg)
+def xp_assert_less(
+    x: Array,
+    y: Array,
+    *,
+    err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
+) -> None:
+    """
+    Array-API compatible version of `np.testing.assert_array_less`.
+
+    Parameters
+    ----------
+    x, y : Array
+        The arrays to compare according to ``x < y`` (elementwise).
+    err_msg : str, optional
+        Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
+
+    See Also
+    --------
+    xp_assert_close : Similar function for inexact equality checks.
+    numpy.testing.assert_array_equal : Similar function for NumPy arrays.
+    """
+    xp = _check_ns_shape_dtype(x, y, check_dtype, check_shape, check_scalar)
+    x_np = as_numpy_array(x, xp=xp)
+    y_np = as_numpy_array(y, xp=xp)
+    np.testing.assert_array_less(x_np, y_np, err_msg=err_msg)
 
 
 def xp_assert_close(
@@ -132,6 +193,9 @@ def xp_assert_close(
     rtol: float | None = None,
     atol: float = 0,
     err_msg: str = "",
+    check_dtype: bool = True,
+    check_shape: bool = True,
+    check_scalar: bool = False,
 ) -> None:
     """
     Array-API compatible version of `np.testing.assert_allclose`.
@@ -148,6 +212,11 @@ def xp_assert_close(
         Absolute tolerance. Default: 0.
     err_msg : str, optional
         Error message to display on failure.
+    check_dtype, check_shape : bool, default: True
+        Whether to check agreement between actual and desired dtypes and shapes
+    check_scalar : bool, default: False
+        NumPy only: whether to check agreement between actual and desired types -
+        0d array vs scalar.
 
     See Also
     --------
@@ -159,40 +228,26 @@ def xp_assert_close(
     -----
     The default `atol` and `rtol` differ from `xp.all(xpx.isclose(a, b))`.
     """
-    xp = _check_ns_shape_dtype(actual, desired)
-
-    floating = xp.isdtype(actual.dtype, ("real floating", "complex floating"))
-    if rtol is None and floating:
-        # multiplier of 4 is used as for `np.float64` this puts the default `rtol`
-        # roughly half way between sqrt(eps) and the default for
-        # `numpy.testing.assert_allclose`, 1e-7
-        rtol = xp.finfo(actual.dtype).eps ** 0.5 * 4
-    elif rtol is None:
-        rtol = 1e-7
-
-    actual = _prepare_for_test(actual, xp)
-    desired = _prepare_for_test(desired, xp)
-
-    if is_cupy_namespace(xp):
-        xp.testing.assert_allclose(
-            actual, desired, rtol=rtol, atol=atol, err_msg=err_msg
-        )
-    elif is_torch_namespace(xp):
-        xp.testing.assert_close(
-            actual, desired, rtol=rtol, atol=atol, equal_nan=True, msg=err_msg or None
-        )
-    else:
-        import numpy as np  # pylint: disable=import-outside-toplevel
-
-        # JAX/Dask arrays work directly with `np.testing`
-        assert isinstance(rtol, float)
-        np.testing.assert_allclose(  # type: ignore[call-overload]  # pyright: ignore[reportCallIssue]
-            actual,  # pyright: ignore[reportArgumentType]
-            desired,  # pyright: ignore[reportArgumentType]
-            rtol=rtol,
-            atol=atol,
-            err_msg=err_msg,
-        )
+    xp = _check_ns_shape_dtype(actual, desired, check_dtype, check_shape, check_scalar)
+
+    if rtol is None:
+        if xp.isdtype(actual.dtype, ("real floating", "complex floating")):
+            # multiplier of 4 is used as for `np.float64` this puts the default `rtol`
+            # roughly half way between sqrt(eps) and the default for
+            # `numpy.testing.assert_allclose`, 1e-7
+            rtol = xp.finfo(actual.dtype).eps ** 0.5 * 4
+        else:
+            rtol = 1e-7
+
+    actual_np = as_numpy_array(actual, xp=xp)
+    desired_np = as_numpy_array(desired, xp=xp)
+    np.testing.assert_allclose(  # pyright: ignore[reportCallIssue]
+        actual_np,
+        desired_np,
+        rtol=rtol,  # pyright: ignore[reportArgumentType]
+        atol=atol,
+        err_msg=err_msg,
+    )
 
 
 def xfail(

tests/test_funcs.py

@@ -196,7 +196,6 @@ def test_device(self, xp: ModuleType, device: Device):
         y = apply_where(x % 2 == 0, x, self.f1, fill_value=x)
         assert get_device(y) == device
 
-    @pytest.mark.xfail_xp_backend(Backend.SPARSE, reason="no isdtype")
     @pytest.mark.filterwarnings("ignore::RuntimeWarning")  # overflows, etc.
     @hypothesis.settings(
         # The xp and library fixtures are not regenerated between hypothesis iterations