Merge pull request scipy#21334 from lucascolley/xfail-xp

TST: add `xfail_xp_backends`

Author: ev-br

scipy/conftest.py

@@ -45,6 +45,10 @@ def pytest_configure(config):
         "skip_xp_backends(*backends, reasons=None, np_only=False, cpu_only=False, "
         "exceptions=None): "
         "mark the desired skip configuration for the `skip_xp_backends` fixture.")
+    config.addinivalue_line("markers",
+        "xfail_xp_backends(*backends, reasons=None, np_only=False, cpu_only=False, "
+        "exceptions=None): "
+        "mark the desired xfail configuration for the `xfail_xp_backends` fixture.")
 
 
 def pytest_runtest_setup(item):
@@ -178,8 +182,26 @@ def check_fpu_mode(request):
 
 @pytest.fixture
 def skip_xp_backends(xp, request):
+    """See the `skip_or_xfail_xp_backends` docstring."""
+    if "skip_xp_backends" not in request.keywords:
+        return
+    backends = request.keywords["skip_xp_backends"].args
+    kwargs = request.keywords["skip_xp_backends"].kwargs
+    skip_or_xfail_xp_backends(xp, backends, kwargs, skip_or_xfail='skip')
+
+@pytest.fixture
+def xfail_xp_backends(xp, request):
+    """See the `skip_or_xfail_xp_backends` docstring."""
+    if "xfail_xp_backends" not in request.keywords:
+        return
+    backends = request.keywords["xfail_xp_backends"].args
+    kwargs = request.keywords["xfail_xp_backends"].kwargs
+    skip_or_xfail_xp_backends(xp, backends, kwargs, skip_or_xfail='xfail')
+    
+
+def skip_or_xfail_xp_backends(xp, backends, kwargs, skip_or_xfail='skip'):
     """
-    Skip based on the ``skip_xp_backends`` marker.
+    Skip based on the ``skip_xp_backends`` or ``xfail_xp_backends`` marker.
     
     See the "Support for the array API standard" docs page for usage examples.
 
@@ -201,51 +223,50 @@ def skip_xp_backends(xp, request):
         any ``backends`` in this case. To specify a reason, pass a
         singleton list to ``reasons``. Default: ``False``.
     cpu_only : bool, optional
-        When ``True``, the test is skipped on non-CPU devices.
+        When ``True``, the test is skipped/x-failed on non-CPU devices.
         There is no need to provide any ``backends`` in this case,
         but any ``backends`` will also be skipped on the CPU.
         Default: ``False``.
     exceptions : list, optional
-        A list of exceptions for use with `cpu_only`. This should be provided
-        when delegation is implemented for some, but not all, non-CPU backends.
+        A list of exceptions for use with `cpu_only` or `np_only`.
+        This should be provided when delegation is implemented for some,
+        but not all, non-CPU/non-NumPy backends.
     """
-    if "skip_xp_backends" not in request.keywords:
-        return
-    backends = request.keywords["skip_xp_backends"].args
-    kwargs = request.keywords["skip_xp_backends"].kwargs
+    skip_or_xfail = getattr(pytest, skip_or_xfail)
+
     np_only = kwargs.get("np_only", False)
     cpu_only = kwargs.get("cpu_only", False)
-    exceptions = kwargs.get("exceptions", None)
+    exceptions = kwargs.get("exceptions", [])
 
     # input validation
     if np_only and cpu_only:
         raise ValueError("at most one of `np_only` and `cpu_only` should be provided")
-    if exceptions and not cpu_only:
-        raise ValueError("`exceptions` is only valid alongside `cpu_only`")
+    if exceptions and not (cpu_only or np_only):
+        raise ValueError("`exceptions` is only valid alongside `cpu_only` or `np_only`")
 
     if np_only:
         reasons = kwargs.get("reasons", ["do not run with non-NumPy backends."])
         if len(reasons) > 1:
             raise ValueError("please provide a singleton list to `reasons` "
                              "when using `np_only`")
         reason = reasons[0]
-        if xp.__name__ != 'numpy':
-            pytest.skip(reason=reason)
+        if xp.__name__ != 'numpy' and xp.__name__ not in exceptions:
+            skip_or_xfail(reason=reason)
         return
     if cpu_only:
         reason = ("no array-agnostic implementation or delegation available "
                   "for this backend and device")
         exceptions = [] if exceptions is None else exceptions
         if SCIPY_ARRAY_API and SCIPY_DEVICE != 'cpu':
             if xp.__name__ == 'cupy' and 'cupy' not in exceptions:
-                pytest.skip(reason=reason)
+                skip_or_xfail(reason=reason)
             elif xp.__name__ == 'torch' and 'torch' not in exceptions:
                 if 'cpu' not in xp.empty(0).device.type:
-                    pytest.skip(reason=reason)
+                    skip_or_xfail(reason=reason)
             elif xp.__name__ == 'jax.numpy' and 'jax.numpy' not in exceptions:
                 for d in xp.empty(0).devices():
                     if 'cpu' not in d.device_kind:
-                        pytest.skip(reason=reason)
+                        skip_or_xfail(reason=reason)
 
     if backends is not None:
         reasons = kwargs.get("reasons", False)
@@ -255,7 +276,7 @@ def skip_xp_backends(xp, request):
                     reason = f"do not run with array API backend: {backend}"
                 else:
                     reason = reasons[i]
-                pytest.skip(reason=reason)
+                skip_or_xfail(reason=reason)
 
 
 # Following the approach of NumPy's conftest.py...

scipy/ndimage/tests/test_filters.py

@@ -21,7 +21,9 @@
 
 from scipy.conftest import array_api_compatible
 skip_xp_backends = pytest.mark.skip_xp_backends
+xfail_xp_backends = pytest.mark.xfail_xp_backends
 pytestmark = [array_api_compatible, pytest.mark.usefixtures("skip_xp_backends"),
+              pytest.mark.usefixtures("xfail_xp_backends"),
               skip_xp_backends(cpu_only=True, exceptions=['cupy', 'jax.numpy']),]
 
 
@@ -179,12 +181,10 @@ def test_correlate01(self, xp):
         output = ndimage.convolve1d(array, weights)
         assert_array_almost_equal(output, expected)
 
+    @xfail_xp_backends('cupy', reason="Differs by a factor of two?")
     @skip_xp_backends("jax.numpy", reasons=["output array is read-only."],
                       cpu_only=True, exceptions=['cupy', 'jax.numpy'])
     def test_correlate01_overlap(self, xp):
-        if is_cupy(xp):
-            pytest.xfail("Differs by a factor of two?")
-
         array = xp.reshape(xp.arange(256), (16, 16))
         weights = xp.asarray([2])
         expected = 2 * array
@@ -334,12 +334,12 @@ def test_correlate12(self, xp):
         output = ndimage.convolve(array, kernel)
         assert_array_almost_equal(xp.asarray([[6, 8, 9], [9, 11, 12]]), output)
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     @pytest.mark.parametrize('dtype_kernel', types)
     def test_correlate13(self, dtype_array, dtype_kernel, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
         dtype_kernel = getattr(xp, dtype_kernel)
 
@@ -356,12 +356,12 @@ def test_correlate13(self, dtype_array, dtype_kernel, xp):
         assert_array_almost_equal(xp.asarray([[6, 8, 9], [9, 11, 12]]), output)
         assert output.dtype.type == dtype_kernel
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     @pytest.mark.parametrize('dtype_output', types)
     def test_correlate14(self, dtype_array, dtype_output, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
         dtype_output = getattr(xp, dtype_output)
 
@@ -378,11 +378,11 @@ def test_correlate14(self, dtype_array, dtype_output, xp):
         assert_array_almost_equal(xp.asarray([[6, 8, 9], [9, 11, 12]]), output)
         assert output.dtype.type == dtype_output
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     def test_correlate15(self, dtype_array, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
 
         kernel = xp.asarray([[1, 0],
@@ -397,11 +397,11 @@ def test_correlate15(self, dtype_array, xp):
         assert_array_almost_equal(xp.asarray([[6, 8, 9], [9, 11, 12]]), output)
         assert output.dtype.type == xp.float32
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     def test_correlate16(self, dtype_array, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
 
         kernel = xp.asarray([[0.5, 0],
@@ -429,11 +429,11 @@ def test_correlate17(self, xp):
         output = ndimage.convolve1d(array, kernel, origin=-1)
         assert_array_almost_equal(tcov, output)
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     def test_correlate18(self, dtype_array, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
 
         kernel = xp.asarray([[1, 0],
@@ -460,11 +460,11 @@ def test_correlate_mode_sequence(self, xp):
         with assert_raises(RuntimeError):
             ndimage.convolve(array, kernel, mode=['nearest', 'reflect'])
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     def test_correlate19(self, dtype_array, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
 
         kernel = xp.asarray([[1, 0],
@@ -483,12 +483,12 @@ def test_correlate19(self, dtype_array, xp):
         assert_array_almost_equal(xp.asarray([[3, 5, 6], [6, 8, 9]]), output)
         assert output.dtype.type == xp.float32
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     @pytest.mark.parametrize('dtype_output', types)
     def test_correlate20(self, dtype_array, dtype_output, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
         dtype_output = getattr(xp, dtype_output)
 
@@ -512,12 +512,12 @@ def test_correlate21(self, xp):
         output = ndimage.convolve1d(array, weights, axis=0)
         assert_array_almost_equal(output, expected)
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_array', types)
     @pytest.mark.parametrize('dtype_output', types)
     def test_correlate22(self, dtype_array, dtype_output, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            pytest.xfail("output=dtype is numpy-specific")
-
         dtype_array = getattr(xp, dtype_array)
         dtype_output = getattr(xp, dtype_output)
 
@@ -603,15 +603,14 @@ def test_correlate26(self, xp):
         y = ndimage.correlate1d(xp.ones(1), xp.ones(5), mode='mirror')
         xp_assert_equal(y, xp.asarray([5.]))
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_kernel', complex_types)
     @pytest.mark.parametrize('dtype_input', types)
     @pytest.mark.parametrize('dtype_output', complex_types)
     def test_correlate_complex_kernel(self, dtype_input, dtype_kernel,
                                       dtype_output, xp):
-        if not (is_numpy(xp) or is_cupy(xp)):
-            # XXX: the issue is in _validate_complex, _correlate_complex
-            pytest.xfail("output=dtype are numpy-specific")
-
         dtype_input = getattr(xp, dtype_input)
         dtype_kernel = getattr(xp, dtype_kernel)
         dtype_output = getattr(xp, dtype_output)
@@ -622,6 +621,9 @@ def test_correlate_complex_kernel(self, dtype_input, dtype_kernel,
                             [4, 5, 6]], dtype=dtype_input)
         self._validate_complex(xp, array, kernel, dtype_output)
 
+    @xfail_xp_backends(np_only=True,
+                       reasons=["output=dtype is numpy-specific"],
+                       exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype_kernel', complex_types)
     @pytest.mark.parametrize('dtype_input', types)
     @pytest.mark.parametrize('dtype_output', complex_types)
@@ -634,8 +636,6 @@ def test_correlate_complex_kernel_cval(self, dtype_input, dtype_kernel,
 
         if is_cupy(xp) and mode == 'grid-constant':
             pytest.xfail('https://github.com/cupy/cupy/issues/8404')
-        elif not is_numpy(xp):
-            pytest.xfail("output= arrays are numpy-specific")
 
         # test use of non-zero cval with complex inputs
         # also verifies that mode 'grid-constant' does not segfault
@@ -646,16 +646,14 @@ def test_correlate_complex_kernel_cval(self, dtype_input, dtype_kernel,
         self._validate_complex(xp, array, kernel, dtype_output, mode=mode,
                                cval=5.0)
 
+    @xfail_xp_backends('cupy', reasons=["cupy/cupy#8405"])
     @pytest.mark.parametrize('dtype_kernel', complex_types)
     @pytest.mark.parametrize('dtype_input', types)
     def test_correlate_complex_kernel_invalid_cval(self, dtype_input,
                                                    dtype_kernel, xp):
         dtype_input = getattr(xp, dtype_input)
         dtype_kernel = getattr(xp, dtype_kernel)
 
-        if is_cupy(xp):
-            pytest.xfail("https://github.com/cupy/cupy/issues/8405")
-
         # cannot give complex cval with a real image
         kernel = xp.asarray([[1, 0],
                              [0, 1 + 1j]], dtype=dtype_kernel)
@@ -726,7 +724,10 @@ def test_correlate1d_complex_input(self, dtype_input, dtype_kernel,
         array = xp.asarray([1, 2j, 3, 1 + 4j, 5, 6j], dtype=dtype_input)
         self._validate_complex(xp, array, kernel, dtype_output)
 
-    @skip_xp_backends(np_only=True, reasons=['output=dtype is numpy-specific'])
+    @xfail_xp_backends('cupy', reasons=["cupy/cupy#8405"])
+    @skip_xp_backends(np_only=True,
+                      reasons=['output=dtype is numpy-specific'],
+                      exceptions=['cupy'])
     @pytest.mark.parametrize('dtype_kernel', types)
     @pytest.mark.parametrize('dtype_input', complex_types)
     @pytest.mark.parametrize('dtype_output', complex_types)
@@ -736,9 +737,6 @@ def test_correlate1d_complex_input_cval(self, dtype_input, dtype_kernel,
         dtype_kernel = getattr(xp, dtype_kernel)
         dtype_output = getattr(xp, dtype_output)
 
-        if is_cupy(xp):
-            pytest.xfail("https://github.com/cupy/cupy/issues/8405")
-
         kernel = xp.asarray([1, 0, 1], dtype=dtype_kernel)
         array = xp.asarray([1, 2j, 3, 1 + 4j, 5, 6j], dtype=dtype_input)
         self._validate_complex(xp, array, kernel, dtype_output, mode='constant',
@@ -757,15 +755,14 @@ def test_correlate_complex_input_and_kernel(self, dtype, dtype_output, xp):
                             [1 + 4j, 5, 6j]], dtype=dtype)
         self._validate_complex(xp, array, kernel, dtype_output)
 
+    @xfail_xp_backends('cupy', reasons=["cupy/cupy#8405"])
+    @skip_xp_backends(np_only=True,
+                      reasons=["output=dtype is numpy-specific"],
+                      exceptions=['cupy'],)
     @pytest.mark.parametrize('dtype', complex_types)
     @pytest.mark.parametrize('dtype_output', complex_types)
     def test_correlate_complex_input_and_kernel_cval(self, dtype,
                                                      dtype_output, xp):
-        if not is_numpy(xp):
-            pytest.xfail("output=dtype is numpy-specific")
-        elif is_cupy(xp):
-            pytest.xfail("https://github.com/cupy/cupy/issues/8405")
-
         dtype = getattr(xp, dtype)
         dtype_output = getattr(xp, dtype_output)
 

scipy/ndimage/tests/test_morphology.py

@@ -12,7 +12,9 @@
 
 from scipy.conftest import array_api_compatible
 skip_xp_backends = pytest.mark.skip_xp_backends
+xfail_xp_backends = pytest.mark.xfail_xp_backends
 pytestmark = [array_api_compatible, pytest.mark.usefixtures("skip_xp_backends"),
+              pytest.mark.usefixtures("xfail_xp_backends"),
               skip_xp_backends(cpu_only=True, exceptions=['cupy', 'jax.numpy'],)]
 
 
@@ -2560,9 +2562,8 @@ def test_black_tophat02(self, xp):
                                       structure=structure)
         assert_array_almost_equal(expected, output)
 
+    @xfail_xp_backends('cupy', reasons=["cupy/cupy#8399"])
     def test_black_tophat03(self, xp):
-        if is_cupy(xp):
-            pytest.xfail("https://github.com/cupy/cupy/issues/8399")
 
         array = np.asarray([[1, 0, 0, 0, 0, 0, 0],
                             [0, 1, 1, 1, 1, 1, 0],