[CI] Resotre MacOS test sanity

DeformConv is only implemented for forward on MPS right now, so skip
backward checks

Logic for modifying markers in place are stolen from
`torch.testing._internal.optests`, namely from https://github.com/pytorch/pytorch/blob/310e8361c565ca1602e719e4c812dc3931ec84d7/torch/testing/_internal/optests/generate_tests.py#L269

Author: malfet

test/test_ops.py

@@ -1200,12 +1200,28 @@ def test_forward_scriptability(self):
         # Non-regression test for https://github.com/pytorch/vision/issues/4078
         torch.jit.script(ops.DeformConv2d(in_channels=8, out_channels=8, kernel_size=3))
 
+# NS: Removeme once bacward is implemented
+def xfail_if_mps(x):
+    mps_xfail_param = pytest.param("mps", marks=(pytest.mark.needs_mps, pytest.mark.xfail))
+    new_pytestmark = []
+    for mark in x.pytestmark:
+        if isinstance(mark, pytest.Mark) and mark.name == "parametrize":
+            if mark.args[0] == 'device':
+                params = cpu_and_cuda() + (mps_xfail_param,)
+                new_pytestmark.append(pytest.mark.parametrize('device', params))
+                continue
+        new_pytestmark.append(mark)
+    x.__dict__["pytestmark"] = new_pytestmark
+    return x
+
 
 optests.generate_opcheck_tests(
     testcase=TestDeformConv,
     namespaces=["torchvision"],
     failures_dict_path=os.path.join(os.path.dirname(__file__), "optests_failures_dict.json"),
-    additional_decorators=[],
+    # Skip tests due to unimplemented backward
+    additional_decorators={"test_aot_dispatch_dynamic__test_forward" : [xfail_if_mps],
+                           "test_autograd_registration__test_forward" : [xfail_if_mps]},
     test_utils=OPTESTS,
 )