feat(pytest): added xfail/xpass test marking functionality (#2416)

* Added xfail/xpass test marking functionality into pytest plugin.

* Added changelog entry

* Changelog entry

Co-authored-by: Brett Langdon <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

ddtrace/contrib/pytest/plugin.py

@@ -105,7 +105,6 @@ def pytest_runtest_protocol(item, nextitem):
         markers = [marker.kwargs for marker in item.iter_markers(name="dd_tags")]
         for tags in markers:
             span.set_tags(tags)
-
         _store_span(item, span)
 
         yield
@@ -132,8 +131,27 @@ def pytest_runtest_makereport(item, call):
 
     try:
         result = outcome.get_result()
+
+        if hasattr(result, "wasxfail") or "xfail" in result.keywords:
+            if result.skipped:
+                # XFail tests that fail are recorded skipped by pytest
+                span.set_tag(test.RESULT, test.Status.XFAIL.value)
+                span.set_tag(test.XFAIL_REASON, result.wasxfail)
+            else:
+                span.set_tag(test.RESULT, test.Status.XPASS.value)
+                if result.passed:
+                    # XPass (strict=False) are recorded passed by pytest
+                    span.set_tag(test.XFAIL_REASON, result.wasxfail)
+                else:
+                    # XPass (strict=True) are recorded failed by pytest, longrepr contains reason
+                    span.set_tag(test.XFAIL_REASON, result.longrepr)
+
         if result.skipped:
-            span.set_tag(test.STATUS, test.Status.SKIP.value)
+            if hasattr(result, "wasxfail"):
+                # XFail tests that fail are recorded skipped by pytest, should be passed instead
+                span.set_tag(test.STATUS, test.Status.PASS.value)
+            else:
+                span.set_tag(test.STATUS, test.Status.SKIP.value)
             reason = _extract_reason(call)
             if reason is not None:
                 span.set_tag(test.SKIP_REASON, reason)

ddtrace/ext/test.py

@@ -14,6 +14,9 @@
 # Test Name
 NAME = TEST_NAME = "test.name"
 
+# Pytest Result (XFail, XPass)
+RESULT = TEST_RESULT = "pytest.result"
+
 # Skip Reason
 SKIP_REASON = TEST_SKIP_REASON = "test.skip_reason"
 
@@ -29,8 +32,13 @@
 # Test Type
 TYPE = TEST_TYPE = "test.type"
 
+# XFail Reason
+XFAIL_REASON = TEST_XFAIL_REASON = "pytest.xfail.reason"
+
 
 class Status(Enum):
     PASS = "pass"
     FAIL = "fail"
     SKIP = "skip"
+    XFAIL = "xfail"
+    XPASS = "xpass"

docs/spelling_wordlist.txt

@@ -125,3 +125,4 @@ versioned
 vertica
 whitelist
 wsgi
+xfail

releasenotes/notes/pytest-xfail-c15914f8ae3281c9.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - |
+    The ddtrace pytest plugin can now label spans from test cases marked xfail with the tag "pytest.result"
+     and the reason for being marked xfail under the tag "pytest.xfail.reason".

tests/contrib/pytest/test_pytest.py

@@ -145,6 +145,72 @@ def test_body():
         assert spans[1].get_tag(test.STATUS) == test.Status.SKIP.value
         assert spans[1].get_tag(test.SKIP_REASON) == "body"
 
+    def test_xfail_fails(self):
+        """Test xfail (expected failure) which fails, should be marked as pass."""
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.xfail(reason="test should fail")
+            def test_should_fail():
+                assert 0
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        # pytest records xfail as skipped
+        rec.assertoutcome(skipped=1)
+        spans = self.pop_spans()
+
+        assert len(spans) == 1
+        assert spans[0].get_tag(test.STATUS) == test.Status.PASS.value
+        assert spans[0].get_tag(test.RESULT) == test.Status.XFAIL.value
+        assert spans[0].get_tag(test.XFAIL_REASON) == "test should fail"
+
+    def test_xpass_not_strict(self):
+        """Test xpass (unexpected passing) with strict=False, should be marked as pass."""
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.xfail(reason="test should fail")
+            def test_should_fail():
+                pass
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        rec.assertoutcome(passed=1)
+        spans = self.pop_spans()
+
+        assert len(spans) == 1
+        assert spans[0].get_tag(test.STATUS) == test.Status.PASS.value
+        assert spans[0].get_tag(test.RESULT) == test.Status.XPASS.value
+        assert spans[0].get_tag(test.XFAIL_REASON) == "test should fail"
+
+    def test_xpass_strict(self):
+        """Test xpass (unexpected passing) with strict=True, should be marked as fail."""
+        py_file = self.testdir.makepyfile(
+            """
+            import pytest
+
+            @pytest.mark.xfail(reason="test should fail", strict=True)
+            def test_should_fail():
+                pass
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        rec.assertoutcome(failed=1)
+        spans = self.pop_spans()
+
+        assert len(spans) == 1
+        assert spans[0].get_tag(test.STATUS) == test.Status.FAIL.value
+        assert spans[0].get_tag(test.RESULT) == test.Status.XPASS.value
+        # Note: xpass (strict=True) does not mark the reason with result.wasxfail but into result.longrepr,
+        # however this provides the entire traceback/error into longrepr.
+        assert "test should fail" in spans[0].get_tag(test.XFAIL_REASON)
+
     def test_tags(self):
         """Test ddspan tags."""
         py_file = self.testdir.makepyfile(