fix(pytest): properly propagate failures and skipped tests when used with unittest [backport 1.14] (#6209)

Backport ecaeac8 from #6204 to 1.14.

Pytest: Propagate properly failures and skipped information when used
with unittest.TestCase

## Checklist
- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed. If no release note is required, add label
`changelog/no-changelog`.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] Backport labels are set (if
[applicable](../docs/contributing.rst#release-branch-maintenance))

## Reviewer Checklist
- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](../docs/contributing.rst#release-branch-maintenance)

Co-authored-by: Federico Mon <[email protected]>

Author: github-actions[bot]

ddtrace/contrib/pytest/plugin.py

@@ -67,6 +67,8 @@ def _store_span(item, span):
 
 def _mark_failed(item):
     """Store test failed status at `pytest.Item` instance."""
+    if item.parent:
+        _mark_failed(item.parent)
     setattr(item, "_failed", True)
 
 
@@ -77,6 +79,8 @@ def _check_failed(item):
 
 def _mark_not_skipped(item):
     """Mark test suite/module/session `pytest.Item` as not skipped."""
+    if item.parent:
+        _mark_not_skipped(item.parent)
     setattr(item, "_fully_skipped", False)
 
 

docs/spelling_wordlist.txt

@@ -199,6 +199,7 @@ submodule
 submodules
 substring
 testagent
+TestCase
 timestamp
 tokenizer
 tracestate
@@ -207,6 +208,7 @@ uWSGI
 unbuffered
 unicode
 uninstrumented
+unittest
 unix
 unpatch
 unpatched

releasenotes/notes/fix-pytest-skip-fail-information-if-testcase-used-f8ccd7bf90615d78.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    pytest: This fix resolves an issue where failures and non-skipped tests were not propagated properly when
+    unittest.TestCase classes were used.

tests/contrib/pytest/test_pytest.py

@@ -1067,6 +1067,96 @@ def test_ok():
         assert test_suite_span.get_tag("test.status") == "pass"
         assert test_session_span.get_tag("test.status") == "pass"
 
+    def test_pytest_some_skipped_tests_does_not_propagate_in_testcase(self):
+        """Test that if not all tests are skipped, that status does not propagate upwards."""
+        py_file = self.testdir.makepyfile(
+            """
+            import unittest
+            import pytest
+
+            class MyTest(unittest.TestCase):
+
+                @pytest.mark.skip(reason="Because")
+                def test_not_ok_but_skipped(self):
+                    assert 0
+
+                def test_ok(self):
+                    assert True
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        rec.assertoutcome(skipped=1, passed=1)
+        spans = self.pop_spans()
+        test_span_skipped = spans[0]
+        test_span_ok = spans[1]
+        test_suite_span = spans[3]
+        test_session_span = spans[2]
+        assert test_span_skipped.get_tag("test.status") == "skip"
+        assert test_span_ok.get_tag("test.status") == "pass"
+        assert test_suite_span.get_tag("test.status") == "pass"
+        assert test_session_span.get_tag("test.status") == "pass"
+
+    def test_pytest_all_skipped_tests_does_propagate_in_testcase(self):
+        """Test that if all tests are skipped, that status is propagated upwards."""
+        py_file = self.testdir.makepyfile(
+            """
+            import unittest
+            import pytest
+
+            class MyTest(unittest.TestCase):
+
+                @pytest.mark.skip(reason="Because")
+                def test_not_ok_but_skipped(self):
+                    assert 0
+
+                @pytest.mark.skip(reason="Because")
+                def test_ok_but_skipped(self):
+                    assert True
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        rec.assertoutcome(skipped=2, passed=0)
+        spans = self.pop_spans()
+        test_span_skipped = spans[0]
+        test_span_ok = spans[1]
+        test_suite_span = spans[3]
+        test_session_span = spans[2]
+        assert test_span_skipped.get_tag("test.status") == "skip"
+        assert test_span_ok.get_tag("test.status") == "skip"
+        assert test_suite_span.get_tag("test.status") == "skip"
+        assert test_session_span.get_tag("test.status") == "skip"
+
+    def test_pytest_failed_tests_propagate_in_testcase(self):
+        """Test that if any test fails, that status is propagated upwards."""
+        py_file = self.testdir.makepyfile(
+            """
+            import unittest
+            import pytest
+
+            class MyTest(unittest.TestCase):
+
+                def test_not_ok(self):
+                    assert 0
+
+                def test_ok(self):
+                    assert True
+        """
+        )
+        file_name = os.path.basename(py_file.strpath)
+        rec = self.inline_run("--ddtrace", file_name)
+        rec.assertoutcome(failed=1, passed=1)
+        spans = self.pop_spans()
+        test_span_skipped = spans[0]
+        test_span_ok = spans[1]
+        test_suite_span = spans[3]
+        test_session_span = spans[2]
+        assert test_span_skipped.get_tag("test.status") == "fail"
+        assert test_span_ok.get_tag("test.status") == "pass"
+        assert test_suite_span.get_tag("test.status") == "fail"
+        assert test_session_span.get_tag("test.status") == "fail"
+
     def test_pytest_module(self):
         """Test that running pytest on a test package will generate a test module span."""
         package_a_dir = self.testdir.mkpydir("test_package_a")