fix(ci_visibility): do not propagate ddtrace logs to the root logger (#14121)

When pytest is run with log capture on (which is the default, unless
`-s` or similar is used), it replaces `sys.stdout` and `sys.stderr` with
temporary file descriptors that are closed at the end of the test
session. If a custom logger is created referencing one of those streams,
any logs emitted after the end of the test session will try to write to
the closed file descriptor, leading to an ` I/O operation on closed
file` error. See #12018 for example cases.

Annoyingly, this happens even if ddtrace is not explicitly enabled in
pytest with `--ddtrace`, because before we even have the opportunity to
parse command line arguments, `ddtrace/__init__.py` will load, which
sets up a bunch of things, including loggers.

At the beginning of a pytest session, we call a
`take_over_logger_stream_handler()` function which sets up the `ddtrace`
logger specifically for test runs. This PR:
- Changes `take_over_logger_stream_handler()` to not propagate logs from
the `ddtrace` handler to the root handler. This means the `ddtrace`
handler will not be affected by custom loggers by default.

- Makes `take_over_logger_stream_handler()` run even if `--ddtrace` is
not enabled. This is not the ideal solution in the sky: ideally we would
not enable ddtrace features at all when `--ddtrace` is not activated,
however this is a bigger change to the way ddtrace works at the moment,
so this PR focuses on avoiding the logging issue.

Finally, this change makes the current tests that use `caplog` to check
for logs emitted by `ddtrace` to break, because `caplog` depends on the
logs being propagated to the root logger to capture them, so this PR
adds a wrapping `caplog` fixture in our `conftest.py` that re-enables
log propagation for those tests. (The alternative would be changing
every `caplog.at_level(lvl)` call in our tests to `caplog.at_level(lvl,
"ddtrace")`, but there are _a lot_ of them.)

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: vitor-de-araujo

ddtrace/contrib/internal/pytest/_plugin_v2.py

@@ -237,11 +237,12 @@ def _pytest_load_initial_conftests_pre_yield(early_config, parser, args):
     ModuleCodeCollector has a tangible impact on the time it takes to load modules, so it should only be installed if
     coverage collection is requested by the backend.
     """
+    take_over_logger_stream_handler()
+
     if not _is_enabled_early(early_config, args):
         return
 
     try:
-        take_over_logger_stream_handler()
         dd_config.test_visibility.itr_skipping_level = ITR_SKIPPING_LEVEL.SUITE
         enable_test_visibility(config=dd_config.pytest)
         if InternalTestSession.should_collect_coverage():

ddtrace/internal/ci_visibility/utils.py

@@ -112,11 +112,10 @@ def take_over_logger_stream_handler(remove_ddtrace_stream_handlers=True):
         log.debug("CIVisibility not taking over ddtrace logger because level is set to: %s", level)
         return
 
-    root_logger = logging.getLogger()
+    ddtrace_logger = logging.getLogger("ddtrace")
 
     if remove_ddtrace_stream_handlers:
         log.debug("CIVisibility removing DDTrace logger handler")
-        ddtrace_logger = logging.getLogger("ddtrace")
         for handler in list(ddtrace_logger.handlers):
             ddtrace_logger.removeHandler(handler)
     else:
@@ -136,8 +135,9 @@ def take_over_logger_stream_handler(remove_ddtrace_stream_handlers=True):
         log.warning("Invalid log level: %s", level)
         return
 
-    root_logger.addHandler(ci_visibility_handler)
-    root_logger.setLevel(min(root_logger.level, ci_visibility_handler.level))
+    ddtrace_logger.addHandler(ci_visibility_handler)
+    ddtrace_logger.setLevel(min(ddtrace_logger.level, ci_visibility_handler.level))
+    ddtrace_logger.propagate = False
 
     log.debug("logger setup complete")
 

releasenotes/notes/ci_visibility-fix-pytest-log-capture-b135eb77a4fcd2b9.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    CI Visibility: This fix resolves an issue where setting custom loggers during a test session could cause the tracer
+    to emit logs to a logging stream handler that was already closed by ``pytest``, leading to ``I/O operation on closed
+    file`` errors at the end of the test session.

tests/conftest.py

@@ -6,6 +6,7 @@
 import importlib
 from itertools import product
 import json
+import logging
 import os
 from os.path import split
 from os.path import splitext
@@ -678,3 +679,20 @@ def test_agent_session(telemetry_writer, request):
     finally:
         os.environ["DD_CIVISIBILITY_AGENTLESS_ENABLED"] = p_agentless
         telemetry_writer.reset_queues()
+
+
+@pytest.fixture()
+def caplog(caplog):
+    """
+    During tests, ddtrace logs are not propagated to the root logger by default (see PR #14121).
+    This breaks caplog tests that capture logs from the ddtrace logger, so we need to re-enable propagation for those.
+    """
+    ddtrace_logger = logging.getLogger("ddtrace")
+
+    try:
+        original_propagate = ddtrace_logger.propagate
+        ddtrace_logger.propagate = True
+        yield caplog
+
+    finally:
+        ddtrace_logger.propagate = original_propagate

tests/contrib/pytest/test_pytest.py

@@ -4187,6 +4187,94 @@ def test_dependency_collection_disabled():
         result = self.subprocess_run("--ddtrace", file_name)
         assert result.ret == 0
 
+    def test_pytest_ddtrace_logger_unaffected_by_log_capture(self):
+        py_file = self.testdir.makepyfile(
+            """
+            import os
+            import sys
+            import logging.config
+
+            def test_log_capture():
+                config = {
+                    "version": 1,
+                    "disable_existing_loggers": False,
+                    "formatters": {
+                        "simple": {
+                            "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                            "datefmt": "%Y-%m-%d %H:%M:%S"
+                        }
+                    },
+                    "handlers": {
+                        "stdout": {
+                            "class": "logging.StreamHandler",
+                            "level": "DEBUG",
+                            "formatter": "simple",
+                            "stream": "ext://sys.stdout"
+                        }
+                    },
+                    "loggers": {
+                        "root": {
+                            "level": "DEBUG",
+                            "handlers": [
+                                "stdout"
+                            ]
+                        }
+                    }
+                }
+                logging.config.dictConfig(config)
+
+            """
+        )
+        file_name = os.path.basename(py_file.strpath)
+
+        result = self.subprocess_run("--ddtrace", file_name)
+        assert "I/O operation on closed file" not in result.stderr.str()
+        assert result.ret == 0
+
+    def test_pytest_no_ddtrace_logger_unaffected_by_log_capture(self):
+        py_file = self.testdir.makepyfile(
+            """
+            import os
+            import sys
+            import logging.config
+
+            def test_log_capture():
+                config = {
+                    "version": 1,
+                    "disable_existing_loggers": False,
+                    "formatters": {
+                        "simple": {
+                            "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+                            "datefmt": "%Y-%m-%d %H:%M:%S"
+                        }
+                    },
+                    "handlers": {
+                        "stdout": {
+                            "class": "logging.StreamHandler",
+                            "level": "DEBUG",
+                            "formatter": "simple",
+                            "stream": "ext://sys.stdout"
+                        }
+                    },
+                    "loggers": {
+                        "root": {
+                            "level": "DEBUG",
+                            "handlers": [
+                                "stdout"
+                            ]
+                        }
+                    }
+                }
+                logging.config.dictConfig(config)
+
+            """
+        )
+        file_name = os.path.basename(py_file.strpath)
+
+        result = self.subprocess_run(file_name)
+        assert "I/O operation on closed file" not in result.stderr.str()
+        assert result.ret == 0
+
 
 def test_pytest_coverage_data_format_handling_none_value():
     """Test that coverage data format issues are handled correctly with proper logging for None value."""