fix(telemetry): ensures telemetry threads do block gunicorn workers (backport #5882 to 1.10) (#5921)

Resolves: #5876

## Checklist
- [x] Change(s) are motivated and described in the PR description.
- [x] Testing strategy is described if automated tests are not included
in the PR.
- [x] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines)
are followed.
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] OPTIONAL: PR description includes explicit acknowledgement of the
performance implications of the change as reported in the benchmarks PR
comment.

## Checklist
- [x] Title is accurate.
- [x] No unnecessary changes are introduced.
- [x] Description motivates each change.
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [x] Testing strategy adequately addresses listed risk(s).
- [x] Change is maintainable (easy to change, telemetry, documentation).
- [x] Release note makes sense to a user of the library.
- [x] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.

---------

## Checklist

- [ ] Change(s) are motivated and described in the PR description.
- [ ] Testing strategy is described if automated tests are not included
in the PR.
- [ ] Risk is outlined (performance impact, potential for breakage,
maintainability, etc).
- [ ] Change is maintainable (easy to change, telemetry, documentation).
- [ ] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/contributing.html#Release-Note-Guidelines)
are followed.
- [ ] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/)).
- [x] OPTIONAL: PR description includes explicit acknowledgement of the
performance implications of the change as reported in the benchmarks PR
comment.

## Reviewer Checklist

- [ ] Title is accurate.
- [ ] No unnecessary changes are introduced.
- [ ] Description motivates each change.
- [ ] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes unless absolutely necessary.
- [ ] Testing strategy adequately addresses listed risk(s).
- [ ] Change is maintainable (easy to change, telemetry, documentation).
- [ ] Release note makes sense to a user of the library.
- [ ] Reviewer has explicitly acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment.

Co-authored-by: Gabriele N. Tornetta <[email protected]>

Author: mabdinur

.circleci/config.yml

@@ -768,6 +768,7 @@ jobs:
 
   gunicorn:
     <<: *machine_executor
+    parallelism: 4
     steps:
       - attach_workspace:
           at: .

ddtrace/internal/telemetry/writer.py

@@ -206,7 +206,7 @@ def _restart_sequence(self):
     def _stop_service(self, *args, **kwargs):
         # type: (...) -> None
         super(TelemetryBase, self)._stop_service(*args, **kwargs)
-        self.join()
+        self.join(timeout=2)
 
 
 class TelemetryMetricsWriter(TelemetryBase):

releasenotes/notes/gunicorn-support-debugmode-py27-74c0b8ca63cc7d9f.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    gunicorn: This fix ensures ddtrace threads do not block the master process from spawning  workers when ``DD_TRACE_DEBUG=true``. 
+    This issue impacts gunicorn applications using gevent and ``python<=3.6``.

tests/contrib/gunicorn/test_gunicorn.py

@@ -54,6 +54,7 @@ def _gunicorn_settings_factory(
     import_auto_in_postworkerinit=False,  # type: bool
     import_auto_in_app=None,  # type: Optional[bool]
     enable_module_cloning=False,  # type: bool
+    debug_mode=False,  # type: bool
 ):
     # type: (...) -> GunicornServerSettings
     """Factory for creating gunicorn settings with simple defaults if settings are not defined."""
@@ -65,6 +66,7 @@ def _gunicorn_settings_factory(
     env["DD_REMOTE_CONFIGURATION_ENABLED"] = str(True)
     env["DD_REMOTECONFIG_POLL_INTERVAL_SECONDS"] = str(SERVICE_INTERVAL)
     env["DD_PROFILING_UPLOAD_INTERVAL"] = str(SERVICE_INTERVAL)
+    env["DD_TRACE_DEBUG"] = str(debug_mode)
     return GunicornServerSettings(
         env=env,
         directory=directory,
@@ -151,6 +153,11 @@ def gunicorn_server(gunicorn_server_settings, tmp_path):
     use_ddtracerun=False,
     import_auto_in_postworkerinit=True,
 )
+SETTINGS_GEVENT_DDTRACERUN_DEBUGMODE_MODULE_CLONE = _gunicorn_settings_factory(
+    worker_class="gevent",
+    debug_mode=True,
+    enable_module_cloning=True,
+)
 
 
 @pytest.mark.skipif(sys.version_info >= (3, 11), reason="Gunicorn is only supported up to 3.10")
@@ -159,7 +166,9 @@ def gunicorn_server(gunicorn_server_settings, tmp_path):
     [
         SETTINGS_GEVENT_APPIMPORT,
         SETTINGS_GEVENT_POSTWORKERIMPORT,
+        SETTINGS_GEVENT_DDTRACERUN,
         SETTINGS_GEVENT_DDTRACERUN_MODULE_CLONE,
+        SETTINGS_GEVENT_DDTRACERUN_DEBUGMODE_MODULE_CLONE,
     ],
 )
 def test_no_known_errors_occur(gunicorn_server_settings, tmp_path):

tests/contrib/gunicorn/wsgi_mw_app.py

@@ -2,19 +2,21 @@
 This app exists to replicate and report on failures and degraded behavior that can arise when using ddtrace with
 gunicorn
 """
-import json
 import os
 
+
+if os.getenv("_DD_TEST_IMPORT_AUTO"):
+    import ddtrace.auto  # noqa: F401  # isort: skip
+
+import json
+
 from ddtrace import tracer
 from ddtrace.contrib.wsgi import DDWSGIMiddleware
 from ddtrace.profiling import bootstrap
 import ddtrace.profiling.auto  # noqa
 from tests.webclient import PingFilter
 
 
-if os.getenv("_DD_TEST_IMPORT_AUTO"):
-    import ddtrace.auto  # noqa: F401  # isort: skip
-
 tracer.configure(
     settings={
         "FILTERS": [PingFilter()],