fix(tracer): fix sampling mechanism tag propagation bug (#6579)

This change fixes a bug causing `_dd.p.*` span tags not to be inherited
by child spans when there is a process boundary between the parent and
child spans. This makes cross-process parenting match the behavior of
parenting within a single process with respect to tag inheritance.

This change updates all snapshot files and test expectations affected by
this change to include expectation of all relevant `_dd.p.*` tags. There
are also two style changes that were required to get the CI pre_check
step to pass (I'm not sure why that isn't appearing on 1.x).

Author: emmettbutler

ddtrace/internal/utils/__init__.py

@@ -4,6 +4,8 @@
 from typing import Optional
 from typing import Tuple
 
+import six
+
 
 class ArgumentError(Exception):
     """
@@ -67,3 +69,12 @@ def set_argument_value(
         raise ArgumentError("%s (at position %d) is invalid" % (kw, pos))
 
     return args, kwargs
+
+
+def _get_metas_to_propagate(context):
+    # type: (Any) -> List[Tuple[str, str]]
+    metas_to_propagate = []
+    for k, v in context._meta.items():
+        if isinstance(k, six.string_types) and k.startswith("_dd.p."):
+            metas_to_propagate.append((k, v))
+    return metas_to_propagate

ddtrace/internal/utils/http.py

@@ -12,8 +12,6 @@
 from typing import Tuple
 from typing import Union
 
-import six
-
 from ddtrace.constants import USER_ID_KEY
 from ddtrace.internal import compat
 from ddtrace.internal.compat import parse
@@ -25,6 +23,7 @@
 from ddtrace.internal.http import HTTPConnection
 from ddtrace.internal.http import HTTPSConnection
 from ddtrace.internal.uds import UDSHTTPConnection
+from ddtrace.internal.utils import _get_metas_to_propagate
 from ddtrace.internal.utils.cache import cached
 
 
@@ -173,13 +172,8 @@ def w3c_get_dd_list_member(context):
         tags.append("t.usr.id:{}".format(w3c_encode_tag((_W3C_TRACESTATE_INVALID_CHARS_REGEX_VALUE, "_", usr_id))))
 
     current_tags_len = sum(len(i) for i in tags)
-    for k, v in context._meta.items():
-        if (
-            isinstance(k, six.string_types)
-            and k.startswith("_dd.p.")
-            # we've already added sampling decision and user id
-            and k not in [SAMPLING_DECISION_TRACE_TAG_KEY, USER_ID_KEY]
-        ):
+    for k, v in _get_metas_to_propagate(context):
+        if k not in [SAMPLING_DECISION_TRACE_TAG_KEY, USER_ID_KEY]:
             # for key replace ",", "=", and characters outside the ASCII range 0x20 to 0x7E
             # for value replace ",", ";", "~" and characters outside the ASCII range 0x20 to 0x7E
             k = k.replace("_dd.p.", "t.")

ddtrace/tracer.py

@@ -15,6 +15,7 @@
 from ddtrace.internal.processor.endpoint_call_counter import EndpointCallCounterProcessor
 from ddtrace.internal.sampling import SpanSamplingRule
 from ddtrace.internal.sampling import get_span_sampling_rules
+from ddtrace.internal.utils import _get_metas_to_propagate
 from ddtrace.settings.peer_service import PeerServiceConfig
 from ddtrace.vendor import debtcollector
 
@@ -718,6 +719,8 @@ def _start_span(
 
             if span._local_root is None:
                 span._local_root = span
+            for k, v in _get_metas_to_propagate(context):
+                span._meta[k] = v
         else:
             # this is the root span of a new trace
             span = Span(

releasenotes/notes/tag-inheritance-interprocess-bc5a2bd188fc08db.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    This fix resolves an issue causing span tags used by the Datadog backend not to be inherited by spans
+    that exist in a different process from their parents.

tests/contrib/botocore/test.py

@@ -2853,7 +2853,7 @@ def test_kinesis_put_records_json_trace_injection(self):
     @mock_kinesis
     @TracerTestCase.run_in_subprocess(env_overrides=dict(DD_DATA_STREAMS_ENABLED="True"))
     @unittest.skipIf(BOTOCORE_VERSION < (1, 26, 31), "Kinesis didn't support streamARN till 1.26.31")
-    @mock.patch('time.time', mock.MagicMock(return_value=1642544540))
+    @mock.patch("time.time", mock.MagicMock(return_value=1642544540))
     def test_kinesis_data_streams_enabled_put_records(self):
         # (dict -> json string)[]
         data = json.dumps({"json": "string"})

tests/contrib/django/test_django.py

@@ -1218,6 +1218,7 @@ def test_cache_get_many(test_spans):
         "component": "django",
         "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache",
         "django.cache.key": "missing_key another_key",
+        "_dd.p.dm": "-0",
     }
 
     assert_dict_issuperset(span_get_many.get_tags(), expected_meta)
@@ -1419,12 +1420,14 @@ def test_cached_view(client, test_spans):
         "django.cache.key": (
             "views.decorators.cache.cache_page..GET.03cdc1cc4aab71b038a6764e5fcabb82.d41d8cd98f00b204e9800998ecf8..."
         ),
+        "_dd.p.dm": "-0",
     }
 
     expected_meta_header = {
         "component": "django",
         "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache",
         "django.cache.key": "views.decorators.cache.cache_header..03cdc1cc4aab71b038a6764e5fcabb82.en-us",
+        "_dd.p.dm": "-0",
     }
 
     assert span_view.get_tags() == expected_meta_view
@@ -1463,6 +1466,7 @@ def test_cached_template(client, test_spans):
         "component": "django",
         "django.cache.backend": "django.core.cache.backends.locmem.LocMemCache",
         "django.cache.key": "template.cache.users_list.d41d8cd98f00b204e9800998ecf8427e",
+        "_dd.p.dm": "-0",
     }
 
     assert span_template_cache.get_tags() == expected_meta

tests/contrib/flask/test_blueprint.py

@@ -103,7 +103,7 @@ def test():
         self.assertEqual(span.service, "flask")
         self.assertEqual(span.name, "bp.test")
         self.assertEqual(span.resource, "/")
-        self.assertEqual(span.get_tags(), {"component": "flask"})
+        self.assertEqual(span.get_tags(), {"component": "flask", "_dd.p.dm": "-0"})
 
     def test_blueprint_request_pin_override(self):
         """
@@ -130,7 +130,7 @@ def test():
         self.assertEqual(span.service, "flask-bp")
         self.assertEqual(span.name, "bp.test")
         self.assertEqual(span.resource, "/")
-        self.assertEqual(span.get_tags(), {"component": "flask"})
+        self.assertEqual(span.get_tags(), {"component": "flask", "_dd.p.dm": "-0"})
 
     def test_blueprint_request_pin_disabled(self):
         """

tests/contrib/flask/test_errorhandler.py

@@ -6,7 +6,7 @@
 from . import BaseFlaskTestCase
 
 
-EXPECTED_METADATA = {"component": "flask"}
+EXPECTED_METADATA = {"component": "flask", "_dd.p.dm": "-0"}
 
 
 class FlaskErrorhandlerTestCase(BaseFlaskTestCase):

tests/contrib/flask/test_views.py

@@ -13,7 +13,7 @@
 if PY2:
     base_exception_name = "exceptions.Exception"
 
-EXPECTED_METADATA = {"component": "flask"}
+EXPECTED_METADATA = {"component": "flask", "_dd.p.dm": "-0"}
 
 
 class FlaskViewTestCase(BaseFlaskTestCase):

tests/contrib/patch.py

@@ -7,8 +7,8 @@
 import unittest
 
 from ddtrace.vendor import wrapt
-from tests.subprocesstest import run_in_subprocess
 from tests.subprocesstest import SubprocessTestCase
+from tests.subprocesstest import run_in_subprocess
 from tests.utils import call_program