Fix runtime-id and system.pid tags not being set on distributed traces (#2562)

* Add regression test

* Fix runtime id tag not being set on distributed traces

As per title.

* Use same logic for pid

* Update releasenotes/notes/runtimeid-39dfdf1e0c1af45d.yaml

Co-authored-by: Brett Langdon <[email protected]>

* Fix spelling, grpc snapshot

Co-authored-by: Brett Langdon <[email protected]>
Co-authored-by: Brett Langdon <[email protected]>

Author: Kyle-Verhoog

ddtrace/tracer.py

@@ -550,8 +550,6 @@ def _start_span(
                 on_finish=[self._on_span_finish],
             )
             span._local_root = span
-            span.metrics[system.PID] = self._pid or getpid()
-            span.meta["runtime-id"] = get_runtime_id()
             if config.report_hostname:
                 span.meta[HOSTNAME_KEY] = hostname.get_hostname()
             span.sampled = self.sampler.sample(span)
@@ -581,6 +579,10 @@ def _start_span(
                 # We must always mark the span as sampled so it is forwarded to the agent
                 span.sampled = True
 
+        if not span._parent:
+            span.meta["runtime-id"] = get_runtime_id()
+            span.metrics[system.PID] = self._pid
+
         # Apply default global tags.
         if self.tags:
             span.set_tags(self.tags)

docs/spelling_wordlist.txt

@@ -85,6 +85,7 @@ mysqldb
 namespace
 opentracer
 opentracing
+pid
 plugin
 posix
 postgres
@@ -107,6 +108,7 @@ repo
 respawn
 runnable
 runtime
+runtime-id
 sanic
 sql
 sqlalchemy

releasenotes/notes/runtimeid-39dfdf1e0c1af45d.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fix runtime-id and system.pid tags not being set on distributed traces.

tests/snapshots/tests.contrib.grpc.test_grpc.test_method_service.snap

@@ -6,10 +6,10 @@
    "span_id" 0
    "trace_id" 0
    "parent_id" nil
-   "start" 1619744633550844100
-   "duration" 1911900
-   "meta" {"runtime-id" "04d67a3bd550461295cfd98eedc69d99"
-           "grpc.port" "55945"
+   "start" 1624047273666117000
+   "duration" 2052000
+   "meta" {"runtime-id" "d8bca53d2d0942c69b9144e37b3626c6"
+           "grpc.port" "59087"
            "grpc.host" "::"
            "grpc.method.kind" "unary"
            "span.kind" "client"
@@ -20,7 +20,7 @@
    "metrics" {"_dd.agent_psr" 1.0
               "_dd.measured" 1
               "_sampling_priority_v1" 1
-              "system.pid" 714
+              "system.pid" 18581
               "_dd.tracer_kr" 1.0}}
       {"name" "grpc"
        "service" "grpc-server"
@@ -30,14 +30,16 @@
        "span_id" 1
        "trace_id" 0
        "parent_id" 0
-       "start" 1619744633552296900
-       "duration" 48200
-       "meta" {"grpc.method.path" "/Servicer/Handler"
+       "start" 1624047273667894000
+       "duration" 43000
+       "meta" {"runtime-id" "d8bca53d2d0942c69b9144e37b3626c6"
+               "grpc.method.path" "/Servicer/Handler"
                "grpc.method.service" "Servicer"
                "grpc.method.name" "Handler"
                "grpc.method.kind" "unary"
                "span.kind" "server"}
-       "metrics" {"_dd.measured" 1}}]
+       "metrics" {"_dd.measured" 1
+                  "system.pid" 18581}}]
  [{"name" "grpc"
    "service" "grpc-client"
    "resource" "/pkg.Servicer/Handler"
@@ -46,10 +48,10 @@
    "span_id" 0
    "trace_id" 1
    "parent_id" nil
-   "start" 1619744633553123900
-   "duration" 1079700
-   "meta" {"runtime-id" "04d67a3bd550461295cfd98eedc69d99"
-           "grpc.port" "55945"
+   "start" 1624047273668507000
+   "duration" 601000
+   "meta" {"runtime-id" "d8bca53d2d0942c69b9144e37b3626c6"
+           "grpc.port" "59087"
            "grpc.method.package" "pkg"
            "grpc.host" "::"
            "grpc.method.kind" "unary"
@@ -61,7 +63,7 @@
    "metrics" {"_dd.agent_psr" 1.0
               "_dd.measured" 1
               "_sampling_priority_v1" 1
-              "system.pid" 714
+              "system.pid" 18581
               "_dd.tracer_kr" 1.0}}
       {"name" "grpc"
        "service" "grpc-server"
@@ -71,12 +73,14 @@
        "span_id" 1
        "trace_id" 1
        "parent_id" 0
-       "start" 1619744633553812700
-       "duration" 42900
-       "meta" {"grpc.method.path" "/pkg.Servicer/Handler"
+       "start" 1624047273668929000
+       "duration" 29000
+       "meta" {"runtime-id" "d8bca53d2d0942c69b9144e37b3626c6"
+               "grpc.method.path" "/pkg.Servicer/Handler"
                "grpc.method.package" "pkg"
                "grpc.method.service" "Servicer"
                "grpc.method.name" "Handler"
                "grpc.method.kind" "unary"
                "span.kind" "server"}
-       "metrics" {"_dd.measured" 1}}]]
+       "metrics" {"_dd.measured" 1
+                  "system.pid" 18581}}]]

tests/snapshots/tests.integration.test_integration_snapshots.test_tracer_trace_across_fork.snap

@@ -5,12 +5,12 @@
    "span_id" 0
    "trace_id" 0
    "parent_id" nil
-   "start" 1618357801277945181
-   "duration" 107575512
-   "meta" {"runtime-id" "cb4a438bb8364fa5ac0fec9f7df34c89"}
+   "start" 1624045258581163000
+   "duration" 22147000
+   "meta" {"runtime-id" "afc32eb3be454c03b8a140291d5fd696"}
    "metrics" {"_dd.agent_psr" 1.0
               "_sampling_priority_v1" 1
-              "system.pid" 8786
+              "system.pid" 12507
               "_dd.tracer_kr" 1.0}}
       {"name" "child"
        "service" nil
@@ -19,7 +19,9 @@
        "span_id" 1
        "trace_id" 0
        "parent_id" 0
-       "start" 1618357801293690837
-       "duration" 76002
+       "start" 1624045258591089000
+       "duration" 53000
+       "meta" {"runtime-id" "dc30b84ca01142239a7f2a51d9c7725e"}
        "metrics" {"_sampling_priority_v1" 1
+                  "system.pid" 12517
                   "_dd.tracer_kr" 1.0}}]]

tests/snapshots/tests.integration.test_integration_snapshots.test_tracer_trace_across_multiple_forks.snap

@@ -5,12 +5,12 @@
    "span_id" 0
    "trace_id" 0
    "parent_id" nil
-   "start" 1618357412996234998
-   "duration" 425047115
-   "meta" {"runtime-id" "31d11c17a6c547ccb9169d1347b1379e"}
+   "start" 1624045173599735000
+   "duration" 51650000
+   "meta" {"runtime-id" "120a8105b0c542d889b939d7c91d71fa"}
    "metrics" {"_dd.agent_psr" 1.0
               "_sampling_priority_v1" 1
-              "system.pid" 6435
+              "system.pid" 12266
               "_dd.tracer_kr" 1.0}}
       {"name" "child1"
        "service" nil
@@ -19,9 +19,11 @@
        "span_id" 1
        "trace_id" 0
        "parent_id" 0
-       "start" 1618357413012622804
-       "duration" 315314377
+       "start" 1624045173606288000
+       "duration" 32138000
+       "meta" {"runtime-id" "8fba36822c6846d895a13f5d043543be"}
        "metrics" {"_sampling_priority_v1" 1
+                  "system.pid" 12294
                   "_dd.tracer_kr" 1.0}}
           {"name" "child2"
            "service" nil
@@ -30,7 +32,9 @@
            "span_id" 2
            "trace_id" 0
            "parent_id" 1
-           "start" 1618357413236058471
-           "duration" 75980
+           "start" 1624045173612023000
+           "duration" 86000
+           "meta" {"runtime-id" "859c494affcb4933a1e15b6c4d2561b8"}
            "metrics" {"_sampling_priority_v1" 1
+                      "system.pid" 12295
                       "_dd.tracer_kr" 1.0}}]]

tests/tracer/test_tracer.py

@@ -910,6 +910,15 @@ def task(tracer, q):
     assert children_tag != span.get_tag("runtime-id")
 
 
+def test_tracer_runtime_tags_cross_execution(tracer):
+    ctx = Context(trace_id=12, span_id=21)
+    tracer.context_provider.activate(ctx)
+    with tracer.trace("span") as span:
+        pass
+    assert span.get_tag("runtime-id") is not None
+    assert span.get_metric(system.PID) is not None
+
+
 def test_start_span_hooks():
     t = ddtrace.Tracer()