feat(tracer): add remote config support of tags (#7515)

Following #7489, add remote
configuration support for tags. This feature will enable users to set
custom tags like `team` for their services.


![image](https://github.com/DataDog/dd-trace-py/assets/6321485/fd8c9a60-433b-42bb-9c3b-5985736d14ba)


Since `tracer.set_tags()` exists, there is the risk of some confusion
for customers for where a tag might be coming from when observing a span
in the UI. I am treating `tracer.set_tags()` as tags local to the
tracer, independent of the global config setting `config.tags` /
`DD_TAGS`. This means that tags set via remote configuration will not be
able to override those set by `tracer.set_tags()` which may not be
desirable. In those cases I think we can recommend that users do not use
the `tracer.set_tags()` method.


Co-authored-by: Munir Abdinur <[email protected]>

Author: Kyle-Verhoog

ddtrace/internal/remoteconfig/client.py

@@ -57,6 +57,7 @@ class RemoteConfigClientConfig(En):
 
 class Capabilities(enum.IntFlag):
     APM_TRACING_SAMPLE_RATE = 1 << 12
+    APM_TRACING_CUSTOM_TAGS = 1 << 15
 
 
 class RemoteConfigError(Exception):
@@ -357,7 +358,9 @@ def _parse_target(target, metadata):
     def _build_payload(self, state):
         # type: (Mapping[str, Any]) -> Mapping[str, Any]
         self._client_tracer["extra_services"] = list(ddtrace.config._get_extra_services())
-        capabilities = appsec_rc_capabilities() | Capabilities.APM_TRACING_SAMPLE_RATE
+        capabilities = (
+            appsec_rc_capabilities() | Capabilities.APM_TRACING_SAMPLE_RATE | Capabilities.APM_TRACING_CUSTOM_TAGS
+        )
         return dict(
             client=dict(
                 id=self.id,

ddtrace/internal/telemetry/writer.py

@@ -344,6 +344,9 @@ def _telemetry_entry(self, cfg_name: str) -> Tuple[str, str, _ConfigSource]:
         elif cfg_name == "trace_http_header_tags":
             name = "trace_header_tags"
             value = ",".join(":".join(x) for x in item.value().items())
+        elif cfg_name == "tags":
+            name = "trace_tags"
+            value = ",".join(":".join(x) for x in item.value().items())
         else:
             raise ValueError("Unknown configuration item: %s" % cfg_name)
         return name, value, item.source()
@@ -365,6 +368,7 @@ def _app_started_event(self, register_app_shutdown=True):
                 self._telemetry_entry("_trace_sample_rate"),
                 self._telemetry_entry("logs_injection"),
                 self._telemetry_entry("trace_http_header_tags"),
+                self._telemetry_entry("tags"),
                 (TELEMETRY_TRACING_ENABLED, config._tracing_enabled, "unknown"),
                 (TELEMETRY_STARTUP_LOGS_ENABLED, config._startup_logs_enabled, "unknown"),
                 (TELEMETRY_DSM_ENABLED, config._data_streams_enabled, "unknown"),

ddtrace/settings/config.py

@@ -201,12 +201,15 @@ class _ConfigItem:
     """Configuration item that tracks the value of a setting, and where it came from."""
 
     def __init__(self, name, default, envs):
-        # type: (str, _JSONType, List[Tuple[str, Callable[[str], Any]]]) -> None
+        # type: (str, Union[_JSONType, Callable[[], _JSONType]], List[Tuple[str, Callable[[str], Any]]]) -> None
         self._name = name
-        self._default_value = default
         self._env_value: _JSONType = None
         self._code_value: _JSONType = None
         self._rc_value: _JSONType = None
+        if callable(default):
+            self._default_value = default()
+        else:
+            self._default_value = default
         self._envs = envs
         for env_var, parser in envs:
             if env_var in os.environ:
@@ -261,6 +264,11 @@ def __repr__(self):
         )
 
 
+def _parse_global_tags(s):
+    # cleanup DD_TAGS, because values will be inserted back in the optimal way (via _dd.git.* tags)
+    return gitmetadata.clean_tags(parse_tags_str(s))
+
+
 def _default_config():
     # type: () -> Dict[str, _ConfigItem]
     return {
@@ -276,9 +284,14 @@ def _default_config():
         ),
         "trace_http_header_tags": _ConfigItem(
             name="trace_http_header_tags",
-            default={},
+            default=lambda: {},
             envs=[("DD_TRACE_HEADER_TAGS", parse_tags_str)],
         ),
+        "tags": _ConfigItem(
+            name="tags",
+            default=lambda: {},
+            envs=[("DD_TAGS", _parse_global_tags)],
+        ),
     }
 
 
@@ -393,9 +406,6 @@ def __init__(self):
         self.client_ip_header = os.getenv("DD_TRACE_CLIENT_IP_HEADER")
         self.retrieve_client_ip = asbool(os.getenv("DD_TRACE_CLIENT_IP_ENABLED", default=False))
 
-        # cleanup DD_TAGS, because values will be inserted back in the optimal way (via _dd.git.* tags)
-        self.tags = gitmetadata.clean_tags(parse_tags_str(os.getenv("DD_TAGS") or ""))
-
         self.propagation_http_baggage_enabled = asbool(
             os.getenv("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", default=False)
         )
@@ -714,6 +724,12 @@ def _handle_remoteconfig(self, data, test_tracer=None):
             if "tracing_sampling_rate" in lib_config:
                 updated_items.append(("_trace_sample_rate", lib_config["tracing_sampling_rate"]))
 
+            if "tracing_tags" in lib_config:
+                tags = lib_config["tracing_tags"]
+                if tags:
+                    tags = {k: v for k, v in [t.split(":") for t in lib_config["tracing_tags"]]}
+                updated_items.append(("tags", tags))
+
         self._set_config_items([(k, v, "remote_config") for k, v in updated_items])
 
     def enable_remote_configuration(self):

ddtrace/tracer.py

@@ -280,6 +280,7 @@ def __init__(
 
         self._new_process = False
         config._subscribe(["_trace_sample_rate"], self._on_global_config_update)
+        config._subscribe(["tags"], self._on_global_config_update)
 
     def _atexit(self) -> None:
         key = "ctrl-break" if os.name == "nt" else "ctrl-c"
@@ -1068,3 +1069,5 @@ def _on_global_config_update(self, cfg, items):
                 sample_rate = None
             sampler = DatadogSampler(default_sample_rate=sample_rate)
             self._sampler = sampler
+        elif "tags" in items:
+            self._tags = cfg.tags.copy()

releasenotes/notes/remote-trace-tags-3e4d8c87afb18b86.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    tracer: Add support for remotely configuring trace tags.

tests/integration/test_settings.py

@@ -21,6 +21,7 @@ def test_setting_origin_environment(test_agent_session, run_python_code_in_subpr
             "DD_TRACE_SAMPLE_RATE": "0.1",
             "DD_LOGS_INJECTION": "true",
             "DD_TRACE_HEADER_TAGS": "X-Header-Tag-1:header_tag_1,X-Header-Tag-2:header_tag_2",
+            "DD_TAGS": "team:apm,component:web",
         }
     )
     out, err, status, _ = run_python_code_in_subprocess(
@@ -49,6 +50,11 @@ def test_setting_origin_environment(test_agent_session, run_python_code_in_subpr
         "value": "X-Header-Tag-1:header_tag_1,X-Header-Tag-2:header_tag_2",
         "origin": "env_var",
     }
+    assert _get_latest_telemetry_config_item(events, "trace_tags") == {
+        "name": "trace_tags",
+        "value": "team:apm,component:web",
+        "origin": "env_var",
+    }
 
 
 @pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent")
@@ -59,6 +65,7 @@ def test_setting_origin_code(test_agent_session, run_python_code_in_subprocess):
             "DD_TRACE_SAMPLE_RATE": "0.1",
             "DD_LOGS_INJECTION": "true",
             "DD_TRACE_HEADER_TAGS": "X-Header-Tag-1:header_tag_1,X-Header-Tag-2:header_tag_2",
+            "DD_TAGS": "team:apm,component:web",
         }
     )
     out, err, status, _ = run_python_code_in_subprocess(
@@ -67,6 +74,7 @@ def test_setting_origin_code(test_agent_session, run_python_code_in_subprocess):
 config._trace_sample_rate = 0.2
 config.logs_injection = False
 config.trace_http_header_tags = {"header": "value"}
+config.tags = {"header": "value"}
 with tracer.trace("test") as span:
     pass
         """,
@@ -90,6 +98,11 @@ def test_setting_origin_code(test_agent_session, run_python_code_in_subprocess):
         "value": "header:value",
         "origin": "code",
     }
+    assert _get_latest_telemetry_config_item(events, "trace_tags") == {
+        "name": "trace_tags",
+        "value": "header:value",
+        "origin": "code",
+    }
 
 
 @pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent")

tests/internal/remoteconfig/test_remoteconfig_client_e2e.py

@@ -18,7 +18,7 @@
 
 def _expected_payload(
     rc_client,
-    capabilities="EAA=",
+    capabilities="kAA=",
     has_errors=False,
     targets_version=0,
     backend_client_state=None,

tests/internal/test_settings.py

@@ -16,6 +16,7 @@ def _base_rc_config(cfg):
         "tracing_sampling_rate": None,
         "log_injection_enabled": None,
         "tracing_header_tags": None,
+        "tracing_tags": None,
         "data_streams_enabled": None,
     }
     lib_config.update(cfg)
@@ -96,6 +97,24 @@ def _deleted_rc_config():
             "expected": {"trace_http_header_tags": {"header": "value"}},
             "expected_source": {"trace_http_header_tags": "code"},
         },
+        {
+            "env": {"DD_TAGS": "key:value,key2:value2"},
+            "expected": {"tags": {"key": "value", "key2": "value2"}},
+            "expected_source": {"tags": "env_var"},
+        },
+        {
+            "env": {"DD_TAGS": "key:value,key2:value2"},
+            "code": {"tags": {"k": "v", "k2": "v2"}},
+            "expected": {"tags": {"k": "v", "k2": "v2"}},
+            "expected_source": {"tags": "code"},
+        },
+        {
+            "env": {"DD_TAGS": "key:value,key2:value2"},
+            "code": {"tags": {"k": "v", "k2": "v2"}},
+            "rc": {"tracing_tags": ["key1:val2", "key2:val3"]},
+            "expected": {"tags": {"key1": "val2", "key2": "val3"}},
+            "expected_source": {"tags": "remote_config"},
+        },
     ],
 )
 def test_settings(testcase, config, monkeypatch):
@@ -177,3 +196,31 @@ def test_remoteconfig_sampling_rate_user(run_python_code_in_subprocess):
         env=env,
     )
     assert status == 0, err.decode("utf-8")
+
+
+def test_remoteconfig_custom_tags(run_python_code_in_subprocess):
+    env = os.environ.copy()
+    env.update({"DD_TAGS": "team:apm"})
+    out, err, status, _ = run_python_code_in_subprocess(
+        """
+from ddtrace import config, tracer
+from tests.internal.test_settings import _base_rc_config
+
+with tracer.trace("test") as span:
+    pass
+assert span.get_tag("team") == "apm"
+
+config._handle_remoteconfig(_base_rc_config({"tracing_tags": ["team:onboarding"]}))
+
+with tracer.trace("test") as span:
+    pass
+assert span.get_tag("team") == "onboarding", span._meta
+
+config._handle_remoteconfig(_base_rc_config({"tracing_tags": None}))
+with tracer.trace("test") as span:
+    pass
+assert span.get_tag("team") == "apm"
+        """,
+        env=env,
+    )
+    assert status == 0, f"err={err.decode('utf-8')} out={out.decode('utf-8')}"

tests/telemetry/test_writer.py

@@ -134,6 +134,7 @@ def test_app_started_event(telemetry_writer, test_agent_session, mock_time):
                     {"name": "trace_sample_rate", "origin": "default", "value": "1.0"},
                     {"name": "trace_header_tags", "origin": "default", "value": ""},
                     {"name": "logs_injection_enabled", "origin": "default", "value": "false"},
+                    {"name": "trace_tags", "origin": "default", "value": ""},
                 ],
                 key=lambda x: x["name"],
             ),
@@ -193,6 +194,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
     env["DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES"] = "9999"
     env["DD_TRACE_WRITER_INTERVAL_SECONDS"] = "30"
     env["DD_TRACE_WRITER_REUSE_CONNECTIONS"] = "True"
+    env["DD_TAGS"] = "team:apm,component:web"
 
     file = tmpdir.join("moon_ears.json")
     file.write('[{"service":"xy?","name":"a*c"}]')
@@ -266,6 +268,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python
             {"name": "trace_sample_rate", "origin": "env_var", "value": "0.5"},
             {"name": "logs_injection_enabled", "origin": "env_var", "value": "true"},
             {"name": "trace_header_tags", "origin": "default", "value": ""},
+            {"name": "trace_tags", "origin": "env_var", "value": "team:apm,component:web"},
         ],
         key=lambda x: x["name"],
     )