feat(APM_TRACING): Test new APM_TRACING_MULTICONFIG merging logic

This is still in development for Java and Python here:

- DataDog/dd-trace-java#9360
- DataDog/dd-trace-py#14364

But we want to have system-tests upfront to ensure it's functioning correctly for Sept releases.

Refs: DEBUG-4399

Author: tylfin

tests/test_library_conf.py

@@ -269,10 +269,122 @@ def test_tracing_client_http_header_tags(self):
         else:
             pytest.fail(f"A span with /status in the resource name was not found {spans}")
 
-    def get_rc_params(self, header_tags):
+    def setup_tracing_client_http_header_tags_apm_multiconfig(self):
+        """We need to test that when the APM_TRACING_MULTICONFIG capability is enabled, it
+        takes the lowest priority.
+
+        This follows the principle that the most specific config wins.
+        """
+        path, config = self.get_rc_params(
+            {
+                "tracing_header_tags": [
+                    {"header": "X-Test-Header", "tag_name": "test_header_rc"},
+                    {"header": "X-Test-Header-2", "tag_name": "test_header_rc2"},
+                    {"header": "Content-Length", "tag_name": ""},
+                ]
+            },
+            service_name="*",
+            env="*",
+        )
+        rc.rc_state.reset().set_config(path, config).apply()
+        self.req1 = weblog.get(
+            "/status?code=202",
+            headers={
+                "X-Test-Header": "1",
+                "X-Test-Header-2": "2",
+                "Content-Length": "0",
+                HEADER_NAME_SHORT: HEADER_VAL_BASIC,
+            },
+        )
+
+        path, config = self.get_rc_params(
+            {},
+            service_name="weblog",
+            env="system-tests",
+        )
+        rc.rc_state.reset().set_config(path, config).apply()
+        self.req2 = weblog.get(
+            "/status?code=202",
+            headers={
+                "X-Test-Header": "1",
+                "X-Test-Header-2": "2",
+                "Content-Length": "0",
+                HEADER_NAME_SHORT: HEADER_VAL_BASIC,
+            },
+        )
+
+        path, config = self.get_rc_params(
+            {
+                "tracing_header_tags": [
+                    {"header": "X-Test-Header", "tag_name": "test_header_rc"},
+                    {"header": "X-Test-Header-2", "tag_name": "test_header_rc2"},
+                    {"header": "Content-Length", "tag_name": ""},
+                ]
+            },
+            service_name="*",
+            env="*",
+        )
+        rc.rc_state.reset().set_config(path, config).apply()
+        self.req3 = weblog.get(
+            "/status?code=202",
+            headers={
+                "X-Test-Header": "1",
+                "X-Test-Header-2": "2",
+                "Content-Length": "0",
+                HEADER_NAME_SHORT: HEADER_VAL_BASIC,
+            },
+        )
+
+    @missing_feature(reason="APM_TRACING_MULTICONFIG is not supported in any language yet")
+    def test_tracing_client_http_header_tags_apm_multiconfig(self):
+        """Ensure the tracing http header tags can be set via RC with the APM_TRACING_MULTICONFIG capability."""
+        # Validate the spans generated by the first request
+        spans = [span for _, _, span in interfaces.library.get_spans(request=self.req1, full_trace=True)]
+        for s in spans:
+            if "/status" in s["resource"]:
+                # Header tags set via remote config
+                assert s["meta"].get("test_header_rc")
+                assert s["meta"].get("test_header_rc2")
+                assert s["meta"].get("http.request.headers.content-length")
+                # Does not have headers set via Enviorment variables
+                assert TAG_SHORT not in s["meta"]
+                break
+        else:
+            pytest.fail(f"A span with /status in the resource name was not found {spans}")
+
+        # Validate the spans generated by the second request
+        spans = [span for _, _, span in interfaces.library.get_spans(request=self.req2, full_trace=True)]
+        for s in spans:
+            if "/status" in s["resource"]:
+                # Headers tags set via remote config
+                assert s["meta"].get(TAG_SHORT) == HEADER_VAL_BASIC
+                # Does not have headers set via remote config
+                assert "test_header_rc" not in s["meta"], s["meta"]
+                assert "test_header_rc2" not in s["meta"], s["meta"]
+                assert "http.request.headers.content-length" in s["meta"], s["meta"]
+                break
+        else:
+            pytest.fail(f"A span with /status in the resource name was not found {spans}")
+
+        # Validate the spans generated by the third request. This should be identical to the second request, because
+        # the last RC config has the lowest priority and should be ignored.
+        spans = [span for _, _, span in interfaces.library.get_spans(request=self.req3, full_trace=True)]
+        for s in spans:
+            if "/status" in s["resource"]:
+                # Headers tags set via remote config
+                assert s["meta"].get(TAG_SHORT) == HEADER_VAL_BASIC
+                # Does not have headers set via remote config
+                assert "test_header_rc" not in s["meta"], s["meta"]
+                assert "test_header_rc2" not in s["meta"], s["meta"]
+                assert "http.request.headers.content-length" in s["meta"], s["meta"]
+                break
+        else:
+            pytest.fail(f"A span with /status in the resource name was not found {spans}")
+
+    def get_rc_params(self, header_tags, service_name="weblog", env="system-tests"):
         config = {
             "action": "enable",
-            "service_target": {"service": "weblog", "env": "system-tests"},
+            "service_target": {"service": service_name, "env": env},
             "lib_config": header_tags,
         }
         rc_id = hash(json.dumps(config))