fix(llmobs): ensure DD_APM_TRACING_ENABLED disables APM (#14617)

The existing DD_APM_TRACING_ENABLED functionality is specially
implemented for asm and did not work for llmobs. Spans are still sent
since some spans are required for asm.

This change introduces a trace filter to drop all apm spans when llmobs
is enabled.

Consequently, asm + llmobs with apm disabled will not be possible with
this change. I'm choosing to avoid complexity in this implementation
until we see a concrete ask for this functionality.

Co-authored-by: Yun Kim <[email protected]>

Author: Kyle-Verhoog

ddtrace/_trace/apm_filter.py

@@ -0,0 +1,22 @@
+import os
+from typing import List
+from typing import Optional
+
+from ddtrace._trace.processor import TraceProcessor
+from ddtrace._trace.span import Span
+from ddtrace.internal.utils.formats import asbool
+
+
+class APMTracingEnabledFilter(TraceProcessor):
+    """
+    Trace processor that drops all APM traces when DD_APM_TRACING_ENABLED is set to a falsy value.
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._apm_tracing_enabled = asbool(os.getenv("DD_APM_TRACING_ENABLED", "true"))
+
+    def process_trace(self, trace: List[Span]) -> Optional[List[Span]]:
+        if not self._apm_tracing_enabled:
+            return None
+        return trace

ddtrace/llmobs/_llmobs.py

@@ -20,6 +20,7 @@
 import ddtrace
 from ddtrace import config
 from ddtrace import patch
+from ddtrace._trace.apm_filter import APMTracingEnabledFilter
 from ddtrace._trace.context import Context
 from ddtrace._trace.span import Span
 from ddtrace._trace.tracer import Tracer
@@ -605,6 +606,11 @@ def enable(
 
             # override the default _instance with a new tracer
             cls._instance = cls(tracer=_tracer, span_processor=span_processor)
+
+            # Add APM trace filter to drop all APM traces when DD_APM_TRACING_ENABLED is falsy
+            apm_filter = APMTracingEnabledFilter()
+            cls._instance.tracer._span_aggregator.dd_processors.append(apm_filter)
+
             cls.enabled = True
             cls._instance.start()
 

releasenotes/notes/llmobs-apm-tracing-8cd612f8a3af4960.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    LLM Observability: ensures APM is disabled when DD_APM_TRACING_ENABLED=0 when using LLM Observability.

tests/llmobs/test_llmobs.py

@@ -656,3 +656,27 @@ def test_trace_id_propagation_with_non_llm_parent(llmobs, llmobs_events):
     # LLMObs trace IDs should be different from APM trace ID
     assert first_child_event["trace_id"] != first_child_event["_dd"]["apm_trace_id"]
     assert second_child_event["trace_id"] != second_child_event["_dd"]["apm_trace_id"]
+
+
+@pytest.mark.parametrize("llmobs_env", [{"DD_APM_TRACING_ENABLED": "false"}])
+def test_apm_traces_dropped_when_disabled(llmobs, llmobs_events, tracer, llmobs_env):
+    from tests.utils import DummyWriter
+
+    dummy_writer = DummyWriter()
+    tracer._span_aggregator.writer = dummy_writer
+
+    with tracer.trace("apm_span") as apm_span:
+        apm_span.set_tag("operation", "test")
+
+    # Create an LLMObs span (should be sent to LLMObs but not APM)
+    with llmobs.llm(model_name="test-model") as llm_span:
+        llmobs.annotate(llm_span, input_data="test input", output_data="test output")
+
+    # Check that no APM traces were sent to the writer
+    assert len(dummy_writer.traces) == 0, "APM traces should be dropped when DD_APM_TRACING_ENABLED=false"
+
+    # But LLMObs events should still be sent
+    assert len(llmobs_events) == 1
+    llm_event = llmobs_events[0]
+    assert llm_event["meta"]["span.kind"] == "llm"
+    assert llm_event["meta"]["model_name"] == "test-model"