chore(llmobs): deprecate name of ML app env var [backport 2.9] (#9466)

Backport 0795fd6 from #9459 to 2.9.

This PR changes the environment variable name for the user's ML
application name from `DD_LLMOBS_APP_NAME` to `DD_LLMOBS_ML_APP` to
better reflect LLM Observability's corp docs and guides.

We are deprecating the old name instead of removing it outright so that
users won't break their apps.

## Checklist

- [x] Change(s) are motivated and described in the PR description
- [x] Testing strategy is described if automated tests are not included
in the PR
- [x] Risks are described (performance impact, potential for breakage,
maintainability)
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] [Library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
are followed or label `changelog/no-changelog` is set
- [x] Documentation is included (in-code, generated user docs, [public
corp docs](https://github.com/DataDog/documentation/))
- [x] Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))
- [x] If this PR changes the public interface, I've notified
`@DataDog/apm-tees`.

## Reviewer Checklist

- [x] Title is accurate
- [x] All changes are related to the pull request's stated goal
- [x] Description motivates each change
- [x] Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- [x] Testing strategy adequately addresses listed risks
- [x] Change is maintainable (easy to change, telemetry, documentation)
- [x] Release note makes sense to a user of the library
- [x] Author has acknowledged and discussed the performance implications
of this PR as reported in the benchmarks PR comment
- [x] Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Co-authored-by: Yun Kim <[email protected]>
Co-authored-by: erikayasuda <[email protected]>
Co-authored-by: Brett Langdon <[email protected]>

Author: 4 people

ddtrace/llmobs/_llmobs.py

@@ -133,9 +133,12 @@ def enable(
         # grab required values for LLMObs
         config._dd_site = site or config._dd_site
         config._dd_api_key = api_key or config._dd_api_key
-        config._llmobs_ml_app = ml_app or config._llmobs_ml_app
         config.env = env or config.env
         config.service = service or config.service
+        if os.getenv("DD_LLMOBS_APP_NAME"):
+            log.warning("`DD_LLMOBS_APP_NAME` is deprecated. Use `DD_LLMOBS_ML_APP` instead.")
+            config._llmobs_ml_app = ml_app or os.getenv("DD_LLMOBS_APP_NAME")
+        config._llmobs_ml_app = ml_app or config._llmobs_ml_app
 
         # validate required values for LLMObs
         if not config._dd_api_key:
@@ -150,7 +153,7 @@ def enable(
             )
         if not config._llmobs_ml_app:
             raise ValueError(
-                "DD_LLMOBS_APP_NAME is required for sending LLMObs data. "
+                "DD_LLMOBS_ML_APP is required for sending LLMObs data. "
                 "Ensure this configuration is set before running your application."
             )
 
@@ -288,7 +291,7 @@ def llm(
                                    If not provided, a default value of "custom" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -314,7 +317,7 @@ def tool(
         :param str name: The name of the traced operation. If not provided, a default value of "tool" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -332,7 +335,7 @@ def task(
         :param str name: The name of the traced operation. If not provided, a default value of "task" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -350,7 +353,7 @@ def agent(
         :param str name: The name of the traced operation. If not provided, a default value of "agent" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -368,7 +371,7 @@ def workflow(
         :param str name: The name of the traced operation. If not provided, a default value of "workflow" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -394,7 +397,7 @@ def embedding(
                                    If not provided, a default value of "custom" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -425,7 +428,7 @@ def retrieval(
         :param str name: The name of the traced operation. If not provided, a default value of "workflow" will be set.
         :param str session_id: The ID of the underlying user session. Required for tracking sessions.
         :param str ml_app: The name of the ML application that the agent is orchestrating. If not provided, the default
-                           value DD_LLMOBS_APP_NAME will be set.
+                           value will be set to the value of `DD_LLMOBS_ML_APP`.
 
         :returns: The Span object representing the traced operation.
         """
@@ -698,7 +701,7 @@ def submit_evaluation(
         # initialize tags with default values that will be overridden by user-provided tags
         evaluation_tags = {
             "ddtrace.version": ddtrace.__version__,
-            "ml_app": config._llmobs_ml_app if config._llmobs_ml_app else "unknown",
+            "ml_app": config._llmobs_ml_app or "unknown",
         }
 
         if tags:

ddtrace/settings/config.py

@@ -565,7 +565,7 @@ def __init__(self):
 
         self._llmobs_enabled = asbool(os.getenv("DD_LLMOBS_ENABLED", False))
         self._llmobs_sample_rate = float(os.getenv("DD_LLMOBS_SAMPLE_RATE", 1.0))
-        self._llmobs_ml_app = os.getenv("DD_LLMOBS_APP_NAME")
+        self._llmobs_ml_app = os.getenv("DD_LLMOBS_ML_APP")
 
     def __getattr__(self, name) -> Any:
         if name in self._config:

releasenotes/notes/llmobs-deprecate-ml-app-name-71c71132c700f3ac.yaml

@@ -0,0 +1,6 @@
+---
+deprecations:
+  - |
+    LLM Observability: ``DD_LLMOBS_APP_NAME`` is deprecated and will be removed in the next major version of ddtrace.
+    As an alternative to ``DD_LLMOBS_APP_NAME``, you can use ``DD_LLMOBS_ML_APP`` instead. See the 
+    `SDK setup documentation <https://docs.datadoghq.com/tracing/llm_observability/sdk/#setup>`_ for more details on how to configure the LLM Observability SDK.

tests/llmobs/test_llmobs_decorators.py

@@ -367,7 +367,7 @@ def f():
 
 
 def test_ml_app_override(LLMObs, mock_llmobs_span_writer):
-    """Test that setting ml_app kwarg on the LLMObs decorators will override the DD_LLMOBS_APP_NAME value."""
+    """Test that setting ml_app kwarg on the LLMObs decorators will override the DD_LLMOBS_ML_APP value."""
     for decorator_name, decorator in [("task", task), ("workflow", workflow), ("tool", tool)]:
 
         @decorator(ml_app="test_ml_app")

tests/llmobs/test_llmobs_service.py

@@ -110,6 +110,18 @@ def test_service_enable_no_ml_app_specified():
         assert llmobs_service._instance._llmobs_span_writer.status.value == "stopped"
 
 
+def test_service_enable_deprecated_ml_app_name(monkeypatch, mock_logs):
+    with override_global_config(dict(_dd_api_key="<not-a-real-key>", _llmobs_ml_app="")):
+        dummy_tracer = DummyTracer()
+        monkeypatch.setenv("DD_LLMOBS_APP_NAME", "test_ml_app")
+        llmobs_service.enable(_tracer=dummy_tracer)
+        assert llmobs_service.enabled is True
+        assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running"
+        assert llmobs_service._instance._llmobs_span_writer.status.value == "running"
+        mock_logs.warning.assert_called_once_with("`DD_LLMOBS_APP_NAME` is deprecated. Use `DD_LLMOBS_ML_APP` instead.")
+        llmobs_service.disable()
+
+
 def test_service_enable_already_enabled(mock_logs):
     with override_global_config(dict(_dd_api_key="<not-a-real-api-key>", _llmobs_ml_app="<ml-app-name>")):
         dummy_tracer = DummyTracer()

tests/llmobs/test_llmobs_trace_processor.py

@@ -172,7 +172,7 @@ def test_session_id_propagates_ignore_non_llmobs_spans():
 
 
 def test_ml_app_tag_defaults_to_env_var():
-    """Test that no ml_app defaults to the environment variable DD_LLMOBS_APP_NAME."""
+    """Test that no ml_app defaults to the environment variable DD_LLMOBS_ML_APP."""
     dummy_tracer = DummyTracer()
     with override_global_config(dict(_llmobs_ml_app="<not-a-real-app-name>")):
         with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span:
@@ -184,7 +184,7 @@ def test_ml_app_tag_defaults_to_env_var():
 
 
 def test_ml_app_tag_overrides_env_var():
-    """Test that when ml_app is set on the span, it overrides the environment variable DD_LLMOBS_APP_NAME."""
+    """Test that when ml_app is set on the span, it overrides the environment variable DD_LLMOBS_ML_APP."""
     dummy_tracer = DummyTracer()
     with override_global_config(dict(_llmobs_ml_app="<not-a-real-app-name>")):
         with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: