refactor: otel settings have stable (de)serialization (lastmile-ai#551)

alienzach · saqadri · andrew-lastmile · commit 7b62b90f1a34 · 2025-11-04T13:14:08.000-05:00
* feat: v3 otel settings

* fix: reserialization and compatability with tests

* fix: align examples with new otel schema

* fix: exporter list merge instead of replace\nfeat: replace {} with string\n feat: test cases

* fix: unused import

* minor fixes

* update schema

* minor fixes to langfuse

* formatter

---------

Co-authored-by: Sarmad Qadri &lt;sarmad@lastmileai.dev&gt;
diff --git a/examples/tracing/agent/README.md b/examples/tracing/agent/README.md
@@ -16,10 +16,10 @@ If desired, [install Jaeger locally](https://www.jaegertracing.io/docs/2.5/getti
 otel:
   enabled: true
   exporters:
-    - type: console
-    - type: file
-    - type: otlp
-      endpoint: "http://localhost:4318/v1/traces"
+    - console
+    - file
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
 ```
 
 <img width="2160" alt="Image" src="https://github.com/user-attachments/assets/93ffc4e5-f255-43a9-be3a-755994fec809" />
diff --git a/examples/tracing/agent/mcp_agent.config.yaml b/examples/tracing/agent/mcp_agent.config.yaml
@@ -26,11 +26,10 @@ openai:
 
 otel:
   enabled: true
-  exporters: [
-      { type: console },
-      { type: file },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" },
-
-    ]
+  exporters:
+    - console
+    - file
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "BasicTracingAgentExample"
diff --git a/examples/tracing/langfuse/README.md b/examples/tracing/langfuse/README.md
@@ -46,23 +46,25 @@ Obtain a secret and public API key for your desired Langfuse project and then ge
 echo -n "pk-your-public-key:sk-your-secret-key" | base64
 ```
 
-In `mcp_agent.secrets.yaml` set the Authorization header for OTLP (merged automatically with the typed exporter):
+In `mcp_agent.secrets.yaml` set the OTLP exporter with the Authorization header (this fully defines the exporter for Langfuse):
 
 ```yaml
 otel:
-  otlp_settings:
-    headers:
-      Authorization: "Basic AUTH_STRING"
+  exporters:
+    - otlp:
+        endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces"
+        headers:
+          Authorization: "Basic AUTH_STRING"
 ```
 
-Lastly, ensure the proper trace endpoint is configured in the typed exporter in `mcp_agent.config.yaml` for your Langfuse region, e.g.:
+The default `mcp_agent.config.yaml` leaves the exporters list commented out so this secrets entry is the only OTLP exporter (preventing a duplicate without headers). For non-authenticated collectors, you can instead define the exporter directly in `mcp_agent.config.yaml` and omit it from `mcp_agent.secrets.yaml`, e.g.:
 
 ```yaml
 otel:
   enabled: true
   exporters:
-    - type: otlp
-      endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces"
+    - otlp:
+        endpoint: "https://some.other.tracing.com"
 ```
 
 ## `4` Run locally
diff --git a/examples/tracing/langfuse/mcp_agent.config.yaml b/examples/tracing/langfuse/mcp_agent.config.yaml
@@ -26,6 +26,10 @@ openai:
 
 otel:
   enabled: true
-  exporters: [{ type: otlp, endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces" }]
+  # OTLP exporter (with headers) is defined in mcp_agent.secrets.yaml.
+  # For non-authenticated collectors, uncomment and configure below:
+  # exporters:
+  #   - otlp:
+  #       endpoint: "https://some.other.tracing.com"
   # Set Authorization header with API key in mcp_agent.secrets.yaml
   service_name: "BasicTracingLangfuseExample"
diff --git a/examples/tracing/langfuse/mcp_agent.secrets.yaml.example b/examples/tracing/langfuse/mcp_agent.secrets.yaml.example
@@ -7,7 +7,10 @@ anthropic:
   api_key: anthropic_api_key
 
 otel:
-  # Headers are merged with typed OTLP exporter settings
-  otlp_settings:
-    headers:
-      Authorization: "Basic <Langfuse AUTH_STRING>"
+  # Define the Langfuse OTLP exporter (including headers) here so
+  # mcp_agent.config.yaml does not need a duplicate entry.
+  exporters:
+    - otlp:
+        endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces"
+        headers:
+          Authorization: "Basic AUTH_STRING"
diff --git a/examples/tracing/llm/README.md b/examples/tracing/llm/README.md
@@ -29,10 +29,10 @@ Then update the `mcp_agent.config.yaml` to include a typed OTLP exporter with th
 otel:
   enabled: true
   exporters:
-    - type: console
-    - type: file
-    - type: otlp
-      endpoint: "http://localhost:4318/v1/traces"
+    - console
+    - file
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
 ```
 
 <img width="2160" alt="Image" src="https://github.com/user-attachments/assets/f2d1cedf-6729-4ce1-9530-ec9d5653103d" />
diff --git a/examples/tracing/llm/mcp_agent.config.yaml b/examples/tracing/llm/mcp_agent.config.yaml
@@ -26,11 +26,10 @@ openai:
 
 otel:
   enabled: true
-  exporters: [
-      { type: console },
-      { type: file },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" },
-    ]
-
+  exporters:
+    - console
+    - file
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "BasicTracingLLMExample"
diff --git a/examples/tracing/mcp/README.md b/examples/tracing/mcp/README.md
@@ -54,8 +54,8 @@ Then open `mcp_agent.secrets.yaml` and add your api key for your preferred LLM f
 otel:
   enabled: true
   exporters:
-    - type: otlp
-      endpoint: "http://localhost:4318/v1/traces"
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
 ```
 
 ## `4` Run locally
diff --git a/examples/tracing/mcp/mcp_agent.config.yaml b/examples/tracing/mcp/mcp_agent.config.yaml
@@ -17,5 +17,7 @@ openai:
 
 otel:
   enabled: true
-  exporters: [{ type: otlp, endpoint: "http://localhost:4318/v1/traces" }]
+  exporters:
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
   service_name: "MCPAgentSSEExample"
diff --git a/examples/tracing/temporal/README.md b/examples/tracing/temporal/README.md
@@ -53,8 +53,8 @@ To run any of these examples, you'll need to:
 otel:
   enabled: true
   exporters:
-    - type: otlp
-      endpoint: "http://localhost:4318/v1/traces"
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
 ```
 
 4. In a separate terminal, start the worker:
diff --git a/examples/tracing/temporal/mcp_agent.config.yaml b/examples/tracing/temporal/mcp_agent.config.yaml
@@ -46,8 +46,7 @@ openai:
 otel:
   enabled: true
   exporters:
-    [
-      { type: file },
-      { type: otlp, endpoint: "http://localhost:4318/v1/traces" },
-    ]
+    - file
+    - otlp:
+        endpoint: "http://localhost:4318/v1/traces"
   service_name: "TemporalTracingExample"
diff --git a/examples/workflows/workflow_deep_orchestrator/mcp_agent.config.yaml b/examples/workflows/workflow_deep_orchestrator/mcp_agent.config.yaml
@@ -24,17 +24,13 @@ openai:
 
 otel:
   enabled: true
-  exporters: [
-      {
-        type: file,
+  exporters:
+    - file:
         path_settings:
-          {
-            path_pattern: "traces/mcp-agent-trace-{unique_id}.jsonl",
-            unique_id: "timestamp",
-            timestamp_format: "%Y%m%d_%H%M%S",
-          },
-      },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" },
-    ]
+          path_pattern: "traces/mcp-agent-trace-{unique_id}.jsonl"
+          unique_id: "timestamp"
+          timestamp_format: "%Y%m%d_%H%M%S"
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "AdaptiveWorkflowExample"
diff --git a/examples/workflows/workflow_evaluator_optimizer/mcp_agent.config.yaml b/examples/workflows/workflow_evaluator_optimizer/mcp_agent.config.yaml
@@ -41,9 +41,9 @@ openai:
 # OpenTelemetry (OTEL) configuration for distributed tracing
 otel:
   enabled: false
-  exporters: [
-      { type: console },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" }
-    ]
+  exporters:
+    - console
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "WorkflowEvaluatorOptimizerExample"
diff --git a/examples/workflows/workflow_intent_classifier/mcp_agent.config.yaml b/examples/workflows/workflow_intent_classifier/mcp_agent.config.yaml
@@ -21,9 +21,9 @@ openai:
 
 otel:
   enabled: false
-  exporters: [
-      { type: console },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" }
-    ]
+  exporters:
+    - console
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "WorkflowIntentClassifierExample"
diff --git a/examples/workflows/workflow_orchestrator_worker/mcp_agent.config.yaml b/examples/workflows/workflow_orchestrator_worker/mcp_agent.config.yaml
@@ -26,9 +26,9 @@ openai:
 
 otel:
   enabled: false
-  exporters: [
-      { type: console },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" }
-    ]
+  exporters:
+    - console
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "WorkflowOrchestratorWorkerExample"
diff --git a/examples/workflows/workflow_parallel/mcp_agent.config.yaml b/examples/workflows/workflow_parallel/mcp_agent.config.yaml
@@ -25,10 +25,9 @@ openai:
 
 otel:
   enabled: false
-  exporters: [
-      { type: console },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" }
-    ]
-
+  exporters:
+    - console
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "WorkflowParallelExample"
diff --git a/examples/workflows/workflow_router/mcp_agent.config.yaml b/examples/workflows/workflow_router/mcp_agent.config.yaml
@@ -21,9 +21,9 @@ openai:
 
 otel:
   enabled: false
-  exporters: [
-      { type: console },
-      # To export to a collector, also include:
-      # { type: otlp, endpoint: "http://localhost:4318/v1/traces" }
-    ]
+  exporters:
+    - console
+    # To export to a collector, also include:
+    # - otlp:
+    #     endpoint: "http://localhost:4318/v1/traces"
   service_name: "WorkflowRouterExample"
diff --git a/schema/mcp-agent.config.schema.json b/schema/mcp-agent.config.schema.json
diff --git a/src/mcp_agent/config.py b/src/mcp_agent/config.py
diff --git a/src/mcp_agent/tracing/tracer.py b/src/mcp_agent/tracing/tracer.py
diff --git a/tests/test_config_exporters.py b/tests/test_config_exporters.py
