added spend metrics

Author: mubashir1osmani

litellm/integrations/datadog/datadog_llm_obs.py

@@ -185,7 +185,6 @@ def create_llm_obs_payload(
 
         messages = standard_logging_payload["messages"]
         messages = self._ensure_string_content(messages=messages)
-        response_obj = standard_logging_payload.get("response")
 
         metadata = kwargs.get("litellm_params", {}).get("metadata", {})
 
@@ -495,6 +494,12 @@ def _get_dd_llm_obs_payload_metadata(
         latency_metrics = self._get_latency_metrics(standard_logging_payload)
         _metadata.update({"latency_metrics": dict(latency_metrics)})
 
+        #########################################################
+        # Add spend metrics to metadata
+        #########################################################
+        spend_metrics = self._get_spend_metrics(standard_logging_payload)
+        _metadata.update({"spend_metrics": dict(spend_metrics)})
+
         ## extract tool calls and add to metadata
         tool_call_metadata = self._extract_tool_call_metadata(standard_logging_payload)
         _metadata.update(tool_call_metadata)
@@ -543,6 +548,47 @@ def _get_latency_metrics(
                 )
 
         return latency_metrics
+    
+    def _get_spend_metrics(
+            self, standard_logging_payload: StandardLoggingPayload
+    ) -> DDLLMObsSpendMetrics:
+        """
+        Get the spend metrics from the standard logging payload
+        """
+        spend_metrics: DDLLMObsSpendMetrics = DDLLMObsSpendMetrics()
+        
+        # Get response cost for litellm_spend_metric
+        response_cost = standard_logging_payload.get("response_cost", 0.0)
+        if response_cost > 0:
+            spend_metrics["litellm_spend_metric"] = response_cost
+
+        # Get budget information from metadata
+        metadata = standard_logging_payload.get("metadata", {})
+        
+        # API key max budget
+        user_api_key_max_budget = metadata.get("user_api_key_max_budget")
+        if user_api_key_max_budget is not None:
+            spend_metrics["litellm_api_key_max_budget_metric"] = user_api_key_max_budget
+
+        # API key budget remaining hours
+        user_api_key_budget_reset_at = metadata.get("user_api_key_budget_reset_at")
+        if user_api_key_budget_reset_at is not None:
+            try:
+                from datetime import datetime
+                if isinstance(user_api_key_budget_reset_at, str):
+                    # Parse ISO string if it's a string
+                    budget_reset_at = datetime.fromisoformat(user_api_key_budget_reset_at.replace('Z', '+00:00'))
+                else:
+                    budget_reset_at = user_api_key_budget_reset_at
+                
+                remaining_hours = (
+                    budget_reset_at - datetime.now(budget_reset_at.tzinfo)
+                ).total_seconds() / 3600
+                spend_metrics["litellm_api_key_budget_remaining_hours_metric"] = max(0, remaining_hours)
+            except Exception as e:
+                verbose_logger.debug(f"Error calculating remaining hours for budget reset: {e}")
+
+        return spend_metrics
 
     def _process_input_messages_preserving_tool_calls(
         self, messages: List[Any]

litellm/types/integrations/datadog_llm_obs.py

@@ -81,3 +81,9 @@ class DDLLMObsLatencyMetrics(TypedDict, total=False):
     time_to_first_token_ms: float
     litellm_overhead_time_ms: float
     guardrail_overhead_time_ms: float
+
+class DDLLMObsSpendMetrics(TypedDict, total=False):
+    litellm_spend_metric: float
+    litellm_api_key_max_budget_metric: float
+    litellm_remaining_api_key_budget_metric: float
+    litellm_api_key_budget_remaining_hours_metric: float

tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py

@@ -657,21 +657,109 @@ def test_guardrail_information_in_metadata(mock_env_vars):
         assert guardrail_info["guardrail_response"]["score"] == 0.1
 
 
+def create_standard_logging_payload_with_spend_metrics() -> StandardLoggingPayload:
+    """Create a StandardLoggingPayload object with spend metrics for testing"""
+    from datetime import datetime, timezone
+    
+    # Create a budget reset time 24 hours from now
+    budget_reset_at = datetime.now(timezone.utc) + timedelta(hours=24)
+    
+    return {
+        "id": "test-request-id-spend",
+        "trace_id": "test-trace-id-spend",
+        "call_type": "completion",
+        "stream": None,
+        "response_cost": 0.15,
+        "response_cost_failure_debug_info": None,
+        "status": "success",
+        "custom_llm_provider": "openai",
+        "total_tokens": 30,
+        "prompt_tokens": 10,
+        "completion_tokens": 20,
+        "startTime": 1234567890.0,
+        "endTime": 1234567891.0,
+        "completionStartTime": 1234567890.5,
+        "response_time": 1.0,
+        "model_map_information": {
+            "model_map_key": "gpt-4",
+            "model_map_value": None
+        },
+        "model": "gpt-4",
+        "model_id": "model-123",
+        "model_group": "openai-gpt",
+        "api_base": "https://api.openai.com",
+        "metadata": {
+            "user_api_key_hash": "test_hash",
+            "user_api_key_org_id": None,
+            "user_api_key_alias": "test_alias",
+            "user_api_key_team_id": "test_team",
+            "user_api_key_user_id": "test_user",
+            "user_api_key_team_alias": "test_team_alias",
+            "user_api_key_user_email": None,
+            "user_api_key_end_user_id": None,
+            "user_api_key_request_route": None,
+            "user_api_key_max_budget": 10.0,  # $10 max budget
+            "user_api_key_budget_reset_at": budget_reset_at.isoformat(),
+            "spend_logs_metadata": None,
+            "requester_ip_address": "127.0.0.1",
+            "requester_metadata": None,
+            "requester_custom_headers": None,
+            "prompt_management_metadata": None,
+            "mcp_tool_call_metadata": None,
+            "vector_store_request_metadata": None,
+            "applied_guardrails": None,
+            "usage_object": None,
+            "cold_storage_object_key": None,
+        },
+        "cache_hit": False,
+        "cache_key": None,
+        "saved_cache_cost": 0.0,
+        "request_tags": [],
+        "end_user": None,
+        "requester_ip_address": "127.0.0.1",
+        "messages": [{"role": "user", "content": "Hello, world!"}],
+        "response": {"choices": [{"message": {"content": "Hi there!"}}]},
+        "error_str": None,
+        "error_information": None,
+        "model_parameters": {"stream": False},
+        "hidden_params": {
+            "model_id": "model-123",
+            "cache_key": None,
+            "api_base": "https://api.openai.com",
+            "response_cost": "0.15",
+            "litellm_overhead_time_ms": None,
+            "additional_headers": None,
+            "batch_models": None,
+            "litellm_model_name": None,
+            "usage_object": None,
+        },
+        "guardrail_information": None,
+        "standard_built_in_tools_params": None,
+    }  # type: ignore
+
+
 def create_standard_logging_payload_with_tool_calls() -> StandardLoggingPayload:
     """Create a StandardLoggingPayload object with tool calls for testing"""
     return {
         "id": "test-request-id-tool-calls",
+        "trace_id": "test-trace-id-tool-calls",
         "call_type": "completion",
+        "stream": None,
         "response_cost": 0.05,
         "response_cost_failure_debug_info": None,
         "status": "success",
+        "custom_llm_provider": "openai",
         "total_tokens": 50,
         "prompt_tokens": 20,
         "completion_tokens": 30,
         "startTime": 1234567890.0,
         "endTime": 1234567891.0,
         "completionStartTime": 1234567890.5,
-        "model_map_information": {"model_map_key": "gpt-4", "model_map_value": None},
+        "response_time": 1.0,
+        "model_map_information": {
+            "model_map_key": "gpt-4",
+            "model_map_value": None
+        },
         "model": "gpt-4",
         "model_id": "model-123",
         "model_group": "openai-gpt",
@@ -746,6 +834,7 @@ def create_standard_logging_payload_with_tool_calls() -> StandardLoggingPayload:
             ]
         },
         "error_str": None,
+        "error_information": None,
         "model_parameters": {"temperature": 0.7},
         "hidden_params": {
             "model_id": "model-123",
@@ -758,14 +847,9 @@ def create_standard_logging_payload_with_tool_calls() -> StandardLoggingPayload:
             "litellm_model_name": None,
             "usage_object": None,
         },
-        "stream": None,
-        "response_time": 1.0,
-        "error_information": None,
         "guardrail_information": None,
         "standard_built_in_tools_params": None,
-        "trace_id": "test-trace-id-tool-calls",
-        "custom_llm_provider": "openai",
-    }
+    }  # type: ignore
 
 
 class TestDataDogLLMObsLoggerToolCalls:
@@ -897,3 +981,51 @@ def test_tool_call_response_handling(self, mock_env_vars):
             assert len(output_tool_calls) == 1
             output_function_info = output_tool_calls[0].get("function", {})
             assert output_function_info.get("name") == "format_response"
+
+
+def test_spend_metrics_in_datadog_payload(mock_env_vars):
+    """Test that spend metrics are correctly included in DataDog LLM Observability payloads"""
+    with patch(
+        "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client"
+    ), patch("asyncio.create_task"):
+        logger = DataDogLLMObsLogger()
+
+        standard_payload = create_standard_logging_payload_with_spend_metrics()
+
+        kwargs = {
+            "standard_logging_object": standard_payload,
+            "litellm_params": {"metadata": {}},
+        }
+
+        start_time = datetime.now()
+        end_time = datetime.now()
+
+        payload = logger.create_llm_obs_payload(kwargs, start_time, end_time)
+
+        # Verify basic payload structure
+        assert payload.get("name") == "litellm_llm_call"
+        assert payload.get("status") == "ok"
+
+        # Verify spend metrics are included in metadata
+        meta = payload.get("meta", {})
+        assert meta is not None, "Meta section should exist in payload"
+        
+        metadata = meta.get("metadata", {})
+        assert metadata is not None, "Metadata section should exist in meta"
+        
+        spend_metrics = metadata.get("spend_metrics", {})
+        assert spend_metrics, "Spend metrics should exist in metadata"
+
+        # Check that all three spend metrics are present
+        assert "litellm_spend_metric" in spend_metrics
+        assert "litellm_api_key_max_budget_metric" in spend_metrics
+        assert "litellm_api_key_budget_remaining_hours_metric" in spend_metrics
+
+        # Verify the values are correct
+        assert spend_metrics["litellm_spend_metric"] == 0.15  # response_cost
+        assert spend_metrics["litellm_api_key_max_budget_metric"] == 10.0  # max budget
+
+        # Verify remaining hours is a reasonable value (should be close to 24 since we set it to 24 hours from now)
+        remaining_hours = spend_metrics["litellm_api_key_budget_remaining_hours_metric"]
+        assert isinstance(remaining_hours, (int, float))
+        assert 20 <= remaining_hours <= 25  # Should be close to 24 hours