Add renamed metric (#21877)

* renamed metrics

* changelog

* fix

* rename

* changelog

* Update litellm/changelog.d/21877.fixed

Co-authored-by: Juanpe Araque <[email protected]>

---------

Co-authored-by: Juanpe Araque <[email protected]>

Author: steveny91

litellm/changelog.d/21877.fixed

@@ -0,0 +1 @@
+Add a renamed metric that was missed from the [1.75 release of LiteLLM](https://github.com/BerriAI/litellm/pull/13271)

litellm/datadog_checks/litellm/metrics.py

@@ -78,6 +78,7 @@
     'litellm_team_budget_remaining_hours_metric': "team.budget.remaining_hours.metric",
     'litellm_team_max_budget_metric': "team.max_budget.metric",
     'litellm_total_tokens': "total.tokens",
+    'litellm_total_tokens_metric': "total.tokens",
     'process_start_time_seconds': {"name": "process.uptime.seconds", "type": "time_elapsed"},
     # Deprecated
     'litellm_llm_api_failed_requests_metric': "llm.api.failed_requests.metric",

litellm/tests/common.py

@@ -119,3 +119,9 @@ def get_fixture_path(filename):
     'litellm.endpoint.healthy_count',
     'litellm.endpoint.unhealthy_count',
 ]
+
+RENAMED_METRICS_V1_75 = [
+    'litellm.total.tokens.count',
+    'litellm.input_tokens.metric.count',
+    'litellm.output_tokens.metric.count',
+]

litellm/tests/fixtures/renamed_metrics.txt

@@ -0,0 +1,9 @@
+# HELP litellm_total_tokens_metric_total Total number of input + output tokens from LLM requests
+# TYPE litellm_total_tokens_metric_total counter
+litellm_total_tokens_metric_total 0.0
+# HELP litellm_input_tokens_metric_total Total number of input tokens from LLM requests
+# TYPE litellm_input_tokens_metric_total counter
+litellm_input_tokens_metric_total 0.0
+# HELP litellm_output_tokens_metric_total Total number of output tokens from LLM requests
+# TYPE litellm_output_tokens_metric_total counter
+litellm_output_tokens_metric_total 0.0

litellm/tests/test_unit.py

@@ -12,6 +12,7 @@
     ENDPOINT_METRICS,
     METRICS,
     OM_MOCKED_INSTANCE,
+    RENAMED_METRICS_V1_75,
     get_fixture_path,
 )
 
@@ -89,3 +90,14 @@ def test__extract_error_type_not_found():
     check = LitellmCheck('litellm', {}, [{}])
     error_msg = 'no litellm error here'
     assert check._extract_error_type(error_msg) == 'unknown'
+
+
+def test_litellm_renamed_metrics(dd_run_check, aggregator, mock_http_response):
+    # Some metrics were renamed here: https://github.com/BerriAI/litellm/pull/13271
+    mock_http_response(file_path=get_fixture_path('renamed_metrics.txt'))
+    check = LitellmCheck('litellm', {}, [OM_MOCKED_INSTANCE])
+    dd_run_check(check)
+    for metric in RENAMED_METRICS_V1_75:
+        aggregator.assert_metric(metric)
+    aggregator.assert_all_metrics_covered()
+    aggregator.assert_service_check('litellm.openmetrics.health', ServiceCheck.OK)