chore(ai): Add metric for tracking ai cost calculation

CHANGELOG.md

@@ -14,6 +14,7 @@
 - Use new processor architecture to process transactions. ([#5379](https://github.com/getsentry/relay/pull/5379))
 - Add `gen_ai_response_time_to_first_token` as a `SpanData` attribute. ([#5575](https://github.com/getsentry/relay/pull/5575))
 - Add sampling to expensive envelope buffer statsd metrics. ([#5576](https://github.com/getsentry/relay/pull/5576))
+- Add `gen_ai.cost_calculation.result` metric to track AI cost calculation outcomes by integration and platform. ([#5560](https://github.com/getsentry/relay/pull/5560))
 
 ## 26.1.0
 

relay-event-normalization/src/eap/ai.rs

@@ -6,6 +6,7 @@ use relay_protocol::Annotated;
 
 use crate::ModelCosts;
 use crate::span::ai;
+use crate::statsd::{Counters, map_origin_to_integration, platform_tag};
 
 /// Normalizes AI attributes.
 ///
@@ -110,6 +111,9 @@ fn normalize_tokens_per_second(attributes: &mut Attributes, duration: Option<Dur
 
 /// Calculates model costs and serializes them into attributes.
 fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCosts>) {
+    let origin = extract_string_value(attributes, ORIGIN);
+    let platform = extract_string_value(attributes, PLATFORM);
+
     let model_cost = attributes
         .get_value(GEN_AI_REQUEST_MODEL)
         .or_else(|| attributes.get_value(GEN_AI_RESPONSE_MODEL))
@@ -133,16 +137,28 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCos
         output_reasoning_tokens: get_tokens(GEN_AI_USAGE_OUTPUT_REASONING_TOKENS),
     };
 
-    let Some(costs) = ai::calculate_costs(model_cost, tokens) else {
+    let integration = map_origin_to_integration(origin);
+    let platform = platform_tag(platform);
+
+    let Some(costs) = ai::calculate_costs(model_cost, tokens, integration, platform) else {
+        relay_statsd::metric!(
+            counter(Counters::GenAiCostCalculationResult) += 1,
+            result = "calculation_none",
+            integration = integration,
+            platform = platform,
+        );
         return;
     };
 
-    // Overwrite all values, the attributes should reflect the values we used to calculate the total.
     attributes.insert(GEN_AI_COST_INPUT_TOKENS, costs.input);
     attributes.insert(GEN_AI_COST_OUTPUT_TOKENS, costs.output);
     attributes.insert(GEN_AI_COST_TOTAL_TOKENS, costs.total());
 }
 
+fn extract_string_value<'a>(attributes: &'a Attributes, key: &str) -> Option<&'a str> {
+    attributes.get_value(key).and_then(|v| v.as_str())
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;

relay-event-normalization/src/normalize/span/ai.rs

@@ -1,5 +1,6 @@
 //! AI cost calculation.
 
+use crate::statsd::Counters;
 use crate::{ModelCostV2, ModelCosts};
 use relay_event_schema::protocol::{
     Event, Measurements, OperationType, Span, SpanData, TraceContext,
@@ -84,7 +85,12 @@ impl CalculatedCost {
 /// Calculates the total cost for a model call.
 ///
 /// Returns `None` if no tokens were used.
-pub fn calculate_costs(model_cost: &ModelCostV2, tokens: UsedTokens) -> Option<CalculatedCost> {
+pub fn calculate_costs(
+    model_cost: &ModelCostV2,
+    tokens: UsedTokens,
+    integration: &str,
+    platform: &str,
+) -> Option<CalculatedCost> {
     if !tokens.has_usage() {
         return None;
     }
@@ -103,6 +109,19 @@ pub fn calculate_costs(model_cost: &ModelCostV2, tokens: UsedTokens) -> Option<C
     let output = (tokens.raw_output_tokens() * model_cost.output_per_token)
         + (tokens.output_reasoning_tokens * reasoning_cost);
 
+    let metric_label = match (input.signum(), output.signum()) {
+        (-1.0, _) | (_, -1.0) => "calculation_negative",
+        (0.0, 0.0) => "calculation_zero",
+        _ => "calculation_positive",
+    };
+
+    relay_statsd::metric!(
+        counter(Counters::GenAiCostCalculationResult) += 1,
+        result = metric_label,
+        integration = integration,
+        platform = platform,
+    );
+
     Some(CalculatedCost { input, output })
 }
 
@@ -162,7 +181,7 @@ fn extract_ai_model_cost_data(model_cost: Option<&ModelCostV2>, data: &mut SpanD
     let Some(model_cost) = model_cost else { return };
 
     let used_tokens = UsedTokens::from_span_data(&*data);
-    let Some(costs) = calculate_costs(model_cost, used_tokens) else {
+    let Some(costs) = calculate_costs(model_cost, used_tokens, "unknown", "unknown") else {
         return;
     };
 
@@ -378,6 +397,8 @@ mod tests {
                 input_cache_write_per_token: 1.0,
             },
             UsedTokens::from_span_data(&SpanData::default()),
+            "test",
+            "test",
         );
         assert!(cost.is_none());
     }
@@ -399,6 +420,8 @@ mod tests {
                 output_tokens: 15.0,
                 output_reasoning_tokens: 9.0,
             },
+            "test",
+            "test",
         )
         .unwrap();
 
@@ -428,6 +451,8 @@ mod tests {
                 output_tokens: 15.0,
                 output_reasoning_tokens: 9.0,
             },
+            "test",
+            "test",
         )
         .unwrap();
 
@@ -459,6 +484,8 @@ mod tests {
                 output_tokens: 1.0,
                 output_reasoning_tokens: 9.0,
             },
+            "test",
+            "test",
         )
         .unwrap();
 
@@ -487,6 +514,8 @@ mod tests {
                 output_tokens: 50.0,
                 output_reasoning_tokens: 10.0,
             },
+            "test",
+            "test",
         )
         .unwrap();
 
@@ -523,6 +552,8 @@ mod tests {
                 input_cache_write_per_token: 0.75,
             },
             tokens,
+            "test",
+            "test",
         )
         .unwrap();
 

relay-event-normalization/src/statsd.rs

@@ -1,4 +1,25 @@
-use relay_statsd::TimerMetric;
+use relay_statsd::{CounterMetric, TimerMetric};
+
+pub enum Counters {
+    GenAiCostCalculationResult,
+}
+
+impl CounterMetric for Counters {
+    fn name(&self) -> &'static str {
+        match *self {
+            /// Records the status of the AI cost calculation.
+            /// The metric is tagged with:
+            ///  - `result`: The outcome of the cost calculation. Possible values are:
+            ///     `calculation_negative`,
+            ///     `calculation_zero`,
+            ///     `calculation_positive`,
+            ///     `calculation_none`
+            ///  - `integration`: The integration used for the cost calculation.
+            ///  - `platform`: The platform used for the cost calculation.
+            Self::GenAiCostCalculationResult => "gen_ai.cost_calculation.result",
+        }
+    }
+}
 
 pub enum Timers {
     /// Measures how log normalization of SQL queries in span description take.
@@ -15,3 +36,51 @@ impl TimerMetric for Timers {
         }
     }
 }
+
+/// Maps a span origin to a well-known AI integration name for metrics.
+///
+/// Origins follow the pattern `auto.<integration>.<source>` or `auto.<category>.<protocol>.<source>`.
+/// This function extracts recognized AI integrations for cleaner metric tagging.
+pub fn map_origin_to_integration(origin: Option<&str>) -> &'static str {
+    match origin {
+        Some(o) if o.starts_with("auto.ai.openai_agents") => "openai_agents",
+        Some(o) if o.starts_with("auto.ai.openai") => "openai",
+        Some(o) if o.starts_with("auto.ai.anthropic") => "anthropic",
+        Some(o) if o.starts_with("auto.ai.cohere") => "cohere",
+        Some(o) if o.starts_with("auto.vercelai.") => "vercelai",
+        Some(o) if o.starts_with("auto.ai.langchain") => "langchain",
+        Some(o) if o.starts_with("auto.ai.langgraph") => "langgraph",
+        Some(o) if o.starts_with("auto.ai.google_genai") => "google_genai",
+        Some(o) if o.starts_with("auto.ai.pydantic_ai") => "pydantic_ai",
+        Some(o) if o.starts_with("auto.ai.huggingface_hub") => "huggingface_hub",
+        Some(o) if o.starts_with("auto.ai.litellm") => "litellm",
+        Some(o) if o.starts_with("auto.ai.mcp_server") => "mcp_server",
+        Some(o) if o.starts_with("auto.ai.mcp") => "mcp",
+        Some(o) if o.starts_with("auto.ai.claude_agent_sdk") => "claude_agent_sdk",
+        Some(o) if o.starts_with("auto.ai.") => "other",
+        Some(_) => "other",
+        None => "unknown",
+    }
+}
+
+pub fn platform_tag(platform: Option<&str>) -> &'static str {
+    match platform {
+        Some("cocoa") => "cocoa",
+        Some("csharp") => "csharp",
+        Some("edge") => "edge",
+        Some("go") => "go",
+        Some("java") => "java",
+        Some("javascript") => "javascript",
+        Some("julia") => "julia",
+        Some("native") => "native",
+        Some("node") => "node",
+        Some("objc") => "objc",
+        Some("perl") => "perl",
+        Some("php") => "php",
+        Some("python") => "python",
+        Some("ruby") => "ruby",
+        Some("swift") => "swift",
+        Some(_) => "other",
+        None => "unknown",
+    }
+}