Skip to content

chore(ai): Add metric for tracking ai cost calculation #5560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shellmayr merged 15 commits into from
Feb 2, 2026
Merged

chore(ai): Add metric for tracking ai cost calculation #5560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
d49be0f
chore(ai): add metric for tracking ai cost calculation
shellmayr Jan 20, 2026
807f933
add fixed platforms and ai integrations
shellmayr Jan 22, 2026
735fe55
wip
shellmayr Jan 26, 2026
a10f08d
fix ownership issues
shellmayr Jan 28, 2026
01deeb5
Merge branch 'master' into shellmayr/chore/add-metric-for-cost-calcul…
shellmayr Jan 28, 2026
a0f7607
address review comments
shellmayr Jan 28, 2026
374de6c
address review comments
shellmayr Jan 29, 2026
8c68e59
add changelog
shellmayr Jan 29, 2026
444eadb
changelog format
shellmayr Jan 29, 2026
ff67e57
describe metric
shellmayr Jan 30, 2026
0483d3f
wip
shellmayr Jan 30, 2026
a53fe30
wip
shellmayr Jan 30, 2026
1b4b705
emit none result if tokens none, set verbose tags
shellmayr Jan 30, 2026
60c4897
fix comments
shellmayr Feb 2, 2026
ed6ecd3
Merge branch 'master' into shellmayr/chore/add-metric-for-cost-calcul…
shellmayr Feb 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
- Use new processor architecture to process transactions. ([#5379](https://github.com/getsentry/relay/pull/5379))
- Add `gen_ai_response_time_to_first_token` as a `SpanData` attribute. ([#5575](https://github.com/getsentry/relay/pull/5575))
- Add sampling to expensive envelope buffer statsd metrics. ([#5576](https://github.com/getsentry/relay/pull/5576))
- Add `gen_ai.cost_calculation.result` metric to track AI cost calculation outcomes by integration and platform. ([#5560](https://github.com/getsentry/relay/pull/5560))
- Normalizes and validates trace metric names. ([#5589](https://github.com/getsentry/relay/pull/5589))

## 26.1.0
Expand Down
13 changes: 12 additions & 1 deletion relay-event-normalization/src/eap/ai.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use relay_protocol::Annotated;

use crate::ModelCosts;
use crate::span::ai;
use crate::statsd::{map_origin_to_integration, platform_tag};

/// Normalizes AI attributes.
///
Expand Down Expand Up @@ -110,6 +111,9 @@ fn normalize_tokens_per_second(attributes: &mut Attributes, duration: Option<Dur

/// Calculates model costs and serializes them into attributes.
fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCosts>) {
let origin = extract_string_value(attributes, ORIGIN);
let platform = extract_string_value(attributes, PLATFORM);

let model_cost = attributes
.get_value(GEN_AI_REQUEST_MODEL)
.or_else(|| attributes.get_value(GEN_AI_RESPONSE_MODEL))
Expand All @@ -133,7 +137,10 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCos
output_reasoning_tokens: get_tokens(GEN_AI_USAGE_OUTPUT_REASONING_TOKENS),
};

let Some(costs) = ai::calculate_costs(model_cost, tokens) else {
let integration = map_origin_to_integration(origin);
let platform = platform_tag(platform);

let Some(costs) = ai::calculate_costs(model_cost, tokens, integration, platform) else {
return;
};

Expand All @@ -143,6 +150,10 @@ fn normalize_ai_costs(attributes: &mut Attributes, model_costs: Option<&ModelCos
attributes.insert(GEN_AI_COST_TOTAL_TOKENS, costs.total());
}

fn extract_string_value<'a>(attributes: &'a Attributes, key: &str) -> Option<&'a str> {
attributes.get_value(key).and_then(|v| v.as_str())
}

#[cfg(test)]
mod tests {
use std::collections::HashMap;
Expand Down
76 changes: 70 additions & 6 deletions relay-event-normalization/src/normalize/span/ai.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! AI cost calculation.

use crate::statsd::{Counters, map_origin_to_integration, platform_tag};
use crate::{ModelCostV2, ModelCosts};
use relay_event_schema::protocol::{
Event, Measurements, OperationType, Span, SpanData, TraceContext,
Expand Down Expand Up @@ -84,8 +85,19 @@ impl CalculatedCost {
/// Calculates the total cost for a model call.
///
/// Returns `None` if no tokens were used.
pub fn calculate_costs(model_cost: &ModelCostV2, tokens: UsedTokens) -> Option<CalculatedCost> {
pub fn calculate_costs(
model_cost: &ModelCostV2,
tokens: UsedTokens,
integration: &str,
platform: &str,
) -> Option<CalculatedCost> {
if !tokens.has_usage() {
relay_statsd::metric!(
counter(Counters::GenAiCostCalculationResult) += 1,
result = "calculation_none",
integration = integration,
platform = platform,
);
return None;
}

Expand All @@ -103,6 +115,19 @@ pub fn calculate_costs(model_cost: &ModelCostV2, tokens: UsedTokens) -> Option<C
let output = (tokens.raw_output_tokens() * model_cost.output_per_token)
+ (tokens.output_reasoning_tokens * reasoning_cost);

let metric_label = match (input, output) {
(x, y) if x < 0.0 || y < 0.0 => "calculation_negative",
(0.0, 0.0) => "calculation_zero",
_ => "calculation_positive",
};

relay_statsd::metric!(
counter(Counters::GenAiCostCalculationResult) += 1,
result = metric_label,
integration = integration,
platform = platform,
);

Some(CalculatedCost { input, output })
}

Expand Down Expand Up @@ -158,11 +183,18 @@ pub fn infer_ai_operation_type(op_name: &str) -> Option<&'static str> {

/// Calculates the cost of an AI model based on the model cost and the tokens used.
/// Calculated cost is in US dollars.
fn extract_ai_model_cost_data(model_cost: Option<&ModelCostV2>, data: &mut SpanData) {
fn extract_ai_model_cost_data(
model_cost: Option<&ModelCostV2>,
data: &mut SpanData,
origin: Option<&str>,
platform: Option<&str>,
) {
let Some(model_cost) = model_cost else { return };

let used_tokens = UsedTokens::from_span_data(&*data);
let Some(costs) = calculate_costs(model_cost, used_tokens) else {
let integration = map_origin_to_integration(origin);
let platform = platform_tag(platform);
let Some(costs) = calculate_costs(model_cost, used_tokens, integration, platform) else {
return;
};

Expand Down Expand Up @@ -220,7 +252,13 @@ fn set_total_tokens(data: &mut SpanData) {
}

/// Extract the additional data into the span
fn extract_ai_data(data: &mut SpanData, duration: f64, ai_model_costs: &ModelCosts) {
fn extract_ai_data(
data: &mut SpanData,
duration: f64,
ai_model_costs: &ModelCosts,
origin: Option<&str>,
platform: Option<&str>,
) {
// Extracts the response tokens per second
if data.gen_ai_response_tokens_per_second.value().is_none()
&& duration > 0.0
Expand All @@ -244,7 +282,12 @@ fn extract_ai_data(data: &mut SpanData, duration: f64, ai_model_costs: &ModelCos
.and_then(|val| val.as_str())
})
{
extract_ai_model_cost_data(ai_model_costs.cost_per_token(model_id), data)
extract_ai_model_cost_data(
ai_model_costs.cost_per_token(model_id),
data,
origin,
platform,
)
}
}

Expand All @@ -255,6 +298,8 @@ fn enrich_ai_span_data(
measurements: &Annotated<Measurements>,
duration: f64,
model_costs: Option<&ModelCosts>,
origin: Option<&str>,
platform: Option<&str>,
) {
if !is_ai_span(span_data, span_op.value()) {
return;
Expand All @@ -267,7 +312,7 @@ fn enrich_ai_span_data(
set_total_tokens(data);

if let Some(model_costs) = model_costs {
extract_ai_data(data, duration, model_costs);
extract_ai_data(data, duration, model_costs, origin, platform);
}

let ai_op_type = data
Expand All @@ -294,6 +339,8 @@ pub fn enrich_ai_span(span: &mut Span, model_costs: Option<&ModelCosts>) {
&span.measurements,
duration,
model_costs,
span.origin.as_str(),
span.platform.as_str(),
);
}

Expand All @@ -316,6 +363,8 @@ pub fn enrich_ai_event_data(event: &mut Event, model_costs: Option<&ModelCosts>)
&event.measurements,
event_duration,
model_costs,
trace_context.origin.as_str(),
event.platform.as_str(),
);
}
let spans = event.spans.value_mut().iter_mut().flatten();
Expand All @@ -326,13 +375,16 @@ pub fn enrich_ai_event_data(event: &mut Event, model_costs: Option<&ModelCosts>)
.get_value("span.duration")
.and_then(|v| v.as_f64())
.unwrap_or(0.0);
let span_platform = span.platform.as_str().or_else(|| event.platform.as_str());

enrich_ai_span_data(
&mut span.data,
&span.op,
&span.measurements,
span_duration,
model_costs,
span.origin.as_str(),
span_platform,
);
}
}
Expand Down Expand Up @@ -378,6 +430,8 @@ mod tests {
input_cache_write_per_token: 1.0,
},
UsedTokens::from_span_data(&SpanData::default()),
"test",
"test",
);
assert!(cost.is_none());
}
Expand All @@ -399,6 +453,8 @@ mod tests {
output_tokens: 15.0,
output_reasoning_tokens: 9.0,
},
"test",
"test",
)
.unwrap();

Expand Down Expand Up @@ -428,6 +484,8 @@ mod tests {
output_tokens: 15.0,
output_reasoning_tokens: 9.0,
},
"test",
"test",
)
.unwrap();

Expand Down Expand Up @@ -459,6 +517,8 @@ mod tests {
output_tokens: 1.0,
output_reasoning_tokens: 9.0,
},
"test",
"test",
)
.unwrap();

Expand Down Expand Up @@ -487,6 +547,8 @@ mod tests {
output_tokens: 50.0,
output_reasoning_tokens: 10.0,
},
"test",
"test",
)
.unwrap();

Expand Down Expand Up @@ -523,6 +585,8 @@ mod tests {
input_cache_write_per_token: 0.75,
},
tokens,
"test",
"test",
)
.unwrap();

Expand Down
72 changes: 71 additions & 1 deletion relay-event-normalization/src/statsd.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,26 @@
use relay_statsd::TimerMetric;
use relay_statsd::{CounterMetric, TimerMetric};

pub enum Counters {
/// Records the status of the AI cost calculation.
///
/// The metric is tagged with:
/// - `result`: The outcome of the cost calculation. Possible values are:
/// `calculation_negative`,
/// `calculation_zero`,
/// `calculation_positive`,
/// `calculation_none`
/// - `integration`: The integration used for the cost calculation.
/// - `platform`: The platform used for the cost calculation.
GenAiCostCalculationResult,
}

impl CounterMetric for Counters {
fn name(&self) -> &'static str {
match *self {
Self::GenAiCostCalculationResult => "gen_ai.cost_calculation.result",
}
}
}

pub enum Timers {
/// Measures how log normalization of SQL queries in span description take.
Expand All @@ -15,3 +37,51 @@ impl TimerMetric for Timers {
}
}
}

/// Maps a span origin to a well-known AI integration name for metrics.
///
/// Origins follow the pattern `auto.<integration>.<source>` or `auto.<category>.<protocol>.<source>`.
/// This function extracts recognized AI integrations for cleaner metric tagging.
pub fn map_origin_to_integration(origin: Option<&str>) -> &'static str {
match origin {
Some(o) if o.starts_with("auto.ai.openai_agents") => "openai_agents",
Some(o) if o.starts_with("auto.ai.openai") => "openai",
Some(o) if o.starts_with("auto.ai.anthropic") => "anthropic",
Some(o) if o.starts_with("auto.ai.cohere") => "cohere",
Some(o) if o.starts_with("auto.vercelai.") => "vercelai",
Some(o) if o.starts_with("auto.ai.langchain") => "langchain",
Some(o) if o.starts_with("auto.ai.langgraph") => "langgraph",
Some(o) if o.starts_with("auto.ai.google_genai") => "google_genai",
Some(o) if o.starts_with("auto.ai.pydantic_ai") => "pydantic_ai",
Some(o) if o.starts_with("auto.ai.huggingface_hub") => "huggingface_hub",
Some(o) if o.starts_with("auto.ai.litellm") => "litellm",
Some(o) if o.starts_with("auto.ai.mcp_server") => "mcp_server",
Some(o) if o.starts_with("auto.ai.mcp") => "mcp",
Some(o) if o.starts_with("auto.ai.claude_agent_sdk") => "claude_agent_sdk",
Some(o) if o.starts_with("auto.ai.") => "other",
Some(_) => "other",
None => "unknown",
}
}

pub fn platform_tag(platform: Option<&str>) -> &'static str {
match platform {
Some("cocoa") => "cocoa",
Some("csharp") => "csharp",
Some("edge") => "edge",
Some("go") => "go",
Some("java") => "java",
Some("javascript") => "javascript",
Some("julia") => "julia",
Some("native") => "native",
Some("node") => "node",
Some("objc") => "objc",
Some("perl") => "perl",
Some("php") => "php",
Some("python") => "python",
Some("ruby") => "ruby",
Some("swift") => "swift",
Some(_) => "other",
None => "unknown",
}
}
Loading