Skip to content

Use aggregate metric tables for dashboards #7218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bendk merged 1 commit into from
Feb 12, 2026
+65 −161
Merged

Use aggregate metric tables for dashboards #7218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions tools/generate-rust-dashboards/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,6 @@ pub struct DistributionMetric {
// * Memory distributions are always stored in bytes, regardless of the unit listed in
// `metrics.yaml`
pub value_divisor: Option<u64>,
// Filter out values lower than this amount (takes effect before the divisor)
pub value_filter: Option<u64>,
// Link to an extra dashboard, the inner value is the name of the dashboard
pub link_to: Option<&'static str>,
}
Expand Down Expand Up @@ -142,8 +140,6 @@ pub struct LabeledDistributionMetric {
// * Memory distributions are always stored in bytes, regardless of the unit listed in
// `metrics.yaml`
pub value_divisor: Option<u64>,
// Filter out values lower than this amount (takes effect before the divisor)
pub value_filter: Option<u64>,
}

#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
Expand Down Expand Up @@ -184,9 +180,9 @@ impl TeamConfig {
impl Application {
pub fn slug(&self) -> &'static str {
match self {
Self::Android => "android",
Self::Ios => "ios",
Self::Desktop => "desktop",
Self::Android => "firefox_android",
Self::Ios => "firefox_ios",
Self::Desktop => "firefox_desktop",
}
}

Expand Down
18 changes: 8 additions & 10 deletions tools/generate-rust-dashboards/src/metrics/counter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,25 +34,23 @@ fn count_panel(application: Application, channel: ReleaseChannel, metric: &Count
..
} = *metric;

let mut query = Query {
select: vec!["$__timeGroup(submission_timestamp, $__interval) as time".into()],
from: format!("`mozdata.{}.{ping}`", application.bigquery_dataset()),
let query = Query {
select: vec!["TIMESTAMP(submission_date) as time".into(), "count".into()],
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
where_: vec![
"$__timeFilter(submission_timestamp)".into(),
format!("{ping}.counter.{category}_{metric} IS NOT NULL"),
format!("normalized_channel = '{channel}'"),
"$__timeFilter(TIMESTAMP(submission_date))".into(),
format!("application = `{}`", application.slug()),
format!("channel = '{channel}'"),
],
group_by: Some("1".into()),
order_by: Some("time asc".into()),
order_by: Some("submission_date asc".into()),
..Query::default()
};
query.add_count_per_day_column(format!("SUM({ping}.counter.{category}_{metric})"), metric);

TimeSeriesPanel {
title: application.display_name(channel),
grid_pos: GridPos::height(8),
datasource: Datasource::bigquery(),
interval: "1h".into(),
interval: "1d".into(),
targets: vec![Target::table(query.sql())],
field_config: FieldConfig {
defaults: FieldConfigDefaults {
Expand Down
83 changes: 23 additions & 60 deletions tools/generate-rust-dashboards/src/metrics/distribution.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use crate::{
config::{Application, DistributionMetric, DistributionMetricKind, TeamConfig},
config::{Application, DistributionMetric, TeamConfig},
schema::{
DashboardBuilder, DataLink, Datasource, FieldConfig, FieldConfigCustom,
FieldConfigDefaults, GridPos, Panel, Target, TimeSeriesPanel, Transformation,
Expand All @@ -20,84 +20,47 @@ pub fn add_to_dashboard(
) -> Result<()> {
builder.add_panel_title(metric.display_name);
for app in metric.applications.iter().cloned() {
builder.add_panel_third(count_panel(app, metric, 5));
builder.add_panel_third(count_panel(app, metric, 50));
builder.add_panel_third(count_panel(app, metric, 95));
builder.add_panel_third(count_panel(app, metric, "q05", "5th percentile"));
builder.add_panel_third(count_panel(app, metric, "q50", "50th percentile"));
builder.add_panel_third(count_panel(app, metric, "q95", "95th percentile"));
}
Ok(())
}

fn count_panel(application: Application, metric: &DistributionMetric, percentile: u32) -> Panel {
fn count_panel(
application: Application,
metric: &DistributionMetric,
quantile: &str,
quantile_label: &str,
) -> Panel {
let DistributionMetric {
kind,
ping,
category,
metric,
value_divisor,
value_filter,
axis_label,
link_to,
..
} = *metric;
let dataset = application.bigquery_dataset();
let metric_table = match kind {
DistributionMetricKind::Memory => "memory_distribution",
DistributionMetricKind::Timing => "timing_distribution",
DistributionMetricKind::Custom => "custom_distribution",
};

// Group metrics and calculate quantiles.
// q is a 20-quantile array (0%, 5%, ..., 95%, 100%)
let mut subquery = Query {
let query = Query {
select: vec![
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
"normalized_channel".into(),
"APPROX_QUANTILES(CAST(values.key AS INT64), 20) as q".into(),
],
from: format!("`mozdata.{dataset}.{ping}`"),
joins: vec![
format!("CROSS JOIN UNNEST(metrics.{metric_table}.{category}_{metric}.values) as values"),
// Cross join with an array with length=values.value to make the APPROX_QUANTILES statement above work.
// Histogram metrics are stored in bigquery as a struct of key/value pairs.
// The key is the measurement value, while the value is the count.
// APPROX_QUANTILES expects to count single values,
// so use this CROSS JOIN to repeat each key `value` times.
"CROSS JOIN UNNEST(GENERATE_ARRAY(1, values.value)) AS repeat_number".into(),
"TIMESTAMP(submission_date) as time".into(),
"channel".into(),
match value_divisor {
None => format!("{quantile} as amount"),
Some(amount) => format!("{quantile} / {amount} as amount"),
},
],
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
order_by: Some("submission_date asc, channel asc".into()),
where_: vec![
"$__timeFilter(submission_timestamp)".into(),
"(normalized_channel = 'nightly' OR normalized_channel = 'beta' OR normalized_channel = 'release')".into(),
"$__timeFilter(TIMESTAMP(submission_date))".into(),
"channel IS NOT NULL".into(),
],
group_by: Some("1, 2".into()),
..Query::default()
};

if let Some(amount) = value_filter {
subquery
.where_
.push(format!("CAST(values.key AS INT64) >= {amount}"));
}
let mut query = Query {
select: vec!["time".into(), "normalized_channel".into()],
from: subquery.as_subquery(),
order_by: Some("time desc, normalized_channel asc".into()),
..Query::default()
};

let quantile_index = percentile / 5;
match value_divisor {
None => {
query
.select
.extend([format!("q[OFFSET({quantile_index})] as amount")]);
}
Some(amount) => {
query
.select
.extend([format!("q[OFFSET({quantile_index})] / {amount} as amount")]);
}
}

let mut links = vec![];
if let Some(link_to) = link_to {
links.push(DataLink {
Expand All @@ -111,7 +74,7 @@ fn count_panel(application: Application, metric: &DistributionMetric, percentile
}

TimeSeriesPanel {
title: format!("{application} ({percentile}th percentile)"),
title: format!("{application} ({quantile_label})"),
grid_pos: GridPos::height(8),
datasource: Datasource::bigquery(),
interval: "1d".into(),
Expand All @@ -127,7 +90,7 @@ fn count_panel(application: Application, metric: &DistributionMetric, percentile
},
transformations: vec![
Transformation::PartitionByValues {
fields: vec!["normalized_channel".into()],
fields: vec!["channel".into()],
keep_fields: true,
},
// Fixup the field names for better legend labels
Expand Down
25 changes: 11 additions & 14 deletions tools/generate-rust-dashboards/src/metrics/labeled_counter.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,32 +37,29 @@ fn count_panel(
metric,
..
} = *metric;
let dataset = application.bigquery_dataset();

let mut query = Query {
let query = Query {
select: vec![
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
"counter.key as label".into(),
"TIMESTAMP(submission_date) as time".into(),
"label".into(),
"count".into(),
],
from: format!("`mozdata.{dataset}.{ping}`"),
joins: vec![format!(
"CROSS JOIN UNNEST(metrics.labeled_counter.{category}_{metric}) as counter"
)],
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
where_: vec![
"$__timeFilter(submission_timestamp)".into(),
format!("normalized_channel = '{channel}'"),
"$__timeFilter(TIMESTAMP(submission_date))".into(),
format!("application = `{}`", application.slug()),
format!("channel = '{channel}'"),
"label IS NOT NULL".into(),
],
group_by: Some("1, 2".into()),
order_by: Some("time asc, label asc".into()),
order_by: Some("submission_date asc".into()),
..Query::default()
};
query.add_count_per_day_column("SUM(counter.value)", metric);

TimeSeriesPanel {
title: application.display_name(channel),
grid_pos: GridPos::height(8),
datasource: Datasource::bigquery(),
interval: "1h".into(),
interval: "1d".into(),
targets: vec![Target::table(query.sql())],
field_config: FieldConfig {
defaults: FieldConfigDefaults {
Expand Down
80 changes: 19 additions & 61 deletions tools/generate-rust-dashboards/src/metrics/labeled_distribution.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use crate::{
config::{Application, DistributionMetricKind, LabeledDistributionMetric, TeamConfig},
config::{Application, LabeledDistributionMetric, TeamConfig},
schema::{
DashboardBuilder, Datasource, FieldConfig, FieldConfigCustom, FieldConfigDefaults, GridPos,
Panel, Target, TimeSeriesPanel, Transformation,
Expand All @@ -19,90 +19,48 @@ pub fn add_to_dashboard(
) -> Result<()> {
builder.add_panel_title(metric.display_name);
for app in metric.applications.iter().cloned() {
builder.add_panel_third(count_panel(app, metric, 5));
builder.add_panel_third(count_panel(app, metric, 50));
builder.add_panel_third(count_panel(app, metric, 95));
builder.add_panel_third(count_panel(app, metric, "q05", "5th percentile"));
builder.add_panel_third(count_panel(app, metric, "q50", "50th percentile"));
builder.add_panel_third(count_panel(app, metric, "q95", "95th percentile"));
}
Ok(())
}

fn count_panel(
application: Application,
metric: &LabeledDistributionMetric,
percentile: u32,
quantile: &str,
quantile_label: &str,
) -> Panel {
let LabeledDistributionMetric {
kind,
ping,
category,
metric,
value_divisor,
value_filter,
axis_label,
..
} = *metric;
let dataset = application.bigquery_dataset();
let metric_table = match kind {
DistributionMetricKind::Memory => "labeled_memory_distribution",
DistributionMetricKind::Timing => "labeled_timing_distribution",
DistributionMetricKind::Custom => "labeled_custom_distribution",
};

// Group metrics and calculate quantiles.
// q is a 20-quantile array (0%, 5%, ..., 95%, 100%)
let mut subquery = Query {
let query = Query {
select: vec![
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
"CONCAT(metric.key, ' ', normalized_channel) as group_name".into(),
"APPROX_QUANTILES(CAST(values.key AS INT64), 20) as q".into(),
],
from: format!("`mozdata.{dataset}.{ping}`"),
joins: vec![
format!("CROSS JOIN UNNEST(metrics.{metric_table}.{category}_{metric}) as metric"),
"CROSS JOIN UNNEST(metric.value.values) as values".into(),
// Cross join with an array with length=values.value to make the APPROX_QUANTILES statement above work.
// Histogram metrics are stored in bigquery as a struct of key/value pairs.
// The key is the measurement value, while the value is the count.
// APPROX_QUANTILES expects to count single values,
// so use this CROSS JOIN to repeat each key `value` times.
"CROSS JOIN UNNEST(GENERATE_ARRAY(1, values.value)) AS repeat_number".into(),
"TIMESTAMP(submission_date) as time".into(),
"CONCAT(label, ' ', channel) as group_name".into(),
match value_divisor {
None => format!("{quantile} as amount"),
Some(amount) => format!("{quantile} / {amount} as amount"),
},
],
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
order_by: Some("submission_date asc, channel asc".into()),
where_: vec![
"$__timeFilter(submission_timestamp)".into(),
"(normalized_channel = 'nightly' OR normalized_channel = 'beta' OR normalized_channel = 'release')".into(),
"$__timeFilter(TIMESTAMP(submission_date))".into(),
"label IS NOT NULL".into(),
"channel IS NOT NULL".into(),
],
group_by: Some("1, 2".into()),
..Query::default()
};

if let Some(amount) = value_filter {
subquery
.where_
.push(format!("CAST(values.key AS INT64) >= {amount}"));
}
let mut query = Query {
select: vec!["time".into(), "group_name".into()],
from: subquery.as_subquery(),
order_by: Some("time desc, group_name asc".into()),
..Query::default()
};

let quantile_index = percentile / 5;
match value_divisor {
None => {
query
.select
.extend([format!("q[OFFSET({quantile_index})] as amount")]);
}
Some(amount) => {
query
.select
.extend([format!("q[OFFSET({quantile_index})] / {amount} as amount")]);
}
}

TimeSeriesPanel {
title: format!("{application} ({percentile}th percentile)"),
title: format!("{application} ({quantile_label})"),
grid_pos: GridPos::height(8),
datasource: Datasource::bigquery(),
interval: "1d".into(),
Expand Down
Loading