Skip to content

Commit a5d3050

Browse files
bendkbendk
authored
Use aggregate metric tables for dashboards (#7218)
Switched to using the aggregate tables that we create using bigquery-etl. These are way faster than quering the source data. One drawback is we can't get more granular than a day, but that seems okay. Aggregate table generation code: https://github.com/mozilla/bigquery-etl/tree/main/sql_generators/rust_component_metrics
1 parent 11e01b2 commit a5d3050

File tree

6 files changed

+65
-161
lines changed

6 files changed

+65
-161
lines changed

‎tools/generate-rust-dashboards/src/config.rs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,6 @@ pub struct DistributionMetric {
110110
// * Memory distributions are always stored in bytes, regardless of the unit listed in
111111
// `metrics.yaml`
112112
pub value_divisor: Option<u64>,
113-
// Filter out values lower than this amount (takes effect before the divisor)
114-
pub value_filter: Option<u64>,
115113
// Link to an extra dashboard, the inner value is the name of the dashboard
116114
pub link_to: Option<&'static str>,
117115
}
@@ -142,8 +140,6 @@ pub struct LabeledDistributionMetric {
142140
// * Memory distributions are always stored in bytes, regardless of the unit listed in
143141
// `metrics.yaml`
144142
pub value_divisor: Option<u64>,
145-
// Filter out values lower than this amount (takes effect before the divisor)
146-
pub value_filter: Option<u64>,
147143
}
148144

149145
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
@@ -184,9 +180,9 @@ impl TeamConfig {
184180
impl Application {
185181
pub fn slug(&self) -> &'static str {
186182
match self {
187-
Self::Android => "android",
188-
Self::Ios => "ios",
189-
Self::Desktop => "desktop",
183+
Self::Android => "firefox_android",
184+
Self::Ios => "firefox_ios",
185+
Self::Desktop => "firefox_desktop",
190186
}
191187
}
192188

‎tools/generate-rust-dashboards/src/metrics/counter.rs‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,25 +34,23 @@ fn count_panel(application: Application, channel: ReleaseChannel, metric: &Count
3434
..
3535
} = *metric;
3636

37-
let mut query = Query {
38-
select: vec!["$__timeGroup(submission_timestamp, $__interval) as time".into()],
39-
from: format!("`mozdata.{}.{ping}`", application.bigquery_dataset()),
37+
let query = Query {
38+
select: vec!["TIMESTAMP(submission_date) as time".into(), "count".into()],
39+
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
4040
where_: vec![
41-
"$__timeFilter(submission_timestamp)".into(),
42-
format!("{ping}.counter.{category}_{metric} IS NOT NULL"),
43-
format!("normalized_channel = '{channel}'"),
41+
"$__timeFilter(TIMESTAMP(submission_date))".into(),
42+
format!("application = `{}`", application.slug()),
43+
format!("channel = '{channel}'"),
4444
],
45-
group_by: Some("1".into()),
46-
order_by: Some("time asc".into()),
45+
order_by: Some("submission_date asc".into()),
4746
..Query::default()
4847
};
49-
query.add_count_per_day_column(format!("SUM({ping}.counter.{category}_{metric})"), metric);
5048

5149
TimeSeriesPanel {
5250
title: application.display_name(channel),
5351
grid_pos: GridPos::height(8),
5452
datasource: Datasource::bigquery(),
55-
interval: "1h".into(),
53+
interval: "1d".into(),
5654
targets: vec![Target::table(query.sql())],
5755
field_config: FieldConfig {
5856
defaults: FieldConfigDefaults {

‎tools/generate-rust-dashboards/src/metrics/distribution.rs‎

Lines changed: 23 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
44

55
use crate::{
6-
config::{Application, DistributionMetric, DistributionMetricKind, TeamConfig},
6+
config::{Application, DistributionMetric, TeamConfig},
77
schema::{
88
DashboardBuilder, DataLink, Datasource, FieldConfig, FieldConfigCustom,
99
FieldConfigDefaults, GridPos, Panel, Target, TimeSeriesPanel, Transformation,
@@ -20,84 +20,47 @@ pub fn add_to_dashboard(
2020
) -> Result<()> {
2121
builder.add_panel_title(metric.display_name);
2222
for app in metric.applications.iter().cloned() {
23-
builder.add_panel_third(count_panel(app, metric, 5));
24-
builder.add_panel_third(count_panel(app, metric, 50));
25-
builder.add_panel_third(count_panel(app, metric, 95));
23+
builder.add_panel_third(count_panel(app, metric, "q05", "5th percentile"));
24+
builder.add_panel_third(count_panel(app, metric, "q50", "50th percentile"));
25+
builder.add_panel_third(count_panel(app, metric, "q95", "95th percentile"));
2626
}
2727
Ok(())
2828
}
2929

30-
fn count_panel(application: Application, metric: &DistributionMetric, percentile: u32) -> Panel {
30+
fn count_panel(
31+
application: Application,
32+
metric: &DistributionMetric,
33+
quantile: &str,
34+
quantile_label: &str,
35+
) -> Panel {
3136
let DistributionMetric {
32-
kind,
3337
ping,
3438
category,
3539
metric,
3640
value_divisor,
37-
value_filter,
3841
axis_label,
3942
link_to,
4043
..
4144
} = *metric;
42-
let dataset = application.bigquery_dataset();
43-
let metric_table = match kind {
44-
DistributionMetricKind::Memory => "memory_distribution",
45-
DistributionMetricKind::Timing => "timing_distribution",
46-
DistributionMetricKind::Custom => "custom_distribution",
47-
};
4845

49-
// Group metrics and calculate quantiles.
50-
// q is a 20-quantile array (0%, 5%, ..., 95%, 100%)
51-
let mut subquery = Query {
46+
let query = Query {
5247
select: vec![
53-
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
54-
"normalized_channel".into(),
55-
"APPROX_QUANTILES(CAST(values.key AS INT64), 20) as q".into(),
56-
],
57-
from: format!("`mozdata.{dataset}.{ping}`"),
58-
joins: vec![
59-
format!("CROSS JOIN UNNEST(metrics.{metric_table}.{category}_{metric}.values) as values"),
60-
// Cross join with an array with length=values.value to make the APPROX_QUANTILES statement above work.
61-
// Histogram metrics are stored in bigquery as a struct of key/value pairs.
62-
// The key is the measurement value, while the value is the count.
63-
// APPROX_QUANTILES expects to count single values,
64-
// so use this CROSS JOIN to repeat each key `value` times.
65-
"CROSS JOIN UNNEST(GENERATE_ARRAY(1, values.value)) AS repeat_number".into(),
48+
"TIMESTAMP(submission_date) as time".into(),
49+
"channel".into(),
50+
match value_divisor {
51+
None => format!("{quantile} as amount"),
52+
Some(amount) => format!("{quantile} / {amount} as amount"),
53+
},
6654
],
55+
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
56+
order_by: Some("submission_date asc, channel asc".into()),
6757
where_: vec![
68-
"$__timeFilter(submission_timestamp)".into(),
69-
"(normalized_channel = 'nightly' OR normalized_channel = 'beta' OR normalized_channel = 'release')".into(),
58+
"$__timeFilter(TIMESTAMP(submission_date))".into(),
59+
"channel IS NOT NULL".into(),
7060
],
71-
group_by: Some("1, 2".into()),
72-
..Query::default()
73-
};
74-
75-
if let Some(amount) = value_filter {
76-
subquery
77-
.where_
78-
.push(format!("CAST(values.key AS INT64) >= {amount}"));
79-
}
80-
let mut query = Query {
81-
select: vec!["time".into(), "normalized_channel".into()],
82-
from: subquery.as_subquery(),
83-
order_by: Some("time desc, normalized_channel asc".into()),
8461
..Query::default()
8562
};
8663

87-
let quantile_index = percentile / 5;
88-
match value_divisor {
89-
None => {
90-
query
91-
.select
92-
.extend([format!("q[OFFSET({quantile_index})] as amount")]);
93-
}
94-
Some(amount) => {
95-
query
96-
.select
97-
.extend([format!("q[OFFSET({quantile_index})] / {amount} as amount")]);
98-
}
99-
}
100-
10164
let mut links = vec![];
10265
if let Some(link_to) = link_to {
10366
links.push(DataLink {
@@ -111,7 +74,7 @@ fn count_panel(application: Application, metric: &DistributionMetric, percentile
11174
}
11275

11376
TimeSeriesPanel {
114-
title: format!("{application} ({percentile}th percentile)"),
77+
title: format!("{application} ({quantile_label})"),
11578
grid_pos: GridPos::height(8),
11679
datasource: Datasource::bigquery(),
11780
interval: "1d".into(),
@@ -127,7 +90,7 @@ fn count_panel(application: Application, metric: &DistributionMetric, percentile
12790
},
12891
transformations: vec![
12992
Transformation::PartitionByValues {
130-
fields: vec!["normalized_channel".into()],
93+
fields: vec!["channel".into()],
13194
keep_fields: true,
13295
},
13396
// Fixup the field names for better legend labels

‎tools/generate-rust-dashboards/src/metrics/labeled_counter.rs‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -37,32 +37,29 @@ fn count_panel(
3737
metric,
3838
..
3939
} = *metric;
40-
let dataset = application.bigquery_dataset();
4140

42-
let mut query = Query {
41+
let query = Query {
4342
select: vec![
44-
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
45-
"counter.key as label".into(),
43+
"TIMESTAMP(submission_date) as time".into(),
44+
"label".into(),
45+
"count".into(),
4646
],
47-
from: format!("`mozdata.{dataset}.{ping}`"),
48-
joins: vec![format!(
49-
"CROSS JOIN UNNEST(metrics.labeled_counter.{category}_{metric}) as counter"
50-
)],
47+
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
5148
where_: vec![
52-
"$__timeFilter(submission_timestamp)".into(),
53-
format!("normalized_channel = '{channel}'"),
49+
"$__timeFilter(TIMESTAMP(submission_date))".into(),
50+
format!("application = `{}`", application.slug()),
51+
format!("channel = '{channel}'"),
52+
"label IS NOT NULL".into(),
5453
],
55-
group_by: Some("1, 2".into()),
56-
order_by: Some("time asc, label asc".into()),
54+
order_by: Some("submission_date asc".into()),
5755
..Query::default()
5856
};
59-
query.add_count_per_day_column("SUM(counter.value)", metric);
6057

6158
TimeSeriesPanel {
6259
title: application.display_name(channel),
6360
grid_pos: GridPos::height(8),
6461
datasource: Datasource::bigquery(),
65-
interval: "1h".into(),
62+
interval: "1d".into(),
6663
targets: vec![Target::table(query.sql())],
6764
field_config: FieldConfig {
6865
defaults: FieldConfigDefaults {

‎tools/generate-rust-dashboards/src/metrics/labeled_distribution.rs‎

Lines changed: 19 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
44

55
use crate::{
6-
config::{Application, DistributionMetricKind, LabeledDistributionMetric, TeamConfig},
6+
config::{Application, LabeledDistributionMetric, TeamConfig},
77
schema::{
88
DashboardBuilder, Datasource, FieldConfig, FieldConfigCustom, FieldConfigDefaults, GridPos,
99
Panel, Target, TimeSeriesPanel, Transformation,
@@ -19,90 +19,48 @@ pub fn add_to_dashboard(
1919
) -> Result<()> {
2020
builder.add_panel_title(metric.display_name);
2121
for app in metric.applications.iter().cloned() {
22-
builder.add_panel_third(count_panel(app, metric, 5));
23-
builder.add_panel_third(count_panel(app, metric, 50));
24-
builder.add_panel_third(count_panel(app, metric, 95));
22+
builder.add_panel_third(count_panel(app, metric, "q05", "5th percentile"));
23+
builder.add_panel_third(count_panel(app, metric, "q50", "50th percentile"));
24+
builder.add_panel_third(count_panel(app, metric, "q95", "95th percentile"));
2525
}
2626
Ok(())
2727
}
2828

2929
fn count_panel(
3030
application: Application,
3131
metric: &LabeledDistributionMetric,
32-
percentile: u32,
32+
quantile: &str,
33+
quantile_label: &str,
3334
) -> Panel {
3435
let LabeledDistributionMetric {
35-
kind,
3636
ping,
3737
category,
3838
metric,
3939
value_divisor,
40-
value_filter,
4140
axis_label,
4241
..
4342
} = *metric;
44-
let dataset = application.bigquery_dataset();
45-
let metric_table = match kind {
46-
DistributionMetricKind::Memory => "labeled_memory_distribution",
47-
DistributionMetricKind::Timing => "labeled_timing_distribution",
48-
DistributionMetricKind::Custom => "labeled_custom_distribution",
49-
};
50-
51-
// Group metrics and calculate quantiles.
52-
// q is a 20-quantile array (0%, 5%, ..., 95%, 100%)
53-
let mut subquery = Query {
43+
let query = Query {
5444
select: vec![
55-
"$__timeGroup(submission_timestamp, $__interval) as time".into(),
56-
"CONCAT(metric.key, ' ', normalized_channel) as group_name".into(),
57-
"APPROX_QUANTILES(CAST(values.key AS INT64), 20) as q".into(),
58-
],
59-
from: format!("`mozdata.{dataset}.{ping}`"),
60-
joins: vec![
61-
format!("CROSS JOIN UNNEST(metrics.{metric_table}.{category}_{metric}) as metric"),
62-
"CROSS JOIN UNNEST(metric.value.values) as values".into(),
63-
// Cross join with an array with length=values.value to make the APPROX_QUANTILES statement above work.
64-
// Histogram metrics are stored in bigquery as a struct of key/value pairs.
65-
// The key is the measurement value, while the value is the count.
66-
// APPROX_QUANTILES expects to count single values,
67-
// so use this CROSS JOIN to repeat each key `value` times.
68-
"CROSS JOIN UNNEST(GENERATE_ARRAY(1, values.value)) AS repeat_number".into(),
45+
"TIMESTAMP(submission_date) as time".into(),
46+
"CONCAT(label, ' ', channel) as group_name".into(),
47+
match value_divisor {
48+
None => format!("{quantile} as amount"),
49+
Some(amount) => format!("{quantile} / {amount} as amount"),
50+
},
6951
],
52+
from: format!("`mozdata.rust_components.{ping}_{category}_{metric}`"),
53+
order_by: Some("submission_date asc, channel asc".into()),
7054
where_: vec![
71-
"$__timeFilter(submission_timestamp)".into(),
72-
"(normalized_channel = 'nightly' OR normalized_channel = 'beta' OR normalized_channel = 'release')".into(),
55+
"$__timeFilter(TIMESTAMP(submission_date))".into(),
56+
"label IS NOT NULL".into(),
57+
"channel IS NOT NULL".into(),
7358
],
74-
group_by: Some("1, 2".into()),
75-
..Query::default()
76-
};
77-
78-
if let Some(amount) = value_filter {
79-
subquery
80-
.where_
81-
.push(format!("CAST(values.key AS INT64) >= {amount}"));
82-
}
83-
let mut query = Query {
84-
select: vec!["time".into(), "group_name".into()],
85-
from: subquery.as_subquery(),
86-
order_by: Some("time desc, group_name asc".into()),
8759
..Query::default()
8860
};
8961

90-
let quantile_index = percentile / 5;
91-
match value_divisor {
92-
None => {
93-
query
94-
.select
95-
.extend([format!("q[OFFSET({quantile_index})] as amount")]);
96-
}
97-
Some(amount) => {
98-
query
99-
.select
100-
.extend([format!("q[OFFSET({quantile_index})] / {amount} as amount")]);
101-
}
102-
}
103-
10462
TimeSeriesPanel {
105-
title: format!("{application} ({percentile}th percentile)"),
63+
title: format!("{application} ({quantile_label})"),
10664
grid_pos: GridPos::height(8),
10765
datasource: Datasource::bigquery(),
10866
interval: "1d".into(),

0 commit comments

Comments
 (0)