Add a build script to the site package that pre-loads compile benchmarks metadata

Use this preloaded metadata to find stable benchmarks.

Author: Kobzol

Cargo.lock

@@ -22,7 +22,7 @@ fn default_runs() -> usize {
     3
 }
 
-#[derive(Debug, Default, Clone, serde::Deserialize)]
+#[derive(Debug, Default, PartialEq, Copy, Clone, serde::Deserialize)]
 #[serde(rename_all = "lowercase")]
 pub enum ArtifactType {
     Binary,
@@ -68,6 +68,10 @@ impl BenchmarkConfig {
     pub fn category(&self) -> Category {
         self.category
     }
+
+    pub fn artifact(&self) -> ArtifactType {
+        self.artifact
+    }
 }
 
 #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Hash)]

collector/src/compile/benchmark/mod.rs

@@ -46,7 +46,7 @@ mime = "0.3"
 prometheus = "0.13"
 uuid = { version = "1.3.0", features = ["v4"] }
 tera = "1.18"
-rust-embed = { version = "6.6.0", features = ["include-exclude"] }
+rust-embed = { version = "6.6.0", features = ["include-exclude", "interpolate-folder-path"] }
 
 [target.'cfg(unix)'.dependencies]
 jemallocator = "0.5"
@@ -58,3 +58,8 @@ path = "../collector"
 [dev-dependencies]
 lazy_static = "1"
 pretty_assertions = "1.3"
+
+[build-dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+toml = "0.7"

site/Cargo.toml

@@ -0,0 +1,84 @@
+//! This build script goes through compile-benchmarks located in the `collector` crate
+//! and gathers some metadata from them (contents of perf-config.json, useful data out of
+//! Cargo.toml).
+//!
+//! It then serializes this data into a JSON, which is stored into the OUT_DIR.
+
+use std::collections::HashMap;
+use std::error::Error;
+use std::path::PathBuf;
+
+use crate::benchmark_metadata::{
+    CompileBenchmarkMetadata, CompileBenchmarkSuite, ProfileMetadata, SERIALIZED_SUITE_NAME,
+};
+
+#[path = "src/benchmark_metadata/metadata.rs"]
+mod benchmark_metadata;
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let site_dir = PathBuf::from(&std::env::var("CARGO_MANIFEST_DIR")?);
+    let root_dir = site_dir.parent().unwrap();
+    let compile_benchmarks_dir = root_dir.join("collector/compile-benchmarks");
+    assert!(
+        compile_benchmarks_dir.is_dir(),
+        "Compile benchmarks directory not found"
+    );
+
+    println!(
+        "cargo:rerun-if-changed={}",
+        compile_benchmarks_dir.display()
+    );
+
+    let mut suite = HashMap::new();
+
+    for compile_benchmark in std::fs::read_dir(compile_benchmarks_dir)? {
+        let compile_benchmark = compile_benchmark?;
+        if !compile_benchmark.file_type()?.is_dir() {
+            continue;
+        }
+
+        let benchmark_name = compile_benchmark
+            .path()
+            .file_name()
+            .and_then(|n| n.to_str())
+            .map(|s| s.to_string())
+            .unwrap();
+
+        // Load perf-config.json
+        let config_path = compile_benchmark.path().join("perf-config.json");
+        let config_contents = std::fs::read_to_string(config_path)?;
+        let config = serde_json::from_str::<serde_json::Value>(&config_contents)?;
+
+        // Load Cargo manifest to find profile information
+        let manifest_path = compile_benchmark.path().join("Cargo.toml");
+        let manifest_contents = std::fs::read_to_string(manifest_path)?;
+        let manifest: toml::Value = toml::from_str(&manifest_contents)?;
+
+        let table = manifest.as_table().unwrap();
+        let profile = table.get("profile");
+        let release = profile
+            .and_then(|p| p.as_table())
+            .and_then(|t| t.get("release"))
+            .and_then(|t| t.as_table());
+        let debug = release.and_then(|t| t.get("debug"));
+        let lto = release.and_then(|t| t.get("lto"));
+        let codegen_units = release.and_then(|t| t.get("codegen-units"));
+
+        let metadata = CompileBenchmarkMetadata {
+            perf_config: config,
+            release_metadata: ProfileMetadata {
+                debug: debug.map(|v| v.to_string()),
+                lto: lto.map(|v| v.to_string()),
+                codegen_units: codegen_units.and_then(|v| v.as_integer().map(|v| v as u32)),
+            },
+        };
+        suite.insert(benchmark_name, metadata);
+    }
+
+    // Write the serialized benchmarks metadata to OUT_DIR.
+    let serialized = serde_json::to_string(&CompileBenchmarkSuite { benchmarks: suite })?;
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
+    std::fs::write(out_dir.join(SERIALIZED_SUITE_NAME), serialized)?;
+
+    Ok(())
+}

site/build.rs

@@ -0,0 +1,30 @@
+//! This file is also included from the crate's build script.
+//! It contains "bare-bones" structs designed for easy (de)serialization,
+//! to minimize the dependencies of the build script.
+//!
+//! For example, the perf-config is stored simply as an opaque JSON, not as `BenchmarkConfig`,
+//! so that the build script doesn't have to depend on the `collector` crate.
+
+use std::collections::HashMap;
+
+/// File in `OUT_DIR` into which the JSON serialized metadata of compile benchmarks will be stored
+/// by the build script of this crate.
+pub const SERIALIZED_SUITE_NAME: &str = "compile-benchmarks.json";
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct ProfileMetadata {
+    pub debug: Option<String>,
+    pub lto: Option<String>,
+    pub codegen_units: Option<u32>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct CompileBenchmarkMetadata {
+    pub perf_config: serde_json::Value,
+    pub release_metadata: ProfileMetadata,
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct CompileBenchmarkSuite {
+    pub benchmarks: HashMap<String, CompileBenchmarkMetadata>,
+}

site/src/benchmark_metadata/metadata.rs

@@ -0,0 +1,73 @@
+use hashbrown::HashMap;
+use rust_embed::RustEmbed;
+
+use collector::compile::benchmark::category::Category;
+use collector::compile::benchmark::BenchmarkConfig;
+
+use crate::benchmark_metadata::metadata::{ProfileMetadata, SERIALIZED_SUITE_NAME};
+
+mod metadata;
+
+#[derive(Debug, Clone)]
+pub struct CompileBenchmarkMetadata {
+    pub perf_config: BenchmarkConfig,
+    pub release_metadata: ProfileMetadata,
+}
+
+/// The metadata of compile-time benchmarks is embedded directly within the binary using
+/// the `EmbeddedCompileBenchmarks` struct.
+#[derive(RustEmbed)]
+#[folder = "$OUT_DIR"]
+struct EmbeddedCompileBenchmarks;
+
+pub fn get_compile_benchmarks_metadata() -> &'static HashMap<String, CompileBenchmarkMetadata> {
+    lazy_static::lazy_static! {
+        static ref METADATA: HashMap<String, CompileBenchmarkMetadata> = load_compile_benchmark_metadata();
+    }
+    &METADATA
+}
+
+pub fn get_stable_benchmark_names() -> Vec<String> {
+    get_compile_benchmarks_metadata()
+        .iter()
+        .filter(|(_, metadata)| metadata.perf_config.category() == Category::Stable)
+        .map(|(key, _)| key.clone())
+        .collect()
+}
+
+fn load_compile_benchmark_metadata() -> HashMap<String, CompileBenchmarkMetadata> {
+    let serialized = EmbeddedCompileBenchmarks::get(SERIALIZED_SUITE_NAME)
+        .unwrap()
+        .data
+        .to_vec();
+    let metadata: metadata::CompileBenchmarkSuite = serde_json::from_slice(&serialized)
+        .expect("Cannot deserialize compile benchmarks metadata");
+    metadata
+        .benchmarks
+        .into_iter()
+        .map(|(name, metadata)| {
+            let metadata::CompileBenchmarkMetadata {
+                perf_config,
+                release_metadata,
+            } = metadata;
+            let perf_config: BenchmarkConfig =
+                serde_json::from_value(perf_config).expect("Cannot deserialize perf-config.json");
+
+            let metadata = CompileBenchmarkMetadata {
+                perf_config,
+                release_metadata,
+            };
+            (name, metadata)
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::benchmark_metadata::get_compile_benchmarks_metadata;
+
+    #[test]
+    fn check_embedded_compile_benchmarks_metadata() {
+        assert!(!get_compile_benchmarks_metadata().is_empty());
+    }
+}

site/src/benchmark_metadata/mod.rs

@@ -16,6 +16,7 @@ pub mod load;
 pub mod server;
 
 mod average;
+mod benchmark_metadata;
 mod comparison;
 mod interpolate;
 mod request_handlers;

site/src/lib.rs

@@ -1,11 +1,9 @@
-use collector::compile::benchmark::category::Category;
-use collector::compile::benchmark::BenchmarkConfig;
-use lazy_static::lazy_static;
-use rust_embed::RustEmbed;
-use std::path::Path;
 use std::sync::Arc;
 
+use lazy_static::lazy_static;
+
 use crate::api::{dashboard, ServerResult};
+use crate::benchmark_metadata::get_stable_benchmark_names;
 use crate::comparison::Metric;
 use crate::db::{self, ArtifactId, Profile, Scenario};
 use crate::load::SiteCtxt;
@@ -81,7 +79,7 @@ pub async fn handle_dashboard(ctxt: Arc<SiteCtxt>) -> ServerResult<dashboard::Re
     );
 
     lazy_static! {
-        static ref STABLE_BENCHMARKS: Vec<String> = get_stable_benchmarks();
+        static ref STABLE_BENCHMARKS: Vec<String> = get_stable_benchmark_names();
     }
 
     let query = selector::CompileBenchmarkQuery::default()
@@ -147,44 +145,6 @@ pub async fn handle_dashboard(ctxt: Arc<SiteCtxt>) -> ServerResult<dashboard::Re
     })
 }
 
-#[derive(RustEmbed)]
-#[folder = "../collector/compile-benchmarks"]
-#[include = "*/perf-config.json"]
-struct EmbeddedBenchmarks;
-
-/// The configurations of compile-time benchmarks are embedded directly within the binary using
-/// the `Benchmarks` struct.
-///
-/// Here we parse the benchmarks configurations and return only stable benchmarks.
-fn get_stable_benchmarks() -> Vec<String> {
-    EmbeddedBenchmarks::iter()
-        .filter_map(|path| EmbeddedBenchmarks::get(&path).map(|file| (file, path)))
-        .filter_map(|(file, path)| {
-            let config: BenchmarkConfig = match serde_json::from_slice(&file.data) {
-                Ok(config) => config,
-                Err(error) => {
-                    log::error!(
-                        "Cannot deserialized stored perf-config.json from {path}: {error:?}"
-                    );
-                    return None;
-                }
-            };
-            if config.category() == Category::Stable {
-                Some(
-                    Path::new(path.as_ref())
-                        .parent()
-                        .unwrap()
-                        .to_str()
-                        .unwrap()
-                        .to_string(),
-                )
-            } else {
-                None
-            }
-        })
-        .collect()
-}
-
 pub struct ByProfile<T> {
     pub check: T,
     pub debug: T,