Merge pull request #1515 from Kobzol/runtime-db-sqlite

Store runtime benchmark results into SQLite

Author: Kobzol

collector/src/bin/collector.rs

@@ -118,7 +118,7 @@ fn bench(
             }
             let mut tx = rt.block_on(conn.transaction());
             let (supports_stable, category) = category.db_representation();
-            rt.block_on(tx.conn().record_benchmark(
+            rt.block_on(tx.conn().record_compile_benchmark(
                 &benchmark_name.0,
                 Some(supports_stable),
                 category,
@@ -571,6 +571,9 @@ enum Commands {
         /// How many iterations of each benchmark should be executed.
         #[clap(long, default_value = "5")]
         iterations: u32,
+
+        #[clap(flatten)]
+        db: DbOption,
     },
     /// Benchmarks a local rustc
     BenchLocal {
@@ -703,7 +706,11 @@ fn main_result() -> anyhow::Result<i32> {
     let target_triple = format!("{}-unknown-linux-gnu", std::env::consts::ARCH);
 
     match args.command {
-        Commands::BenchRuntimeLocal { local, iterations } => {
+        Commands::BenchRuntimeLocal {
+            local,
+            iterations,
+            db,
+        } => {
             let toolchain = get_local_toolchain(
                 &[Profile::Opt],
                 &local.rustc,
@@ -712,12 +719,17 @@ fn main_result() -> anyhow::Result<i32> {
                 local.id.as_deref(),
                 "",
             )?;
-            bench_runtime(
+            let pool = Pool::open(&db.db);
+
+            let fut = bench_runtime(
+                pool,
+                ArtifactId::Tag(toolchain.id.clone()),
                 toolchain,
                 BenchmarkFilter::new(local.exclude, local.include),
                 runtime_benchmark_dir,
                 iterations,
-            )?;
+            );
+            rt.block_on(fut)?;
             Ok(0)
         }
         Commands::BenchLocal {

collector/src/execute/bencher.rs

@@ -7,7 +7,7 @@ use crate::execute::{
     SelfProfileFiles, Stats, Upload,
 };
 use crate::toolchain::Compiler;
-use anyhow::Context;
+use crate::utils::git::get_rustc_perf_commit;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use std::path::PathBuf;
@@ -81,17 +81,7 @@ impl<'a> BenchProcessor<'a> {
         profile: Profile,
         stats: (Stats, Option<SelfProfile>, Option<SelfProfileFiles>),
     ) {
-        let version = String::from_utf8(
-            Command::new("git")
-                .arg("rev-parse")
-                .arg("HEAD")
-                .output()
-                .context("git rev-parse HEAD")
-                .unwrap()
-                .stdout,
-        )
-        .context("utf8")
-        .unwrap();
+        let version = get_rustc_perf_commit();
 
         let collection = self.rt.block_on(self.conn.collection_id(&version));
         let profile = match profile {

collector/src/execute/rustc.rs

@@ -8,6 +8,7 @@
 //! having to think about how to deduplicate results.
 
 use crate::toolchain::Compiler;
+use crate::utils::git::get_rustc_perf_commit;
 use anyhow::Context;
 use database::ArtifactId;
 use std::env;
@@ -140,17 +141,7 @@ fn record(
         }
     }
 
-    let version = String::from_utf8(
-        Command::new("git")
-            .arg("rev-parse")
-            .arg("HEAD")
-            .output()
-            .context("git rev-parse HEAD")
-            .unwrap()
-            .stdout,
-    )
-    .context("utf8")
-    .unwrap();
+    let version = get_rustc_perf_commit();
     let collection = rt.block_on(conn.collection_id(&version));
 
     for (krate, timing) in timing_data {

collector/src/runtime/benchmark.rs

@@ -45,7 +45,7 @@ impl BenchmarkSuite {
             .count() as u64
     }
 
-    fn benchmark_names(&self) -> impl Iterator<Item = &str> {
+    pub fn benchmark_names(&self) -> impl Iterator<Item = &str> {
         self.groups
             .iter()
             .flat_map(|suite| suite.benchmark_names.iter().map(|n| n.as_ref()))

collector/src/runtime/mod.rs

@@ -1,26 +1,38 @@
 mod benchmark;
 
 use crate::toolchain::LocalToolchain;
-use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use thousands::Separable;
 
+use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 pub use benchmark::BenchmarkFilter;
+use database::{ArtifactId, ArtifactIdNumber, Connection, Pool};
+
+use crate::utils::git::get_rustc_perf_commit;
 
 /// Perform a series of runtime benchmarks using the provided `rustc` compiler.
 /// The runtime benchmarks are looked up in `benchmark_dir`, which is expected to be a path
 /// to a Cargo crate. All binaries built by that crate will are expected to be runtime benchmark
 /// groups that leverage `benchlib`.
-pub fn bench_runtime(
+pub async fn bench_runtime(
+    db: Pool,
+    artifact_id: ArtifactId,
     toolchain: LocalToolchain,
     filter: BenchmarkFilter,
     benchmark_dir: PathBuf,
     iterations: u32,
 ) -> anyhow::Result<()> {
     let suite = benchmark::discover_benchmarks(&toolchain, &benchmark_dir)?;
 
+    let conn = db.connection().await;
+    for benchmark in suite.benchmark_names() {
+        conn.record_runtime_benchmark(benchmark).await;
+    }
+
+    let artifact_id = conn.artifact_id(&artifact_id).await;
+
     let total_benchmark_count = suite.total_benchmark_count();
     let filtered = suite.filtered_benchmark_count(&filter);
     println!(
@@ -29,6 +41,8 @@ pub fn bench_runtime(
         total_benchmark_count - filtered
     );
 
+    let rustc_perf_version = get_rustc_perf_commit();
+
     let mut benchmark_index = 0;
     for binary in suite.groups {
         for message in execute_runtime_benchmark_binary(&binary.binary, &filter, iterations)? {
@@ -48,7 +62,9 @@ pub fn bench_runtime(
                         benchmark_index,
                         filtered
                     );
+
                     print_stats(&result);
+                    record_stats(&conn, artifact_id, &rustc_perf_version, result).await;
                 }
             }
         }
@@ -57,6 +73,29 @@ pub fn bench_runtime(
     Ok(())
 }
 
+/// Records the results (stats) of a benchmark into the database.
+async fn record_stats(
+    conn: &Box<dyn Connection>,
+    artifact_id: ArtifactIdNumber,
+    rustc_perf_version: &str,
+    result: BenchmarkResult,
+) {
+    for stat in result.stats {
+        let collection_id = conn.collection_id(rustc_perf_version).await;
+
+        if let Some(value) = stat.instructions {
+            conn.record_runtime_statistic(
+                collection_id,
+                artifact_id,
+                &result.name,
+                "instructions:u",
+                value as f64,
+            )
+            .await;
+        }
+    }
+}
+
 /// Starts executing a single runtime benchmark group defined in a binary crate located in
 /// `runtime-benchmarks`. The binary is expected to use benchlib's `BenchmarkGroup` to execute
 /// a set of runtime benchmarks and print `BenchmarkMessage`s encoded as JSON, one per line.
@@ -87,8 +126,7 @@ fn execute_runtime_benchmark_binary(
 
     let reader = BufReader::new(stdout);
     let iterator = reader.lines().map(|line| {
-        line.and_then(|line| Ok(serde_json::from_str::<BenchmarkMessage>(&line)?))
-            .map_err(|err| err.into())
+        Ok(line.and_then(|line| Ok(serde_json::from_str::<BenchmarkMessage>(&line)?))?)
     });
     Ok(iterator)
 }

collector/src/utils/git.rs

@@ -0,0 +1,16 @@
+use anyhow::Context;
+use std::process::Command;
+
+pub fn get_rustc_perf_commit() -> String {
+    String::from_utf8(
+        Command::new("git")
+            .arg("rev-parse")
+            .arg("HEAD")
+            .output()
+            .context("git rev-parse HEAD")
+            .unwrap()
+            .stdout,
+    )
+    .context("utf8")
+    .unwrap()
+}

collector/src/utils/mod.rs

@@ -1,2 +1,3 @@
 pub mod fs;
+pub mod git;
 pub mod read2;

database/src/bin/import-sqlite.rs

@@ -1,4 +1,4 @@
-use database::{BenchmarkData, Lookup, Pool};
+use database::{CompileBenchmark, Lookup, Pool};
 use hashbrown::HashMap;
 use std::collections::HashSet;
 
@@ -25,8 +25,8 @@ async fn main() {
     let cid = postgres_conn.collection_id(&cid_name).await;
 
     let mut benchmarks = HashSet::new();
-    let benchmark_data: HashMap<String, BenchmarkData> = sqlite_conn
-        .get_benchmarks()
+    let benchmark_data: HashMap<String, CompileBenchmark> = sqlite_conn
+        .get_compile_benchmarks()
         .await
         .into_iter()
         .map(|benchmark| (benchmark.name.clone(), benchmark))
@@ -48,7 +48,7 @@ async fn main() {
         for &(benchmark, profile, scenario, metric) in sqlite_idx.all_statistic_descriptions() {
             if benchmarks.insert(benchmark) {
                 postgres_conn
-                    .record_benchmark(
+                    .record_compile_benchmark(
                         benchmark.as_str(),
                         None,
                         benchmark_data[benchmark.as_str()].category.clone(),

database/src/lib.rs

@@ -748,7 +748,7 @@ impl fmt::Display for CollectionId {
 }
 
 #[derive(Debug, Clone, Serialize)]
-pub struct BenchmarkData {
+pub struct CompileBenchmark {
     pub name: String,
     pub category: String,
 }

database/src/pool.rs

@@ -1,4 +1,4 @@
-use crate::{ArtifactId, ArtifactIdNumber, BenchmarkData};
+use crate::{ArtifactId, ArtifactIdNumber, CompileBenchmark};
 use crate::{CollectionId, Index, Profile, QueuedCommit, Scenario, Step};
 use chrono::{DateTime, Utc};
 use hashbrown::HashMap;
@@ -15,7 +15,18 @@ pub trait Connection: Send + Sync {
     async fn transaction(&mut self) -> Box<dyn Transaction + '_>;
 
     async fn load_index(&mut self) -> Index;
-    async fn get_benchmarks(&self) -> Vec<BenchmarkData>;
+
+    /// None means that the caller doesn't know; it should be left alone if
+    /// known or set to false if unknown.
+    async fn record_compile_benchmark(
+        &self,
+        krate: &str,
+        supports_stable: Option<bool>,
+        category: String,
+    );
+    async fn get_compile_benchmarks(&self) -> Vec<CompileBenchmark>;
+
+    async fn record_runtime_benchmark(&self, name: &str);
 
     async fn artifact_by_name(&self, artifact: &str) -> Option<ArtifactId>;
 
@@ -25,9 +36,7 @@ pub trait Connection: Send + Sync {
 
     async fn collection_id(&self, version: &str) -> CollectionId;
     async fn artifact_id(&self, artifact: &ArtifactId) -> ArtifactIdNumber;
-    /// None means that the caller doesn't know; it should be left alone if
-    /// known or set to false if unknown.
-    async fn record_benchmark(&self, krate: &str, supports_stable: Option<bool>, category: String);
+
     async fn record_statistic(
         &self,
         collection: CollectionId,
@@ -38,6 +47,14 @@ pub trait Connection: Send + Sync {
         metric: &str,
         value: f64,
     );
+    async fn record_runtime_statistic(
+        &self,
+        collection: CollectionId,
+        artifact: ArtifactIdNumber,
+        benchmark: &str,
+        metric: &str,
+        value: f64,
+    );
     /// Records a self-profile artifact in S3.
     ///
     /// The upload is a separate step (which may fail or be canceled, but that's