Store runtime benchmark results into the DB

Author: Kobzol

collector/src/bin/collector.rs

@@ -720,8 +720,10 @@ fn main_result() -> anyhow::Result<i32> {
                 "",
             )?;
             let pool = Pool::open(&db.db);
+
             let fut = bench_runtime(
                 pool,
+                ArtifactId::Tag(toolchain.id.clone()),
                 toolchain,
                 BenchmarkFilter::new(local.exclude, local.include),
                 runtime_benchmark_dir,

collector/src/execute/bencher.rs

@@ -7,7 +7,7 @@ use crate::execute::{
     SelfProfileFiles, Stats, Upload,
 };
 use crate::toolchain::Compiler;
-use anyhow::Context;
+use crate::utils::git::get_rustc_perf_commit;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use std::path::PathBuf;
@@ -81,17 +81,7 @@ impl<'a> BenchProcessor<'a> {
         profile: Profile,
         stats: (Stats, Option<SelfProfile>, Option<SelfProfileFiles>),
     ) {
-        let version = String::from_utf8(
-            Command::new("git")
-                .arg("rev-parse")
-                .arg("HEAD")
-                .output()
-                .context("git rev-parse HEAD")
-                .unwrap()
-                .stdout,
-        )
-        .context("utf8")
-        .unwrap();
+        let version = get_rustc_perf_commit();
 
         let collection = self.rt.block_on(self.conn.collection_id(&version));
         let profile = match profile {

collector/src/execute/rustc.rs

@@ -8,6 +8,7 @@
 //! having to think about how to deduplicate results.
 
 use crate::toolchain::Compiler;
+use crate::utils::git::get_rustc_perf_commit;
 use anyhow::Context;
 use database::ArtifactId;
 use std::env;
@@ -140,17 +141,7 @@ fn record(
         }
     }
 
-    let version = String::from_utf8(
-        Command::new("git")
-            .arg("rev-parse")
-            .arg("HEAD")
-            .output()
-            .context("git rev-parse HEAD")
-            .unwrap()
-            .stdout,
-    )
-    .context("utf8")
-    .unwrap();
+    let version = get_rustc_perf_commit();
     let collection = rt.block_on(conn.collection_id(&version));
 
     for (krate, timing) in timing_data {

collector/src/runtime/mod.rs

@@ -1,33 +1,38 @@
 mod benchmark;
 
 use crate::toolchain::LocalToolchain;
-use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use thousands::Separable;
 
+use benchlib::comm::messages::{BenchmarkMessage, BenchmarkResult, BenchmarkStats};
 pub use benchmark::BenchmarkFilter;
-use database::Pool;
+use database::{ArtifactId, ArtifactIdNumber, Connection, Pool};
+
+use crate::utils::git::get_rustc_perf_commit;
 
 /// Perform a series of runtime benchmarks using the provided `rustc` compiler.
 /// The runtime benchmarks are looked up in `benchmark_dir`, which is expected to be a path
 /// to a Cargo crate. All binaries built by that crate will are expected to be runtime benchmark
 /// groups that leverage `benchlib`.
 pub async fn bench_runtime(
     db: Pool,
+    artifact_id: ArtifactId,
     toolchain: LocalToolchain,
     filter: BenchmarkFilter,
     benchmark_dir: PathBuf,
     iterations: u32,
 ) -> anyhow::Result<()> {
     let suite = benchmark::discover_benchmarks(&toolchain, &benchmark_dir)?;
 
-    let connection = db.connection().await;
+    let conn = db.connection().await;
     for benchmark in suite.benchmark_names() {
-        connection.record_runtime_benchmark(benchmark).await;
+        conn.record_runtime_benchmark(benchmark).await;
     }
 
+    let artifact_id = conn.artifact_id(&artifact_id).await;
+
     let total_benchmark_count = suite.total_benchmark_count();
     let filtered = suite.filtered_benchmark_count(&filter);
     println!(
@@ -36,6 +41,8 @@ pub async fn bench_runtime(
         total_benchmark_count - filtered
     );
 
+    let rustc_perf_version = get_rustc_perf_commit();
+
     let mut benchmark_index = 0;
     for binary in suite.groups {
         for message in execute_runtime_benchmark_binary(&binary.binary, &filter, iterations)? {
@@ -55,7 +62,9 @@ pub async fn bench_runtime(
                         benchmark_index,
                         filtered
                     );
+
                     print_stats(&result);
+                    record_stats(&conn, artifact_id, &rustc_perf_version, result).await;
                 }
             }
         }
@@ -64,6 +73,29 @@ pub async fn bench_runtime(
     Ok(())
 }
 
+/// Records the results (stats) of a benchmark into the database.
+async fn record_stats(
+    conn: &Box<dyn Connection>,
+    artifact_id: ArtifactIdNumber,
+    rustc_perf_version: &str,
+    result: BenchmarkResult,
+) {
+    for stat in result.stats {
+        let collection_id = conn.collection_id(rustc_perf_version).await;
+
+        if let Some(value) = stat.instructions {
+            conn.record_runtime_statistic(
+                collection_id,
+                artifact_id,
+                &result.name,
+                "instructions:u",
+                value as f64,
+            )
+            .await;
+        }
+    }
+}
+
 /// Starts executing a single runtime benchmark group defined in a binary crate located in
 /// `runtime-benchmarks`. The binary is expected to use benchlib's `BenchmarkGroup` to execute
 /// a set of runtime benchmarks and print `BenchmarkMessage`s encoded as JSON, one per line.
@@ -94,8 +126,7 @@ fn execute_runtime_benchmark_binary(
 
     let reader = BufReader::new(stdout);
     let iterator = reader.lines().map(|line| {
-        line.and_then(|line| Ok(serde_json::from_str::<BenchmarkMessage>(&line)?))
-            .map_err(|err| err.into())
+        Ok(line.and_then(|line| Ok(serde_json::from_str::<BenchmarkMessage>(&line)?))?)
     });
     Ok(iterator)
 }

collector/src/utils/git.rs

@@ -0,0 +1,16 @@
+use anyhow::Context;
+use std::process::Command;
+
+pub fn get_rustc_perf_commit() -> String {
+    String::from_utf8(
+        Command::new("git")
+            .arg("rev-parse")
+            .arg("HEAD")
+            .output()
+            .context("git rev-parse HEAD")
+            .unwrap()
+            .stdout,
+    )
+    .context("utf8")
+    .unwrap()
+}

collector/src/utils/mod.rs

@@ -1,2 +1,3 @@
 pub mod fs;
+pub mod git;
 pub mod read2;

database/src/pool.rs

@@ -47,6 +47,14 @@ pub trait Connection: Send + Sync {
         metric: &str,
         value: f64,
     );
+    async fn record_runtime_statistic(
+        &self,
+        collection: CollectionId,
+        artifact: ArtifactIdNumber,
+        benchmark: &str,
+        metric: &str,
+        value: f64,
+    );
     /// Records a self-profile artifact in S3.
     ///
     /// The upload is a separate step (which may fail or be canceled, but that's

database/src/pool/postgres.rs

@@ -739,6 +739,16 @@ where
             .await
             .unwrap();
     }
+    async fn record_runtime_statistic(
+        &self,
+        _collection: CollectionId,
+        _artifact: ArtifactIdNumber,
+        _benchmark: &str,
+        _metric: &str,
+        _value: f64,
+    ) {
+        unimplemented!()
+    }
 
     async fn record_rustc_crate(
         &self,

database/src/pool/sqlite.rs

@@ -734,6 +734,35 @@ impl Connection for SqliteConnection {
             )
             .unwrap();
     }
+    async fn record_runtime_statistic(
+        &self,
+        collection: CollectionId,
+        artifact: ArtifactIdNumber,
+        benchmark: &str,
+        metric: &str,
+        value: f64,
+    ) {
+        self.raw_ref()
+            .execute(
+                "insert or ignore into runtime_pstat_series (benchmark, metric) VALUES (?, ?)",
+                params![&benchmark, &metric,],
+            )
+            .unwrap();
+        let sid: i32 = self
+            .raw_ref()
+            .query_row(
+                "select id from runtime_pstat_series where benchmark = ? and metric = ?",
+                params![&benchmark, &metric,],
+                |r| r.get(0),
+            )
+            .unwrap();
+        self.raw_ref()
+            .execute(
+                "insert into runtime_pstat (series, aid, cid, value) VALUES (?, ?, ?, ?)",
+                params![&sid, &artifact.0, &collection.0, &value],
+            )
+            .unwrap();
+    }
 
     async fn record_rustc_crate(
         &self,