Add codegen diff for runtime benchmarks

Author: Kobzol

Cargo.lock

@@ -35,6 +35,8 @@ rayon = "1.5.2"
 cargo_metadata = "0.15.0"
 thousands = "0.2.0"
 rustc-demangle = { version = "0.1", features = ["std"] }
+similar = "2.2"
+console = "0.15"
 
 benchlib = { path = "benchlib" }
 

collector/Cargo.toml

@@ -490,6 +490,16 @@ It is also possible to profile runtime benchmarks using the following command:
 Currently, a `<PROFILER>` can be `cachegrind`, which will run the runtime benchmark under
 `Cachegrind`.
 
+## Codegen diff
+You can use the `codegen_diff` command to display the assembly, LLVM IR or MIR difference between two
+versions of `rustc` for individual functions of a single runtime benchmark group:
+
+```
+./target/release/collector codegen_diff <asm|asm-source|llvm|mir> <benchmark-name> <rustc> <rustc2>
+```
+
+Codegen diff is currently only implemented for runtime benchmarks.
+
 ## How `rustc` wrapping works
 When a crate is benchmarked or profiled, the real `rustc` is replaced with the `rustc-fake` binary,
 which parses commands passed from the `collector` and invokes the actual profiling or benchmarking

collector/README.md

@@ -4,6 +4,7 @@ use anyhow::Context;
 use clap::builder::{PossibleValue, TypedValueParser};
 use clap::{Arg, Parser, ValueEnum};
 use collector::api::next_artifact::NextArtifact;
+use collector::codegen::{codegen_diff, CodegenType};
 use collector::compile::benchmark::category::Category;
 use collector::compile::benchmark::profile::Profile;
 use collector::compile::benchmark::scenario::Scenario;
@@ -31,9 +32,9 @@ use tokio::runtime::Runtime;
 use collector::compile::execute::bencher::BenchProcessor;
 use collector::compile::execute::profiler::{ProfileProcessor, Profiler};
 use collector::runtime::{
-    bench_runtime, prepare_runtime_benchmark_suite, runtime_benchmark_dir, BenchmarkFilter,
-    BenchmarkSuite, BenchmarkSuiteCompilation, CargoIsolationMode, RuntimeProfiler,
-    DEFAULT_RUNTIME_ITERATIONS,
+    bench_runtime, get_runtime_benchmark_groups, prepare_runtime_benchmark_suite,
+    runtime_benchmark_dir, BenchmarkFilter, BenchmarkSuite, BenchmarkSuiteCompilation,
+    CargoIsolationMode, RuntimeProfiler, DEFAULT_RUNTIME_ITERATIONS,
 };
 use collector::runtime::{profile_runtime, RuntimeCompilationOpts};
 use collector::toolchain::{
@@ -452,6 +453,21 @@ enum Commands {
         benchmark: String,
     },
 
+    /// Displays the diff between assembly, LLVM or MIR for a runtime benchmark group.
+    CodegenDiff {
+        /// Profiler to use
+        codegen_type: CodegenType,
+
+        /// Runtime benchmark group to diff (name of a directory in `collector/runtime-benchmarks`)
+        group: String,
+
+        /// The path to the local rustc used to compile the runtime benchmark
+        rustc1: String,
+
+        /// The path to a second rustc used to compile with the baseline
+        rustc2: String,
+    },
+
     /// Benchmarks a local rustc
     BenchLocal {
         #[command(flatten)]
@@ -659,7 +675,7 @@ fn main_result() -> anyhow::Result<i32> {
                     // generated profiles.
                     RuntimeCompilationOpts::default().debug_info("1"),
                 )?
-                .suite;
+                .extract_suite();
                 Ok::<_, anyhow::Error>((toolchain, suite))
             };
 
@@ -692,6 +708,37 @@ fn main_result() -> anyhow::Result<i32> {
 
             Ok(0)
         }
+        Commands::CodegenDiff {
+            codegen_type,
+            group,
+            rustc1: rustc,
+            rustc2,
+        } => {
+            let get_toolchain = |rustc: &str, id: &str| {
+                let toolchain = get_local_toolchain(
+                    &[Profile::Opt],
+                    rustc,
+                    None,
+                    None,
+                    None,
+                    id,
+                    target_triple.clone(),
+                )?;
+                Ok::<_, anyhow::Error>(toolchain)
+            };
+
+            let toolchain1 = get_toolchain(&rustc, "1")?;
+            let toolchain2 = get_toolchain(&rustc2, "2")?;
+
+            let mut benchmark_groups =
+                get_runtime_benchmark_groups(&runtime_benchmark_dir, Some(group))?;
+            let group = benchmark_groups.pop().unwrap();
+            assert!(benchmark_groups.is_empty());
+
+            codegen_diff(codegen_type, toolchain1, toolchain2, group)?;
+            Ok(0)
+        }
+
         Commands::BenchLocal {
             local,
             opts,