Add rustc-perf to metrics

Author: matklad

crates/ra_prof/src/memory_usage.rs

@@ -31,6 +31,12 @@ impl fmt::Display for MemoryUsage {
 #[derive(Default, PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy)]
 pub struct Bytes(usize);
 
+impl Bytes {
+    pub fn megabytes(self) -> usize {
+        self.0 / 1024 / 1024
+    }
+}
+
 impl fmt::Display for Bytes {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let bytes = self.0;

crates/rust-analyzer/src/cli.rs

@@ -74,3 +74,10 @@ fn read_stdin() -> Result<String> {
     std::io::stdin().read_to_string(&mut buff)?;
     Ok(buff)
 }
+
+fn report_metric(metric: &str, value: u64, unit: &str) {
+    if std::env::var("RA_METRICS").is_err() {
+        return;
+    }
+    println!("METRIC:{}:{}:{}", metric, value, unit)
+}

crates/rust-analyzer/src/cli/analysis_stats.rs

@@ -3,26 +3,27 @@
 
 use std::{path::Path, time::Instant};
 
-use itertools::Itertools;
-use rand::{seq::SliceRandom, thread_rng};
-use rayon::prelude::*;
-use rustc_hash::FxHashSet;
-
 use hir::{
     db::{AstDatabase, DefDatabase, HirDatabase},
     original_range, AssocItem, Crate, HasSource, HirDisplay, ModuleDef,
 };
 use hir_def::FunctionId;
 use hir_ty::{Ty, TypeWalk};
+use itertools::Itertools;
 use ra_db::{
     salsa::{self, ParallelDatabase},
     SourceDatabaseExt,
 };
 use ra_syntax::AstNode;
+use rand::{seq::SliceRandom, thread_rng};
+use rayon::prelude::*;
+use rustc_hash::FxHashSet;
 use stdx::format_to;
 
 use crate::{
-    cli::{load_cargo::load_cargo, progress_report::ProgressReport, Result, Verbosity},
+    cli::{
+        load_cargo::load_cargo, progress_report::ProgressReport, report_metric, Result, Verbosity,
+    },
     print_memory_usage,
 };
 
@@ -48,7 +49,7 @@ pub fn analysis_stats(
     let db_load_time = Instant::now();
     let (host, vfs) = load_cargo(path, load_output_dirs, with_proc_macro)?;
     let db = host.raw_database();
-    println!("Database loaded {:?}", db_load_time.elapsed());
+    eprintln!("Database loaded {:?}", db_load_time.elapsed());
     let analysis_time = Instant::now();
     let mut num_crates = 0;
     let mut visited_modules = FxHashSet::default();
@@ -74,7 +75,7 @@ pub fn analysis_stats(
         visit_queue.shuffle(&mut thread_rng());
     }
 
-    println!("Crates in this dir: {}", num_crates);
+    eprintln!("Crates in this dir: {}", num_crates);
     let mut num_decls = 0;
     let mut funcs = Vec::new();
     while let Some(module) = visit_queue.pop() {
@@ -98,10 +99,15 @@ pub fn analysis_stats(
             }
         }
     }
-    println!("Total modules found: {}", visited_modules.len());
-    println!("Total declarations: {}", num_decls);
-    println!("Total functions: {}", funcs.len());
-    println!("Item Collection: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+    eprintln!("Total modules found: {}", visited_modules.len());
+    eprintln!("Total declarations: {}", num_decls);
+    eprintln!("Total functions: {}", funcs.len());
+    let item_collection_memory = ra_prof::memory_usage();
+    eprintln!(
+        "Item Collection: {:?}, {}",
+        analysis_time.elapsed(),
+        item_collection_memory.allocated
+    );
 
     if randomize {
         funcs.shuffle(&mut thread_rng());
@@ -123,7 +129,11 @@ pub fn analysis_stats(
                 snap.0.infer(f_id.into());
             })
             .count();
-        println!("Parallel Inference: {:?}, {}", inference_time.elapsed(), ra_prof::memory_usage());
+        eprintln!(
+            "Parallel Inference: {:?}, {}",
+            inference_time.elapsed(),
+            ra_prof::memory_usage().allocated
+        );
     }
 
     let inference_time = Instant::now();
@@ -260,20 +270,31 @@ pub fn analysis_stats(
         bar.inc(1);
     }
     bar.finish_and_clear();
-    println!("Total expressions: {}", num_exprs);
-    println!(
+    eprintln!("Total expressions: {}", num_exprs);
+    eprintln!(
         "Expressions of unknown type: {} ({}%)",
         num_exprs_unknown,
         if num_exprs > 0 { num_exprs_unknown * 100 / num_exprs } else { 100 }
     );
-    println!(
+    eprintln!(
         "Expressions of partially unknown type: {} ({}%)",
         num_exprs_partially_unknown,
         if num_exprs > 0 { num_exprs_partially_unknown * 100 / num_exprs } else { 100 }
     );
-    println!("Type mismatches: {}", num_type_mismatches);
-    println!("Inference: {:?}, {}", inference_time.elapsed(), ra_prof::memory_usage());
-    println!("Total: {:?}, {}", analysis_time.elapsed(), ra_prof::memory_usage());
+    eprintln!("Type mismatches: {}", num_type_mismatches);
+
+    let inference_time = inference_time.elapsed();
+    let total_memory = ra_prof::memory_usage();
+    eprintln!(
+        "Inference: {:?}, {}",
+        inference_time,
+        total_memory.allocated - item_collection_memory.allocated
+    );
+
+    let analysis_time = analysis_time.elapsed();
+    eprintln!("Total: {:?}, {}", analysis_time, total_memory);
+    report_metric("total time", analysis_time.as_millis() as u64, "ms");
+    report_metric("total memory", total_memory.allocated.megabytes() as u64, "MB");
 
     if memory_usage {
         print_memory_usage(host, vfs);

crates/rust-analyzer/src/lib.rs

@@ -86,6 +86,6 @@ fn print_memory_usage(mut host: AnalysisHost, vfs: Vfs) {
     mem.push(("Remaining".into(), ra_prof::memory_usage().allocated));
 
     for (name, bytes) in mem {
-        println!("{:>8} {}", bytes, name);
+        eprintln!("{:>8} {}", bytes, name);
     }
 }

xtask/src/metrics.rs

@@ -3,14 +3,15 @@ use std::{
     env,
     fmt::{self, Write as _},
     io::Write as _,
+    path::Path,
     time::{Instant, SystemTime, UNIX_EPOCH},
 };
 
 use anyhow::{bail, format_err, Result};
 
-use crate::not_bash::{fs2, pushd, rm_rf, run};
+use crate::not_bash::{fs2, pushd, pushenv, rm_rf, run};
 
-type Unit = &'static str;
+type Unit = String;
 
 pub struct MetricsCmd {
     pub dry_run: bool,
@@ -22,9 +23,21 @@ impl MetricsCmd {
         if !self.dry_run {
             rm_rf("./target/release")?;
         }
+        if !Path::new("./target/rustc-perf").exists() {
+            fs2::create_dir_all("./target/rustc-perf")?;
+            run!("git clone https://github.com/rust-lang/rustc-perf.git ./target/rustc-perf")?;
+        }
+        {
+            let _d = pushd("./target/rustc-perf");
+            run!("git reset --hard 1d9288b0da7febf2599917da1b57dc241a1af033")?;
+        }
+
+        let _env = pushenv("RA_METRICS", "1");
 
         metrics.measure_build()?;
         metrics.measure_analysis_stats_self()?;
+        metrics.measure_analysis_stats("ripgrep")?;
+        metrics.measure_analysis_stats("webrender")?;
 
         if !self.dry_run {
             let _d = pushd("target");
@@ -46,23 +59,47 @@ impl MetricsCmd {
 
 impl Metrics {
     fn measure_build(&mut self) -> Result<()> {
+        eprintln!("\nMeasuring build");
         run!("cargo fetch")?;
 
         let time = Instant::now();
         run!("cargo build --release --package rust-analyzer --bin rust-analyzer")?;
         let time = time.elapsed();
-        self.report("build", time.as_millis() as u64, "ms");
+        self.report("build", time.as_millis() as u64, "ms".into());
         Ok(())
     }
     fn measure_analysis_stats_self(&mut self) -> Result<()> {
-        let time = Instant::now();
-        run!("./target/release/rust-analyzer analysis-stats .")?;
-        let time = time.elapsed();
-        self.report("analysis-stats/self", time.as_millis() as u64, "ms");
+        self.measure_analysis_stats_path("self", &".")
+    }
+    fn measure_analysis_stats(&mut self, bench: &str) -> Result<()> {
+        self.measure_analysis_stats_path(
+            bench,
+            &format!("./target/rustc-perf/collector/benchmarks/{}", bench),
+        )
+    }
+    fn measure_analysis_stats_path(&mut self, name: &str, path: &str) -> Result<()> {
+        eprintln!("\nMeasuring analysis-stats/{}", name);
+        let output = run!("./target/release/rust-analyzer analysis-stats --quiet {}", path)?;
+        for (metric, value, unit) in parse_metrics(&output) {
+            self.report(&format!("analysis-stats/{}/{}", name, metric), value, unit.into());
+        }
         Ok(())
     }
 }
 
+fn parse_metrics(output: &str) -> Vec<(&str, u64, &str)> {
+    output
+        .lines()
+        .filter_map(|it| {
+            let entry = it.split(':').collect::<Vec<_>>();
+            match entry.as_slice() {
+                ["METRIC", name, value, unit] => Some((*name, value.parse().unwrap(), *unit)),
+                _ => None,
+            }
+        })
+        .collect()
+}
+
 #[derive(Debug)]
 struct Metrics {
     host: Host,
@@ -111,11 +148,11 @@ impl Metrics {
             json.field("metrics");
             json.begin_object();
             {
-                for (k, &(value, unit)) in &self.metrics {
+                for (k, (value, unit)) in &self.metrics {
                     json.field(k);
                     json.begin_array();
                     {
-                        json.number(value as f64);
+                        json.number(*value as f64);
                         json.string(unit);
                     }
                     json.end_array();

xtask/src/not_bash.rs

@@ -186,7 +186,8 @@ impl Env {
     fn pushd(&mut self, dir: PathBuf) {
         let dir = self.cwd().join(dir);
         self.pushd_stack.push(dir);
-        env::set_current_dir(self.cwd()).unwrap();
+        env::set_current_dir(self.cwd())
+            .unwrap_or_else(|err| panic!("Failed to set cwd to {}: {}", self.cwd().display(), err));
     }
     fn popd(&mut self) {
         self.pushd_stack.pop().unwrap();