internal: switch from CLI to internal benchmarking

We have a CLI for benchmarking, but no one actually uses it it seems.
Let's try switching to "internal" benchmarks, implemented as rust tests.
They should be easier to "script" to automate tracking of perf
regressions.

Author: matklad

crates/rust-analyzer/src/benchmarks.rs

@@ -0,0 +1,69 @@
+//! Fully integrated benchmarks for rust-analyzer, which load real cargo
+//! projects.
+//!
+//! The benchmark here is used to debug specific performance regressions. If you
+//! notice that, eg, completion is slow in some specific case, you can  modify
+//! code here exercise this specific completion, and thus have a fast
+//! edit/compile/test cycle.
+//!
+//! Note that "Rust Analyzer: Run" action does not allow running a single test
+//! in release mode in VS Code. There's however "Rust Analyzer: Copy Run Command Line"
+//! which you can use to paste the command in terminal and add `--release` manually.
+
+use std::sync::Arc;
+
+use ide::Change;
+use test_utils::project_root;
+use vfs::{AbsPathBuf, VfsPath};
+
+use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig};
+
+#[test]
+fn benchmark_integrated_highlighting() {
+    // Don't run slow benchmark by default
+    if true {
+        return;
+    }
+
+    // Load rust-analyzer itself.
+    let workspace_to_load = project_root();
+    let file = "./crates/ide_db/src/apply_change.rs";
+
+    let cargo_config = Default::default();
+    let load_cargo_config =
+        LoadCargoConfig { load_out_dirs_from_check: true, with_proc_macro: false };
+
+    let (mut host, vfs, _proc_macro) = {
+        let _it = stdx::timeit("workspace loading");
+        load_workspace_at(&workspace_to_load, &cargo_config, &load_cargo_config, &|_| {}).unwrap()
+    };
+
+    let file_id = {
+        let file = workspace_to_load.join(file);
+        let path = VfsPath::from(AbsPathBuf::assert(file));
+        vfs.file_id(&path).unwrap_or_else(|| panic!("can't find virtual file for {}", path))
+    };
+
+    {
+        let _it = stdx::timeit("initial");
+        let analysis = host.analysis();
+        analysis.highlight_as_html(file_id, false).unwrap();
+    }
+
+    profile::init_from("*>100");
+
+    {
+        let _it = stdx::timeit("change");
+        let mut text = host.analysis().file_text(file_id).unwrap().to_string();
+        text.push_str("\npub fn _dummy() {}\n");
+        let mut change = Change::new();
+        change.change_file(file_id, Some(Arc::new(text)));
+        host.apply_change(change);
+    }
+
+    {
+        let _it = stdx::timeit("after change");
+        let analysis = host.analysis();
+        analysis.highlight_as_html(file_id, false).unwrap();
+    }
+}

crates/rust-analyzer/src/bin/flags.rs

@@ -1,10 +1,9 @@
 //! Grammar for the command-line arguments.
 #![allow(unreachable_pub)]
-use std::{env, path::PathBuf};
+use std::path::PathBuf;
 
 use ide_ssr::{SsrPattern, SsrRule};
-use rust_analyzer::cli::{BenchWhat, Position, Verbosity};
-use vfs::AbsPathBuf;
+use rust_analyzer::cli::Verbosity;
 
 xflags::xflags! {
     src "./src/bin/flags.rs"
@@ -74,27 +73,6 @@ xflags::xflags! {
             optional --with-proc-macro
         }
 
-        /// Benchmark specific analysis operation
-        cmd analysis-bench
-            /// Directory with Cargo.toml.
-            required path: PathBuf
-        {
-            /// Collect memory usage statistics.
-            optional --memory-usage
-
-            /// Compute syntax highlighting for this file
-            optional --highlight path: PathBuf
-            /// Compute completions at file:line:column location.
-            optional --complete location: Position
-            /// Compute goto definition at file:line:column location.
-            optional --goto-def location: Position
-
-            /// Load OUT_DIR values by running `cargo check` before analysis.
-            optional --load-output-dirs
-            /// Use proc-macro-srv for proc-macro expanding.
-            optional --with-proc-macro
-        }
-
         cmd diagnostics
             /// Directory with Cargo.toml.
             required path: PathBuf
@@ -142,7 +120,6 @@ pub enum RustAnalyzerCmd {
     Symbols(Symbols),
     Highlight(Highlight),
     AnalysisStats(AnalysisStats),
-    AnalysisBench(AnalysisBench),
     Diagnostics(Diagnostics),
     Ssr(Ssr),
     Search(Search),
@@ -183,18 +160,6 @@ pub struct AnalysisStats {
     pub with_proc_macro: bool,
 }
 
-#[derive(Debug)]
-pub struct AnalysisBench {
-    pub path: PathBuf,
-
-    pub memory_usage: bool,
-    pub highlight: Option<PathBuf>,
-    pub complete: Option<Position>,
-    pub goto_def: Option<Position>,
-    pub load_output_dirs: bool,
-    pub with_proc_macro: bool,
-}
-
 #[derive(Debug)]
 pub struct Diagnostics {
     pub path: PathBuf,
@@ -239,17 +204,3 @@ impl RustAnalyzer {
         }
     }
 }
-
-impl AnalysisBench {
-    pub(crate) fn what(&self) -> BenchWhat {
-        match (&self.highlight, &self.complete, &self.goto_def) {
-            (Some(path), None, None) => {
-                let path = env::current_dir().unwrap().join(path);
-                BenchWhat::Highlight { path: AbsPathBuf::assert(path) }
-            }
-            (None, Some(position), None) => BenchWhat::Complete(position.clone()),
-            (None, None, Some(position)) => BenchWhat::GotoDef(position.clone()),
-            _ => panic!("exactly one of  `--highlight`, `--complete` or `--goto-def` must be set"),
-        }
-    }
-}

crates/rust-analyzer/src/bin/main.rs

@@ -9,7 +9,7 @@ use std::{convert::TryFrom, env, fs, path::Path, process};
 use lsp_server::Connection;
 use project_model::ProjectManifest;
 use rust_analyzer::{
-    cli::{self, AnalysisStatsCmd, BenchCmd},
+    cli::{self, AnalysisStatsCmd},
     config::Config,
     from_json,
     lsp_ext::supports_utf8,
@@ -80,17 +80,6 @@ fn try_main() -> Result<()> {
             with_proc_macro: cmd.with_proc_macro,
         }
         .run(verbosity)?,
-        flags::RustAnalyzerCmd::AnalysisBench(cmd) => {
-            let what = cmd.what();
-            BenchCmd {
-                memory_usage: cmd.memory_usage,
-                path: cmd.path,
-                load_output_dirs: cmd.load_output_dirs,
-                with_proc_macro: cmd.with_proc_macro,
-                what,
-            }
-            .run(verbosity)?
-        }
 
         flags::RustAnalyzerCmd::Diagnostics(cmd) => {
             cli::diagnostics(&cmd.path, cmd.load_output_dirs, cmd.with_proc_macro)?

crates/rust-analyzer/src/cli.rs

@@ -1,8 +1,7 @@
 //! Various batch processing tasks, intended primarily for debugging.
 
-mod load_cargo;
+pub(crate) mod load_cargo;
 mod analysis_stats;
-mod analysis_bench;
 mod diagnostics;
 mod progress_report;
 mod ssr;
@@ -15,7 +14,6 @@ use syntax::{AstNode, SourceFile};
 use vfs::Vfs;
 
 pub use self::{
-    analysis_bench::{BenchCmd, BenchWhat, Position},
     analysis_stats::AnalysisStatsCmd,
     diagnostics::diagnostics,
     load_cargo::{load_workspace, load_workspace_at, LoadCargoConfig},