tests: Add an automatic perf profiler (#38543)

Add an auto-profiler for our tests, to hopefully allow better triage of
performance impacts resulting from code changes. Comprehensive usage
docs are in the code.

Currently, it uses hyperfine under the hood and prints markdown to the
command line for all crates with relevant tests enabled. We may want to
expand this to allow outputting json in the future to allow e.g.
automatically comparing the difference between two runs on different
commits, and in general a lot of functionality could be added (maybe
measuring memory usage?).

It's enabled (mostly as an example) on two tests inside `gpui` and a
bunch of those inside `vim`. I'd have happily used `cargo bench`, but that's nightly-only.

Release Notes:

- N/A

Author: nia-e

.cargo/config.toml

@@ -4,6 +4,7 @@ rustflags = ["-C", "symbol-mangling-version=v0", "--cfg", "tokio_unstable"]
 
 [alias]
 xtask = "run --package xtask --"
+perf-test = ["test", "--profile", "release-fast", "--lib", "--bins", "--tests", "--config", "target.'cfg(true)'.runner='target/release/perf'", "--config", "target.'cfg(true)'.rustflags=[\"--cfg\", \"perf_enabled\"]"]
 
 [target.x86_64-unknown-linux-gnu]
 linker = "clang"

Cargo.lock

@@ -220,7 +220,7 @@ members = [
     #
 
     "tooling/workspace-hack",
-    "tooling/xtask",
+    "tooling/xtask", "tooling/perf",
 ]
 default-members = ["crates/zed"]
 

Cargo.toml

@@ -110,6 +110,7 @@ resvg = { version = "0.45.0", default-features = false, features = [
     "memmap-fonts",
 ] }
 usvg = { version = "0.45.0", default-features = false }
+util_macros.workspace = true
 schemars.workspace = true
 seahash = "4.1"
 semantic_version.workspace = true

crates/gpui/Cargo.toml

@@ -1300,7 +1300,9 @@ mod tests {
 
     use super::*;
 
-    #[test]
+    use util_macros::perf;
+
+    #[perf]
     fn test_basic_highlight_style_combination() {
         let style_a = HighlightStyle::default();
         let style_b = HighlightStyle::default();
@@ -1385,7 +1387,7 @@ mod tests {
         );
     }
 
-    #[test]
+    #[perf]
     fn test_combine_highlights() {
         assert_eq!(
             combine_highlights(

crates/gpui/src/style.rs

@@ -17,3 +17,6 @@ doctest = false
 quote.workspace = true
 syn.workspace = true
 workspace-hack.workspace = true
+
+[features]
+perf-enabled = []

crates/util_macros/Cargo.toml

@@ -1,8 +1,9 @@
 #![cfg_attr(not(target_os = "windows"), allow(unused))]
+#![allow(clippy::test_attr_in_doctest)]
 
 use proc_macro::TokenStream;
-use quote::quote;
-use syn::{LitStr, parse_macro_input};
+use quote::{ToTokens, quote};
+use syn::{ItemFn, LitStr, parse_macro_input, parse_quote};
 
 /// A macro used in tests for cross-platform path string literals in tests. On Windows it replaces
 /// `/` with `\\` and adds `C:` to the beginning of absolute paths. On other platforms, the path is
@@ -87,3 +88,145 @@ pub fn line_endings(input: TokenStream) -> TokenStream {
         #text
     })
 }
+
+/// Inner data for the perf macro.
+struct PerfArgs {
+    /// How many times to loop a test before rerunning the test binary.
+    /// If left empty, the test harness will auto-determine this value.
+    iterations: Option<syn::Expr>,
+}
+
+impl syn::parse::Parse for PerfArgs {
+    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+        if input.is_empty() {
+            return Ok(PerfArgs { iterations: None });
+        }
+
+        let mut iterations = None;
+        // In principle we only have one possible argument, but leave this as
+        // a loop in case we expand this in the future.
+        for meta in
+            syn::punctuated::Punctuated::<syn::Meta, syn::Token![,]>::parse_terminated(input)?
+        {
+            match &meta {
+                syn::Meta::NameValue(meta_name_value) => {
+                    if meta_name_value.path.is_ident("iterations") {
+                        iterations = Some(meta_name_value.value.clone());
+                    } else {
+                        return Err(syn::Error::new_spanned(
+                            &meta_name_value.path,
+                            "unexpected argument, expected 'iterations'",
+                        ));
+                    }
+                }
+                _ => {
+                    return Err(syn::Error::new_spanned(
+                        meta,
+                        "expected name-value argument like 'iterations = 1'",
+                    ));
+                }
+            }
+        }
+
+        Ok(PerfArgs { iterations })
+    }
+}
+
+/// Marks a test as perf-sensitive, to be triaged when checking the performance
+/// of a build. This also automatically applies `#[test]`.
+///
+/// By default, the number of iterations when profiling this test is auto-determined.
+/// If this needs to be overwritten, pass the desired iteration count to the macro
+/// as a parameter (`#[perf(iterations = n)]`). Note that the actual profiler may still
+/// run the test an arbitrary number times; this flag just sets the number of executions
+/// before the process is restarted and global state is reset.
+///
+/// # Usage notes
+/// This should probably not be applied to tests that do any significant fs IO, as
+/// locks on files may not be released in time when repeating a test many times. This
+/// might lead to spurious failures.
+///
+/// # Examples
+/// ```rust
+/// use util_macros::perf;
+///
+/// #[perf]
+/// fn expensive_computation_test() {
+///     // Test goes here.
+/// }
+/// ```
+///
+/// This also works with `#[gpui::test]`s, though in most cases it shouldn't
+/// be used with automatic iterations.
+/// ```rust,ignore
+/// use util_macros::perf;
+///
+/// #[perf(iterations = 1)]
+/// #[gpui::test]
+/// fn oneshot_test(_cx: &mut gpui::TestAppContext) {
+///     // Test goes here.
+/// }
+/// ```
+#[proc_macro_attribute]
+pub fn perf(our_attr: TokenStream, input: TokenStream) -> TokenStream {
+    // If any of the below constants are changed, make sure to also update the perf
+    // profiler to match!
+
+    /// The suffix on tests marked with `#[perf]`.
+    const SUF_NORMAL: &str = "__ZED_PERF";
+    /// The suffix on tests marked with `#[perf(iterations = n)]`.
+    const SUF_FIXED: &str = "__ZED_PERF_FIXEDITER";
+    /// The env var in which we pass the iteration count to our tests.
+    const ITER_ENV_VAR: &str = "ZED_PERF_ITER";
+
+    let iter_count = parse_macro_input!(our_attr as PerfArgs).iterations;
+
+    let ItemFn {
+        mut attrs,
+        vis,
+        mut sig,
+        block,
+    } = parse_macro_input!(input as ItemFn);
+    attrs.push(parse_quote!(#[test]));
+    attrs.push(parse_quote!(#[allow(non_snake_case)]));
+
+    let block: Box<syn::Block> = if cfg!(perf_enabled) {
+        // Make the ident obvious when calling, for the test parser.
+        let mut new_ident = sig.ident.to_string();
+        if iter_count.is_some() {
+            new_ident.push_str(SUF_FIXED);
+        } else {
+            new_ident.push_str(SUF_NORMAL);
+        }
+
+        let new_ident = syn::Ident::new(&new_ident, sig.ident.span());
+        sig.ident = new_ident;
+        // If we have a preset iteration count, just use that.
+        if let Some(iter_count) = iter_count {
+            parse_quote!({
+               for _ in 0..#iter_count {
+                   #block
+               }
+            })
+        } else {
+            // Otherwise, the perf harness will pass us the value in an env var.
+            parse_quote!({
+                let iter_count = std::env::var(#ITER_ENV_VAR).unwrap().parse::<usize>().unwrap();
+                for _ in 0..iter_count {
+                    #block
+                }
+            })
+        }
+    } else {
+        block
+    };
+
+    ItemFn {
+        attrs,
+        vis,
+        sig,
+        block,
+    }
+    .into_token_stream()
+    .into()
+}

crates/util_macros/src/util_macros.rs

@@ -46,6 +46,7 @@ theme.workspace = true
 tokio = { version = "1.15", features = ["full"], optional = true }
 ui.workspace = true
 util.workspace = true
+util_macros.workspace = true
 vim_mode_setting.workspace = true
 workspace.workspace = true
 zed_actions.workspace = true