Benchmark runner + #[bench] infrastructure

Author: Bromeon

godot-macros/src/bench.rs

@@ -0,0 +1,70 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+use proc_macro2::TokenStream;
+use quote::quote;
+use venial::{Declaration, Error, Function};
+
+use crate::util::{bail, KvParser};
+use crate::ParseResult;
+
+const DEFAULT_REPETITIONS: usize = 100;
+
+pub fn attribute_bench(input_decl: Declaration) -> ParseResult<TokenStream> {
+    let func = match input_decl {
+        Declaration::Function(f) => f,
+        _ => return bail!(&input_decl, "#[bench] can only be applied to functions"),
+    };
+
+    // Note: allow attributes for things like #[rustfmt] or #[clippy]
+    if func.generic_params.is_some() || !func.params.is_empty() || func.where_clause.is_some() {
+        return bad_signature(&func);
+    }
+
+    // Ignore -> (), as no one does that by accident.
+    // We need `ret` to make sure the type is correct and to avoid unused imports (by IDEs).
+    let Some(ret) = func.return_ty else {
+        return bail!(
+            func,
+            "#[bench] function must return a value from its computation, to prevent optimizing the operation away"
+        );
+    };
+
+    let mut attr = KvParser::parse_required(&func.attributes, "bench", &func.name)?;
+    let repetitions = attr.handle_usize("repeat")?.unwrap_or(DEFAULT_REPETITIONS);
+    attr.finish()?;
+
+    let bench_name = &func.name;
+    let bench_name_str = func.name.to_string();
+
+    let body = &func.body;
+
+    Ok(quote! {
+        pub fn #bench_name() {
+            for _ in 0..#repetitions {
+                let __ret: #ret = #body;
+                crate::common::bench_used(__ret);
+            }
+        }
+
+        ::godot::sys::plugin_add!(__GODOT_BENCH in crate::framework; crate::framework::RustBenchmark {
+            name: #bench_name_str,
+            file: std::file!(),
+            line: std::line!(),
+            function: #bench_name,
+            repetitions: #repetitions,
+        });
+    })
+}
+
+fn bad_signature(func: &Function) -> Result<TokenStream, Error> {
+    bail!(
+        func,
+        "#[bench] function must have one of these signatures:\
+        \n  fn {f}() {{ ... }}",
+        f = func.name,
+    )
+}

godot-macros/src/lib.rs

@@ -4,18 +4,20 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
+mod bench;
 mod class;
 mod derive;
 mod gdextension;
 mod itest;
 mod util;
 
-use crate::util::ident;
 use proc_macro::TokenStream;
 use proc_macro2::TokenStream as TokenStream2;
 use quote::quote;
 use venial::Declaration;
 
+use crate::util::ident;
+
 // Below intra-doc link to the trait only works as HTML, not as symbol link.
 /// Derive macro for [the `GodotClass` trait](../obj/trait.GodotClass.html) on structs.
 ///
@@ -528,6 +530,14 @@ pub fn itest(meta: TokenStream, input: TokenStream) -> TokenStream {
     translate_meta("itest", meta, input, itest::attribute_itest)
 }
 
+/// Similar to `#[test]`, but runs an benchmark with Godot.
+///
+/// Calls the `fn` many times and gathers statistics from its execution time.
+#[proc_macro_attribute]
+pub fn bench(meta: TokenStream, input: TokenStream) -> TokenStream {
+    translate_meta("bench", meta, input, bench::attribute_bench)
+}
+
 /// Proc-macro attribute to be used in combination with the [`ExtensionLibrary`] trait.
 ///
 /// [`ExtensionLibrary`]: trait.ExtensionLibrary.html

godot-macros/src/util/kv_parser.rs

@@ -125,6 +125,36 @@ impl KvParser {
         }
     }
 
+    pub fn handle_usize(&mut self, key: &str) -> ParseResult<Option<usize>> {
+        let Some(expr) = self.handle_expr(key)? else {
+            return Ok(None);
+        };
+
+        let mut tokens = expr.into_iter();
+        let Some(TokenTree::Literal(lit)) = tokens.next() else {
+            return bail!(
+                key,
+                "missing value for '{key}' (must be unsigned integer literal)"
+            );
+        };
+
+        if let Some(surplus) = tokens.next() {
+            return bail!(
+                key,
+                "value for '{key}' must be unsigned integer literal; found extra {surplus:?}"
+            );
+        }
+
+        let Ok(int) = lit.to_string().parse() else {
+            return bail!(
+                key,
+                "value for '{key}' must be unsigned integer literal; found {lit:?}"
+            );
+        };
+
+        Ok(Some(int))
+    }
+
     /// Handles a key that must be provided and must have an identifier as the value.
     pub fn handle_ident_required(&mut self, key: &str) -> ParseResult<Ident> {
         self.handle_ident(key)?

godot-macros/src/util/mod.rs

@@ -36,6 +36,9 @@ pub fn bail_fn<R, T>(msg: impl AsRef<str>, tokens: T) -> ParseResult<R>
 where
     T: Spanned,
 {
+    // TODO: using T: Spanned often only highlights the first tokens of the symbol, e.g. #[attr] in a function.
+    // Could use function.name; possibly our own trait to get a more meaningful span... or change upstream in venial.
+
     Err(error_fn(msg, tokens))
 }
 

godot/src/lib.rs

@@ -175,7 +175,7 @@ pub mod bind {
 /// Testing facilities (unstable).
 #[doc(hidden)]
 pub mod test {
-    pub use godot_macros::itest;
+    pub use godot_macros::{bench, itest};
 }
 
 #[doc(hidden)]

itest/godot/TestRunner.gd

@@ -61,6 +61,9 @@ func _ready():
 		filters
 	)
 
+	if success:
+		rust_runner.run_all_benchmarks(self)
+
 	var exit_code: int = 0 if success else 1
 	get_tree().quit(exit_code)
 

itest/rust/src/benchmarks/mod.rs

@@ -0,0 +1,98 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+// File can be split once this grows.
+
+use std::hint::black_box;
+
+use godot::bind::GodotClass;
+use godot::builtin::inner::InnerRect2i;
+use godot::builtin::{GodotString, Rect2i, StringName, Vector2i};
+use godot::engine::{Node3D, Os, RefCounted};
+use godot::obj::{Gd, InstanceId};
+
+use crate::framework::bench;
+
+#[bench]
+fn builtin_string_ctor() -> GodotString {
+    GodotString::from("some test string")
+}
+
+#[bench]
+fn builtin_stringname_ctor() -> StringName {
+    StringName::from("some test string")
+}
+
+#[bench]
+fn builtin_rust_call() -> bool {
+    let point = black_box(Vector2i::new(50, 60));
+
+    let rect = Rect2i::from_components(0, 0, 100, 100);
+
+    rect.contains_point(point)
+}
+
+#[bench]
+fn builtin_ffi_call() -> bool {
+    let point = black_box(Vector2i::new(50, 60));
+
+    let rect = Rect2i::from_components(0, 0, 100, 100);
+    let rect = InnerRect2i::from_outer(&rect);
+
+    rect.has_point(point)
+}
+
+#[bench(repeat = 25)]
+fn class_node_life() -> InstanceId {
+    let node = Node3D::new_alloc();
+    let instance_id = node.instance_id();
+
+    node.free();
+    instance_id // No longer valid, but enough for compiler to assume it's used.
+}
+
+#[bench(repeat = 25)]
+fn class_refcounted_life() -> Gd<RefCounted> {
+    RefCounted::new()
+}
+
+#[bench(repeat = 25)]
+fn class_user_refc_life() -> Gd<MyBenchType> {
+    Gd::<MyBenchType>::new_default()
+}
+
+#[bench]
+fn class_singleton_access() -> Gd<Os> {
+    Os::singleton()
+}
+
+#[bench]
+fn utilities_allocate_rid() -> i64 {
+    godot::engine::utilities::rid_allocate_id()
+}
+
+#[bench]
+fn utilities_rust_call() -> f64 {
+    let base = black_box(5.678);
+    let exponent = black_box(3.456);
+
+    f64::powf(base, exponent)
+}
+
+#[bench]
+fn utilities_ffi_call() -> f64 {
+    let base = black_box(5.678);
+    let exponent = black_box(3.456);
+
+    godot::engine::utilities::pow(base, exponent)
+}
+
+// ----------------------------------------------------------------------------------------------------------------------------------------------
+// Helpers for benchmarks above
+
+#[derive(GodotClass)]
+#[class(init)]
+struct MyBenchType {}

itest/rust/src/common.rs

@@ -20,3 +20,13 @@ where
 
     assert_eq!(value, back);
 }
+
+/// Signal to the compiler that a value is used (to avoid optimization).
+pub fn bench_used<T: Sized>(value: T) {
+    // The following check would be used to prevent `()` arguments, ensuring that a value from the bench is actually going into the blackbox.
+    // However, we run into this issue, despite no array being used: https://github.com/rust-lang/rust/issues/43408.
+    //   error[E0401]: can't use generic parameters from outer function
+    // sys::static_assert!(std::mem::size_of::<T>() != 0, "returned unit value in benchmark; make sure to use a real value");
+
+    std::hint::black_box(value);
+}

itest/rust/src/framework/bencher.rs

@@ -0,0 +1,77 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+// This is a very minimalistic measurement tool for micro-benchmarks. Its goal is to provide coarse overview of performance improvements
+// or regressions, NOT a statistically rigorous analysis. We simply measure wall time (not CPU time) and don't consider specifics of
+// the hardware or architecture. There are more sophisticated benchmarking tools, but at the moment there is no need for them:
+// - https://github.com/bheisler/criterion.rs
+// - https://github.com/Canop/glassbench
+// - https://github.com/sharkdp/hyperfine
+
+// We currently avoid mean or max, as we're not that interested in outliers (e.g. CPU spike).
+// This may of course obscure bad performance in only small number of cases, but that's something we take into account.
+// Instead, we focus on min (fastest run) and median -- even median may vary quite a bit between runs; but it gives an idea of the distribution.
+// See also https://easyperf.net/blog/2019/12/30/Comparing-performance-measurements#average-median-minimum.
+
+use std::time::{Duration, Instant};
+
+const WARMUP_RUNS: usize = 200;
+const TEST_RUNS: usize = 501; // uneven, so median need not be interpolated.
+const METRIC_COUNT: usize = 2;
+
+pub struct BenchResult {
+    pub stats: [Duration; METRIC_COUNT],
+}
+
+pub fn metrics() -> [&'static str; METRIC_COUNT] {
+    ["min", "median"]
+}
+
+pub fn run_benchmark(code: fn(), inner_repetitions: usize) -> BenchResult {
+    for _ in 0..WARMUP_RUNS {
+        code();
+    }
+
+    let mut times = Vec::with_capacity(TEST_RUNS);
+    for _ in 0..TEST_RUNS {
+        let start = Instant::now();
+        code();
+        let duration = start.elapsed();
+
+        times.push(duration / inner_repetitions as u32);
+    }
+    times.sort();
+
+    calculate_stats(times)
+}
+
+fn calculate_stats(times: Vec<Duration>) -> BenchResult {
+    // See top of file for rationale.
+
+    /*let mean = {
+        let total = times.iter().sum::<Duration>();
+        total / TEST_RUNS as u32
+    };
+    let std_dev = {
+        let mut variance = 0;
+        for time in times.iter() {
+            let diff = time.as_nanos() as i128 - mean.as_nanos() as i128;
+            variance += (diff * diff) as u128;
+        }
+        Duration::from_nanos((variance as f64 / TEST_RUNS as f64).sqrt() as u64)
+    };
+    let max = times[TEST_RUNS - 1];
+    let percentile05 = times[(TEST_RUNS as f64 * 0.05) as usize];
+    */
+
+    // Interpolating percentiles is not that important.
+    let min = times[0];
+    let median = times[TEST_RUNS / 2];
+
+    BenchResult {
+        stats: [min, median],
+    }
+}