Merge pull request #97 from gadget-inc/nicer-benchmarking

nicer benchmarking

Author: airhorns

.vscode/launch.json

@@ -0,0 +1,35 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "name": "vscode-jest-tests.v2",
+      "request": "launch",
+      "args": ["--runInBand", "--watchAll=false", "--testNamePattern", "${jest.testNamePattern}", "--runTestsByPath", "${jest.testFile}"],
+      "cwd": "${workspaceFolder}",
+      "runtimeExecutable": "bash",
+      "console": "integratedTerminal",
+      "internalConsoleOptions": "neverOpen",
+      "disableOptimisticBPs": true,
+      "program": "${workspaceFolder}/node_modules/.bin/jest",
+      "windows": {
+        "program": "${workspaceFolder}/node_modules/jest/bin/jest"
+      }
+    },
+    {
+      "type": "node",
+      "name": "run-benchmark",
+      "request": "launch",
+      "args": ["--transpile-only", "bench/create-large-root.ts"],
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "runtimeExecutable": "bash",
+      "internalConsoleOptions": "neverOpen",
+      "disableOptimisticBPs": true,
+      "program": "${workspaceFolder}/node_modules/.bin/ts-node"
+    }
+  ]
+}

Benchmarking.md

@@ -7,18 +7,30 @@
 You can run a benchmark file with `pnpm x <the file>`:
 
 ```shell
-pnpm x bench/create-large-root.ts
+pnpm x bench/instantiation.benchmark.ts
 ```
 
 This will run the file and output a speed measurement for comparison between git branches.
 
+You can also run all the benchmarks with:
+
+```shell
+pnpm x bench/all.ts
+```
+
 ## Profiling
 
 It's nice to use the benchmarks for profiling to identify optimization candidates.
 
 ### CPU profiling
 
-You can run a benchmark to generate a profile using node.js' built in sampling profiler
+The benchmark framework supports a `--profile` option for writing a profile of the benchmark loop, excluding setup and teardown code. Run a benchmark with profile, then open the created `.cpuprofile` file:
+
+```shell
+pnpm x bench/instantiation.benchmark.ts --profile
+```
+
+You can also run a benchmark to generate a profile using node.js' built in sampling profiler
 
 ```shell
 node -r ts-node/register/transpile-only --prof bench/create-large-root.ts

bench/all.ts

@@ -1,53 +1,46 @@
-import { Bench } from "tinybench";
+import globby from "globby";
+import { hideBin } from "yargs/helpers";
+import yargs from "yargs/yargs";
+import { benchTable, createSuite } from "./benchmark";
 import { withCodSpeed } from "@codspeed/tinybench-plugin";
-import findRoot from "find-root";
-import fs from "fs";
-import { FruitAisle } from "../spec/fixtures/FruitAisle";
-import { LargeRoot } from "../spec/fixtures/LargeRoot";
-import { TestClassModel } from "../spec/fixtures/TestClassModel";
-import { BigTestModelSnapshot, TestModelSnapshot } from "../spec/fixtures/TestModel";
-import { registerPropertyAccess } from "./property-access-model-class";
-
-const root = findRoot(__dirname);
-const largeRoot = JSON.parse(fs.readFileSync(root + "/spec/fixtures/large-root-snapshot.json", "utf8"));
-const fruitAisle = JSON.parse(fs.readFileSync(root + "/spec/fixtures/fruit-aisle-snapshot.json", "utf8"));
-
-void (async () => {
-  let suite = new Bench();
+
+export const runAll = async () => {
+  const argv = await yargs(hideBin(process.argv))
+    .option("benchmarks", {
+      alias: ["b", "t"],
+      type: "string",
+      describe: "Benchmark file pattern to match",
+    })
+    .usage("Usage: run.ts [options]")
+    .help().argv;
+
+  let benchmarkFiles = await globby(__dirname + "/**/*.benchmark.ts");
+  let suite = createSuite();
+
   if (process.env.CI) {
     suite = withCodSpeed(suite);
   }
 
-  suite
-    .add("instantiating a small root", function () {
-      TestClassModel.createReadOnly(TestModelSnapshot);
-    })
-    .add("instantiating a large root", function () {
-      LargeRoot.createReadOnly(largeRoot);
-    })
-    .add("instantiating a large union", function () {
-      FruitAisle.createReadOnly(fruitAisle);
-    })
-    .add("instantiating a diverse root", function () {
-      TestClassModel.createReadOnly(BigTestModelSnapshot);
-    })
-    .add("instantiating a small root (mobx-state-tree)", function () {
-      TestClassModel.create(TestModelSnapshot);
-    })
-    .add("instantiating a large root (mobx-state-tree)", function () {
-      LargeRoot.create(largeRoot);
-    })
-    .add("instantiating a large union (mobx-state-tree)", function () {
-      FruitAisle.create(fruitAisle);
-    })
-    .add("instantiating a diverse root (mobx-state-tree)", function () {
-      TestClassModel.create(BigTestModelSnapshot);
-    });
+  if (argv.benchmarks) {
+    benchmarkFiles = benchmarkFiles.filter((file) => file.includes(argv.benchmarks!));
+  }
+  console.info("running benchmarks", { benchmarkFiles });
 
-  suite = registerPropertyAccess(suite);
+  for (const file of benchmarkFiles) {
+    let benchmark = await import(file);
+    if (benchmark.default) {
+      benchmark = benchmark.default;
+    }
+    suite = await benchmark.fn(suite);
+  }
 
   await suite.warmup();
   await suite.run();
 
-  console.table(suite.table());
-})();
+  console.table(benchTable(suite));
+};
+
+if (require.main === module) {
+  void runAll();
+}
+

bench/benchmark.ts

@@ -0,0 +1,163 @@
+import { writeFile } from "fs-extra";
+import { compact } from "lodash";
+import { Bench, type Options } from "tinybench";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import type { Profiler } from "inspector";
+import { Session } from "inspector";
+
+export const newInspectorSession = () => {
+  const session = new Session();
+  const post = (method: string, params?: Record<string, unknown>): any =>
+    new Promise((resolve, reject) => {
+      session.post(method, params, (err: Error | null, result: any) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(result);
+        }
+      });
+    });
+
+  session.connect();
+  return { session, post };
+};
+
+export type BenchmarkGenerator = ((suite: Bench) => Bench | Promise<Bench>) & { options?: Options };
+
+/**
+ * Set up a new benchmark in our library of benchmarks
+ * If this file is executed directly, it will run the benchmark
+ * Otherwise, it will export the benchmark for use in other files
+ *
+ * @example
+ * export default benchmarker((suite) => {
+ *   return suite.add("My Benchmark", async () => {
+ *      // something expensive
+ *   });
+ * });
+ **/
+export const benchmarker = (fn: BenchmarkGenerator, options?: Options) => {
+  fn.options = options;
+
+  const err = new NiceStackError();
+  const callerFile = (err.stack as unknown as NodeJS.CallSite[])[2].getFileName();
+
+  if (require.main?.filename === callerFile) {
+    void runBenchmark(fn);
+  } else {
+    return { fn };
+  }
+};
+
+/** Wrap a plain old async function in the weird deferred management code benchmark.js requires */
+export const asyncBench = (fn: () => Promise<void>) => {
+  return {
+    defer: true,
+    fn: async (deferred: any) => {
+      await fn();
+      deferred.resolve();
+    },
+  };
+};
+
+/** Boot up a benchmark suite for registering new cases on */
+export const createSuite = (options: Options = { iterations: 100 }) => {
+  const suite = new Bench(options);
+
+  suite.addEventListener("error", (event: any) => {
+    console.error(event);
+  });
+
+  return suite;
+};
+
+/** Run one benchmark function in isolation */
+const runBenchmark = async (fn: BenchmarkGenerator) => {
+  const args = await yargs(hideBin(process.argv))
+    .option("p", {
+      alias: "profile",
+      default: false,
+      describe: "profile each benchmarked case as it runs, writing a CPU profile to disk for each",
+      type: "boolean",
+    })
+    .option("b", {
+      alias: "blocking",
+      default: false,
+      describe: "track event loop blocking time during each iteration, which changes the stats",
+      type: "boolean",
+    }).argv;
+
+  let suite = createSuite(fn.options);
+
+  if (args.profile) {
+    const key = formatDateForFile();
+
+    const { post } = newInspectorSession();
+    await post("Profiler.enable");
+    await post("Profiler.setSamplingInterval", { interval: 20 });
+
+    suite.addEventListener("add", (event) => {
+      const oldBeforeAll = event.task.opts.beforeAll;
+      const oldAfterAll = event.task.opts.beforeAll;
+
+      event.task.opts.beforeAll = async function () {
+        await post("Profiler.start");
+        await oldBeforeAll?.call(this);
+      };
+      event.task.opts.afterAll = async function () {
+        await oldAfterAll?.call(this);
+        const { profile } = (await post("Profiler.stop")) as Profiler.StopReturnType;
+        await writeFile(`./bench-${event.task.name}-${key}.cpuprofile`, JSON.stringify(profile));
+      };
+    });
+  }
+
+  suite = await fn(suite);
+
+  console.log("running benchmark");
+
+  await suite.warmup();
+  await suite.run();
+
+  console.table(benchTable(suite));
+};
+
+class NiceStackError extends Error {
+  constructor() {
+    super();
+    const oldStackTrace = Error.prepareStackTrace;
+    try {
+      Error.prepareStackTrace = (err, structuredStackTrace) => structuredStackTrace;
+
+      Error.captureStackTrace(this);
+
+      this.stack; // Invoke the getter for `stack`.
+    } finally {
+      Error.prepareStackTrace = oldStackTrace;
+    }
+  }
+}
+
+const formatDateForFile = () => {
+  const now = new Date();
+  return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}_${String(
+    now.getHours()
+  ).padStart(2, "0")}-${String(now.getMinutes()).padStart(2, "0")}-${String(now.getSeconds()).padStart(2, "0")}`;
+};
+
+export const benchTable = (bench: Bench) => {
+  return compact(
+    bench.tasks.map(({ name: t, result: e }) => {
+      if (!e) return null;
+      return {
+        "Task Name": t,
+        "ops/sec": e.error ? "NaN" : parseInt(e.hz.toString(), 10).toLocaleString(),
+        "Average Time (ms)": e.error ? "NaN" : e.mean,
+        "p99 Time (ms)": e.error ? "NaN" : e.p99,
+        Margin: e.error ? "NaN" : `\xB1${e.rme.toFixed(2)}%`,
+        Samples: e.error ? "NaN" : e.samples.length,
+      };
+    })
+  );
+};

bench/create-large-root.ts

@@ -1,19 +1,15 @@
-import { Bench } from "tinybench";
 import findRoot from "find-root";
 import fs from "fs";
 import { FruitAisle } from "../spec/fixtures/FruitAisle";
+import { benchmarker } from "./benchmark";
 
 const root = findRoot(__dirname);
 const fruitBasket = JSON.parse(fs.readFileSync(root + "/spec/fixtures/fruit-aisle-snapshot.json", "utf8"));
 
-void (async () => {
-  const suite = new Bench();
-
+export default benchmarker(async (suite) => {
   suite.add("instantiating a large union", function () {
     FruitAisle.createReadOnly(fruitBasket);
   });
 
-  await suite.warmup();
-  await suite.run();
-  console.table(suite.table());
-})();
+  return suite
+});

bench/create-union.ts renamed to bench/create-union.benchmark.ts

@@ -1,8 +1,8 @@
-import { Bench } from "tinybench";
 import { TestClassModel } from "../spec/fixtures/TestClassModel";
 import { TestModel } from "../spec/fixtures/TestModel";
 import { TestPlainModel } from "./reference/plain-class";
 import { ObservablePlainModel } from "./reference/mobx";
+import { benchmarker } from "./benchmark";
 
 const TestModelSnapshot: (typeof TestModel)["InputType"] = {
   bool: true,
@@ -21,9 +21,7 @@ const TestModelSnapshot: (typeof TestModel)["InputType"] = {
   },
 };
 
-void (async () => {
-  const suite = new Bench();
-
+export default benchmarker(async (suite) => {
   suite
     .add("mobx-state-tree", function () {
       TestModel.create(TestModelSnapshot);
@@ -41,7 +39,5 @@ void (async () => {
       new TestPlainModel(TestModelSnapshot);
     });
 
-  await suite.warmup();
-  await suite.run();
-  console.table(suite.table());
-})();
+  return suite
+});