feat!: update V8 flags and force optimization

Author: adriencaccia

packages/benchmark.js-plugin/src/index.ts

@@ -196,6 +196,7 @@ async function runBenchmarks({
     if (isAsync) {
       await optimizeFunction(benchPayload);
       await mongoMeasurement.start(uri);
+      global.gc?.();
       await (async function __codspeed_root_frame__() {
         Measurement.startInstrumentation();
         await benchPayload();

packages/core/src/introspection.ts

@@ -12,6 +12,10 @@ export const getV8Flags = () => {
     "--predictable",
     "--predictable-gc-schedule",
     "--interpreted-frames-native-stack",
+    "--allow-natives-syntax",
+    "--expose-gc",
+    "--no-concurrent-sweeping",
+    "--max-old-space-size=4096",
   ];
   if (nodeVersionMajor < 18) {
     flags.push("--no-randomize-hashes");

packages/core/src/optimization.ts

@@ -1,34 +1,30 @@
-const skipOptimization = process.env.CODSPEED_FORCE_OPTIMIZATION !== "true";
-
 export const initOptimization = () => {
-  if (!skipOptimization) {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    require("v8").setFlagsFromString("--allow-natives-syntax");
-  }
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  require("v8").setFlagsFromString("--allow-natives-syntax");
 };
 
 export const optimizeFunction = async (fn: CallableFunction) => {
-  if (skipOptimization) {
-    // warmup V8 symbols generation of the performance map
-    await fn();
-    return;
-  }
   // Source: https://github.com/petkaantonov/bluebird/wiki/Optimization-killers#optimization-killers
-  await fn(); //Fill type-info
-  await fn(); // 2 calls are needed to go from uninitialized -> pre-monomorphic -> monomorphic
+  // a total of 7 calls seems to be the sweet spot
+  await fn();
+  await fn();
+  await fn();
+  await fn();
+  await fn();
+  await fn();
   eval("%OptimizeFunctionOnNextCall(fn)");
   await fn(); // optimize
 };
 
 export const optimizeFunctionSync = (fn: CallableFunction) => {
-  if (skipOptimization) {
-    // warmup V8 symbols generation of the performance map
-    fn();
-    return;
-  }
   // Source: https://github.com/petkaantonov/bluebird/wiki/Optimization-killers#optimization-killers
-  fn(); //Fill type-info
-  fn(); // 2 calls are needed to go from uninitialized -> pre-monomorphic -> monomorphic
+  // a total of 7 calls seems to be the sweet spot
+  fn();
+  fn();
+  fn();
+  fn();
+  fn();
+  fn();
   eval("%OptimizeFunctionOnNextCall(fn)");
   fn(); // optimize
 };

packages/tinybench-plugin/src/index.ts

@@ -70,6 +70,7 @@ export function withCodSpeed(bench: Bench): Bench {
       await task.opts.beforeEach?.call(task);
 
       await mongoMeasurement.start(uri);
+      global.gc?.();
       await (async function __codspeed_root_frame__() {
         Measurement.startInstrumentation();
         await task.fn();

packages/vitest-plugin/src/__tests__/index.test.ts

@@ -89,6 +89,10 @@ describe("codSpeedPlugin", () => {
               "--predictable",
               "--predictable-gc-schedule",
               "--interpreted-frames-native-stack",
+              "--allow-natives-syntax",
+              "--expose-gc",
+              "--no-concurrent-sweeping",
+              "--max-old-space-size=4096",
               "--no-scavenge-task",
             ],
           },

packages/vitest-plugin/src/__tests__/runner.test.ts

@@ -18,23 +18,22 @@ const coreMocks = vi.hoisted(() => {
   };
 });
 
+global.eval = vi.fn();
+
 vi.mock("@codspeed/core", async (importOriginal) => {
   const mod = await importOriginal<typeof import("@codspeed/core")>();
   return { ...mod, ...coreMocks };
 });
 
 console.log = vi.fn();
 
-vi.mock("vitest/suite", () => ({
-  getBenchFn: vi.fn(),
-  // wrapping the value in vi.fn(...) here will not work for some reason
-  getHooks: () => ({
-    beforeAll: [],
-    beforeEach: [],
-    afterEach: [],
-    afterAll: [],
-  }),
-}));
+vi.mock("vitest/suite", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("vitest/suite")>();
+  return {
+    ...actual,
+    getBenchFn: vi.fn(),
+  };
+});
 const mockedGetBenchFn = vi.mocked(getBenchFn);
 
 describe("CodSpeedRunner", () => {
@@ -62,7 +61,7 @@ describe("CodSpeedRunner", () => {
       "packages/vitest-plugin/src/__tests__/runner.test.ts::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
-    expect(benchFn).toHaveBeenCalledTimes(2);
+    expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(
@@ -116,7 +115,7 @@ describe("CodSpeedRunner", () => {
       "packages/vitest-plugin/src/__tests__/runner.test.ts::nested suite::test bench"
     );
     expect(coreMocks.Measurement.startInstrumentation).toHaveBeenCalledTimes(1);
-    expect(benchFn).toHaveBeenCalledTimes(2);
+    expect(benchFn).toHaveBeenCalledTimes(8);
     expect(coreMocks.Measurement.stopInstrumentation).toHaveBeenCalledTimes(1);
     expect(coreMocks.mongoMeasurement.stop).toHaveBeenCalledTimes(1);
     expect(console.log).toHaveBeenCalledWith(

packages/vitest-plugin/src/runner.ts

@@ -15,23 +15,23 @@ import { getBenchFn, getHooks } from "vitest/suite";
 type SuiteHooks = ReturnType<typeof getHooks>;
 
 function getSuiteHooks(suite: Suite, name: keyof SuiteHooks) {
-  return getHooks(suite)[name];
+  return getHooks(suite)?.[name] ?? [];
 }
 
 export async function callSuiteHook<T extends keyof SuiteHooks>(
   suite: Suite,
   currentTask: Task,
   name: T
 ): Promise<void> {
-  if (name === "beforeEach" && suite.suite) {
+  if (name === "beforeEach" && suite?.suite) {
     await callSuiteHook(suite.suite, currentTask, name);
   }
 
   const hooks = getSuiteHooks(suite, name);
 
   await Promise.all(hooks.map((fn) => fn()));
 
-  if (name === "afterEach" && suite.suite) {
+  if (name === "afterEach" && suite?.suite) {
     await callSuiteHook(suite.suite, currentTask, name);
   }
 }
@@ -69,6 +69,7 @@ async function runBench(benchmark: Benchmark, currentSuiteName: string) {
 
   await callSuiteHook(benchmark.suite, benchmark, "beforeEach");
   await mongoMeasurement.start(uri);
+  global.gc?.();
   await (async function __codspeed_root_frame__() {
     Measurement.startInstrumentation();
     // @ts-expect-error we do not need to bind the function to an instance of tinybench's Bench