feat(core): introduce new "accurate" engine

Author: boneskull

eslint.config.js

@@ -166,6 +166,7 @@ export default defineConfig(
       'site',
       'astro.config.js',
       '.astro/**/*',
+      'PROTOTYPE_CUSTOM_ENGINE.ts',
     ],
   },
 );

examples/accurate-engine-example.js

@@ -0,0 +1,86 @@
+/**
+ * AccurateEngine Example
+ *
+ * This example demonstrates how to use the AccurateEngine for high-accuracy
+ * benchmarking with V8 optimization guards.
+ *
+ * Run with: node --allow-natives-syntax examples/accurate-engine-example.js
+ */
+
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { ModestBenchConfigurationManager } from '../src/config/manager.js';
+import { ModestBenchErrorManager } from '../src/core/error-manager.js';
+import { BenchmarkFileLoader } from '../src/core/loader.js';
+import { AccurateEngine } from '../src/index.js';
+import { ModestBenchProgressManager } from '../src/progress/manager.js';
+import { ModestBenchReporterRegistry } from '../src/reporters/registry.js';
+import { FileHistoryStorage } from '../src/storage/history.js';
+
+const main = async () => {
+  console.log('🎯 AccurateEngine Example\n');
+
+  // Create engine with all dependencies
+  const engine = new AccurateEngine({
+    configManager: new ModestBenchConfigurationManager(),
+    errorManager: new ModestBenchErrorManager(),
+    fileLoader: new BenchmarkFileLoader(),
+    historyStorage: new FileHistoryStorage({
+      storageDir: join(tmpdir(), '.modestbench-example'),
+    }),
+    progressManager: new ModestBenchProgressManager(),
+    reporterRegistry: new ModestBenchReporterRegistry(),
+  });
+
+  // Configuration for accurate measurement
+  const config = {
+    config: {
+      iterations: 100, // Minimum iterations per task
+      limitBy: 'any', // Stop at first limit reached
+      quiet: false, // Show progress
+      time: 1000, // Maximum time per task (1 second)
+      verbose: true, // Detailed output
+      warmup: 20, // 20 warmup iterations
+    },
+    // Point to example benchmark files
+    pattern: 'examples/benchmarks/*.bench.js',
+    reporters: ['human'],
+  };
+
+  try {
+    console.log('Running benchmarks with AccurateEngine...\n');
+
+    const result = await engine.execute(config);
+
+    console.log('\n✅ Benchmark complete!\n');
+    console.log('Summary:');
+    console.log(`  Files: ${result.files.length}`);
+    console.log(`  Suites: ${result.summary.totalSuites}`);
+    console.log(`  Tasks: ${result.summary.totalTasks}`);
+    console.log(`  Total operations: ${result.summary.totalOperations}`);
+    console.log(`  Duration: ${(result.duration / 1000).toFixed(2)}s`);
+
+    if (result.summary.fastest) {
+      console.log(`\n⚡ Fastest: ${result.summary.fastest.name}`);
+      console.log(
+        `   ${result.summary.fastest.opsPerSecond.toFixed(2)} ops/sec`,
+      );
+    }
+
+    if (result.summary.slowest) {
+      console.log(`\n🐌 Slowest: ${result.summary.slowest.name}`);
+      console.log(
+        `   ${result.summary.slowest.opsPerSecond.toFixed(2)} ops/sec`,
+      );
+    }
+  } catch (error) {
+    console.error(
+      '❌ Error running benchmarks:',
+      error instanceof Error ? error.message : String(error),
+    );
+    process.exit(1);
+  }
+};
+
+void main();

site/src/content/docs/getting-started/index.md

@@ -122,6 +122,45 @@ Run benchmarks with options:
 modestbench run --iterations 5000 --reporters human,json
 ```
 
+### Choosing an Engine
+
+modestbench provides two engines with different trade-offs:
+
+```bash
+# Default: tinybench (fast, good for development)
+modestbench run
+
+# Accurate engine (higher precision, recommended for production benchmarks)
+node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate
+```
+
+**Engine Comparison:**
+
+| Feature | `tinybench` (default) | `accurate` |
+|---------|----------------------|------------|
+| Speed | ⚡ Very fast | 🐢 Slower (more thorough) |
+| Statistical Quality | ✅ Good | ⭐ Excellent |
+| Outlier Removal | ✅ IQR-based | ✅ IQR-based |
+| V8 Optimization Guards | ❌ No | ✅ Yes (prevents JIT interference) |
+| Requirements | None | `--allow-natives-syntax` flag |
+| Best For | Development, CI | Production benchmarks, publications |
+
+:::tip[When to Use Accurate Engine]
+Use the `accurate` engine when:
+
+- Publishing benchmark results
+- Making critical performance decisions
+- Comparing micro-optimizations
+- Needing the highest statistical quality
+
+Use the default `tinybench` engine when:
+
+- Rapid iteration during development
+- CI/CD performance regression tests
+- General performance comparisons
+
+:::
+
 Run specific files or directories:
 
 ```bash

site/src/content/docs/guides/advanced.md

@@ -3,6 +3,107 @@ title: Advanced Usage
 description: Advanced features and patterns for modestbench
 ---
 
+## Benchmark Engines
+
+ModestBench provides two engines with different performance characteristics and statistical approaches.
+
+### Engine Selection
+
+Choose an engine based on your requirements:
+
+```bash
+# Tinybench engine (default) - fast development iteration
+modestbench run --engine tinybench
+
+# Accurate engine - high-precision measurements
+node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate
+```
+
+### Statistical Improvements
+
+Both engines now use **IQR (Interquartile Range) outlier removal** to filter extreme values caused by:
+
+- Garbage collection pauses
+- System interruptions
+- Background processes
+- OS scheduler variations
+
+This results in more stable and reliable measurements compared to raw statistical analysis.
+
+#### AccurateEngine Statistical Features
+
+The `accurate` engine provides enhanced statistical analysis:
+
+1. **V8 Optimization Guards**: Uses V8 intrinsics (`%NeverOptimizeFunction`) to prevent JIT compiler interference with measurements
+2. **IQR Outlier Removal**: Automatically removes extreme outliers (beyond Q1 - 1.5×IQR and Q3 + 1.5×IQR)
+3. **Comprehensive Statistics**:
+   - Mean, min, max execution times
+   - Standard deviation and variance
+   - **Coefficient of Variation (CV)**: Measures relative variability (`stdDev / mean × 100`)
+   - 95th and 99th percentiles
+   - Margin of error (95% confidence interval)
+
+#### Coefficient of Variation (CV)
+
+The CV metric helps assess benchmark quality:
+
+```text
+CV < 5%    → Excellent (very stable)
+CV 5-10%   → Good (acceptable variance)
+CV 10-20%  → Fair (consider more samples)
+CV > 20%   → Poor (investigate noise sources)
+```
+
+Example output showing CV:
+
+```bash
+$ modestbench run --engine accurate --allow-natives-syntax --reporters json
+{
+  "name": "Array.push()",
+  "mean": 810050,  // nanoseconds
+  "stdDev": 19842,
+  "cv": 2.45,      // 2.45% - excellent stability
+  "marginOfError": 0.024,
+  "p95": 845200,
+  "p99": 862100
+}
+```
+
+### Performance Comparison
+
+Real-world comparison using `examples/benchmarks`:
+
+```bash
+# Tinybench (fast iteration)
+$ modestbench run --engine tinybench --reporters json
+# Typical run time: 3-5 seconds for 5 benchmark files
+
+# Accurate (high precision)
+$ node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate --reporters json
+# Typical run time: 8-12 seconds for 5 benchmark files
+```
+
+The `accurate` engine takes ~2-3x longer but provides:
+
+- More consistent results between runs
+- Better outlier filtering with V8 guards
+- Higher confidence in micro-optimizations
+
+### Choosing the Right Engine
+
+| Use Case | Recommended Engine |
+|----------|-------------------|
+| Development iteration | `tinybench` |
+| CI/CD regression tests | `tinybench` |
+| Blog post/publication | `accurate` |
+| Library optimization | `accurate` |
+| Micro-benchmark comparison | `accurate` |
+| Algorithm selection | Either (results typically consistent) |
+
+:::tip[Best Practice]
+Start with `tinybench` during development for fast feedback. Switch to `accurate` for final measurements and critical decisions.
+:::
+
 ## Multiple Suites
 
 Organize related benchmarks into separate suites with independent setup and teardown:

site/src/content/docs/guides/cli.md

@@ -98,6 +98,43 @@ modestbench run --warmup 100
 
 Helps stabilize JIT compilation for more consistent results.
 
+##### `--engine <name>`
+
+Select the benchmark engine. Options: `tinybench` (default) or `accurate`.
+
+```bash
+# Use the accurate engine for high-precision measurements
+node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate
+
+# Use tinybench engine (default)
+modestbench run --engine tinybench
+```
+
+**Engine Differences:**
+
+- **`tinybench`** (default): Fast, lightweight engine suitable for development and CI. Uses IQR-based outlier removal.
+- **`accurate`**: High-precision engine with V8 optimization guards to prevent JIT compiler interference. Requires `--allow-natives-syntax` flag. Recommended for production benchmarks and critical performance measurements.
+
+See the [Getting Started](/getting-started/#choosing-an-engine) guide for detailed comparison.
+
+:::caution[Node.js Flag Required]
+The `accurate` engine requires running Node.js with the `--allow-natives-syntax` flag. This flag must be passed to the Node.js runtime, not to modestbench:
+
+```bash
+# Using Node.js directly
+node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate
+
+# Using npx (pass flag to Node.js)
+npx --node-arg=--allow-natives-syntax modestbench run --engine accurate
+
+# Using package.json script
+# package.json: "bench": "node --allow-natives-syntax ./node_modules/.bin/modestbench run --engine accurate"
+npm run bench
+```
+
+If the flag is not present, AccurateEngine will fall back to a less accurate mode and display a warning.
+:::
+
 ##### `--timeout <number>`
 
 Maximum time in milliseconds for a single task before timing out.

site/src/content/docs/index.mdx

@@ -22,7 +22,7 @@ import { Card, CardGrid } from '@astrojs/starlight/components';
 
 <CardGrid stagger>
   <Card title="Fast & Accurate" icon="rocket">
-    High-precision timing with statistical analysis powered by `tinybench`. Get reliable measurements you can trust.
+    High-precision timing with advanced statistical analysis. Choose between the fast `tinybench` engine or the `accurate` engine with V8 optimization guards and IQR outlier removal.
   </Card>
 
   <Card title="Multiple Output Formats" icon="document">
@@ -84,8 +84,10 @@ modestbench run --iterations 5000 --reporters human,json
 
 ## Why modestbench?
 
-modestbench wraps [tinybench](https://github.com/tinylibs/tinybench) and enhances it with a bunch of features so you don't have to think:
+modestbench provides a powerful benchmarking framework with dual engines and rich features:
 
+- **Dual engines** - Fast `tinybench` for development, `accurate` for precision measurements
+- **Advanced statistics** - IQR outlier removal, coefficient of variation, percentiles
 - **Project structure** - Organized benchmark files and suites
 - **Configuration management** - Multiple formats (JSON, YAML, JS, TS)
 - **Rich CLI** - Discover and run benchmarks with powerful options

src/cli/commands/run.ts

@@ -19,6 +19,7 @@ interface RunOptions {
   bail?: boolean | undefined;
   config?: string | undefined;
   cwd: string;
+  engine?: 'accurate' | 'tinybench' | undefined;
   exclude?: string[] | undefined;
   excludeTags?: string[] | undefined;
   iterations?: number | undefined;

src/cli/index.ts

@@ -23,6 +23,7 @@ import type {
 } from '../types/index.js';
 
 import { bootstrap } from '../bootstrap.js';
+import { AccurateEngine, TinybenchEngine } from '../core/engines/index.js';
 import {
   CsvReporter,
   HumanReporter,
@@ -262,6 +263,14 @@ export const main = async (
               description: 'Exclude benchmarks with any of these tags',
               type: 'array',
             })
+            .option('engine', {
+              alias: 'e',
+              choices: ['tinybench', 'accurate'] as const,
+              default: 'tinybench' as const,
+              description:
+                'Benchmark engine: tinybench (default) or accurate (requires --allow-natives-syntax)',
+              type: 'string',
+            })
             .example([
               ['$0 run', 'Run benchmarks in current directory and bench/'],
               ['$0 run benchmarks/', 'Run all benchmarks in a directory'],
@@ -271,15 +280,21 @@ export const main = async (
               ['$0 run benchmarks/ tests/perf/', 'Run multiple directories'],
               ['$0 run --reporters json,csv', 'Use multiple reporters'],
               ['$0 run --iterations 1000', 'Set iteration count'],
+              ['$0 run --engine accurate', 'Use high-accuracy engine'],
               ['$0 run --bail', 'Stop on first failure'],
             ]);
         },
         async (argv) => {
-          const context = await createCliContext(argv, abortController!);
+          const context = await createCliContext(
+            argv,
+            abortController!,
+            argv.engine,
+          );
           const exitCode = await runCommand(context, {
             bail: argv.bail,
             config: argv.config,
             cwd: argv.cwd,
+            engine: argv.engine,
             exclude: argv.exclude,
             excludeTags: argv['exclude-tags'],
             iterations: argv.iterations,
@@ -503,9 +518,16 @@ export const main = async (
 const createCliContext = async (
   options: GlobalOptions,
   abortController: AbortController,
+  engineType: 'accurate' | 'tinybench' = 'tinybench',
 ): Promise<CliContext> => {
   try {
-    const engine = bootstrap();
+    const dependencies = bootstrap();
+
+    // Select engine based on type
+    const engine =
+      engineType === 'accurate'
+        ? new AccurateEngine(dependencies)
+        : new TinybenchEngine(dependencies);
 
     // Register built-in reporters
     engine.registerReporter(