Bump node, bump tinybench, and add support of walltime runs for tinybench

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
     needs: list-examples
     strategy:
       matrix:
-        node-version: ["16", "18", "20.5.1"]
+        node-version: ["18", "20.5.1"]
         example: ${{ fromJson(needs.list-examples.outputs.examples) }}
       fail-fast: false
     steps:

.github/workflows/codspeed.yml

@@ -7,8 +7,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  codspeed:
-    name: Run CodSpeed
+  codspeed-instrumented:
+    name: Run CodSpeed instrumented
     runs-on: "ubuntu-latest"
     steps:
       - uses: "actions/checkout@v4"
@@ -35,3 +35,31 @@ jobs:
             pnpm --workspace-concurrency 1 -r bench-tinybench
             pnpm --workspace-concurrency 1 -r bench-benchmark-js
             pnpm --workspace-concurrency 1 -r bench-vitest
+
+  codspeed-walltime:
+    name: Run CodSpeed walltime
+    runs-on: "codspeed-macro"
+    steps:
+      - uses: "actions/checkout@v4"
+        with:
+          fetch-depth: 0
+      - name: Install valgrind
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y valgrind
+      - uses: pnpm/action-setup@v2
+      - uses: actions/setup-node@v3
+        with:
+          cache: pnpm
+          node-version-file: .nvmrc
+      - run: pnpm install --frozen-lockfile --prefer-offline
+      - run: pnpm moon run :build
+
+      - name: Run benchmarks
+        # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2`
+        uses: CodSpeedHQ/action@main
+        with:
+          # Only tinybench supports walltime for now
+          run: |
+            pnpm moon run tinybench-plugin:bench
+            pnpm --workspace-concurrency 1 -r bench-tinybench

.moon/tasks.yml

@@ -25,6 +25,15 @@ tasks:
       - "@globs(sources)"
       - "@globs(tests)"
       - "@globs(configs)"
+  fix-format:
+    local: true
+    command: "prettier --config @in(0) --ignore-path @in(1) --write ."
+    inputs:
+      - "/.prettierrc.json"
+      - "/.prettierignore"
+      - "@globs(sources)"
+      - "@globs(tests)"
+      - "@globs(configs)"
   lint:
     command: "eslint ."
     inputs:

.moon/toolchain.yml

@@ -4,7 +4,7 @@ $schema: "https://moonrepo.dev/schemas/toolchain.json"
 node:
   packageManager: "pnpm"
   pnpm:
-    version: "8.6.3"
+    version: "10.12.4"
 
   dedupeOnLockfileChange: false
   dependencyVersionFormat: "workspace"

.nvmrc

@@ -1 +1 @@
-18.16.0
+20.5.1

CLAUDE.md

@@ -0,0 +1,91 @@
+# CodSpeed Node Repository Layout
+
+## Repository Structure
+
+This is a monorepo containing CodSpeed plugins for various Node.js benchmarking frameworks.
+
+### Root Level
+- `package.json` - Root package configuration
+- `pnpm-workspace.yaml` - PNPM workspace configuration
+- `lerna.json` - Lerna monorepo configuration
+- `tsconfig.base.json` - Base TypeScript configuration
+- `rollup.options.js` - Rollup bundler configuration
+- `scripts/` - Build and release scripts
+- `docs/` - Documentation files
+- `examples/` - Example projects using the plugins
+
+### Packages (`packages/`)
+
+#### Core Package (`packages/core/`)
+- **Purpose**: Core measurement and instrumentation functionality
+- **Key files**:
+  - `src/index.ts` - Main exports, setupCore/teardownCore functions
+  - `src/mongoMeasurement.ts` - MongoDB measurement handling
+  - `src/optimization.ts` - Function optimization utilities
+  - `src/native_core/` - Native C++ bindings for performance measurement
+  - `src/introspection.ts` - V8 flags and runtime introspection
+
+#### Tinybench Plugin (`packages/tinybench-plugin/`)
+- **Purpose**: CodSpeed integration for tinybench framework
+- **Key files**:
+  - `src/index.ts` - Main plugin implementation with `withCodSpeed()` function
+  - `tests/index.integ.test.ts` - Integration tests
+  - `benches/` - Benchmark examples
+
+#### Benchmark.js Plugin (`packages/benchmark.js-plugin/`)
+- **Purpose**: CodSpeed integration for benchmark.js framework
+- **Key files**:
+  - `src/index.ts` - Main plugin implementation
+  - `src/buildSuiteAdd.ts` - Suite building utilities
+
+#### Vitest Plugin (`packages/vitest-plugin/`)
+- **Purpose**: CodSpeed integration for Vitest framework
+- **Key files**:
+  - `src/index.ts` - Main plugin implementation
+  - `src/runner.ts` - Custom test runner
+  - `src/globalSetup.ts` - Global setup configuration
+
+### Examples Directory (`examples/`)
+- `with-javascript-cjs/` - CommonJS JavaScript examples
+- `with-javascript-esm/` - ESM JavaScript examples
+- `with-typescript-cjs/` - CommonJS TypeScript examples
+- `with-typescript-esm/` - ESM TypeScript examples
+- `with-typescript-simple-cjs/` - Simple CommonJS TypeScript examples
+- `with-typescript-simple-esm/` - Simple ESM TypeScript examples
+
+## Tinybench Plugin Architecture
+
+### Current Stats/Measurements Access
+
+The tinybench plugin currently has **limited stats exposure**:
+
+1. **No direct stats API**: The `withCodSpeed()` function wraps a tinybench instance but doesn't expose measurement results
+2. **Console-only output**: Results are only printed to console via `console.log()`
+3. **Core measurement**: Uses `@codspeed/core` for actual measurement via:
+   - `mongoMeasurement.start(uri)` / `mongoMeasurement.stop(uri)`
+   - `Measurement.startInstrumentation()` / `Measurement.stopInstrumentation(uri)`
+
+### Current Workflow
+1. User calls `withCodSpeed(new Bench())` to wrap their tinybench instance
+2. Plugin intercepts `bench.run()` to add CodSpeed instrumentation
+3. Each benchmark task runs with measurement instrumentation
+4. Results are logged to console but not returned as structured data
+
+### Key Functions in tinybench plugin
+- `withCodSpeed(bench: Bench): Bench` - Main wrapper function
+- `setupInstruments(body)` - Dynamic instrument setup
+- `getCallingFile()` - Helper to generate unique URIs for benchmarks
+
+## Potential Enhancement Areas
+
+Based on the codebase analysis, to add stats access features:
+
+1. **Extend return value**: Modify `bench.run()` to return structured measurement data
+2. **Add stats methods**: Add methods like `getStats()`, `getResults()`, `getLastRunStats()`
+3. **Integrate with core**: Leverage `@codspeed/core` measurement data
+4. **Maintain tinybench compatibility**: Ensure existing `bench.table()` still works
+
+## Repository Management Memories
+
+- Use pnpm instead of npm
+- To run tests in a package use moon <package-name>:test

examples/with-javascript-esm/package.json

@@ -10,6 +10,6 @@
     "@codspeed/benchmark.js-plugin": "workspace:*",
     "@codspeed/tinybench-plugin": "workspace:*",
     "benchmark": "^2.1.4",
-    "tinybench": "^2.5.0"
+    "tinybench": "^4.0.1"
   }
 }

examples/with-typescript-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-esm/package.json

@@ -14,7 +14,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3",
     "vitest": "^1.2.2"
   }

examples/with-typescript-simple-cjs/package.json

@@ -11,7 +11,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

examples/with-typescript-simple-esm/package.json

@@ -12,7 +12,7 @@
     "@types/benchmark": "^2.1.2",
     "benchmark": "^2.1.4",
     "esbuild-register": "^3.4.2",
-    "tinybench": "^2.5.0",
+    "tinybench": "^4.0.1",
     "typescript": "^5.1.3"
   }
 }

flake.nix

@@ -0,0 +1,53 @@
+{
+  description = "CodSpeed Node development environment";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs =
+    {
+      self,
+      nixpkgs,
+      flake-utils,
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+        commonBuildInputs = with pkgs; [
+          # Needed for node-gyp
+          (python314.withPackages (
+            ps: with ps; [
+              setuptools
+            ]
+          ))
+          valgrind # TODO: Remove this in favor of codspeed's valgrind
+        ];
+
+      in
+      {
+        devShells = {
+          default = pkgs.mkShell {
+            buildInputs = commonBuildInputs;
+            shellHook = ''
+              echo "CodSpeed Node development environment"
+            '';
+          };
+
+          lsp = pkgs.mkShell {
+            buildInputs =
+              with pkgs;
+              [
+                typescript-language-server
+              ]
+              ++ commonBuildInputs;
+            shellHook = ''
+              echo "CodSpeed Node development environment with LSP"
+            '';
+          };
+        };
+      }
+    );
+}