refactor: switch benchmarks to tinybench

Signed-off-by: Jérôme Benoit <[email protected]>

Author: jerome-benoit

.github/workflows/internal-benchmark.yml

@@ -43,4 +43,4 @@ jobs:
           --file benchmark-report.json \
           --err \
           --github-actions ${{ secrets.GITHUB_TOKEN }} \
-          "deno task benchmark:tatami-ng"
+          "deno task benchmark:tinybench"

.serena/memories/suggested_commands.md

@@ -2,7 +2,7 @@
 
 ## Development Commands
 
-- `deno task test` - Run all tests
+- `deno task test` - Run tests
 - `deno task test:parallel` - Run tests in parallel
 - `deno task test:coverage` - Run tests with coverage
 - `deno test -A tests/path/to/specific.test.mjs` - Run single test file
@@ -16,8 +16,9 @@
 ## Benchmarking Commands
 
 - `deno task benchmark:deno` - Run Deno benchmarks
-- `deno task benchmark:tatami-ng` - Run Tatami-ng benchmarks
-- `deno task benchmark:tatami-ng:debug` - Run benchmarks with debugging
+- `deno task benchmark:tinybench` - Run Tinybench benchmarks
+- `deno task benchmark:tinybench:debug` - Run Tinybench benchmarks with
+  debugging
 
 ## Documentation
 

.vscode/launch.json

@@ -17,10 +17,10 @@
     {
       "type": "node",
       "request": "launch",
-      "name": "Launch Tatami NG Benchmark Debug",
+      "name": "Launch Tinybench Benchmark Debug",
       "cwd": "${workspaceFolder}",
       "runtimeExecutable": "deno",
-      "runtimeArgs": ["task", "benchmark:tatami-ng:debug"],
+      "runtimeArgs": ["task", "benchmark:tinybench:debug"],
       "skipFiles": ["<node_internals>/**"],
       "stopOnEntry": true
     }

benchmarks/README.md

@@ -30,11 +30,11 @@ poolifier cloned repository and run:
 
 #### Deno
 
-- `deno task benchmark:tatami-ng` or
+- `deno task benchmark:tinybench` or
 - `deno task benchmark:deno`
 
 #### Bun
 
-- `bun ./benchmarks/internal/bench.mjs -t tatami-ng`
+- `bun ./benchmarks/internal/bench.mjs -t tinybench`
 
 ### [Results](https://bencher.dev/perf/poolifier-web-worker)

benchmarks/benchmarks-utils.mjs

@@ -7,7 +7,7 @@ import {
   rmSync,
   writeFileSync,
 } from 'node:fs'
-import { bench, group, run } from '@poolifier/tatami-ng'
+import { Bench } from 'tinybench'
 import {
   DynamicThreadPool,
   FixedThreadPool,
@@ -144,87 +144,6 @@ export const runPoolifierBenchmarkDenoBench = (
   }
 }
 
-export const runPoolifierBenchmarkTatamiNg = async (
-  name,
-  workerType,
-  poolType,
-  poolSize,
-  benchmarkReporter,
-  { taskExecutions, workerData },
-) => {
-  try {
-    const pool = buildPoolifierPool(workerType, poolType, poolSize)
-    for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) {
-      for (const enableTasksQueue of [false, true]) {
-        if (workerChoiceStrategy === WorkerChoiceStrategies.FAIR_SHARE) {
-          for (const measurement of [Measurements.runTime]) {
-            group(name, () => {
-              bench(
-                `${name} with ${workerChoiceStrategy}, with ${measurement} and ${
-                  enableTasksQueue ? 'with' : 'without'
-                } tasks queue`,
-                async () => {
-                  await runPoolifierPool(pool, {
-                    taskExecutions,
-                    workerData,
-                  })
-                },
-                {
-                  before: () => {
-                    pool.setWorkerChoiceStrategy(workerChoiceStrategy, {
-                      measurement,
-                    })
-                    pool.enableTasksQueue(enableTasksQueue)
-                    strictEqual(
-                      pool.opts.workerChoiceStrategy,
-                      workerChoiceStrategy,
-                    )
-                    strictEqual(pool.opts.enableTasksQueue, enableTasksQueue)
-                    strictEqual(
-                      pool.opts.workerChoiceStrategyOptions.measurement,
-                      measurement,
-                    )
-                  },
-                },
-              )
-            })
-          }
-        } else {
-          group(name, () => {
-            bench(
-              `${name} with ${workerChoiceStrategy} and ${
-                enableTasksQueue ? 'with' : 'without'
-              } tasks queue`,
-              async () => {
-                await runPoolifierPool(pool, {
-                  taskExecutions,
-                  workerData,
-                })
-              },
-              {
-                before: () => {
-                  pool.setWorkerChoiceStrategy(workerChoiceStrategy)
-                  pool.enableTasksQueue(enableTasksQueue)
-                  strictEqual(
-                    pool.opts.workerChoiceStrategy,
-                    workerChoiceStrategy,
-                  )
-                  strictEqual(pool.opts.enableTasksQueue, enableTasksQueue)
-                },
-              },
-            )
-          })
-        }
-      }
-    }
-    const report = await run({ reporter: benchmarkReporter })
-    await pool.destroy()
-    return report
-  } catch (error) {
-    console.error(error)
-  }
-}
-
 const jsonIntegerSerialization = (n) => {
   for (let i = 0; i < n; i++) {
     const o = {
@@ -330,3 +249,104 @@ export const runtime = (() => {
   if (isBrowser) return JSRuntime.browser
   throw new Error('Unsupported JavaScript runtime environment')
 })()
+
+export const runPoolifierBenchmarkTinyBench = async (
+  name,
+  workerType,
+  poolType,
+  poolSize,
+  { taskExecutions, workerData },
+) => {
+  try {
+    const bench = new Bench()
+    const pool = buildPoolifierPool(workerType, poolType, poolSize)
+
+    for (const workerChoiceStrategy of Object.values(WorkerChoiceStrategies)) {
+      for (const enableTasksQueue of [false, true]) {
+        if (workerChoiceStrategy === WorkerChoiceStrategies.FAIR_SHARE) {
+          for (const measurement of [Measurements.runTime]) {
+            const taskName =
+              `${name} with ${workerChoiceStrategy}, with ${measurement} and ${
+                enableTasksQueue ? 'with' : 'without'
+              } tasks queue`
+
+            bench.add(
+              taskName,
+              async () => {
+                await runPoolifierPool(pool, {
+                  taskExecutions,
+                  workerData,
+                })
+              },
+              {
+                beforeAll: () => {
+                  pool.setWorkerChoiceStrategy(workerChoiceStrategy, {
+                    measurement,
+                  })
+                  pool.enableTasksQueue(enableTasksQueue)
+                  strictEqual(
+                    pool.opts.workerChoiceStrategy,
+                    workerChoiceStrategy,
+                  )
+                  strictEqual(pool.opts.enableTasksQueue, enableTasksQueue)
+                  strictEqual(
+                    pool.opts.workerChoiceStrategyOptions.measurement,
+                    measurement,
+                  )
+                },
+              },
+            )
+          }
+        } else {
+          const taskName = `${name} with ${workerChoiceStrategy} and ${
+            enableTasksQueue ? 'with' : 'without'
+          } tasks queue`
+
+          bench.add(
+            taskName,
+            async () => {
+              await runPoolifierPool(pool, {
+                taskExecutions,
+                workerData,
+              })
+            },
+            {
+              beforeAll: () => {
+                pool.setWorkerChoiceStrategy(workerChoiceStrategy)
+                pool.enableTasksQueue(enableTasksQueue)
+                strictEqual(
+                  pool.opts.workerChoiceStrategy,
+                  workerChoiceStrategy,
+                )
+                strictEqual(pool.opts.enableTasksQueue, enableTasksQueue)
+              },
+            },
+          )
+        }
+      }
+    }
+
+    const tasks = await bench.run()
+    console.table(bench.table())
+    await pool.destroy()
+
+    const bmfResults = {}
+    for (const task of tasks) {
+      bmfResults[task.name] = {
+        latency: {
+          value: task.result.latency.mean,
+          lower_value: task.result.latency.mean - task.result.latency.sd,
+          upper_value: task.result.latency.mean + task.result.latency.sd,
+        },
+        throughput: {
+          value: task.result.throughput.mean,
+          lower_value: task.result.throughput.mean - task.result.throughput.sd,
+          upper_value: task.result.throughput.mean + task.result.throughput.sd,
+        },
+      }
+    }
+    return bmfResults
+  } catch (error) {
+    console.error(error)
+  }
+}

benchmarks/internal/bench.mjs

@@ -1,9 +1,8 @@
-import { bmf } from '@poolifier/tatami-ng'
 import { availableParallelism, PoolTypes, WorkerTypes } from '../../src/mod.ts'
 import { TaskFunctions } from '../benchmarks-types.mjs'
 import {
   runPoolifierBenchmarkDenoBench,
-  runPoolifierBenchmarkTatamiNg,
+  runPoolifierBenchmarkTinyBench,
   runtime,
 } from '../benchmarks-utils.mjs'
 
@@ -26,26 +25,24 @@ const runBenchmark = async () => {
     deno: async () => {
       const { parseArgs } = await import('@std/cli/parse-args')
       switch (parseArgs(Deno.args).t) {
-        case 'tatami-ng':
-          benchmarkReport = await runPoolifierBenchmarkTatamiNg(
+        case 'tinybench':
+          benchmarkReport = await runPoolifierBenchmarkTinyBench(
             fixedThreadPoolGroupname,
             WorkerTypes.web,
             PoolTypes.fixed,
             poolSize,
-            bmf,
             {
               taskExecutions,
               workerData,
             },
           )
           benchmarkReport = {
             ...benchmarkReport,
-            ...(await runPoolifierBenchmarkTatamiNg(
+            ...(await runPoolifierBenchmarkTinyBench(
               dynamicThreadPoolGroupname,
               WorkerTypes.web,
               PoolTypes.dynamic,
               poolSize,
-              bmf,
               {
                 taskExecutions,
                 workerData,
@@ -97,27 +94,25 @@ const runBenchmark = async () => {
           allowPositionals: true,
         }).values.type
       ) {
-        case 'tatami-ng':
+        case 'tinybench':
         default:
-          benchmarkReport = await runPoolifierBenchmarkTatamiNg(
+          benchmarkReport = await runPoolifierBenchmarkTinyBench(
             fixedThreadPoolGroupname,
             WorkerTypes.web,
             PoolTypes.fixed,
             poolSize,
-            bmf,
             {
               taskExecutions,
               workerData,
             },
           )
           benchmarkReport = {
             ...benchmarkReport,
-            ...(await runPoolifierBenchmarkTatamiNg(
+            ...(await runPoolifierBenchmarkTinyBench(
               dynamicThreadPoolGroupname,
               WorkerTypes.web,
               PoolTypes.dynamic,
               poolSize,
-              bmf,
               {
                 taskExecutions,
                 workerData,

deno.json

@@ -9,8 +9,8 @@
   "tasks": {
     "benchmark:deno": "deno bench -A ./benchmarks/internal/bench.mjs",
     "benchmark:deno:json": "deno bench -A --json ./benchmarks/internal/bench.mjs",
-    "benchmark:tatami-ng": "deno run -A ./benchmarks/internal/bench.mjs -t tatami-ng",
-    "benchmark:tatami-ng:debug": "deno run -A --inspect ./benchmarks/internal/bench.mjs -t tatami-ng",
+    "benchmark:tinybench": "deno run -A ./benchmarks/internal/bench.mjs -t tinybench",
+    "benchmark:tinybench:debug": "deno run -A --inspect ./benchmarks/internal/bench.mjs -t tinybench",
     "bundle": "deno run -A bundle.ts",
     "test": "ENVIRONMENT=test deno test -A",
     "test:parallel": "ENVIRONMENT=test deno test -A --parallel",
@@ -32,8 +32,8 @@
     "singleQuote": true
   },
   "imports": {
-    "@poolifier/tatami-ng": "jsr:@poolifier/tatami-ng@^0.8.18",
-    "@std/cli": "jsr:@std/cli@^1.0.22",
+    "tinybench": "npm:tinybench@^5.1.0",
+    "@std/cli": "jsr:@std/cli@^1.0.23",
     "@std/expect": "jsr:@std/expect@^1.0.17",
     "@std/testing": "jsr:@std/testing@^1.0.15"
   },