fix: improve validation for custom benchmarks

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## Unreleased
 
+- **fix** improve parsing for custom benchmarks (#323)
+
 <a name="v1.20.5"></a>
 # [v1.20.5](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.20.5) - 02 Sep 2025
 - **feat** allow to parse generic cargo bench/criterion units (#280)

package.json

@@ -34,7 +34,8 @@
     "@actions/core": "^1.10.0",
     "@actions/exec": "^1.1.1",
     "@actions/github": "^5.1.1",
-    "@actions/io": "^1.1.2"
+    "@actions/io": "^1.1.2",
+    "zod": "^4.1.5"
   },
   "devDependencies": {
     "@types/acorn": "^4.0.5",
@@ -43,7 +44,7 @@
     "@types/deep-equal": "^1.0.1",
     "@types/jest": "^29.5.11",
     "@types/markdown-it": "^12.2.3",
-    "@types/node": "^13.9.1",
+    "@types/node": "^24.3.1",
     "@types/rimraf": "^2.0.3",
     "@typescript-eslint/eslint-plugin": "^5.4.0",
     "@typescript-eslint/parser": "^5.4.0",
@@ -61,6 +62,6 @@
     "prettier": "^2.4.1",
     "rimraf": "^3.0.2",
     "ts-jest": "^29.1.2",
-    "typescript": "^4.5.2"
+    "typescript": "^5.9.2"
   }
 }

src/extract.ts

@@ -2,14 +2,19 @@
 import { promises as fs } from 'fs';
 import * as github from '@actions/github';
 import { Config, ToolType } from './config';
+import { z } from 'zod';
 
-export interface BenchmarkResult {
-    name: string;
-    value: number;
-    range?: string;
-    unit: string;
-    extra?: string;
-}
+export const BenchmarkResult = z.object({
+    name: z.coerce.string(),
+    value: z.coerce.number(),
+    range: z.coerce.string().optional(),
+    unit: z.coerce.string(),
+    extra: z.coerce.string().optional(),
+});
+
+export type BenchmarkResult = z.infer<typeof BenchmarkResult>;
+
+export const BenchmarkResults = z.array(BenchmarkResult);
 
 interface GitHubUser {
     email?: string;
@@ -29,7 +34,7 @@
 }
 
 interface PullRequest {
     [key: string]: any;
     number: number;
     html_url?: string;
     body?: string;
@@ -436,7 +441,7 @@
             const extra = `mean: ${mean} ${meanUnit}\nrounds: ${stats.rounds}`;
             return { name, value, unit, range, extra };
         });
     } catch (err: any) {
         throw new Error(
             `Output file for 'pytest' must be JSON file generated by --benchmark-json option: ${err.message}`,
         );
@@ -447,7 +452,7 @@
     let json: GoogleCppBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'googlecpp' must be JSON file generated by --benchmark_format=json option: ${err.message}`,
         );
@@ -575,7 +580,7 @@
     return ret;
 }
 
 function extractJuliaBenchmarkHelper([_, bench]: JuliaBenchmarkGroup, labels: string[] = []): BenchmarkResult[] {
     const res: BenchmarkResult[] = [];
     for (const key in bench.data) {
         const value = bench.data[key];
@@ -606,7 +611,7 @@
     let json: JuliaBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'julia' must be JSON file generated by BenchmarkTools.save("output.json", suit::BenchmarkGroup) :  ${err.message}`,
         );
@@ -624,7 +629,7 @@
     let json: JmhBenchmarkJson[];
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(`Output file for 'jmh' must be JSON file generated by -rf json option: ${err.message}`);
     }
     return json.map((b) => {
@@ -641,7 +646,7 @@
     let json: BenchmarkDotNetBenchmarkJson;
     try {
         json = JSON.parse(output);
     } catch (err: any) {
         throw new Error(
             `Output file for 'benchmarkdotnet' must be JSON file generated by '--exporters json' option or by adding the JsonExporter to your run config: ${err.message}`,
         );
@@ -658,13 +663,11 @@
 
 function extractCustomBenchmarkResult(output: string): BenchmarkResult[] {
     try {
-        const json: BenchmarkResult[] = JSON.parse(output);
-        return json.map(({ name, value, unit, range, extra }) => {
-            return { name, value, unit, range, extra };
-        });
-    } catch (err: any) {
+        return BenchmarkResults.parse(JSON.parse(output));
+    } catch (err: unknown) {
+        const errMessage = err instanceof Error ? err.message : String(err);
         throw new Error(
-            `Output file for 'custom-(bigger|smaller)-is-better' must be JSON file containing an array of entries in BenchmarkResult format: ${err.message}`,
+            `Output file for 'custom-(bigger|smaller)-is-better' must be JSON file containing an array of entries in BenchmarkResult format: ${errMessage}`,
         );
     }
 }
@@ -673,7 +676,6 @@
     const lines = output.split(/\n/);
     const results: BenchmarkResult[] = [];
 
-    output;
     for (const line of lines) {
         if (!line.startsWith('SUCCESS')) continue;
         const [_0, name, _2, valueStr, _4, range, _6, extra] = line.split(/\s+/);

test/__snapshots__/extract.spec.ts.snap

@@ -427,9 +427,7 @@ exports[`extractResult() extracts benchmark output from customBiggerIsBetter - c
 {
   "benches": [
     {
-      "extra": undefined,
       "name": "My Custom Bigger Is Better Benchmark - Throughput",
-      "range": undefined,
       "unit": "req/s",
       "value": 70,
     },
@@ -467,9 +465,7 @@ exports[`extractResult() extracts benchmark output from customSmallerIsBetter -
       "value": 50,
     },
     {
-      "extra": undefined,
       "name": "My Custom Smaller Is Better Benchmark - Memory Used",
-      "range": undefined,
       "unit": "Megabytes",
       "value": 100,
     },
@@ -487,6 +483,47 @@ exports[`extractResult() extracts benchmark output from customSmallerIsBetter -
 }
 `;
 
+exports[`extractResult() extracts benchmark output from customSmallerIsBetter - customSmallerIsBetter_output2.json 1`] = `
+{
+  "benches": [
+    {
+      "name": "No instrumentation",
+      "range": "0.519165",
+      "unit": "ns",
+      "value": 90.9439,
+    },
+    {
+      "name": "Deactivated probe",
+      "range": "8.21371",
+      "unit": "ns",
+      "value": 445.661,
+    },
+    {
+      "name": "No logging",
+      "range": "10.2008",
+      "unit": "ns",
+      "value": 1847.38,
+    },
+    {
+      "name": "Binary file",
+      "range": "87.1657",
+      "unit": "ns",
+      "value": 3886.75,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "customSmallerIsBetter",
+}
+`;
+
 exports[`extractResult() extracts benchmark output from go - go_fiber_output.txt 1`] = `
 {
   "benches": [

test/data/extract/customSmallerIsBetter_output2.json

@@ -0,0 +1,26 @@
+[
+  {
+    "name": "No instrumentation",
+    "unit": "ns",
+    "value": 90.9439,
+    "range": 0.519165
+  },
+  {
+    "name": "Deactivated probe",
+    "unit": "ns",
+    "value": 445.661,
+    "range": 8.21371
+  },
+  {
+    "name": "No logging",
+    "unit": "ns",
+    "value": 1847.38,
+    "range": 10.2008
+  },
+  {
+    "name": "Binary file",
+    "unit": "ns",
+    "value": 3886.75,
+    "range": 87.1657
+  }
+]

test/extract.spec.ts

@@ -143,6 +143,10 @@ describe('extractResult()', function () {
             tool: 'customSmallerIsBetter',
             file: 'customSmallerIsBetter_output.json',
         },
+        {
+            tool: 'customSmallerIsBetter',
+            file: 'customSmallerIsBetter_output2.json',
+        },
     ];
 
     it.each(normalCases)(`extracts benchmark output from $tool - $file`, async function (test) {