Add time CLI utility to supported benchmark tools

.github/workflows/time.yml

@@ -0,0 +1,51 @@
+name: Time (CLI utility) Example
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Run Some silly benchmark under `time`
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run some silly benchmark with time
+        run: (time (head --bytes=1000000000 /dev/random  >/dev/null)) 2>&1 | tee output.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Time Benchmark
+          tool: 'time'
+          output-file-path: output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@ktrz','@henrikingo'
+
+      - name: Store benchmark result - separate results repo
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Time Benchmark
+          tool: 'time'
+          output-file-path: output.txt
+          github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: '@ktrz','@henrikingo'
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'

README.md

@@ -26,6 +26,7 @@ This action currently supports the following tools:
 - [Benchmark.Net][benchmarkdotnet] for .Net projects
 - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects
 - [JMH][jmh] for Java projects
+- [time][time-cli-util] CLI utility to measure runtime time of any Linux executable
 - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter'
 
 Multiple languages in the same repository are supported for polyglot projects.
@@ -51,6 +52,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor
 | .Net         | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) |
 | Java         | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) |
 | Luau         | Coming soon | Coming soon |
+| Time (CLI utility) | [`time`][time-workflow-example]
 
 All benchmark charts from above workflows are gathered in GitHub pages:
 
@@ -651,6 +653,7 @@ Every release will appear on your GitHub notifications page.
 [examples-page]: https://benchmark-action.github.io/github-action-benchmark/dev/bench/
 [pytest-benchmark]: https://pypi.org/project/pytest-benchmark/
 [pytest]: https://pypi.org/project/pytest/
+[time-cli-util]: https://man7.org/linux/man-pages/man1/time.1.html
 [alert-comment-example]: https://github.com/benchmark-action/github-action-benchmark/commit/077dde1c236baba9244caad4d9e82ea8399dae20#commitcomment-36047186
 [rust-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22
 [go-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Go+Example%22
@@ -660,6 +663,7 @@ Every release will appear on your GitHub notifications page.
 [catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22
 [julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22
 [java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22
+[time-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22time+Example%22
 [help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository
 [help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication
 [minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22

action-types.yml

@@ -14,6 +14,7 @@ inputs:
       - jmh
       - benchmarkdotnet
       - benchmarkluau
+      - time
       - customBiggerIsBetter
       - customSmallerIsBetter
   output-file-path:

action.yml

@@ -11,7 +11,7 @@ inputs:
     required: true
     default: 'Benchmark'
   tool:
-    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "customBiggerIsBetter", "customSmallerIsBetter"'
+    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "time", "customBiggerIsBetter", "customSmallerIsBetter"'
     required: true
   output-file-path:
     description: 'A path to file which contains the benchmark output'

examples/time/README.md

@@ -0,0 +1,41 @@
+Example for parsing output of the `time` CLI utility
+====================================================
+
+- [Workflow for this example](../../.github/workflows/time.yml)
+- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22)
+- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/)
+
+This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark)
+with [`cargo bench`](https://doc.rust-lang.org/cargo/commands/cargo-bench.html).
+
+## Run benchmarks
+
+Official documentation for usage of `time`:
+
+https://man7.org/linux/man-pages/man1/time.1.html
+
+So to run a benchmark, simply call time first:
+
+```yaml
+- name: Measure execution time with `time`
+  run: (time (head --bytes=1000000000 /dev/random  >/dev/null)) 2>&1 | tee output.txt
+```
+
+Note the `2>&1` too capture also stderr output. By default that's where `time` will direct its output.
+
+
+## Process benchmark results
+
+Store the benchmark results with step using the action. Please set `cargo` to `tool` input.
+
+```yaml
+- name: Store benchmark result
+  uses: benchmark-action/github-action-benchmark@v1
+  with:
+    tool: 'time'
+    output-file-path: output.txt
+```
+
+Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage.
+
+

src/config.ts

@@ -38,6 +38,7 @@ export const VALID_TOOLS = [
     'julia',
     'jmh',
     'benchmarkdotnet',
+    'time',
     'customBiggerIsBetter',
     'customSmallerIsBetter',
 ] as const;

src/extract.ts

@@ -2,6 +2,7 @@
 import { promises as fs } from 'fs';
 import * as github from '@actions/github';
 import { Config, ToolType } from './config';
+import * as core from '@actions/core';
 
 export interface BenchmarkResult {
     name: string;
@@ -690,6 +691,93 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] {
     return results;
 }
 
+function parseTimeOutput(line: string): number | undefined {
+    core.debug(`parse time ${line}`);
+    const t = line.split('\t')[1];
+    if (t === undefined) return;
+    const tparts = t.split('m');
+
+    core.debug(tparts[0]);
+    if (tparts[1] === undefined || !tparts[1].endsWith('s')) return;
+    return parseFloat(tparts[0]) * 60.0 + parseFloat(tparts[1].substring(-1));
+}
+/* Expect to process text with the following structure:
+
+        $ time echo Hello there
+        Hello there
+
+        real    0m0,100s
+        user    0m0,020s
+        sys     0m0,003s
+        $ time echo Hello there
+        Hello again
+
+        real    0m0,100s
+        user    0m0,020s
+        sys     0m0,003s
+
+ Into:
+        [{"name": "real",
+         "value": 0,1,
+         "unit": "s",
+         "extra": "testName: Hello there"},
+         ...
+*/
+function extractTimeBenchmarkResult(output: string): BenchmarkResult[] {
+    const lines = output.split(/\n/);
+    const results: BenchmarkResult[] = [];
+    let firstline = true;
+    let name: string | undefined = undefined;
+
+    for (const line of lines) {
+        core.debug(line);
+        if (firstline) {
+            name = line;
+            firstline = false;
+            core.debug(`firstline: ${name}`);
+            continue;
+        }
+        if (line.startsWith('real')) {
+            const v = parseTimeOutput(line);
+            core.debug(`v: ${v}`);
+            if (v !== undefined)
+                results.push({
+                    extra: name,
+                    name: 'real',
+                    value: v,
+                    unit: 's',
+                });
+        }
+        if (line.startsWith('user')) {
+            const v = parseTimeOutput(line);
+
+            core.debug(`v: ${v}`);
+            if (v !== undefined)
+                results.push({
+                    extra: name,
+                    name: 'user',
+                    value: v,
+                    unit: 's',
+                });
+        }
+        if (line.startsWith('sys')) {
+            const v = parseTimeOutput(line);
+            core.debug(`v: ${v}`);
+            if (v !== undefined)
+                results.push({
+                    extra: name,
+                    name: 'sys',
+                    value: v,
+                    unit: 's',
+                });
+            firstline = true;
+        }
+        core.debug('loop');
+    }
+
+    return results;
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool, githubToken, ref } = config;
@@ -723,6 +811,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'benchmarkdotnet':
             benches = extractBenchmarkDotnetResult(output);
             break;
+        case 'time':
+            benches = extractTimeBenchmarkResult(output);
+            break;
         case 'customBiggerIsBetter':
             benches = extractCustomBenchmarkResult(output);
             break;

src/write.ts

@@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'benchmarkdotnet':
             return false;
+        case 'time':
+            return false;
         case 'customBiggerIsBetter':
             return true;
         case 'customSmallerIsBetter':

test/__snapshots__/extract.spec.ts.snap

@@ -968,3 +968,56 @@ rounds: 5",
   "tool": "pytest",
 }
 `;
+
+exports[`extractResult() extracts benchmark output from time - time_output.txt 1`] = `
+{
+  "benches": [
+    {
+      "extra": "Hello there",
+      "name": "real",
+      "unit": "s",
+      "value": 0.1,
+    },
+    {
+      "extra": "Hello there",
+      "name": "user",
+      "unit": "s",
+      "value": 0.02,
+    },
+    {
+      "extra": "Hello there",
+      "name": "sys",
+      "unit": "s",
+      "value": 0.003,
+    },
+    {
+      "extra": "Hello again",
+      "name": "real",
+      "unit": "s",
+      "value": 0.1,
+    },
+    {
+      "extra": "Hello again",
+      "name": "user",
+      "unit": "s",
+      "value": 0.02,
+    },
+    {
+      "extra": "Hello again",
+      "name": "sys",
+      "unit": "s",
+      "value": 0.003,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "time",
+}
+`;

test/data/extract/time_output.txt

@@ -0,0 +1,10 @@
+Hello there
+
+real	0m0.100s
+user	0m0.020s
+sys	0m0.003s
+Hello again
+
+real	0m0.100s
+user	0m0.020s
+sys	0m0.003s

test/extract.spec.ts

@@ -135,6 +135,10 @@ describe('extractResult()', function () {
             tool: 'benchmarkdotnet',
             file: 'benchmarkdotnet.json',
         },
+        {
+            tool: 'time',
+            file: 'time_output.txt',
+        },
         {
             tool: 'customBiggerIsBetter',
             file: 'customBiggerIsBetter_output.json',