diff --git a/.github/workflows/time.yml b/.github/workflows/time.yml new file mode 100644 index 000000000..6e6427cef --- /dev/null +++ b/.github/workflows/time.yml @@ -0,0 +1,51 @@ +name: Time (CLI utility) Example + +on: + push: + branches: + - master + pull_request: + branches: + - master + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run Some silly benchmark under `time` + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Run some silly benchmark with time + run: (time (head --bytes=1000000000 /dev/random >/dev/null)) 2>&1 | tee output.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Time Benchmark + tool: 'time' + output-file-path: output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz','@henrikingo' + + - name: Store benchmark result - separate results repo + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Time Benchmark + tool: 'time' + output-file-path: output.txt + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz','@henrikingo' + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' diff --git a/README.md b/README.md index 7dacb6c4d..39dd6b27c 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This action currently supports the following tools: - [Benchmark.Net][benchmarkdotnet] for .Net projects - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects - [JMH][jmh] for Java projects +- [time][time-cli-util] CLI utility to measure runtime time of any Linux executable - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter' Multiple languages in the same repository are supported for polyglot projects. @@ -51,6 +52,7 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor | .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) | | Java | [![Java Example Workflow][java-badge]][java-workflow-example] | [examples/java](./examples/java) | | Luau | Coming soon | Coming soon | +| Time (CLI utility) | [`time`][time-workflow-example] All benchmark charts from above workflows are gathered in GitHub pages: @@ -651,6 +653,7 @@ Every release will appear on your GitHub notifications page. [examples-page]: https://benchmark-action.github.io/github-action-benchmark/dev/bench/ [pytest-benchmark]: https://pypi.org/project/pytest-benchmark/ [pytest]: https://pypi.org/project/pytest/ +[time-cli-util]: https://man7.org/linux/man-pages/man1/time.1.html [alert-comment-example]: https://github.com/benchmark-action/github-action-benchmark/commit/077dde1c236baba9244caad4d9e82ea8399dae20#commitcomment-36047186 [rust-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22 [go-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Go+Example%22 @@ -660,6 +663,7 @@ Every release will appear on your GitHub notifications page. [catch2-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Catch2+C%2B%2B+Example%22 [julia-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Julia+Example+with+BenchmarkTools.jl%22 [java-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22JMH+Example%22 +[time-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22time+Example%22 [help-watch-release]: https://docs.github.com/en/github/receiving-notifications-about-activity-on-github/watching-and-unwatching-releases-for-a-repository [help-github-token]: https://docs.github.com/en/actions/security-guides/automatic-token-authentication [minimal-workflow-example]: https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Example+for+minimal+setup%22 diff --git a/action-types.yml b/action-types.yml index e199a6627..daf8c04a9 100644 --- a/action-types.yml +++ b/action-types.yml @@ -14,6 +14,7 @@ inputs: - jmh - benchmarkdotnet - benchmarkluau + - time - customBiggerIsBetter - customSmallerIsBetter output-file-path: diff --git a/action.yml b/action.yml index daa6cd343..a76d8934d 100644 --- a/action.yml +++ b/action.yml @@ -11,7 +11,7 @@ inputs: required: true default: 'Benchmark' tool: - description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "customBiggerIsBetter", "customSmallerIsBetter"' + description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest", "googlecpp", "catch2", "julia", "benchmarkdotnet", "time", "customBiggerIsBetter", "customSmallerIsBetter"' required: true output-file-path: description: 'A path to file which contains the benchmark output' diff --git a/examples/time/README.md b/examples/time/README.md new file mode 100644 index 000000000..712bed986 --- /dev/null +++ b/examples/time/README.md @@ -0,0 +1,41 @@ +Example for parsing output of the `time` CLI utility +==================================================== + +- [Workflow for this example](../../.github/workflows/time.yml) +- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/actions?query=workflow%3A%22Rust+Example%22) +- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) + +This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) +with [`cargo bench`](https://doc.rust-lang.org/cargo/commands/cargo-bench.html). + +## Run benchmarks + +Official documentation for usage of `time`: + +https://man7.org/linux/man-pages/man1/time.1.html + +So to run a benchmark, simply call time first: + +```yaml +- name: Measure execution time with `time` + run: (time (head --bytes=1000000000 /dev/random >/dev/null)) 2>&1 | tee output.txt +``` + +Note the `2>&1` too capture also stderr output. By default that's where `time` will direct its output. + + +## Process benchmark results + +Store the benchmark results with step using the action. Please set `cargo` to `tool` input. + +```yaml +- name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'time' + output-file-path: output.txt +``` + +Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. + + diff --git a/src/config.ts b/src/config.ts index 0ce3bd5c0..837f881b3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -38,6 +38,7 @@ export const VALID_TOOLS = [ 'julia', 'jmh', 'benchmarkdotnet', + 'time', 'customBiggerIsBetter', 'customSmallerIsBetter', ] as const; diff --git a/src/extract.ts b/src/extract.ts index ec7bc35e2..470f7974e 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -2,6 +2,7 @@ import { promises as fs } from 'fs'; import * as github from '@actions/github'; import { Config, ToolType } from './config'; +import * as core from '@actions/core'; export interface BenchmarkResult { name: string; @@ -690,6 +691,93 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] { return results; } +function parseTimeOutput(line: string): number | undefined { + core.debug(`parse time ${line}`); + const t = line.split('\t')[1]; + if (t === undefined) return; + const tparts = t.split('m'); + + core.debug(tparts[0]); + if (tparts[1] === undefined || !tparts[1].endsWith('s')) return; + return parseFloat(tparts[0]) * 60.0 + parseFloat(tparts[1].substring(-1)); +} +/* Expect to process text with the following structure: + + $ time echo Hello there + Hello there + + real 0m0,100s + user 0m0,020s + sys 0m0,003s + $ time echo Hello there + Hello again + + real 0m0,100s + user 0m0,020s + sys 0m0,003s + + Into: + [{"name": "real", + "value": 0,1, + "unit": "s", + "extra": "testName: Hello there"}, + ... +*/ +function extractTimeBenchmarkResult(output: string): BenchmarkResult[] { + const lines = output.split(/\n/); + const results: BenchmarkResult[] = []; + let firstline = true; + let name: string | undefined = undefined; + + for (const line of lines) { + core.debug(line); + if (firstline) { + name = line; + firstline = false; + core.debug(`firstline: ${name}`); + continue; + } + if (line.startsWith('real')) { + const v = parseTimeOutput(line); + core.debug(`v: ${v}`); + if (v !== undefined) + results.push({ + extra: name, + name: 'real', + value: v, + unit: 's', + }); + } + if (line.startsWith('user')) { + const v = parseTimeOutput(line); + + core.debug(`v: ${v}`); + if (v !== undefined) + results.push({ + extra: name, + name: 'user', + value: v, + unit: 's', + }); + } + if (line.startsWith('sys')) { + const v = parseTimeOutput(line); + core.debug(`v: ${v}`); + if (v !== undefined) + results.push({ + extra: name, + name: 'sys', + value: v, + unit: 's', + }); + firstline = true; + } + core.debug('loop'); + } + + return results; +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool, githubToken, ref } = config; @@ -723,6 +811,9 @@ export async function extractResult(config: Config): Promise { case 'benchmarkdotnet': benches = extractBenchmarkDotnetResult(output); break; + case 'time': + benches = extractTimeBenchmarkResult(output); + break; case 'customBiggerIsBetter': benches = extractCustomBenchmarkResult(output); break; diff --git a/src/write.ts b/src/write.ts index e1ad4eafd..42dfe9501 100644 --- a/src/write.ts +++ b/src/write.ts @@ -81,6 +81,8 @@ function biggerIsBetter(tool: ToolType): boolean { return false; case 'benchmarkdotnet': return false; + case 'time': + return false; case 'customBiggerIsBetter': return true; case 'customSmallerIsBetter': diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index 76d1e91a9..61afea752 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -968,3 +968,56 @@ rounds: 5", "tool": "pytest", } `; + +exports[`extractResult() extracts benchmark output from time - time_output.txt 1`] = ` +{ + "benches": [ + { + "extra": "Hello there", + "name": "real", + "unit": "s", + "value": 0.1, + }, + { + "extra": "Hello there", + "name": "user", + "unit": "s", + "value": 0.02, + }, + { + "extra": "Hello there", + "name": "sys", + "unit": "s", + "value": 0.003, + }, + { + "extra": "Hello again", + "name": "real", + "unit": "s", + "value": 0.1, + }, + { + "extra": "Hello again", + "name": "user", + "unit": "s", + "value": 0.02, + }, + { + "extra": "Hello again", + "name": "sys", + "unit": "s", + "value": 0.003, + }, + ], + "commit": { + "author": null, + "committer": null, + "id": "123456789abcdef", + "message": "this is dummy", + "timestamp": "dummy timestamp", + "url": "https://github.com/dummy/repo", + }, + "date": 1712131503296, + "tool": "time", +} +`; diff --git a/test/data/extract/time_output.txt b/test/data/extract/time_output.txt new file mode 100644 index 000000000..c66672ae3 --- /dev/null +++ b/test/data/extract/time_output.txt @@ -0,0 +1,10 @@ +Hello there + +real 0m0.100s +user 0m0.020s +sys 0m0.003s +Hello again + +real 0m0.100s +user 0m0.020s +sys 0m0.003s diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 48a5c271a..c82a972ce 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -135,6 +135,10 @@ describe('extractResult()', function () { tool: 'benchmarkdotnet', file: 'benchmarkdotnet.json', }, + { + tool: 'time', + file: 'time_output.txt', + }, { tool: 'customBiggerIsBetter', file: 'customBiggerIsBetter_output.json',