From bca2c71515c64c4a67c743fa50b5eda7176e1c04 Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Fri, 10 Mar 2023 14:17:53 +0700 Subject: [PATCH 1/6] New benchmark workflow --- .github/workflows/new-benchmark.yml | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .github/workflows/new-benchmark.yml diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml new file mode 100644 index 00000000000..c1688776a01 --- /dev/null +++ b/.github/workflows/new-benchmark.yml @@ -0,0 +1,55 @@ +name: New Benchmark + +on: + push: + branches: + - master + +permissions: + # Deployments permission to deploy GitHub pages website + deployments: write + # Contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + new-benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3.3.0 + + - name: Run benchmark + run: | + # TODO run the actual benchmarks + cat << EOF> ./output.json + [ + { + "name": "My Custom Smaller Is Better Benchmark - CPU Load", + "unit": "Percent", + "value": 8 + }, + { + "name": "My Custom Smaller Is Better Benchmark - Memory Used", + "unit": "Megabytes", + "value": 5, + "range": "3", + "extra": "Value for Tooltip: 25\nOptional Num #2: 100\nAnything Else!" + } + ] + EOF + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.16.2 + with: + name: My Project Go Benchmark + tool: 'customSmallerIsBetter' + output-file-path: output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # Workflow will fail when an alert happens + fail-on-alert: true + # Push and deploy GitHub pages branch automatically + auto-push: true + # Enable alert commit comment + comment-on-alert: true + # Mention @rhysd in the commit comment + alert-comment-cc-users: '@zeme-iohk' From cb955de5bdf5afcb15e02ca4385c310584f25c3e Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Fri, 10 Mar 2023 14:35:18 +0700 Subject: [PATCH 2/6] alert threshold --- .github/workflows/new-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml index c1688776a01..727adaa2a86 100644 --- a/.github/workflows/new-benchmark.yml +++ b/.github/workflows/new-benchmark.yml @@ -53,3 +53,4 @@ jobs: comment-on-alert: true # Mention @rhysd in the commit comment alert-comment-cc-users: '@zeme-iohk' + alert-threshold: '110%' From f91ee9cb7d2723a14132ee202378b64260fee609 Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Fri, 10 Mar 2023 14:36:40 +0700 Subject: [PATCH 3/6] upd --- .github/workflows/new-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml index 727adaa2a86..75a9e400caa 100644 --- a/.github/workflows/new-benchmark.yml +++ b/.github/workflows/new-benchmark.yml @@ -51,6 +51,6 @@ jobs: auto-push: true # Enable alert commit comment comment-on-alert: true - # Mention @rhysd in the commit comment + # Mention @zeme-iohk in the commit comment alert-comment-cc-users: '@zeme-iohk' alert-threshold: '110%' From 80a2c2ce9fe4cc40bfa88e88d3d7181894707a4a Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Mon, 13 Mar 2023 20:31:47 +0700 Subject: [PATCH 4/6] Addressed some comments --- .github/workflows/new-benchmark.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml index 75a9e400caa..f54c356f038 100644 --- a/.github/workflows/new-benchmark.yml +++ b/.github/workflows/new-benchmark.yml @@ -1,4 +1,4 @@ -name: New Benchmark +name: Longitudinal Benchmarking on: push: @@ -45,12 +45,13 @@ jobs: tool: 'customSmallerIsBetter' output-file-path: output.json github-token: ${{ secrets.GITHUB_TOKEN }} - # Workflow will fail when an alert happens - fail-on-alert: true # Push and deploy GitHub pages branch automatically auto-push: true # Enable alert commit comment comment-on-alert: true - # Mention @zeme-iohk in the commit comment - alert-comment-cc-users: '@zeme-iohk' - alert-threshold: '110%' + # Mention @input-output-hk/plutus-core in the commit comment + alert-comment-cc-users: '@input-output-hk/plutus-core' + # Percentage value like "110%". + # It is a ratio indicating how worse the current benchmark result is. + # For example, if we now get 110 ns/iter and previously got 100 ns/iter, it gets 110% worse. + alert-threshold: '105%' From ab91a7d014ba76f5e934e2fab2e33c4013aa3006 Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Sat, 18 Mar 2023 18:57:49 +0700 Subject: [PATCH 5/6] WIP --- .github/workflows/longitudinal-benchmark.yml | 75 ++++++++++++++++++++ .github/workflows/new-benchmark.yml | 57 --------------- 2 files changed, 75 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/longitudinal-benchmark.yml delete mode 100644 .github/workflows/new-benchmark.yml diff --git a/.github/workflows/longitudinal-benchmark.yml b/.github/workflows/longitudinal-benchmark.yml new file mode 100644 index 00000000000..c1946951149 --- /dev/null +++ b/.github/workflows/longitudinal-benchmark.yml @@ -0,0 +1,75 @@ +# Longitudinal Benchmarking +# +# This workflow will run the benchmarks defined in the environment variable BENCHMARKS. +# It will collect and aggreate the benchmark output, format it and feed it to github-action-benchmark. +# +# The benchmark charts are live at https://input-output-hk.github.io/plutus/dev/bench +# The benchmark data is available at https://input-output-hk.github.io/plutus/dev/bench/data.js + +name: Longitudinal Benchmarking + +on: + push: + branches: + - master + +permissions: + # Deployments permission to deploy GitHub pages website + deployments: write + # Contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + new-benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3.3.0 + + - name: Run benchmarks + env: + BENCHMARKS: "validation validation-decode" + run: | + for bench in $BENCHMARKS; do + 2>&1 cabal run "$bench" | tee "$bench-output.txt" + done + + read -r -d '' PYTHON_SCRIPT <<- END_SCRIPT + import json + result = [] + for benchmark in "$BENCHMARKS".split(): + with open(f"{benchmark}-output.txt", "r") as file: + name = "" + for line in file.readlines(): + if line.startswith("benchmarking"): + name = line.split()[1] + elif line.startswith("mean"): + parts = line.split() + mean = parts[1] + unit = parts[2] + result.append({ + "name": f"{benchmark}-{name}", + "unit": unit, + "value": float(mean) + }) + with open("output.json", "w") as file: + json.dump(result, file) + END_SCRIPT + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1.16.2 + with: + name: My Project Go Benchmark + tool: 'customSmallerIsBetter' + output-file-path: output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true + # Enable alert commit comment + comment-on-alert: true + # Mention @input-output-hk/plutus-core in the commit comment + alert-comment-cc-users: '@input-output-hk/plutus-core' + # Percentage value like "110%". + # It is a ratio indicating how worse the current benchmark result is. + # For example, if we now get 110 ns/iter and previously got 100 ns/iter, it gets 110% worse. + alert-threshold: '105%' diff --git a/.github/workflows/new-benchmark.yml b/.github/workflows/new-benchmark.yml deleted file mode 100644 index f54c356f038..00000000000 --- a/.github/workflows/new-benchmark.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Longitudinal Benchmarking - -on: - push: - branches: - - master - -permissions: - # Deployments permission to deploy GitHub pages website - deployments: write - # Contents permission to update benchmark contents in gh-pages branch - contents: write - -jobs: - new-benchmark: - name: Performance regression check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3.3.0 - - - name: Run benchmark - run: | - # TODO run the actual benchmarks - cat << EOF> ./output.json - [ - { - "name": "My Custom Smaller Is Better Benchmark - CPU Load", - "unit": "Percent", - "value": 8 - }, - { - "name": "My Custom Smaller Is Better Benchmark - Memory Used", - "unit": "Megabytes", - "value": 5, - "range": "3", - "extra": "Value for Tooltip: 25\nOptional Num #2: 100\nAnything Else!" - } - ] - EOF - - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1.16.2 - with: - name: My Project Go Benchmark - tool: 'customSmallerIsBetter' - output-file-path: output.json - github-token: ${{ secrets.GITHUB_TOKEN }} - # Push and deploy GitHub pages branch automatically - auto-push: true - # Enable alert commit comment - comment-on-alert: true - # Mention @input-output-hk/plutus-core in the commit comment - alert-comment-cc-users: '@input-output-hk/plutus-core' - # Percentage value like "110%". - # It is a ratio indicating how worse the current benchmark result is. - # For example, if we now get 110 ns/iter and previously got 100 ns/iter, it gets 110% worse. - alert-threshold: '105%' From 47006c6fcda5bcbc3f946c842a7d98b990d097b0 Mon Sep 17 00:00:00 2001 From: zeme-iohk Date: Mon, 20 Mar 2023 17:23:21 +0700 Subject: [PATCH 6/6] Applied requested changes --- .github/workflows/longitudinal-benchmark.yml | 32 ++++---------------- scripts/format-benchmark-output.py | 23 ++++++++++++++ 2 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 scripts/format-benchmark-output.py diff --git a/.github/workflows/longitudinal-benchmark.yml b/.github/workflows/longitudinal-benchmark.yml index c1946951149..498e6196963 100644 --- a/.github/workflows/longitudinal-benchmark.yml +++ b/.github/workflows/longitudinal-benchmark.yml @@ -1,4 +1,4 @@ -# Longitudinal Benchmarking +# Longitudinal Benchmarks # # This workflow will run the benchmarks defined in the environment variable BENCHMARKS. # It will collect and aggreate the benchmark output, format it and feed it to github-action-benchmark. @@ -6,7 +6,7 @@ # The benchmark charts are live at https://input-output-hk.github.io/plutus/dev/bench # The benchmark data is available at https://input-output-hk.github.io/plutus/dev/bench/data.js -name: Longitudinal Benchmarking +name: Longitudinal Benchmarks on: push: @@ -20,9 +20,9 @@ permissions: contents: write jobs: - new-benchmark: + longitudinal-benchmarks: name: Performance regression check - runs-on: ubuntu-latest + runs-on: [self-hosted, plutus-benchmark] steps: - uses: actions/checkout@v3.3.0 @@ -33,33 +33,13 @@ jobs: for bench in $BENCHMARKS; do 2>&1 cabal run "$bench" | tee "$bench-output.txt" done + python ./scripts/format-benchmark-output.py - read -r -d '' PYTHON_SCRIPT <<- END_SCRIPT - import json - result = [] - for benchmark in "$BENCHMARKS".split(): - with open(f"{benchmark}-output.txt", "r") as file: - name = "" - for line in file.readlines(): - if line.startswith("benchmarking"): - name = line.split()[1] - elif line.startswith("mean"): - parts = line.split() - mean = parts[1] - unit = parts[2] - result.append({ - "name": f"{benchmark}-{name}", - "unit": unit, - "value": float(mean) - }) - with open("output.json", "w") as file: - json.dump(result, file) - END_SCRIPT - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1.16.2 with: - name: My Project Go Benchmark + name: Plutus Benchmarks tool: 'customSmallerIsBetter' output-file-path: output.json github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/format-benchmark-output.py b/scripts/format-benchmark-output.py new file mode 100644 index 00000000000..0aee2bafe8f --- /dev/null +++ b/scripts/format-benchmark-output.py @@ -0,0 +1,23 @@ +import json +import os + +result = [] + +for benchmark in os.getenv("BENCHMARKS").split(): + with open(f"{benchmark}-output.txt", "r") as file: + name = "" + for line in file.readlines(): + if line.startswith("benchmarking"): + name = line.split()[1] + elif line.startswith("mean"): + parts = line.split() + mean = parts[1] + unit = parts[2] + result.append({ + "name": f"{benchmark}-{name}", + "unit": unit, + "value": float(mean) + }) + +with open("output.json", "w") as file: + json.dump(result, file) \ No newline at end of file