From e0db2d035e8ae2df52b2b8d010ec5f50b95a4053 Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Wed, 15 Dec 2021 19:46:36 -0500 Subject: [PATCH 1/5] Change performance test to auto-commit results to gh-pages branch. --- .github/workflows/CI.yml | 4 +- .github/workflows/perf.yml | 42 +++++------- docs/performance_model.md | 92 -------------------------- e2etest/get-performance-model-table.py | 2 +- 4 files changed, 20 insertions(+), 120 deletions(-) delete mode 100644 docs/performance_model.md diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a5fcfb554b..ea582fa2cc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1033,12 +1033,12 @@ jobs: version: - 'VERSION' - - name: Trigger soaking + - name: Trigger performance test if: steps.filter.outputs.version == 'true' uses: peter-evans/repository-dispatch@v1.1.1 with: token: "${{ secrets.REPO_WRITE_ACCESS_TOKEN }}" - event-type: bump-version + event-type: trigger-perf client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' clean: diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 1e72b4f2b3..806b65663e 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -13,12 +13,9 @@ name: 'Performance test' on: - schedule: - - cron: '0 9 * * 0' # Sunday at 9 am UTC: pst 1 am. - - # we can manually trigger this workflow by using dispatch for debuging + # we can manually trigger this workflow by using dispatch for debugging repository_dispatch: - types: [manual-perf] + types: [trigger-perf] workflow_dispatch: inputs: sha: @@ -135,6 +132,10 @@ jobs: needs: [get-testing-version, run-perf-test] steps: - uses: actions/checkout@v2 + - uses: actions/checkout@v2 + with: + ref: gh-pages + path: gh-pages - name: Setup Python uses: actions/setup-python@v2.1.4 @@ -147,28 +148,19 @@ jobs: with: path: artifacts - - name: Produce performance model table + - name: Produce performance report run: python e2etest/get-performance-model-table.py - - name: Create a new branch - uses: peterjgrainger/action-create-branch@v2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: "perf-test-${{ github.run_id }}" - - - name: Commit to a branch + - name: Copy performance report + run: cp doc/performance-report.md gh-pages/benchmark/report.md + + - name: Copy artifacts + run: rsync -avv artifacts/ gh-pages/benchmark/data + + - name: Commit to gh-pages branch uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: "Update benchmarking" - branch: "perf-test-${{ github.run_id }}" - file_pattern: docs/performance_model.md - - - name: pull-request - uses: repo-sync/pull-request@v2 - with: - source_branch: "perf-test-${{ github.run_id }}" - destination_branch: "main" - github_token: ${{ secrets.GITHUB_TOKEN }} - pr_title: "Update Performance Model" - pr_body: "Generated by performance test workflow [#${{ github.run_number }}](https://github.com/aws-observability/aws-otel-collector/actions/runs/${{ github.run_id }}) using https://github.com/aws-observability/aws-otel-collector/commit/${{ needs.get-testing-version.outputs.commit_id }}." + branch: gh-pages + repository: gh-pages + file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/* diff --git a/docs/performance_model.md b/docs/performance_model.md deleted file mode 100644 index aafd0b0553..0000000000 --- a/docs/performance_model.md +++ /dev/null @@ -1,92 +0,0 @@ -## Performance Report - -**Commit ID:** [6306f81c2fa65cce889149df1c5b775e50775b2a](https://github.com/aws-observability/aws-otel-collector/commit/6306f81c2fa65cce889149df1c5b775e50775b2a) - -**Collection Period:** 10 minutes - -**Testing AMI:** soaking_linux - - -### Metric (TPS: 100) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| otlp | batch | datadog | datadog_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 61.83 | 0.30 | 64.05 | -| otlp | batch | dynatrace | dynatrace_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 58.99 | 0.20 | 59.20 | -| otlp | batch | otlp | otlp_grpc_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 59.28 | 0.20 | 59.99 | -| otlp | batch | otlphttp | otlp_http_exporter_metric_mock | otlp | m5.2xlarge | 0.05 | 58.37 | 0.20 | 58.62 | -| otlp | batch | awsemf, logging | otlp_metric | otlp | m5.2xlarge | 0.04 | 58.90 | 0.20 | 59.05 | -| otlp | batch | signalfx | signalfx_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 58.56 | 0.20 | 59.27 | -| prometheus | | awsprometheusremotewrite | prometheus_mock | prometheus | m5.2xlarge | 0.13 | 71.32 | 0.40 | 72.61 | -| prometheus | | awsprometheusremotewrite | prometheus_static | prometheus | m5.2xlarge | 0.13 | 72.39 | 0.40 | 73.61 | -| statsd | | awsemf, logging | statsd | statsd | m5.2xlarge | 0.67 | 66.82 | 0.90 | 67.43 | -| statsd | | otlphttp | statsd_mock | statsd | m5.2xlarge | 0.06 | 58.88 | 0.20 | 59.15 | - -### Metric (TPS: 1000) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| otlp | batch | datadog | datadog_exporter_metric_mock | otlp | m5.2xlarge | 0.05 | 61.11 | 0.40 | 63.34 | -| otlp | batch | dynatrace | dynatrace_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 58.56 | 0.20 | 58.74 | -| otlp | batch | otlp | otlp_grpc_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 59.30 | 0.20 | 59.68 | -| otlp | batch | otlphttp | otlp_http_exporter_metric_mock | otlp | m5.2xlarge | 0.05 | 59.38 | 0.20 | 59.86 | -| otlp | batch | awsemf, logging | otlp_metric | otlp | m5.2xlarge | 0.05 | 58.15 | 0.20 | 58.25 | -| otlp | batch | signalfx | signalfx_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 59.20 | 0.20 | 59.60 | -| prometheus | | awsprometheusremotewrite | prometheus_mock | prometheus | m5.2xlarge | 1.23 | 114.72 | 2.40 | 120.74 | -| prometheus | | awsprometheusremotewrite | prometheus_static | prometheus | m5.2xlarge | 1.22 | 114.92 | 2.70 | 119.64 | -| statsd | | awsemf, logging | statsd | statsd | m5.2xlarge | 5.50 | 67.17 | 6.00 | 67.76 | -| statsd | | otlphttp | statsd_mock | statsd | m5.2xlarge | 0.02 | 59.64 | 0.10 | 59.76 | - -### Metric (TPS: 5000) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| otlp | batch | datadog | datadog_exporter_metric_mock | otlp | m5.2xlarge | 0.05 | 61.48 | 0.30 | 63.53 | -| otlp | batch | dynatrace | dynatrace_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 58.89 | 0.20 | 59.13 | -| otlp | batch | otlp | otlp_grpc_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 59.94 | 0.20 | 60.52 | -| otlp | batch | otlphttp | otlp_http_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 58.20 | 0.20 | 58.46 | -| otlp | batch | awsemf, logging | otlp_metric | otlp | m5.2xlarge | 0.04 | 59.63 | 0.20 | 60.36 | -| otlp | batch | signalfx | signalfx_exporter_metric_mock | otlp | m5.2xlarge | 0.04 | 60.26 | 0.20 | 60.38 | -| prometheus | | awsprometheusremotewrite | prometheus_mock | prometheus | m5.2xlarge | 7.26 | 273.35 | 13.70 | 293.58 | -| prometheus | | awsprometheusremotewrite | prometheus_static | prometheus | m5.2xlarge | 6.82 | 280.98 | 11.30 | 292.55 | -| statsd | | awsemf, logging | statsd | statsd | m5.2xlarge | 26.39 | 67.91 | 27.69 | 68.26 | -| statsd | | otlphttp | statsd_mock | statsd | m5.2xlarge | 0.02 | 59.35 | 0.20 | 59.56 | - -### Trace (TPS: 100) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| awsxray | batch | awsxray | xrayreceiver | xray | m5.2xlarge | 4.56 | 142.60 | 5.90 | 206.99 | -| jaeger | batch | logging, otlphttp | jaeger_mock | jaeger | m5.2xlarge | 2.37 | 74.13 | 2.70 | 75.72 | -| otlp | batch | datadog | datadog_exporter_trace_mock | otlp | m5.2xlarge | 5.83 | 72.96 | 6.60 | 74.72 | -| otlp | batch | logzio | logzio_exporter_trace_mock | otlp | m5.2xlarge | 3.42 | 88.16 | 3.90 | 88.84 | -| otlp | batch | otlp | otlp_grpc_exporter_trace_mock | otlp | m5.2xlarge | 2.88 | 126.30 | 4.30 | 179.29 | -| otlp | batch | otlphttp | otlp_http_exporter_trace_mock | otlp | m5.2xlarge | 3.39 | 69.30 | 3.60 | 69.71 | -| otlp | batch | awsxray | otlp_mock | otlp | m5.2xlarge | 3.99 | 70.78 | 4.60 | 71.79 | -| otlp | batch | awsxray, logging | otlp_trace | otlp | m5.2xlarge | 5.31 | 72.04 | 5.60 | 73.34 | -| otlp | batch | sapm | sapm_exporter_trace_mock | otlp | m5.2xlarge | 3.13 | 83.88 | 3.30 | 84.05 | -| zipkin | batch | logging, otlphttp | zipkin_mock | zipkin | m5.2xlarge | 5.43 | 76.17 | 6.60 | 80.14 | - -### Trace (TPS: 1000) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| awsxray | batch | awsxray | xrayreceiver | xray | m5.2xlarge | 31.35 | 571.19 | 42.29 | 986.86 | -| jaeger | batch | logging, otlphttp | jaeger_mock | jaeger | m5.2xlarge | 17.23 | 149.72 | 22.60 | 180.76 | -| otlp | batch | datadog | datadog_exporter_trace_mock | otlp | m5.2xlarge | 31.61 | 73.73 | 32.31 | 74.86 | -| otlp | batch | logzio | logzio_exporter_trace_mock | otlp | m5.2xlarge | 27.60 | 101.18 | 28.50 | 105.62 | -| otlp | batch | otlp | otlp_grpc_exporter_trace_mock | otlp | m5.2xlarge | 28.28 | 658.23 | 41.92 | 1163.26 | -| otlp | batch | otlphttp | otlp_http_exporter_trace_mock | otlp | m5.2xlarge | 28.97 | 70.74 | 29.79 | 71.54 | -| otlp | batch | awsxray | otlp_mock | otlp | m5.2xlarge | 34.53 | 76.25 | 44.69 | 77.38 | -| otlp | batch | awsxray, logging | otlp_trace | otlp | m5.2xlarge | 35.93 | 77.61 | 46.28 | 79.54 | -| otlp | batch | sapm | sapm_exporter_trace_mock | otlp | m5.2xlarge | 26.49 | 85.00 | 27.30 | 85.77 | -| zipkin | batch | logging, otlphttp | zipkin_mock | zipkin | m5.2xlarge | 30.67 | 477.05 | 35.92 | 558.60 | - -### Trace (TPS: 5000) -| Receivers | Processors | Exporters | Test Case | Data Type | Instance Type | Avg CPU Usage (Percent) | Avg Memory Usage (Megabytes) | Max CPU Usage (Percent) | Max Memory Usage (Megabytes) | -|:---------:|:----------:|:---------:|:---------:|:---------:|:-------------:|:-----------------------:|:----------------------------:|:-----------------------:|:----------------------------:| -| awsxray | batch | awsxray | xrayreceiver | xray | m5.2xlarge | 50.30 | 849.34 | 70.32 | 1485.22 | -| jaeger | batch | logging, otlphttp | jaeger_mock | jaeger | m5.2xlarge | 15.87 | 167.25 | 23.40 | 201.14 | -| otlp | batch | datadog | datadog_exporter_trace_mock | otlp | m5.2xlarge | 131.07 | 82.09 | 136.12 | 83.50 | -| otlp | batch | logzio | logzio_exporter_trace_mock | otlp | m5.2xlarge | 118.86 | 123.59 | 120.30 | 131.09 | -| otlp | batch | otlp | otlp_grpc_exporter_trace_mock | otlp | m5.2xlarge | 123.19 | 3360.09 | 189.65 | 6049.49 | -| otlp | batch | otlphttp | otlp_http_exporter_trace_mock | otlp | m5.2xlarge | 122.79 | 76.54 | 124.70 | 78.01 | -| otlp | batch | awsxray | otlp_mock | otlp | m5.2xlarge | 145.06 | 16417.05 | 478.15 | 28533.93 | -| otlp | batch | awsxray, logging | otlp_trace | otlp | m5.2xlarge | 153.77 | 15337.49 | 419.97 | 26340.54 | -| otlp | batch | sapm | sapm_exporter_trace_mock | otlp | m5.2xlarge | 112.55 | 90.70 | 113.80 | 93.22 | -| zipkin | batch | logging, otlphttp | zipkin_mock | zipkin | m5.2xlarge | 28.49 | 511.65 | 36.23 | 557.35 | diff --git a/e2etest/get-performance-model-table.py b/e2etest/get-performance-model-table.py index 010bc6243b..e11d083a33 100644 --- a/e2etest/get-performance-model-table.py +++ b/e2etest/get-performance-model-table.py @@ -93,5 +93,5 @@ def flatten_performance_models(models): print(rendered_result) # write rendered result to docs/performance_model.md - with open("docs/performance_model.md", "w") as f: + with open("docs/performance_model.md", "w+") as f: f.write(rendered_result) From 5fd2662b1a69bf8bd80d0b0cdc4aa31216edf401 Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Thu, 16 Dec 2021 20:09:17 -0500 Subject: [PATCH 2/5] Added benchmarking to performance test. --- .github/workflows/perf.yml | 22 ++++++++++---- e2etest/get-performance-model-table.py | 41 ++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 806b65663e..ee60ae035b 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -149,18 +149,30 @@ jobs: path: artifacts - name: Produce performance report - run: python e2etest/get-performance-model-table.py + run: python e2etest/get-performance-model-table.py ${{ needs.get-testing-version.outputs.testing_version }} - name: Copy performance report - run: cp doc/performance-report.md gh-pages/benchmark/report.md + run: rsync -avv performance-report.md gh-pages/benchmark/report.md - - name: Copy artifacts - run: rsync -avv artifacts/ gh-pages/benchmark/data + - name: Copy performance data + run: rsync -avv performance-data.json gh-pages/benchmark/data + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "customSmallerIsBetter" + output-file-path: performance-data.json + external-data-json-path: gh-pages/benchmark/data/previous-benchmark-data.json + benchmark-data-dir-path: gh-pages/benchmark/trend + skip-fetch-gh-pages: true + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: false - name: Commit to gh-pages branch uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: "Update benchmarking" + commit_options: "--amend" branch: gh-pages repository: gh-pages - file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/* + file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/* gh-pages/benchmark/trend/* diff --git a/e2etest/get-performance-model-table.py b/e2etest/get-performance-model-table.py index e11d083a33..b641342fe6 100644 --- a/e2etest/get-performance-model-table.py +++ b/e2etest/get-performance-model-table.py @@ -36,7 +36,7 @@ def add_performance_model(model): performance_models[data_mode][data_rate].append(model) -def flatten_performance_models(models): +def flatten_performance_models(): """ Flattens performance model into list of grouped models where each group corresponds to a table in the report. @@ -58,9 +58,36 @@ def flatten_performance_models(models): x["data_mode"], x["data_rate"])) return models_list +def get_benchmark_data(): + """ + Splits models by test + """ + benchmark_data = [] + + for data_mode, data_rates in performance_models.items(): + for data_rate, models in data_rates.items(): + for model in models: + benchmark_cpu = {} + benchmark_cpu["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average CPU Usage" + benchmark_cpu["value"] = model["avgCpu"] + benchmark_cpu["unit"] = "Percent" + + benchmark_mem = {} + benchmark_mem["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average Memory Usage" + benchmark_mem["value"] = model["avgMem"] + benchmark_mem["unit"] = "Megabytes" + + benchmark_data.append(benchmark_cpu) + benchmark_data.append(benchmark_mem) + + return benchmark_data if __name__ == "__main__": - aoc_version = Path('VERSION').read_text() + try: + aoc_version = sys.argv[1].rstrip() + except IndexError: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) from jinja2 import Environment, PackageLoader, select_autoescape templateLoader = jinja2.FileSystemLoader(searchpath="e2etest/templates/") @@ -79,7 +106,7 @@ def flatten_performance_models(models): testing_ami = model["testingAmi"] add_performance_model(model) - models_list = flatten_performance_models(performance_models) + models_list = flatten_performance_models() # render performance models into markdown template = env.get_template('performance_model.tpl') @@ -92,6 +119,10 @@ def flatten_performance_models(models): }) print(rendered_result) - # write rendered result to docs/performance_model.md - with open("docs/performance_model.md", "w+") as f: + # write rendered result to report.md + with open("performance-report.md", "w+") as f: f.write(rendered_result) + + # write benchmark-data.json + with open("performance-data.json", "w+") as f: + f.write(json.dumps(get_benchmark_data())) \ No newline at end of file From 17ea1e8a558e77673bc1eeac4e4ae1c2e6b10db0 Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Mon, 20 Dec 2021 15:28:13 -0500 Subject: [PATCH 3/5] Fix auto-commit. --- .github/workflows/perf.yml | 39 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index ee60ae035b..2010499db9 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -24,7 +24,11 @@ on: env: TF_VAR_aws_access_key_id: ${{ secrets.INTEG_TEST_AWS_KEY_ID }} - TF_VAR_aws_secret_access_key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }} + TF_VAR_aws_secret_access_key: ${{ secrets.INTEG_TEST_AWS_KEY_SECRET }} + GH_PAGES_BRANCH: gh-pages + MAX_BENCHMARKS_TO_KEEP: 100 + COMMIT_USER: Github Actions + COMMIT_EMAIL: actions@github.com jobs: get-testing-version: @@ -132,10 +136,6 @@ jobs: needs: [get-testing-version, run-perf-test] steps: - uses: actions/checkout@v2 - - uses: actions/checkout@v2 - with: - ref: gh-pages - path: gh-pages - name: Setup Python uses: actions/setup-python@v2.1.4 @@ -150,29 +150,24 @@ jobs: - name: Produce performance report run: python e2etest/get-performance-model-table.py ${{ needs.get-testing-version.outputs.testing_version }} - - - name: Copy performance report - run: rsync -avv performance-report.md gh-pages/benchmark/report.md - - - name: Copy performance data - run: rsync -avv performance-data.json gh-pages/benchmark/data - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 + uses: jefchien/github-action-benchmark@v1.11.12-alpha2 with: tool: "customSmallerIsBetter" output-file-path: performance-data.json - external-data-json-path: gh-pages/benchmark/data/previous-benchmark-data.json - benchmark-data-dir-path: gh-pages/benchmark/trend - skip-fetch-gh-pages: true + benchmark-data-dir-path: benchmark/trend + max-items-in-chart: ${{ env.MAX_BENCHMARKS_TO_KEEP }} + gh-pages-branch: ${{ env.GH_PAGES_BRANCH }} github-token: ${{ secrets.GITHUB_TOKEN }} + commit-sha: ${{ github.event.inputs.sha }} auto-push: false - name: Commit to gh-pages branch - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "Update benchmarking" - commit_options: "--amend" - branch: gh-pages - repository: gh-pages - file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/* gh-pages/benchmark/trend/* + run: | + git switch ${{ env.GH_PAGES_BRANCH }} + rsync -avv performance-report.md benchmark/report.md + rsync -avv performance-data.json benchmark/data/ + git add benchmark/data/* benchmark/report.md + git -c user.name="${{ env.COMMIT_USER }}" -c user.email="${{ env.COMMIT_EMAIL }}" commit --amend --reset-author -m "Update benchmarking for ${{ github.event.inputs.sha }}" + git push origin ${{ env.GH_PAGES_BRANCH }}:${{ env.GH_PAGES_BRANCH }} From 1042da8dc365a943015949f97928cd2463d02747 Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Mon, 20 Dec 2021 17:39:25 -0500 Subject: [PATCH 4/5] Change benchmark data generation to use the extra property as a group tag. --- e2etest/get-performance-model-table.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/e2etest/get-performance-model-table.py b/e2etest/get-performance-model-table.py index b641342fe6..ea5c5b0e4f 100644 --- a/e2etest/get-performance-model-table.py +++ b/e2etest/get-performance-model-table.py @@ -68,14 +68,16 @@ def get_benchmark_data(): for data_rate, models in data_rates.items(): for model in models: benchmark_cpu = {} - benchmark_cpu["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average CPU Usage" + benchmark_cpu["name"] = model["testcase"] benchmark_cpu["value"] = model["avgCpu"] - benchmark_cpu["unit"] = "Percent" + benchmark_cpu["unit"] = "%" + benchmark_cpu["extra"] = f"{data_mode} (TPS: {data_rate}) - Average CPU Usage" benchmark_mem = {} - benchmark_mem["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average Memory Usage" + benchmark_mem["name"] = model["testcase"] benchmark_mem["value"] = model["avgMem"] - benchmark_mem["unit"] = "Megabytes" + benchmark_mem["unit"] = "MB" + benchmark_mem["extra"] = f"{data_mode} (TPS: {data_rate}) - Average Memory Usage" benchmark_data.append(benchmark_cpu) benchmark_data.append(benchmark_mem) From 3bf3652b8a78e857596b3556b73c15e3dc22bcd9 Mon Sep 17 00:00:00 2001 From: Jeffrey Chien Date: Wed, 22 Dec 2021 15:58:13 -0500 Subject: [PATCH 5/5] Add version flag for get-performance-model-table.py. --- .github/workflows/perf.yml | 4 ++- e2etest/get-performance-model-table.py | 48 +++++++++++--------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 2010499db9..533d3a28ac 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -149,8 +149,10 @@ jobs: path: artifacts - name: Produce performance report - run: python e2etest/get-performance-model-table.py ${{ needs.get-testing-version.outputs.testing_version }} + run: python e2etest/get-performance-model-table.py -v ${{ needs.get-testing-version.outputs.testing_version }} + # Uses github-action-benchmark to update historic benchmark data + # Temporarily using forked action in order to pass in commit SHA - name: Store benchmark result uses: jefchien/github-action-benchmark@v1.11.12-alpha2 with: diff --git a/e2etest/get-performance-model-table.py b/e2etest/get-performance-model-table.py index ea5c5b0e4f..7fd2863255 100644 --- a/e2etest/get-performance-model-table.py +++ b/e2etest/get-performance-model-table.py @@ -1,9 +1,9 @@ -import os -import json -import sys -from pathlib import Path +import argparse import jinja2 +import json +import os import string +import sys # Schema: performance_models[data_mode][tps] = [model] performance_models = {} @@ -58,42 +58,36 @@ def flatten_performance_models(): x["data_mode"], x["data_rate"])) return models_list +def get_benchmark_entry(model, data_mode, data_rate, value_field, unit, subgroup): + benchmark_entry = {} + benchmark_entry["name"] = model["testcase"] + benchmark_entry["value"] = model[value_field] + benchmark_entry["unit"] = unit + benchmark_entry["extra"] = f"{data_mode} (TPS: {data_rate}) - {subgroup}" + return benchmark_entry + def get_benchmark_data(): """ - Splits models by test + Splits models by testcase and groups by data mode, data rate, and field type. """ benchmark_data = [] for data_mode, data_rates in performance_models.items(): for data_rate, models in data_rates.items(): for model in models: - benchmark_cpu = {} - benchmark_cpu["name"] = model["testcase"] - benchmark_cpu["value"] = model["avgCpu"] - benchmark_cpu["unit"] = "%" - benchmark_cpu["extra"] = f"{data_mode} (TPS: {data_rate}) - Average CPU Usage" - - benchmark_mem = {} - benchmark_mem["name"] = model["testcase"] - benchmark_mem["value"] = model["avgMem"] - benchmark_mem["unit"] = "MB" - benchmark_mem["extra"] = f"{data_mode} (TPS: {data_rate}) - Average Memory Usage" - - benchmark_data.append(benchmark_cpu) - benchmark_data.append(benchmark_mem) + benchmark_data.append(get_benchmark_entry(model, data_mode, data_rate, "avgCpu", "%", "Average CPU Usage")) + benchmark_data.append(get_benchmark_entry(model, data_mode, data_rate, "avgMem", "MB", "Average Memory Usage")) return benchmark_data if __name__ == "__main__": - try: - aoc_version = sys.argv[1].rstrip() - except IndexError: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) + parser = argparse.ArgumentParser("Generate performance-report.md and performance-data.json from artifacts") + parser.add_argument('-v', '--version', help="version to tag the report with", required=True) + args = parser.parse_args() + aoc_version = args.version - from jinja2 import Environment, PackageLoader, select_autoescape templateLoader = jinja2.FileSystemLoader(searchpath="e2etest/templates/") - env = Environment(autoescape=select_autoescape(['html', 'xml', 'tpl', 'yaml', 'yml']), loader=templateLoader) + env = jinja2.Environment(autoescape=jinja2.select_autoescape(['html', 'xml', 'tpl', 'yaml', 'yml']), loader=templateLoader) # get performance models from artifacts artifacts_path = "artifacts/" @@ -127,4 +121,4 @@ def get_benchmark_data(): # write benchmark-data.json with open("performance-data.json", "w+") as f: - f.write(json.dumps(get_benchmark_data())) \ No newline at end of file + json.dump(get_benchmark_data(), f, indent=4) \ No newline at end of file