Added benchmarking to performance test.

Author: jefchien

.github/workflows/perf.yml

@@ -149,18 +149,30 @@ jobs:
           path: artifacts
 
       - name: Produce performance report
-        run: python e2etest/get-performance-model-table.py
+        run: python e2etest/get-performance-model-table.py ${{ needs.get-testing-version.outputs.testing_version }}
 
       - name: Copy performance report
-        run: cp doc/performance-report.md gh-pages/benchmark/report.md
+        run: rsync -avv performance-report.md gh-pages/benchmark/report.md
 
-      - name: Copy artifacts
-        run: rsync -avv artifacts/ gh-pages/benchmark/data
+      - name: Copy performance data
+        run: rsync -avv performance-data.json gh-pages/benchmark/data
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: "customSmallerIsBetter"
+          output-file-path: performance-data.json
+          external-data-json-path: gh-pages/benchmark/data/previous-benchmark-data.json
+          benchmark-data-dir-path: gh-pages/benchmark/trend
+          skip-fetch-gh-pages: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: false
 
       - name: Commit to gh-pages branch
         uses: stefanzweifel/git-auto-commit-action@v4
         with:
           commit_message: "Update benchmarking"
+          commit_options: "--amend"
           branch: gh-pages
           repository: gh-pages
-          file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/*
+          file_pattern: gh-pages/benchmark/report.md gh-pages/benchmark/data/* gh-pages/benchmark/trend/*

e2etest/get-performance-model-table.py

@@ -36,7 +36,7 @@ def add_performance_model(model):
     performance_models[data_mode][data_rate].append(model)
 
 
-def flatten_performance_models(models):
+def flatten_performance_models():
     """
     Flattens performance model into list of grouped models where each group
     corresponds to a table in the report.
@@ -58,9 +58,36 @@ def flatten_performance_models(models):
         x["data_mode"], x["data_rate"]))
     return models_list
 
+def get_benchmark_data():
+    """
+    Splits models by test
+    """
+    benchmark_data = []
+
+    for data_mode, data_rates in performance_models.items():
+        for data_rate, models in data_rates.items():
+            for model in models:
+                benchmark_cpu = {}
+                benchmark_cpu["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average CPU Usage"
+                benchmark_cpu["value"] = model["avgCpu"]
+                benchmark_cpu["unit"] = "Percent"
+
+                benchmark_mem = {}
+                benchmark_mem["name"] = f"{model['testcase']} ({data_mode}/{data_rate}) - Average Memory Usage"
+                benchmark_mem["value"] = model["avgMem"]
+                benchmark_mem["unit"] = "Megabytes"
+
+                benchmark_data.append(benchmark_cpu)
+                benchmark_data.append(benchmark_mem)
+
+    return benchmark_data
 
 if __name__ == "__main__":
-    aoc_version = Path('VERSION').read_text()
+    try:
+        aoc_version = sys.argv[1].rstrip()
+    except IndexError:
+        print(f"Usage: {sys.argv[0]} <version>")
+        sys.exit(1)
 
     from jinja2 import Environment, PackageLoader, select_autoescape
     templateLoader = jinja2.FileSystemLoader(searchpath="e2etest/templates/")
@@ -79,7 +106,7 @@ def flatten_performance_models(models):
             testing_ami = model["testingAmi"]
             add_performance_model(model)
 
-    models_list = flatten_performance_models(performance_models)
+    models_list = flatten_performance_models()
 
     # render performance models into markdown
     template = env.get_template('performance_model.tpl')
@@ -92,6 +119,10 @@ def flatten_performance_models(models):
     })
     print(rendered_result)
 
-    # write rendered result to docs/performance_model.md
-    with open("docs/performance_model.md", "w+") as f:
+    # write rendered result to report.md
+    with open("performance-report.md", "w+") as f:
         f.write(rendered_result)
+
+    # write benchmark-data.json
+    with open("performance-data.json", "w+") as f:
+        f.write(json.dumps(get_benchmark_data()))