Merge branch 'perf-baseline' into dev

Author: rayluo

.github/workflows/python-package.yml

@@ -66,6 +66,12 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Setup an updatable cache for Performance Baselines
+      uses: actions/cache@v3
+      with:
+        path: .perf.baseline
+        key: ${{ runner.os }}-performance-${{ hashFiles('tests/test_benchmark.py') }}
+        restore-keys: ${{ runner.os }}-performance-
     - name: Run benchmark
       run: pytest --benchmark-only --benchmark-json benchmark.json --log-cli-level INFO tests/test_benchmark.py
     - name: Render benchmark result

.gitignore

@@ -58,4 +58,5 @@ docs/_build/
 tests/config.json
 
 
-.env
+.env
+.perf.baseline

requirements.txt

@@ -1,3 +1,8 @@
-.
-python-dotenv
-pytest-benchmark
+-e .
+
+# python-dotenv 1.0+ no longer supports Python 3.7
+python-dotenv>=0.21,<2
+
+pytest-benchmark>=4,<5
+perf_baseline>=0.1,<0.2
+

tests/test_benchmark.py

@@ -1,20 +1,28 @@
 from tests.simulator import ClientCredentialGrantSimulator as CcaTester
+from perf_baseline import Baseline
+
+
+baseline = Baseline(".perf.baseline", threshold=1.5)  # Up to 1.5x slower than baseline
 
 # Here come benchmark test cases, powered by pytest-benchmark
 # Func names will become diag names.
 def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_hit(benchmark):
     tester = CcaTester(tokens_per_tenant=10, cache_hit=True)
+    baseline.set_or_compare(tester.run)
     benchmark(tester.run)
 
 def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_hit(benchmark):
     tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=True)
+    baseline.set_or_compare(tester.run)
     benchmark(tester.run)
 
 def test_cca_1_tenant_with_10_tokens_per_tenant_and_cache_miss(benchmark):
     tester = CcaTester(tokens_per_tenant=10, cache_hit=False)
+    baseline.set_or_compare(tester.run)
     benchmark(tester.run)
 
 def test_cca_many_tenants_with_10_tokens_per_tenant_and_cache_miss(benchmark):
     tester = CcaTester(number_of_tenants=1000, tokens_per_tenant=10, cache_hit=False)
+    baseline.set_or_compare(tester.run)
     benchmark(tester.run)