setup codspeed pt 2

Author: bagel897

.github/actions/run_benchmark/action.yml

@@ -17,15 +17,11 @@ inputs:
     description: 'Additional pytest arguments'
     required: false
     default: ''
-  alert_threshold:
-    description: 'Alert threshold percentage'
-    required: false
-    default: '101%'
   codecov_token:
     description: 'Token for uploading to codecov'
     required: false
-  github_token:
-    description: 'GitHub token'
+  codspeed_token:
+    description: 'Token for uploading to codspeed'
     required: false
 
 runs:
@@ -34,31 +30,12 @@ runs:
     - name: Make benchmark directory
       shell: bash
       run: mkdir -p build/test-results/test
-
-    - uses: ./.github/actions/run_pytest
+    - name: Run benchmarks
+      uses: CodSpeedHQ/action@v3
       with:
-        test_path: ${{ inputs.test_path }}
-        timeout: ${{ inputs.timeout }}
-        suite_name: ${{ inputs.suite_name }}
-        extra_args: >-
-          --benchmark-json build/test-results/test/benchmark-data.json
-          ${{ inputs.extra_args }}
-          --benchmark-only
-          --dist=no
-        codecov_token: ${{ inputs.codecov_token }}
-
-    - uses: actions/cache@v4
+        token: ${{ inputs.codspeed_token }}
+        run: uv run pytest ${{ inputs.test_path }} ${{ inputs.extra_args }} -o junit_suite_name=${{ inputs.suite_name }}
+    - uses: ./.github/actions/report
       with:
-        path: ./cache
-        key: ${{ runner.os }}-benchmark
-
-    - uses: benchmark-action/github-action-benchmark@v1
-      with:
-        name: Python Benchmark with pytest-benchmark
-        tool: pytest
-        output-file-path: build/test-results/test/benchmark-data.json
-        external-data-json-path: ./cache/benchmark-data.json
-        alert-threshold: ${{ inputs.alert_threshold }}
-        comment-on-alert: true
-        fail-on-alert: true
-        github-token: ${{ inputs.github_token }}
+        flag: no-flag
+        codecov_token: ${{ inputs.codecov_token }}

.github/workflows/unit-tests.yml

@@ -17,13 +17,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Setup backend
         uses: ./.github/actions/setup-backend
+
       - name: Run benchmark tests
         uses: ./.github/actions/run_benchmark
         with:
           test_path: tests/benchmark
           timeout: 5
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          codspeed_token: ${{ secrets.CODSPEED_TOKEN }}
   unit-tests:
     # changing the following value will significantly affect github's cost. Be careful and consult with the team before changing it.
     runs-on: ubuntu-latest-32
@@ -97,7 +98,7 @@ jobs:
           test_path: tests/integration/codemod/test_parse.py
           timeout: 15
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          codspeed_token: ${{ secrets.CODSPEED_TOKEN }}
       - name: Notify parse tests failure
         uses: slackapi/slack-github-action@v2.0.0
         if: failure() && github.event_name == 'push' && false

pyproject.toml

@@ -140,8 +140,8 @@ dev-dependencies = [
   "black>=24.8.0",
   "isort>=5.13.2",
   "emoji>=2.14.0",
-  "pytest-benchmark[histogram]>=5.1.0",
   "loguru>=0.7.3",
+  "pytest-codspeed>=3.2.0",
 ]
 keyring-provider = "subprocess"
 #extra-index-url = ["https://aws@codegen-922078275900.d.codeartifact.us-east-1.amazonaws.com/pypi/codegen/simple/"]

tests/benchmark/codegen/sdk/codebase/test_reset.py

@@ -31,9 +31,9 @@ def reset_codebase(codebase: Codebase):
 def test_codebase_reset_stress_test(extension: str, tmp_path, benchmark):
     def setup():
         codebase, _ = setup_codebase(NUM_FILES, extension, tmp_path)
-        return ((codebase,), {})
+        return codebase
 
-    benchmark.pedantic(reset_codebase, setup=setup)
+    benchmark(lambda: reset_codebase(setup()))
 
 
 @pytest.mark.skip("Skipping this test for now")