Skip to content

Commit 3794a90

Browse files
committed
Change regression test logic
This commit changes the regression test logic to run benchmarks on main brnach by checking out in the workflow itself instead of a previously uploaded artifact Signed-off-by: Arjun Raja Yogidas <[email protected]>
1 parent 4ecf750 commit 3794a90

File tree

1 file changed

+64
-30
lines changed

1 file changed

+64
-30
lines changed

.github/workflows/benchmark_regression_test.yml

Lines changed: 64 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -28,43 +28,77 @@ jobs:
2828
- run: make
2929
- name: Run benchmark
3030
run: make benchmarks-perf-test
31-
- name: Make previous directory
32-
run: mkdir -v ${{ github.workspace }}/previous
33-
- name: Copy results to previous directory
34-
run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/previous
31+
- name: Upload latest benchmark result
32+
uses: actions/upload-artifact@v3
33+
with:
34+
name: benchmark-result-artifact-main
35+
path: ${{github.workspace}}/benchmark/performanceTest/output/results.json
36+
- name: remove output directory
37+
run: sudo rm -rf ${{ github.workspace }}/benchmark/performanceTest/output
38+
- name: Stash uncommitted changes
39+
run: git stash push --keep-index --include-untracked -m "Stashing changes for tests"
3540
- name: Check out PR
3641
uses: actions/checkout@v3
3742
with:
3843
ref: ${{ github.event.pull_request.head.sha }}
3944
- run: make
4045
- name: Run benchmark
4146
run: make benchmarks-perf-test
42-
- name: Make current directory
43-
run: mkdir -v ${{ github.workspace }}/current
44-
- name: Copy results to current directory
45-
run: cp -r ${{ github.workspace }}/benchmark/performanceTest/output ${{ github.workspace }}/current
46-
- name: Perform Comparison and log results
47-
id: run-compare
48-
run: |
49-
sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh
50-
if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/current/results.json; then
51-
echo "Comparison successful. All P90 values are within the acceptable range."
52-
else
53-
echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values."
54-
echo "regression-detected=true" >> $GITHUB_OUTPUT
55-
fi
56-
- name: Stop the workflow if regression is detected
57-
if: steps.run-compare.outputs.regression-detected == 'true'
58-
uses: actions/github-script@v6
47+
- name: Upload latest benchmark result
48+
uses: actions/upload-artifact@v3
5949
with:
60-
github-token: ${{ secrets.GITHUB_TOKEN }}
61-
script: |
62-
const comment = `
63-
:warning: **Regression Detected** :warning:
50+
name: benchmark-result-artifact-pr
51+
path: ${{github.workspace}}/benchmark/performanceTest/output/results.json
52+
53+
download_and_perform_comparison:
54+
runs-on: ubuntu-20.04
55+
needs: test-twice
56+
steps:
57+
- uses: actions/setup-go@v4
58+
with:
59+
go-version: '1.18.10'
60+
- name: Checkout main
61+
uses: actions/checkout@v3
62+
with:
63+
ref: main
64+
- run: make
65+
66+
- name: Create previous directory
67+
run: mkdir -v ${{ github.workspace }}/previous
68+
- name: Create current directory
69+
run: mkdir -v ${{ github.workspace }}/current
70+
- name: Download previous benchmark result
71+
uses: actions/download-artifact@v3
72+
with:
73+
name: benchmark-result-artifact-main
74+
path: ${{github.workspace}}/previous
75+
- name: Download current benchmark result
76+
uses: actions/download-artifact@v3
77+
with:
78+
name: benchmark-result-artifact-pr
79+
path: ${{github.workspace}}/current
80+
- name: Perform Comparison and log results
81+
id: run-compare
82+
run: |
83+
sudo chmod +x ${{ github.workspace }}/scripts/check_regression.sh
84+
if sudo ${{ github.workspace }}/scripts/check_regression.sh ${{ github.workspace }}/previous/results.json ${{github.workspace}}/current/results.json; then
85+
echo "Comparison successful. All P90 values are within the acceptable range."
86+
else
87+
echo "Comparison failed. Current P90 values exceed 110% of the corresponding past values."
88+
echo "regression-detected=true" >> $GITHUB_OUTPUT
89+
fi
90+
- name: Stop the workflow if regression is detected
91+
if: steps.run-compare.outputs.regression-detected == 'true'
92+
uses: actions/github-script@v6
93+
with:
94+
github-token: ${{ secrets.GITHUB_TOKEN }}
95+
script: |
96+
const comment = `
97+
:warning: **Regression Detected** :warning:
6498
65-
The benchmark comparison indicates that there has been a performance regression.
66-
Please investigate and address the issue.
67-
To Investigate check logs of the previous job above.
68-
`;
99+
The benchmark comparison indicates that there has been a performance regression.
100+
Please investigate and address the issue.
101+
To Investigate check logs of the previous job above.
102+
`;
69103
70-
core.setFailed(comment);
104+
core.setFailed(comment);

0 commit comments

Comments
 (0)