update bench workflows to support re-benching main (#69)

* update bench workflows to support re-benching main

* fix issues

Author: richkcho

.github/workflows/bench-mpmc.yml

@@ -37,7 +37,7 @@ jobs:
           echo "runner_labels_list=[\"self-hosted\",\"bench\"${labels:+,\"$labels\"}]" >> "$GITHUB_OUTPUT"
 
   benchmark:
-    name: Bench PR #${{ inputs.pr_number }}
+    name: ${{ inputs.pr_number == 0 && 'Bench main' || format('Bench PR {0}', inputs.pr_number) }}
     needs: prepare-runner-labels
     runs-on: ${{ fromJson(needs.prepare-runner-labels.outputs.runner_labels_list) }}
     outputs:
@@ -55,8 +55,18 @@ jobs:
           # Parse PR number as integer (handle float inputs like "10.0")
           pr_int=$(printf "%.0f" "${{ inputs.pr_number }}")
           echo "pr_number=${pr_int}" >> "$GITHUB_OUTPUT"
+          if [ "$pr_int" -eq 0 ]; then
+            echo "is_main_run=true" >> "$GITHUB_OUTPUT"
+            echo "target_ref=main" >> "$GITHUB_OUTPUT"
+            echo "run_label=main" >> "$GITHUB_OUTPUT"
+          else
+            echo "is_main_run=false" >> "$GITHUB_OUTPUT"
+            echo "target_ref=refs/pull/${pr_int}/head" >> "$GITHUB_OUTPUT"
+            echo "run_label=pr-${pr_int}" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Comment benchmark start on PR
+        if: steps.parse-pr.outputs.is_main_run != 'true'
         id: progress-comment
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
@@ -72,10 +82,10 @@ jobs:
             });
             core.setOutput('comment_id', comment.id);
 
-      - name: Checkout PR head
+      - name: Checkout target
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
         with:
-          ref: refs/pull/${{ steps.parse-pr.outputs.pr_number }}/head
+          ref: ${{ steps.parse-pr.outputs.target_ref }}
           fetch-depth: 0
 
       - name: Checkout main for baseline
@@ -109,6 +119,8 @@ jobs:
         with:
           path: ${{ env.CARGO_TARGET_DIR }}/criterion
           key: criterion-main-${{ runner.name }}-${{ steps.main_sha.outputs.sha }}
+          restore-keys: |
+            criterion-main-${{ runner.name }}-
 
       - name: Run baseline on main (only if missing)
         if: steps.baseline-cache.outputs.cache-hit != 'true'
@@ -125,7 +137,11 @@ jobs:
       - name: Run benchmarks for PR against main baseline
         id: bench-pr
         run: |
-          cargo bench --bench mpmc-throughput -- --measurement-time 20 --sample-size 50 --baseline main 2>&1 | tee bench_output.txt
+          save_flag=()
+          if [ "${{ steps.parse-pr.outputs.is_main_run }}" = "true" ]; then
+            save_flag=(--save-baseline main)
+          fi
+          cargo bench --bench mpmc-throughput -- --measurement-time 20 --sample-size 50 --baseline main "${save_flag[@]}" 2>&1 | tee bench_output.txt
 
       - name: Upload criterion reports
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
@@ -287,8 +303,12 @@ jobs:
           # Output for next step
           cat results.md
 
-      - name: Comment benchmark artifact on PR
+      - name: Publish job summary
         if: success()
+        run: cat results.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment benchmark artifact on PR
+        if: success() && steps.parse-pr.outputs.is_main_run != 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           script: |
@@ -335,7 +355,7 @@ jobs:
             }
 
       - name: Comment benchmark failure on PR
-        if: failure()
+        if: failure() && steps.parse-pr.outputs.is_main_run != 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           script: |

.github/workflows/bench-spsc.yml

@@ -37,7 +37,7 @@ jobs:
           echo "runner_labels_list=[\"self-hosted\",\"bench\"${labels:+,\"$labels\"}]" >> "$GITHUB_OUTPUT"
 
   benchmark:
-    name: Bench PR #${{ inputs.pr_number }}
+    name: ${{ inputs.pr_number == 0 && 'Bench main' || format('Bench PR {0}', inputs.pr_number) }}
     needs: prepare-runner-labels
     runs-on: ${{ fromJson(needs.prepare-runner-labels.outputs.runner_labels_list) }}
     outputs:
@@ -55,8 +55,18 @@ jobs:
           # Parse PR number as integer (handle float inputs like "10.0")
           pr_int=$(printf "%.0f" "${{ inputs.pr_number }}")
           echo "pr_number=${pr_int}" >> "$GITHUB_OUTPUT"
+          if [ "$pr_int" -eq 0 ]; then
+            echo "is_main_run=true" >> "$GITHUB_OUTPUT"
+            echo "target_ref=main" >> "$GITHUB_OUTPUT"
+            echo "run_label=main" >> "$GITHUB_OUTPUT"
+          else
+            echo "is_main_run=false" >> "$GITHUB_OUTPUT"
+            echo "target_ref=refs/pull/${pr_int}/head" >> "$GITHUB_OUTPUT"
+            echo "run_label=pr-${pr_int}" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Comment benchmark start on PR
+        if: steps.parse-pr.outputs.is_main_run != 'true'
         id: progress-comment
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
@@ -72,10 +82,10 @@ jobs:
             });
             core.setOutput('comment_id', comment.id);
 
-      - name: Checkout PR head
+      - name: Checkout target
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
         with:
-          ref: refs/pull/${{ steps.parse-pr.outputs.pr_number }}/head
+          ref: ${{ steps.parse-pr.outputs.target_ref }}
           fetch-depth: 0
 
       - name: Checkout main for baseline
@@ -109,6 +119,8 @@ jobs:
         with:
           path: ${{ env.CARGO_TARGET_DIR }}/criterion
           key: criterion-main-${{ runner.name }}-${{ steps.main_sha.outputs.sha }}
+          restore-keys: |
+            criterion-main-${{ runner.name }}-
 
       - name: Run baseline on main (only if missing)
         if: steps.baseline-cache.outputs.cache-hit != 'true'
@@ -125,7 +137,12 @@ jobs:
       - name: Run benchmarks for PR against main baseline
         id: bench-pr
         run: |
-          cargo bench --bench spsc-throughput -- --measurement-time 20 --sample-size 50 --baseline main 2>&1 | tee bench_output.txt
+          save_flag=()
+          if [ "${{ steps.parse-pr.outputs.is_main_run }}" = "true" ]; then
+            # Refresh baseline for next main run while comparing against the previous one.
+            save_flag=(--save-baseline main)
+          fi
+          cargo bench --bench spsc-throughput -- --measurement-time 20 --sample-size 50 --baseline main "${save_flag[@]}" 2>&1 | tee bench_output.txt
 
       - name: Upload criterion reports
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
@@ -287,8 +304,12 @@ jobs:
           # Output for next step
           cat results.md
 
-      - name: Comment benchmark artifact on PR
+      - name: Publish job summary
         if: success()
+        run: cat results.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment benchmark artifact on PR
+        if: success() && steps.parse-pr.outputs.is_main_run != 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           script: |
@@ -335,7 +356,7 @@ jobs:
             }
 
       - name: Comment benchmark failure on PR
-        if: failure()
+        if: failure() && steps.parse-pr.outputs.is_main_run != 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           script: |