Enhance CI benchmarks with regression detection (#531)

This commit introduces a new script to detect performance regressions.
It compares the 'http_reqs' rate metric from the PR summary against the
main branch summary and determines if there is a regression of more than
2%.

Also, the commit adds .gitignore entries, a script to detect
regressions, and changes to the CI workflow to execute the benchmarks
and compare the results.

Author: kamilkisiela

.github/workflows/ci.yaml

@@ -128,11 +128,46 @@ jobs:
       - name: Build router
         run: cargo build --release -p hive-router
       - name: Run router
-        run: ./target/release/hive_router & sleep 5
+        run: |
+          ./target/release/hive_router &
+          echo "$!" > router.pid
+          sleep 5
         env:
           SUPERGRAPH_FILE_PATH: bench/supergraph.graphql
-      - name: Run k6 benchmark
+      # run only on main
+      - name: Run k6 benchmark for Pull Request
+        if: github.event_name == 'push'
         run: k6 run bench/k6.js
+      # run only on PR
+      - name: Run k6 benchmark for Pull Request
+        if: github.event_name == 'pull_request'
+        run: k6 run -e SUMMARY_PATH=./bench/results/pr bench/k6.js
+      - name: Checkout main branch in a separate directory
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          ref: main
+          path: main-branch
+      - name: Build router
+        if: github.event_name == 'pull_request'
+        run: cargo build --release -p hive-router
+        working-directory: main-branch
+      - name: Run router
+        if: github.event_name == 'pull_request'
+        run: |
+          kill $(cat router.pid)
+          ./main-branch/target/release/hive_router &
+          sleep 5
+        env:
+          SUPERGRAPH_FILE_PATH: bench/supergraph.graphql
+      - name: Run k6 benchmark for main
+        if: github.event_name == 'pull_request'
+        run: k6 run -e SUMMARY_PATH=./bench/results/main bench/k6.js
+      - name: Compare benchmark results
+        if: github.event_name == 'pull_request'
+        run: |
+          chmod +x ./bench/ci-detect-regression.sh
+          ./bench/ci-detect-regression.sh
 
   graphql-over-http:
     runs-on: ubuntu-latest

bench/.gitignore

@@ -0,0 +1,2 @@
+./results/main
+./results/pr

bench/ci-detect-regression.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# This script is meant to be run in CI (./github/workflows/ci.yaml#router-benchmark).
+#
+# It's a script to detect performance regression between two k6 benchmark summary files.
+# It compares the 'http_reqs' rate metric from the PR summary against the main branch
+# summary and determines if there is a regression of more than 2%.
+# If a regression is detected, the script exits with a non-zero status code.
+
+# Ensure jq and bc are installed
+if ! command -v jq &> /dev/null
+then
+    echo "jq could not be found. Please install jq to run this script."
+    exit 1
+fi
+if ! command -v bc &> /dev/null
+then
+    echo "bc could not be found. Please install bc to run this script."
+    exit 1
+fi
+
+PR_SUMMARY="./bench/results/pr/k6_summary.json"
+MAIN_SUMMARY="./bench/results/main/k6_summary.json"
+
+# Check if the summary files exist
+if [ ! -f "$PR_SUMMARY" ] || [ ! -f "$MAIN_SUMMARY" ]; then
+    echo "Benchmark summary files ($PR_SUMMARY and/or $MAIN_SUMMARY) not found."
+    exit 1
+fi
+
+# Extract the rate values
+MAIN_RATE=$(jq '.metrics.http_reqs.values.rate' "$MAIN_SUMMARY")
+PR_RATE=$(jq '.metrics.http_reqs.values.rate' "$PR_SUMMARY")
+
+# Check if jq successfully extracted the rates
+if [ -z "$MAIN_RATE" ] || [ -z "$PR_RATE" ] || [ "$MAIN_RATE" == "null" ] || [ "$PR_RATE" == "null" ]; then
+    echo "Could not extract rate from one or both summary files."
+    exit 1
+fi
+
+# Handle case where main rate is 0 to avoid division by zero
+if (( $(echo "$MAIN_RATE == 0" | bc -l) )); then
+    echo "Main branch rate is zero, cannot calculate percentage change."
+    # If the main rate is 0, any positive PR rate is an improvement, not a regression.
+    exit 0
+fi
+
+# Calculate the percentage difference using bc
+# scale determines the number of decimal places
+diff=$(echo "scale=4; (($PR_RATE - $MAIN_RATE) / $MAIN_RATE) * 100" | bc)
+
+# Print the results
+echo "Main branch http_reqs rate: $MAIN_RATE"
+echo "PR branch http_reqs rate:   $PR_RATE"
+printf "Difference: %.2f%%\n" "$diff"
+
+# Check if the difference is a regression of more than 5%
+# bc returns 1 for true, 0 for false. We compare with a negative number.
+is_regression=$(echo "$diff < -2" | bc)
+
+if [ "$is_regression" -eq 1 ]; then
+    echo "Performance regression detected! The PR is more than 2% slower than main."
+    exit 1
+else
+    echo "No significant performance regression detected."
+    exit 0
+fi

bench/k6.js

@@ -155,6 +155,9 @@ function handleBenchmarkSummary(data, additionalContext = {}) {
   };
 
   if (__ENV.SUMMARY_PATH) {
+    console.log(
+      `Writing summary to ${__ENV.SUMMARY_PATH}/k6_summary.json and .txt`,
+    );
     out[`${__ENV.SUMMARY_PATH}/k6_summary.json`] = JSON.stringify(
       Object.assign(data, additionalContext),
     );