Skip to content

Commit a7fc318

Browse files
jianzhangbjzjianzhangbjz
committed
use prometheus instead of benchstat
1 parent 001bc32 commit a7fc318

File tree

1 file changed

+231
-7
lines changed

1 file changed

+231
-7
lines changed

‎.github/workflows/benchmark.yaml‎

Lines changed: 231 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
- main
77

88
jobs:
9-
benchmark:
9+
run-benchmark:
1010
runs-on: ubuntu-latest
1111
steps:
1212
- name: Checkout code
@@ -33,14 +33,238 @@ jobs:
3333
mkdir -p /tmp/artifacts/
3434
ARTIFACT_PATH=/tmp/artifacts make test-benchmark
3535
36-
- name: Compare with baseline
36+
- name: Convert Benchmark Output to Prometheus Metrics
3737
run: |
38-
go install golang.org/x/perf/cmd/benchstat@latest
39-
benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
38+
mkdir -p /tmp/artifacts/prometheus/
39+
echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV
40+
cat << 'EOF' > benchmark_to_prometheus.py
41+
import sys
42+
import re
43+
import os
4044
41-
- name: Upload benchmark results
45+
def parse_benchmark_output(benchmark_output):
46+
metrics = []
47+
round = 0
48+
value = os.getenv('RUN_ID') #get the github action run id so that those metrics cannot be overwritten
49+
for line in benchmark_output.split("\n"):
50+
match = re.match(r"Benchmark([\w\d]+)-\d+\s+\d+\s+([\d]+)\s+ns/op\s+([\d]+)\s+B/op\s+([\d]+)\s+allocs/op", line)
51+
if match:
52+
benchmark_name = match.group(1).lower()
53+
time_ns = match.group(2)
54+
memory_bytes = match.group(3)
55+
allocs = match.group(4)
56+
57+
metrics.append(f"benchmark_{benchmark_name}_ns {{run_id=\"{value}\", round=\"{round}\"}} {time_ns}")
58+
metrics.append(f"benchmark_{benchmark_name}_allocs {{run_id=\"{value}\", round=\"{round}\"}} {allocs}")
59+
metrics.append(f"benchmark_{benchmark_name}_mem_bytes {{run_id=\"{value}\", round=\"{round}\"}} {memory_bytes}")
60+
round+=1
61+
62+
return "\n".join(metrics)
63+
64+
if __name__ == "__main__":
65+
benchmark_output = sys.stdin.read()
66+
metrics = parse_benchmark_output(benchmark_output)
67+
print(metrics)
68+
EOF
69+
70+
cat /tmp/artifacts/new.txt | python3 benchmark_to_prometheus.py | tee /tmp/artifacts/prometheus/metrics.txt
71+
72+
# - name: Compare with baseline
73+
# run: |
74+
# go install golang.org/x/perf/cmd/benchstat@latest
75+
# benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
76+
77+
- name: Upload Benchmark Metrics
4278
uses: actions/upload-artifact@v4
4379
with:
44-
name: benchmark-artifacts
45-
path: /tmp/artifacts/
80+
name: benchmark-metrics
81+
path: /tmp/artifacts/prometheus/
82+
83+
run-prometheus:
84+
needs: run-benchmark
85+
runs-on: ubuntu-latest
86+
steps:
87+
- name: Checkout code
88+
uses: actions/checkout@v4
89+
with:
90+
fetch-depth: 0
91+
92+
# ToDo: use GitHub REST API to download artifact across repos
93+
- name: Download Prometheus Snapshot
94+
run: |
95+
echo "Available Artifacts in this run:"
96+
gh run list --repo operator-framework/operator-controller --limit 5
97+
gh run download --repo operator-framework/operator-controller --name prometheus-snapshot --dir .
98+
ls -lh ./
99+
env:
100+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
101+
102+
# #this step is invalid if download the artifacts in a different job
103+
# - name: Download Prometheus Snapshot2
104+
# uses: actions/download-artifact@v4
105+
# with:
106+
# name: prometheus-snapshot
107+
# path: ./
108+
# env:
109+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
110+
111+
- name: Download Benchmark Metrics
112+
uses: actions/download-artifact@v4
113+
with:
114+
name: benchmark-metrics
115+
path: ./
116+
117+
- name: Set Up Prometheus Config
118+
run: |
119+
cat << 'EOF' > prometheus.yml
120+
global:
121+
scrape_interval: 5s
122+
scrape_configs:
123+
- job_name: 'benchmark_metrics'
124+
static_configs:
125+
- targets: ['localhost:9000']
126+
EOF
127+
mkdir -p ${{ github.workspace }}/prometheus-data
128+
sudo chown -R 65534:65534 ${{ github.workspace }}/prometheus-data
129+
sudo chmod -R 777 ${{ github.workspace }}/prometheus-data
130+
131+
- name: Extract and Restore Prometheus Snapshot
132+
run: |
133+
SNAPSHOT_ZIP="${{ github.workspace }}/prometheus-snapshot.zip"
134+
SNAPSHOT_TAR="${{ github.workspace }}/prometheus_snapshot.tar.gz"
135+
SNAPSHOT_DIR="${{ github.workspace }}/prometheus-data/snapshots"
136+
137+
mkdir -p "$SNAPSHOT_DIR"
138+
139+
if [[ -f "$SNAPSHOT_ZIP" ]]; then
140+
echo "📦 Detected ZIP archive: $SNAPSHOT_ZIP"
141+
unzip -o "$SNAPSHOT_ZIP" -d "$SNAPSHOT_DIR"
142+
echo "✅ Successfully extracted ZIP snapshot."
143+
elif [[ -f "$SNAPSHOT_TAR" ]]; then
144+
echo "📦 Detected TAR archive: $SNAPSHOT_TAR"
145+
tar -xzf "$SNAPSHOT_TAR" -C "$SNAPSHOT_DIR"
146+
echo "✅ Successfully extracted TAR snapshot."
147+
else
148+
echo "⚠️ WARNING: No snapshot file found. Skipping extraction."
149+
fi
150+
151+
- name: Run Prometheus
152+
run: |
153+
docker run -d --name prometheus -p 9090:9090 \
154+
--user=root \
155+
-v ${{ github.workspace }}/prometheus.yml:/etc/prometheus/prometheus.yml \
156+
-v ${{ github.workspace }}/prometheus-data:/prometheus \
157+
prom/prometheus --config.file=/etc/prometheus/prometheus.yml \
158+
--storage.tsdb.path=/prometheus \
159+
--storage.tsdb.retention.time=1h \
160+
--web.enable-admin-api
161+
162+
- name: Wait for Prometheus to start
163+
run: sleep 10
164+
165+
- name: Check Prometheus is running
166+
run: |
167+
set -e
168+
curl -s http://localhost:9090/-/ready || (docker logs prometheus && exit 1)
46169
170+
- name: Start HTTP Server to Expose Metrics
171+
run: |
172+
cat << 'EOF' > server.py
173+
from http.server import SimpleHTTPRequestHandler, HTTPServer
174+
175+
class MetricsHandler(SimpleHTTPRequestHandler):
176+
def do_GET(self):
177+
if self.path == "/metrics":
178+
self.send_response(200)
179+
self.send_header("Content-type", "text/plain")
180+
self.end_headers()
181+
with open("metrics.txt", "r") as f:
182+
self.wfile.write(f.read().encode())
183+
else:
184+
self.send_response(404)
185+
self.end_headers()
186+
187+
if __name__ == "__main__":
188+
server = HTTPServer(('0.0.0.0', 9000), MetricsHandler)
189+
print("Serving on port 9000...")
190+
server.serve_forever()
191+
EOF
192+
193+
nohup python3 server.py &
194+
195+
- name: Wait for Prometheus to Collect Data
196+
run: sleep 30
197+
198+
- name: Check Prometheus targets page
199+
run: |
200+
http_status=$(curl -o /dev/null -s -w "%{http_code}" http://localhost:9090/targets)
201+
if [ "$http_status" -eq 200 ]; then
202+
echo "Prometheus targets page is reachable."
203+
else
204+
echo "Error: Prometheus targets page is not reachable. Status code: $http_status"
205+
exit 1
206+
fi
207+
208+
- name: Debug via SSH
209+
uses: mxschmitt/action-tmate@v3
210+
211+
- name: Check Benchmark Metrics Against Threshold
212+
run: |
213+
MAX_TIME_NS=1200000000 # 1.2s
214+
MAX_ALLOCS=4000
215+
MAX_MEM_BYTES=450000
216+
217+
# Query Prometheus Metrics, get the max value
218+
time_ns=$(curl -s "http://localhost:9090/api/v1/query?query=max(benchmark_createclustercatalog_ns)" | jq -r '.data.result[0].value[1]')
219+
allocs=$(curl -s "http://localhost:9090/api/v1/query?query=max(benchmark_createclustercatalog_allocs)" | jq -r '.data.result[0].value[1]')
220+
mem_bytes=$(curl -s "http://localhost:9090/api/v1/query?query=max(benchmark_createclustercatalog_mem_bytes)" | jq -r '.data.result[0].value[1]')
221+
222+
echo "⏳ Benchmark Execution Time: $time_ns ns"
223+
echo "🛠️ Memory Allocations: $allocs"
224+
echo "💾 Memory Usage: $mem_bytes bytes"
225+
226+
# threshold checking
227+
if (( $(echo "$time_ns > $MAX_TIME_NS" | bc -l) )); then
228+
echo "❌ ERROR: Execution time exceeds threshold!"
229+
exit 1
230+
fi
231+
232+
if (( $(echo "$allocs > $MAX_ALLOCS" | bc -l) )); then
233+
echo "❌ ERROR: Too many memory allocations!"
234+
exit 1
235+
fi
236+
237+
if (( $(echo "$mem_bytes > $MAX_MEM_BYTES" | bc -l) )); then
238+
echo "❌ ERROR: Memory usage exceeds threshold!"
239+
exit 1
240+
fi
241+
242+
echo "✅ All benchmarks passed within threshold!"
243+
244+
- name: Trigger Prometheus Snapshot
245+
run: |
246+
set -e
247+
curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot || (docker logs prometheus && exit 1)
248+
249+
- name: Find and Upload Prometheus Snapshot
250+
run: |
251+
SNAPSHOT_PATH=$(ls -td ${{ github.workspace }}/prometheus-data/snapshots/* 2>/dev/null | head -1 || echo "")
252+
if [[ -z "$SNAPSHOT_PATH" ]]; then
253+
echo "❌ No Prometheus snapshot found!"
254+
docker logs prometheus
255+
exit 1
256+
fi
257+
258+
echo "✅ Prometheus snapshot stored in: $SNAPSHOT_PATH"
259+
tar -czf $GITHUB_WORKSPACE/prometheus_snapshot.tar.gz -C "$SNAPSHOT_PATH" .
260+
261+
262+
- name: Stop Prometheus
263+
run: docker stop prometheus
264+
265+
- name: Upload Prometheus Snapshot
266+
uses: actions/upload-artifact@v4
267+
with:
268+
name: prometheus-snapshot
269+
path: prometheus_snapshot.tar.gz
270+

0 commit comments

Comments
 (0)