Skip to content

Commit 60518fb

Browse files
jianzhangbjzjianzhangbjz
committed
use prometheus instead of benchstat
1 parent 001bc32 commit 60518fb

File tree

1 file changed

+204
-7
lines changed

1 file changed

+204
-7
lines changed

‎.github/workflows/benchmark.yaml‎

Lines changed: 204 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66
- main
77

88
jobs:
9-
benchmark:
9+
run-benchmark:
1010
runs-on: ubuntu-latest
1111
steps:
1212
- name: Checkout code
@@ -33,14 +33,211 @@ jobs:
3333
mkdir -p /tmp/artifacts/
3434
ARTIFACT_PATH=/tmp/artifacts make test-benchmark
3535
36-
- name: Compare with baseline
36+
- name: Convert Benchmark Output to Prometheus Metrics
3737
run: |
38-
go install golang.org/x/perf/cmd/benchstat@latest
39-
benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
38+
mkdir -p /tmp/artifacts/prometheus/
39+
cat << 'EOF' > benchmark_to_prometheus.py
40+
import sys
41+
import re
4042
41-
- name: Upload benchmark results
43+
def parse_benchmark_output(benchmark_output):
44+
metrics = []
45+
for line in benchmark_output.split("\n"):
46+
match = re.match(r"Benchmark([\w\d]+)-\d+\s+\d+\s+([\d]+)\s+ns/op\s+([\d]+)\s+B/op\s+([\d]+)\s+allocs/op", line)
47+
if match:
48+
benchmark_name = match.group(1).lower()
49+
time_ns = match.group(2)
50+
memory_bytes = match.group(3)
51+
allocs = match.group(4)
52+
53+
metrics.append(f"benchmark_{benchmark_name}_ns {time_ns}")
54+
metrics.append(f"benchmark_{benchmark_name}_allocs {allocs}")
55+
metrics.append(f"benchmark_{benchmark_name}_mem_bytes {memory_bytes}")
56+
57+
return "\n".join(metrics)
58+
59+
if __name__ == "__main__":
60+
benchmark_output = sys.stdin.read()
61+
metrics = parse_benchmark_output(benchmark_output)
62+
print(metrics)
63+
EOF
64+
65+
cat /tmp/artifacts/new.txt | python3 benchmark_to_prometheus.py > /tmp/artifacts/prometheus/metrics.txt
66+
67+
# - name: Compare with baseline
68+
# run: |
69+
# go install golang.org/x/perf/cmd/benchstat@latest
70+
# benchstat benchmarks/baseline.txt /tmp/artifacts/new.txt | tee /tmp/artifacts/output
71+
72+
- name: Upload Benchmark Metrics
4273
uses: actions/upload-artifact@v4
4374
with:
44-
name: benchmark-artifacts
45-
path: /tmp/artifacts/
75+
name: benchmark-metrics
76+
path: /tmp/artifacts/prometheus/
77+
78+
run-prometheus:
79+
needs: run-benchmark
80+
runs-on: ubuntu-latest
81+
steps:
82+
- name: Checkout code
83+
uses: actions/checkout@v4
84+
with:
85+
fetch-depth: 0
86+
87+
- name: Debug Info
88+
run: |
89+
echo "Available Artifacts in this run:"
90+
gh run list -L 5 --json artifacts --jq '.artifacts[].name'
91+
92+
- name: Download Prometheus Snapshot
93+
run: |
94+
gh run list --repo operator-framework/operator-controller --limit 5
95+
gh run download --repo operator-framework/operator-controller --name prometheus-snapshot --dir .
96+
env:
97+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
98+
99+
- name: Download Prometheus Snapshot2
100+
uses: actions/download-artifact@v4
101+
with:
102+
name: prometheus-snapshot
103+
path: ./
104+
env:
105+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
106+
107+
- name: Download Benchmark Metrics
108+
uses: actions/download-artifact@v4
109+
with:
110+
name: benchmark-metrics
111+
path: ./
112+
113+
- name: Set Up Prometheus Config
114+
run: |
115+
cat << 'EOF' > prometheus.yml
116+
global:
117+
scrape_interval: 5s
118+
scrape_configs:
119+
- job_name: 'benchmark_metrics'
120+
static_configs:
121+
- targets: ['localhost:9000']
122+
EOF
123+
mkdir -p ${{ github.workspace }}/prometheus-data
124+
sudo chown -R 65534:65534 ${{ github.workspace }}/prometheus-data
125+
sudo chmod -R 777 ${{ github.workspace }}/prometheus-data
126+
127+
- name: Extract and Restore Prometheus Snapshot
128+
run: |
129+
SNAPSHOT_FILE="${{ github.workspace }}/prometheus_snapshot.tar.gz"
130+
SNAPSHOT_DIR="${{ github.workspace }}/prometheus-data/snapshots"
131+
132+
if [[ -f "$SNAPSHOT_FILE" ]]; then
133+
mkdir -p "$SNAPSHOT_DIR"
134+
tar -xzf "$SNAPSHOT_FILE" -C "$SNAPSHOT_DIR"
135+
echo "✅ Successfully extracted snapshot to: $SNAPSHOT_DIR"
136+
else
137+
echo "⚠️ WARNING: No snapshot file found. Skipping extraction."
138+
fi
139+
140+
- name: Run Prometheus
141+
run: |
142+
docker run -d --name prometheus -p 9090:9090 \
143+
-v ${{ github.workspace }}/prometheus.yml:/etc/prometheus/prometheus.yml \
144+
-v ${{ github.workspace }}/prometheus-data:/prometheus \
145+
prom/prometheus --config.file=/etc/prometheus/prometheus.yml \
146+
--storage.tsdb.path=/prometheus \
147+
--storage.tsdb.retention.time=1h \
148+
--web.enable-admin-api \
149+
--storage.tsdb.load-wal
150+
151+
- name: Wait for Prometheus to start
152+
run: sleep 10
153+
154+
- name: Check Prometheus is running
155+
run: curl -s http://localhost:9090/-/ready || (docker logs prometheus && exit 1)
156+
157+
- name: Start HTTP Server to Expose Metrics
158+
run: |
159+
cat << 'EOF' > server.py
160+
from http.server import SimpleHTTPRequestHandler, HTTPServer
161+
162+
class MetricsHandler(SimpleHTTPRequestHandler):
163+
def do_GET(self):
164+
if self.path == "/metrics":
165+
self.send_response(200)
166+
self.send_header("Content-type", "text/plain")
167+
self.end_headers()
168+
with open("metrics.txt", "r") as f:
169+
self.wfile.write(f.read().encode())
170+
else:
171+
self.send_response(404)
172+
self.end_headers()
173+
174+
if __name__ == "__main__":
175+
server = HTTPServer(('0.0.0.0', 9000), MetricsHandler)
176+
print("Serving on port 9000...")
177+
server.serve_forever()
178+
EOF
46179
180+
nohup python3 server.py &
181+
182+
- name: Wait for Prometheus to Collect Data
183+
run: sleep 30
184+
185+
- name: Check Benchmark Metrics Against Threshold
186+
run: |
187+
MAX_TIME_NS=1200000000 # 1.2s
188+
MAX_ALLOCS=4000
189+
MAX_MEM_BYTES=450000
190+
191+
# Query Prometheus Metrics
192+
time_ns=$(curl -s "http://localhost:9090/api/v1/query?query=benchmark_create_cluster_catalog_ns" | jq -r '.data.result[0].value[1]')
193+
allocs=$(curl -s "http://localhost:9090/api/v1/query?query=benchmark_create_cluster_catalog_allocs" | jq -r '.data.result[0].value[1]')
194+
mem_bytes=$(curl -s "http://localhost:9090/api/v1/query?query=benchmark_create_cluster_catalog_mem_bytes" | jq -r '.data.result[0].value[1]')
195+
196+
echo "⏳ Benchmark Execution Time: $time_ns ns"
197+
echo "🛠️ Memory Allocations: $allocs"
198+
echo "💾 Memory Usage: $mem_bytes bytes"
199+
200+
# threshold checking
201+
if (( $(echo "$time_ns > $MAX_TIME_NS" | bc -l) )); then
202+
echo "❌ ERROR: Execution time exceeds threshold!"
203+
exit 1
204+
fi
205+
206+
if (( $(echo "$allocs > $MAX_ALLOCS" | bc -l) )); then
207+
echo "❌ ERROR: Too many memory allocations!"
208+
exit 1
209+
fi
210+
211+
if (( $(echo "$mem_bytes > $MAX_MEM_BYTES" | bc -l) )); then
212+
echo "❌ ERROR: Memory usage exceeds threshold!"
213+
exit 1
214+
fi
215+
216+
echo "✅ All benchmarks passed within threshold!"
217+
218+
- name: Trigger Prometheus Snapshot
219+
run: |
220+
curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot || (docker logs prometheus && exit 1)
221+
222+
- name: Find and Upload Prometheus Snapshot
223+
run: |
224+
SNAPSHOT_PATH=$(ls -td ${{ github.workspace }}/prometheus-data/snapshots/* 2>/dev/null | head -1 || echo "")
225+
if [[ -z "$SNAPSHOT_PATH" ]]; then
226+
echo "❌ No Prometheus snapshot found!"
227+
docker logs prometheus
228+
exit 1
229+
fi
230+
231+
echo "✅ Prometheus snapshot stored in: $SNAPSHOT_PATH"
232+
tar -czf $GITHUB_WORKSPACE/prometheus_snapshot.tar.gz -C "$SNAPSHOT_PATH" .
233+
234+
235+
- name: Stop Prometheus
236+
run: docker stop prometheus
237+
238+
- name: Upload Prometheus Snapshot
239+
uses: actions/upload-artifact@v4
240+
with:
241+
name: prometheus-snapshot
242+
path: prometheus_snapshot.tar.gz
243+

0 commit comments

Comments
 (0)