Skip to content

Commit a167004

Browse files
yaroheryaroher
committed
Add TPC-C performance testing workflow with stroppy
Adds automated PR performance benchmarking: - perf-test.yml: matrix-based workflow (build once, bench N times) - perf_build.sh: PG+OrioleDB build with local cache for self-hosted runner - perf_bench.sh: single TPC-C run via stroppy Docker image - perf_compare.py: parse k6 output, generate markdown comparison table
1 parent 5b3b689 commit a167004

File tree

4 files changed

+522
-0
lines changed

4 files changed

+522
-0
lines changed

‎.github/workflows/perf-test.yml‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
name: perf-test
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
inputs:
7+
pg_version:
8+
description: "PostgreSQL version"
9+
required: false
10+
default: "17"
11+
compiler:
12+
description: "Compiler"
13+
required: false
14+
default: "gcc"
15+
bench_duration:
16+
description: "Duration per benchmark run"
17+
required: false
18+
default: "10m"
19+
bench_runs:
20+
description: "Number of benchmark runs"
21+
required: false
22+
default: "5"
23+
scale_factor:
24+
description: "TPC-C scale factor"
25+
required: false
26+
default: "1"
27+
28+
jobs:
29+
setup:
30+
name: Generate run matrix
31+
runs-on: ubuntu-latest
32+
outputs:
33+
run-matrix: ${{ steps.gen.outputs.run-matrix }}
34+
steps:
35+
- id: gen
36+
run: |
37+
RUNS="${{ inputs.bench_runs || '5' }}"
38+
echo "run-matrix=$(python3 -c "import json; print(json.dumps({'run': list(range(1, int('$RUNS')+1))}))")" >> $GITHUB_OUTPUT
39+
40+
bench-base:
41+
name: "Bench base #${{ matrix.run }}"
42+
needs: setup
43+
runs-on: perf-runner
44+
strategy:
45+
matrix: ${{ fromJson(needs.setup.outputs.run-matrix) }}
46+
max-parallel: 1
47+
env:
48+
LLVM_VER: 18
49+
CHECK_TYPE: normal
50+
COMPILER: ${{ inputs.compiler || 'gcc' }}
51+
PG_VERSION: ${{ inputs.pg_version || '17' }}
52+
RUN_NUMBER: ${{ matrix.run }}
53+
BENCH_DURATION: ${{ inputs.bench_duration || '10m' }}
54+
SCALE_FACTOR: ${{ inputs.scale_factor || '1' }}
55+
steps:
56+
- name: Checkout extension code (base branch)
57+
uses: actions/checkout@v4
58+
with:
59+
ref: ${{ github.event.pull_request.base.sha }}
60+
path: orioledb
61+
- name: Get the required tag name
62+
shell: bash
63+
run: |
64+
echo "PGTAG=$(grep '^${{ env.PG_VERSION }}: ' orioledb/.pgtags | cut -d' ' -f2-)" >> $GITHUB_ENV
65+
- name: Checkout PostgreSQL
66+
uses: actions/checkout@v4
67+
with:
68+
repository: orioledb/postgres
69+
ref: ${{ env.PGTAG }}
70+
path: postgresql
71+
- name: Setup prerequisites
72+
run: bash ./orioledb/ci/prerequisites.sh
73+
- name: Build (with local cache)
74+
run: bash ./orioledb/ci/perf_build.sh
75+
- name: Run benchmark
76+
run: bash ./orioledb/ci/perf_bench.sh
77+
- name: Upload results
78+
uses: actions/upload-artifact@v4
79+
with:
80+
name: perf-results-base-${{ matrix.run }}
81+
path: results/
82+
retention-days: 3
83+
84+
bench-head:
85+
name: "Bench head #${{ matrix.run }}"
86+
needs:
87+
- setup
88+
- bench-base
89+
runs-on: perf-runner
90+
strategy:
91+
matrix: ${{ fromJson(needs.setup.outputs.run-matrix) }}
92+
max-parallel: 1
93+
env:
94+
LLVM_VER: 18
95+
CHECK_TYPE: normal
96+
COMPILER: ${{ inputs.compiler || 'gcc' }}
97+
PG_VERSION: ${{ inputs.pg_version || '17' }}
98+
RUN_NUMBER: ${{ matrix.run }}
99+
BENCH_DURATION: ${{ inputs.bench_duration || '10m' }}
100+
SCALE_FACTOR: ${{ inputs.scale_factor || '1' }}
101+
steps:
102+
- name: Checkout extension code (head branch)
103+
uses: actions/checkout@v4
104+
with:
105+
ref: ${{ github.event.pull_request.head.sha }}
106+
path: orioledb
107+
- name: Get the required tag name
108+
shell: bash
109+
run: |
110+
echo "PGTAG=$(grep '^${{ env.PG_VERSION }}: ' orioledb/.pgtags | cut -d' ' -f2-)" >> $GITHUB_ENV
111+
- name: Checkout PostgreSQL
112+
uses: actions/checkout@v4
113+
with:
114+
repository: orioledb/postgres
115+
ref: ${{ env.PGTAG }}
116+
path: postgresql
117+
- name: Setup prerequisites
118+
run: bash ./orioledb/ci/prerequisites.sh
119+
- name: Build (with local cache)
120+
run: bash ./orioledb/ci/perf_build.sh
121+
- name: Run benchmark
122+
run: bash ./orioledb/ci/perf_bench.sh
123+
- name: Upload results
124+
uses: actions/upload-artifact@v4
125+
with:
126+
name: perf-results-head-${{ matrix.run }}
127+
path: results/
128+
retention-days: 3
129+
130+
compare:
131+
name: Compare results
132+
needs:
133+
- bench-base
134+
- bench-head
135+
runs-on: ubuntu-latest
136+
steps:
137+
- name: Checkout extension code
138+
uses: actions/checkout@v4
139+
with:
140+
path: orioledb
141+
- name: Download base results
142+
uses: actions/download-artifact@v4
143+
with:
144+
pattern: perf-results-base-*
145+
path: results-base/
146+
merge-multiple: true
147+
- name: Download head results
148+
uses: actions/download-artifact@v4
149+
with:
150+
pattern: perf-results-head-*
151+
path: results-head/
152+
merge-multiple: true
153+
- name: Compare results
154+
run: |
155+
python3 ./orioledb/ci/perf_compare.py \
156+
--base-dir results-base/ \
157+
--head-dir results-head/ \
158+
--runs "${{ inputs.bench_runs || '5' }}" \
159+
--duration "${{ inputs.bench_duration || '10m' }}" \
160+
--scale-factor "${{ inputs.scale_factor || '1' }}" \
161+
--output comment.md
162+
- name: Post PR comment
163+
if: github.event_name == 'pull_request'
164+
env:
165+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
166+
run: |
167+
PR_NUMBER=${{ github.event.pull_request.number }}
168+
MARKER="<!-- perf-test-results -->"
169+
BODY=$(cat comment.md)
170+
COMMENT="${MARKER}"$'\n'"${BODY}"
171+
172+
# Check if a comment with the marker already exists
173+
EXISTING=$(gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
174+
--jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -1)
175+
176+
if [ -n "$EXISTING" ]; then
177+
gh api repos/${{ github.repository }}/issues/comments/${EXISTING} \
178+
-X PATCH -f body="$COMMENT"
179+
else
180+
gh pr comment "$PR_NUMBER" --body "$COMMENT"
181+
fi

‎ci/perf_bench.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
6+
7+
RUN_NUMBER="${RUN_NUMBER:-1}"
8+
RESULTS_DIR="$GITHUB_WORKSPACE/results"
9+
PGDATA="$GITHUB_WORKSPACE/pgdata"
10+
BENCH_DURATION="${BENCH_DURATION:-10m}"
11+
SCALE_FACTOR="${SCALE_FACTOR:-1}"
12+
13+
mkdir -p "$RESULTS_DIR"
14+
15+
# Initialize PostgreSQL
16+
rm -rf "$PGDATA"
17+
initdb -N --encoding=UTF-8 --locale=C -D "$PGDATA"
18+
19+
# Configure for benchmarks
20+
TOTAL_MEM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}')
21+
SHARED_BUFFERS_MB=$(( TOTAL_MEM_KB / 4 / 1024 ))
22+
23+
cat >> "$PGDATA/postgresql.conf" <<EOF
24+
listen_addresses = 'localhost'
25+
shared_preload_libraries = 'orioledb'
26+
default_table_access_method = 'orioledb'
27+
shared_buffers = '${SHARED_BUFFERS_MB}MB'
28+
max_connections = 200
29+
max_wal_size = '4GB'
30+
checkpoint_completion_target = 0.9
31+
EOF
32+
33+
# Start PostgreSQL
34+
pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" start
35+
36+
# Wait for PostgreSQL to be ready
37+
pg_isready -t 30
38+
39+
# Run single TPC-C benchmark
40+
echo "=== Benchmark run ${RUN_NUMBER} ==="
41+
docker run --rm --network host \
42+
-e DRIVER_URL="postgres://$(whoami)@localhost:5432/postgres" \
43+
-e DURATION="$BENCH_DURATION" \
44+
-e SCALE_FACTOR="$SCALE_FACTOR" \
45+
-e SQL_FILE="/workloads/tpcc/tpcc.sql" \
46+
ghcr.io/stroppy-io/stroppy \
47+
run /workloads/tpcc/tpcc.ts /workloads/tpcc/tpcc.sql \
48+
2>&1 | tee "$RESULTS_DIR/run_${RUN_NUMBER}.log"
49+
50+
# Stop PostgreSQL and clean up
51+
pg_ctl -D "$PGDATA" stop
52+
rm -rf "$PGDATA"

‎ci/perf_build.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
# Local build cache for self-hosted runners.
6+
# Avoids rebuilding PG+OrioleDB in each matrix job.
7+
ORIOLEDB_SHA=$(cd orioledb && git rev-parse HEAD)
8+
CACHE_KEY="${COMPILER}-${PGTAG}-${ORIOLEDB_SHA}"
9+
CACHE_DIR="/tmp/perf-build-cache/${CACHE_KEY}"
10+
11+
if [ -d "$CACHE_DIR/pgsql" ]; then
12+
echo "=== Restoring build from local cache ==="
13+
cp -a "$CACHE_DIR/pgsql" "$GITHUB_WORKSPACE/pgsql"
14+
exit 0
15+
fi
16+
17+
echo "=== Building from scratch ==="
18+
19+
if [ $COMPILER = "clang" ]; then
20+
export CC=clang-$LLVM_VER
21+
else
22+
export CC=gcc
23+
fi
24+
25+
# configure & build PostgreSQL (debug symbols, no asserts)
26+
CONFIG_ARGS="--enable-debug --disable-cassert --with-icu --prefix=$GITHUB_WORKSPACE/pgsql"
27+
28+
cd postgresql
29+
./configure $CONFIG_ARGS
30+
if printf "%s\n" "$PGTAG" | grep -v -Fqe "patches$(sed -n "/PACKAGE_VERSION='\(.*\)'/ s//\1/ p" configure | cut -d'.' -f1 )_"; then \
31+
echo "ORIOLEDB_PATCHSET_VERSION = $PGTAG" >> src/Makefile.global; \
32+
fi ;
33+
make -sj `nproc`
34+
make -sj `nproc` install
35+
make -C contrib -sj `nproc`
36+
make -C contrib -sj `nproc` install
37+
cd ..
38+
39+
export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
40+
41+
# build OrioleDB (no coverage, no sanitizer, no -Werror)
42+
cd orioledb
43+
make -j `nproc` USE_PGXS=1
44+
make -j `nproc` USE_PGXS=1 install
45+
cd ..
46+
47+
# Save to local cache, clean up old entries (keep last 4)
48+
mkdir -p "$CACHE_DIR"
49+
cp -a "$GITHUB_WORKSPACE/pgsql" "$CACHE_DIR/pgsql"
50+
find /tmp/perf-build-cache/ -maxdepth 1 -mindepth 1 -type d \
51+
| sort | head -n -4 | xargs -r rm -rf

0 commit comments

Comments
 (0)