Skip to content

Commit e73069b

Browse files
yaroheryaroher
committed
Add TPC-C performance testing CI workflow
Add perf-test workflow and supporting scripts for TPC-C benchmarks using stroppy-action with performance comparison against base branch.
1 parent 5b3b689 commit e73069b

File tree

5 files changed

+503
-0
lines changed

5 files changed

+503
-0
lines changed

‎.github/workflows/perf-test.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
name: perf-test
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
inputs:
7+
pg_version:
8+
description: "PostgreSQL version"
9+
required: false
10+
default: "17"
11+
compiler:
12+
description: "Compiler"
13+
required: false
14+
default: "gcc"
15+
bench_duration:
16+
description: "Duration per benchmark run"
17+
required: false
18+
default: "5m"
19+
bench_runs:
20+
description: "Number of benchmark runs"
21+
required: false
22+
default: "1"
23+
scale_factor:
24+
description: "TPC-C scale factor"
25+
required: false
26+
default: "1"
27+
28+
jobs:
29+
setup:
30+
name: Generate run matrix
31+
runs-on: ubuntu-latest
32+
outputs:
33+
run-matrix: ${{ steps.gen.outputs.run-matrix }}
34+
steps:
35+
- id: gen
36+
run: |
37+
RUNS="${{ inputs.bench_runs || '1' }}"
38+
echo "run-matrix=$(python3 -c "import json; print(json.dumps({'run': list(range(1, int('$RUNS')+1))}))")" >> $GITHUB_OUTPUT
39+
40+
bench-base:
41+
name: "Bench base #${{ matrix.run }}"
42+
needs: setup
43+
runs-on: perf-runner
44+
strategy:
45+
matrix: ${{ fromJson(needs.setup.outputs.run-matrix) }}
46+
max-parallel: 1
47+
env:
48+
LLVM_VER: 18
49+
CHECK_TYPE: normal
50+
COMPILER: ${{ inputs.compiler || 'gcc' }}
51+
PG_VERSION: ${{ inputs.pg_version || '17' }}
52+
DURATION: ${{ inputs.bench_duration || '5m' }}
53+
SCALE_FACTOR: ${{ inputs.scale_factor || '1' }}
54+
steps:
55+
- name: Checkout extension code (base branch)
56+
uses: actions/checkout@v4
57+
with:
58+
ref: ${{ github.event.pull_request.base.sha }}
59+
path: orioledb
60+
- name: Get the required tag name
61+
shell: bash
62+
run: |
63+
echo "PGTAG=$(grep '^${{ env.PG_VERSION }}: ' orioledb/.pgtags | cut -d' ' -f2-)" >> $GITHUB_ENV
64+
- name: Checkout PostgreSQL
65+
uses: actions/checkout@v4
66+
with:
67+
repository: orioledb/postgres
68+
ref: ${{ env.PGTAG }}
69+
path: postgresql
70+
- name: Setup prerequisites
71+
run: bash ./orioledb/ci/prerequisites.sh
72+
- name: Build (with local cache)
73+
run: bash ./orioledb/ci/perf_build.sh
74+
- name: Start PostgreSQL
75+
run: bash ./orioledb/ci/perf_pg_start.sh
76+
- name: Resolve current user
77+
run: echo "PGUSER=$(whoami)" >> $GITHUB_ENV
78+
- name: Run TPC-C benchmark
79+
uses: stroppy-io/stroppy-action@main
80+
with:
81+
preset: tpcc
82+
driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres
83+
artifact-name: perf-results-base-${{ matrix.run }}
84+
- name: Stop PostgreSQL
85+
if: always()
86+
run: bash ./orioledb/ci/perf_pg_stop.sh
87+
88+
bench-head:
89+
name: "Bench head #${{ matrix.run }}"
90+
needs:
91+
- setup
92+
- bench-base
93+
runs-on: perf-runner
94+
strategy:
95+
matrix: ${{ fromJson(needs.setup.outputs.run-matrix) }}
96+
max-parallel: 1
97+
env:
98+
LLVM_VER: 18
99+
CHECK_TYPE: normal
100+
COMPILER: ${{ inputs.compiler || 'gcc' }}
101+
PG_VERSION: ${{ inputs.pg_version || '17' }}
102+
DURATION: ${{ inputs.bench_duration || '5m' }}
103+
SCALE_FACTOR: ${{ inputs.scale_factor || '1' }}
104+
steps:
105+
- name: Checkout extension code (head branch)
106+
uses: actions/checkout@v4
107+
with:
108+
ref: ${{ github.event.pull_request.head.sha }}
109+
path: orioledb
110+
- name: Get the required tag name
111+
shell: bash
112+
run: |
113+
echo "PGTAG=$(grep '^${{ env.PG_VERSION }}: ' orioledb/.pgtags | cut -d' ' -f2-)" >> $GITHUB_ENV
114+
- name: Checkout PostgreSQL
115+
uses: actions/checkout@v4
116+
with:
117+
repository: orioledb/postgres
118+
ref: ${{ env.PGTAG }}
119+
path: postgresql
120+
- name: Setup prerequisites
121+
run: bash ./orioledb/ci/prerequisites.sh
122+
- name: Build (with local cache)
123+
run: bash ./orioledb/ci/perf_build.sh
124+
- name: Start PostgreSQL
125+
run: bash ./orioledb/ci/perf_pg_start.sh
126+
- name: Resolve current user
127+
run: echo "PGUSER=$(whoami)" >> $GITHUB_ENV
128+
- name: Run TPC-C benchmark
129+
uses: stroppy-io/stroppy-action@main
130+
with:
131+
preset: tpcc
132+
driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres
133+
artifact-name: perf-results-head-${{ matrix.run }}
134+
- name: Stop PostgreSQL
135+
if: always()
136+
run: bash ./orioledb/ci/perf_pg_stop.sh
137+
138+
compare:
139+
name: Compare results
140+
needs:
141+
- bench-base
142+
- bench-head
143+
runs-on: ubuntu-latest
144+
steps:
145+
- name: Checkout extension code
146+
uses: actions/checkout@v4
147+
with:
148+
path: orioledb
149+
- name: Download base results
150+
uses: actions/download-artifact@v4
151+
with:
152+
pattern: perf-results-base-*
153+
path: results-base/
154+
- name: Download head results
155+
uses: actions/download-artifact@v4
156+
with:
157+
pattern: perf-results-head-*
158+
path: results-head/
159+
- name: Compare results
160+
run: |
161+
python3 ./orioledb/ci/perf_compare.py \
162+
--base-dir results-base/ \
163+
--head-dir results-head/ \
164+
--runs "${{ inputs.bench_runs || '1' }}" \
165+
--duration "${{ inputs.bench_duration || '5m' }}" \
166+
--scale-factor "${{ inputs.scale_factor || '1' }}" \
167+
--output comment.md
168+
- name: Post PR comment
169+
if: github.event_name == 'pull_request'
170+
env:
171+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
172+
run: |
173+
PR_NUMBER=${{ github.event.pull_request.number }}
174+
MARKER="<!-- perf-test-results -->"
175+
BODY=$(cat comment.md)
176+
COMMENT="${MARKER}"$'\n'"${BODY}"
177+
178+
# Check if a comment with the marker already exists
179+
EXISTING=$(gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
180+
--jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -1)
181+
182+
if [ -n "$EXISTING" ]; then
183+
gh api repos/${{ github.repository }}/issues/comments/${EXISTING} \
184+
-X PATCH -f body="$COMMENT"
185+
else
186+
gh api repos/${{ github.repository }}/issues/${PR_NUMBER}/comments \
187+
-f body="$COMMENT"
188+
fi

‎ci/perf_build.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
# Local build cache for self-hosted runners.
6+
# Avoids rebuilding PG+OrioleDB in each matrix job.
7+
ORIOLEDB_SHA=$(cd orioledb && git rev-parse HEAD)
8+
CACHE_KEY="${COMPILER}-${PGTAG}-${ORIOLEDB_SHA}"
9+
CACHE_DIR="/tmp/perf-build-cache/${CACHE_KEY}"
10+
11+
if [ -d "$CACHE_DIR/pgsql" ]; then
12+
echo "=== Restoring build from local cache ==="
13+
cp -a "$CACHE_DIR/pgsql" "$GITHUB_WORKSPACE/pgsql"
14+
exit 0
15+
fi
16+
17+
echo "=== Building from scratch ==="
18+
19+
if [ $COMPILER = "clang" ]; then
20+
export CC=clang-$LLVM_VER
21+
else
22+
export CC=gcc
23+
fi
24+
25+
# configure & build PostgreSQL (debug symbols, no asserts)
26+
CONFIG_ARGS="--enable-debug --disable-cassert --with-icu --prefix=$GITHUB_WORKSPACE/pgsql"
27+
28+
cd postgresql
29+
./configure $CONFIG_ARGS
30+
if printf "%s\n" "$PGTAG" | grep -v -Fqe "patches$(sed -n "/PACKAGE_VERSION='\(.*\)'/ s//\1/ p" configure | cut -d'.' -f1 )_"; then \
31+
echo "ORIOLEDB_PATCHSET_VERSION = $PGTAG" >> src/Makefile.global; \
32+
fi ;
33+
make -sj `nproc`
34+
make -sj `nproc` install
35+
make -C contrib -sj `nproc`
36+
make -C contrib -sj `nproc` install
37+
cd ..
38+
39+
export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
40+
41+
# build OrioleDB (no coverage, no sanitizer, no -Werror)
42+
cd orioledb
43+
make -j `nproc` USE_PGXS=1
44+
make -j `nproc` USE_PGXS=1 install
45+
cd ..
46+
47+
# Save to local cache, clean up old entries (keep last 4)
48+
mkdir -p "$CACHE_DIR"
49+
cp -a "$GITHUB_WORKSPACE/pgsql" "$CACHE_DIR/pgsql"
50+
find /tmp/perf-build-cache/ -maxdepth 1 -mindepth 1 -type d \
51+
| sort | head -n -4 | xargs -r rm -rf

0 commit comments

Comments
 (0)