Update perf-test: rename to TPC-C, support warehouses/vus_scale/pool_size

- Rename workflow to "TPC-C Performance Test"
- Replace scale-factor with warehouses throughout
- Pass vus-scale and pool-size to perf_compare.py
- Update artifact name pattern matching for new naming convention
- Disable build cache
- Fail job if PG fails to start
- preserve PG logs for each run
- register Oriole extension

Author: drrrrr900

.github/workflows/perf-test.yml

@@ -1,4 +1,4 @@
-name: perf-test
+name: TPC-C Performance Test
 
 on:
   pull_request:
@@ -94,13 +94,18 @@ jobs:
         run: bash ./orioledb/ci/perf_build.sh
       - name: Start PostgreSQL
         run: bash ./orioledb/ci/perf_pg_start.sh
+      - name: Verify PostgreSQL is ready
+        run: |
+          export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
+          pg_isready -t 5
+          psql -d postgres -c "SELECT orioledb_version();"
       - name: Resolve current user
         run: echo "PGUSER=$(whoami)" >> $GITHUB_ENV
       - name: Run TPC-C benchmark
         uses: stroppy-io/stroppy-action@main
         with:
           preset: tpcc
-          driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres
+          driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres?sslmode=disable
           artifact-name: perf-results-base-${{ matrix.warehouses }}W-${{ matrix.run }}
       - name: Stop PostgreSQL
         if: always()
@@ -146,13 +151,18 @@ jobs:
         run: bash ./orioledb/ci/perf_build.sh
       - name: Start PostgreSQL
         run: bash ./orioledb/ci/perf_pg_start.sh
+      - name: Verify PostgreSQL is ready
+        run: |
+          export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
+          pg_isready -t 5
+          psql -d postgres -c "SELECT orioledb_version();"
       - name: Resolve current user
         run: echo "PGUSER=$(whoami)" >> $GITHUB_ENV
       - name: Run TPC-C benchmark
         uses: stroppy-io/stroppy-action@main
         with:
           preset: tpcc
-          driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres
+          driver-url: postgres://${{ env.PGUSER }}@localhost:5432/postgres?sslmode=disable
           artifact-name: perf-results-head-${{ matrix.warehouses }}W-${{ matrix.run }}
       - name: Stop PostgreSQL
         if: always()
@@ -187,6 +197,8 @@ jobs:
             --runs "${{ inputs.bench_runs || '1' }}" \
             --duration "${{ inputs.bench_duration || '5m' }}" \
             --warehouses "${{ inputs.warehouses || '1' }}" \
+            --vus-scale "${{ inputs.vus_scale || '1' }}" \
+            --pool-size "${{ inputs.pool_size || '100' }}" \
             --output comment.md
       - name: Post PR comment
         if: github.event_name == 'pull_request'

ci/perf_build.sh

@@ -14,6 +14,8 @@ if [ -d "$CACHE_DIR/pgsql" ]; then
 	exit 0
 fi
 
+rm -rf /tmp/perf-build-cache/
+
 echo "=== Building from scratch ==="
 
 if [ $COMPILER = "clang" ]; then

ci/perf_compare.py

@@ -56,26 +56,26 @@ def parse_k6_summary(filepath):
     return metrics
 
 
-def find_result_files(results_dir, num_runs, scale_factor=None):
+def find_result_files(results_dir, num_runs, warehouses=None):
     """Find stroppy JSON result files in the download directory.
 
     stroppy-action artifacts are downloaded as:
-      results-dir/perf-results-{branch}-sf{SF}-{N}/stroppy-results.json
-    If scale_factor is given, only match directories containing that sf.
+      results-dir/perf-results-{branch}-{W}W-{N}/stroppy-results.json
+    If warehouses is given, only match directories containing that tag.
     """
     pattern = os.path.join(results_dir, "**", "stroppy-results.json")
     files = sorted(glob.glob(pattern, recursive=True))
     if not files:
         files = sorted(glob.glob(os.path.join(results_dir, "*.json")))
-    if scale_factor is not None:
-        sf_tag = f"-sf{scale_factor}-"
-        files = [f for f in files if sf_tag in f]
+    if warehouses is not None:
+        wh_tag = f"-{warehouses}W-"
+        files = [f for f in files if wh_tag in f]
     return files[:num_runs]
 
 
-def load_run_results(results_dir, num_runs, scale_factor=None):
+def load_run_results(results_dir, num_runs, warehouses=None):
     """Load and parse all result files from a results directory."""
-    files = find_result_files(results_dir, num_runs, scale_factor)
+    files = find_result_files(results_dir, num_runs, warehouses)
     all_metrics = []
     for filepath in files:
         print(f"Parsing: {filepath}", file=sys.stderr)
@@ -180,7 +180,8 @@ def generate_markdown(base_medians, head_medians, config):
     lines.append("")
     lines.append(
         f"**Config**: {config['runs']} runs, {config['duration']} each, "
-        f"scale_factor={config['scale_factor']}"
+        f"warehouses={config['warehouses']}, vus_scale={config['vus_scale']}, "
+        f"pool_size={config['pool_size']}"
     )
     lines.append("")
 
@@ -193,23 +194,27 @@ def main():
     parser.add_argument("--head-dir", required=True, help="Directory with head branch results")
     parser.add_argument("--runs", type=int, default=5, help="Number of benchmark runs")
     parser.add_argument("--duration", default="10m", help="Duration per run")
-    parser.add_argument("--scale-factor", default="1",
-                        help="TPC-C scale factor (comma-separated for multiple)")
+    parser.add_argument("--warehouses", default="1",
+                        help="Number of warehouses (comma-separated for multiple)")
+    parser.add_argument("--vus-scale", default="1",
+                        help="VU scale multiplier")
+    parser.add_argument("--pool-size", default="100",
+                        help="Connection pool size")
     parser.add_argument("--output", default="comment.md", help="Output markdown file")
     args = parser.parse_args()
 
-    scale_factors = [s.strip() for s in args.scale_factor.split(",")]
+    warehouse_list = [s.strip() for s in args.warehouses.split(",")]
     sections = []
 
-    for sf in scale_factors:
-        base_metrics = load_run_results(args.base_dir, args.runs, scale_factor=sf)
-        head_metrics = load_run_results(args.head_dir, args.runs, scale_factor=sf)
+    for wh in warehouse_list:
+        base_metrics = load_run_results(args.base_dir, args.runs, warehouses=wh)
+        head_metrics = load_run_results(args.head_dir, args.runs, warehouses=wh)
 
         if not base_metrics:
-            print(f"Error: no base branch results found for scale_factor={sf}", file=sys.stderr)
+            print(f"Error: no base branch results found for warehouses={wh}", file=sys.stderr)
             sys.exit(1)
         if not head_metrics:
-            print(f"Error: no head branch results found for scale_factor={sf}", file=sys.stderr)
+            print(f"Error: no head branch results found for warehouses={wh}", file=sys.stderr)
             sys.exit(1)
 
         base_medians = compute_medians(base_metrics)
@@ -218,7 +223,9 @@ def main():
         config = {
             "runs": args.runs,
             "duration": args.duration,
-            "scale_factor": sf,
+            "warehouses": wh,
+            "vus_scale": args.vus_scale,
+            "pool_size": args.pool_size,
         }
 
         sections.append(generate_markdown(base_medians, head_medians, config))

ci/perf_pg_start.sh

@@ -30,3 +30,6 @@ pg_ctl -D "$PGDATA" -l "$PGDATA/postgresql.log" start
 # Wait for PostgreSQL to be ready
 pg_isready -t 30
 
+# Register the OrioleDB extension
+psql -d postgres -c "CREATE EXTENSION IF NOT EXISTS orioledb;"
+

ci/perf_pg_stop.sh

@@ -7,4 +7,22 @@ export PATH="$GITHUB_WORKSPACE/pgsql/bin:$PATH"
 PGDATA="$GITHUB_WORKSPACE/pgdata"
 
 pg_ctl -D "$PGDATA" stop || true
+
+# Save PostgreSQL log before cleanup
+LOG_DIR="$HOME/logs"
+mkdir -p "$LOG_DIR"
+if [ -n "${GITHUB_RUN_ID:-}" ]; then
+  LOG_NAME="${GITHUB_RUN_ID}-postgres.log"
+else
+  # Incrementing counter fallback
+  COUNTER=1
+  while [ -f "$LOG_DIR/${COUNTER}-postgres.log" ]; do
+    COUNTER=$((COUNTER + 1))
+  done
+  LOG_NAME="${COUNTER}-postgres.log"
+fi
+
+echo "Saving PostgreSQL log to $LOG_DIR/$LOG_NAME"
+cp "$PGDATA/postgresql.log" "$LOG_DIR/$LOG_NAME" 2>/dev/null || echo "Warning: no postgresql.log found"
+
 rm -rf "$PGDATA"