More fixes

Signed-off-by: Adam Gutglick <[email protected]>

Author: AdamGS

.github/workflows/bench-pr.yml

@@ -90,7 +90,7 @@ jobs:
         env:
           RUST_BACKTRACE: full
         run: |
-          target/release_debug/${{ matrix.benchmark.id }} -d gh-json
+          target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false
@@ -124,7 +124,7 @@ jobs:
 
           echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
           echo '' >> comment.md
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json target/vortex-bench/${{ matrix.benchmark.id }}/results.json "${{ matrix.benchmark.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
             >> comment.md
 
       - name: Comment PR

.github/workflows/ci.yml

@@ -399,8 +399,8 @@ jobs:
         if: ${{ matrix.suite == 'tpc-h' }}
         # We use i2 to ensure that restarting the duckdb connection succeeds
         run: |
-          cargo run --bin datafusion-bench -- tpch -i2 --formats "vortex,vortex-compact" --options scale-factor=0.1
-          cargo run --bin duckdb-bench -- tpch -i2 --formats "vortex,vortex-compact" --options scale-factor=0.1
+          cargo run --bin datafusion-bench -- tpch -i2 --formats "vortex,vortex-compact" --opt scale-factor=0.1
+          cargo run --bin duckdb-bench -- tpch -i2 --formats "vortex,vortex-compact" --opt scale-factor=0.1
       - name: Run FFI Example
         if: ${{ matrix.suite == 'ffi' }}
         run: |

.github/workflows/sql-benchmarks.yml

@@ -146,7 +146,7 @@ jobs:
           # Build options string if scale_factor is set
           opts=""
           if [ -n "${{ matrix.scale_factor }}" ]; then
-            opts="--options scale_factor=${{ matrix.scale_factor }}"
+            opts="--opt scale_factor=${{ matrix.scale_factor }}"
           fi
 
           # Generate all data formats with a single command
@@ -198,7 +198,7 @@ jobs:
           # Build options string if scale_factor is set
           opts=""
           if [ -n "${{ matrix.scale_factor }}" ]; then
-            opts="--options scale_factor=${{ matrix.scale_factor }}"
+            opts="--opt scale_factor=${{ matrix.scale_factor }}"
           fi
 
           touch results.json
@@ -252,9 +252,9 @@ jobs:
           df_formats=$(echo "${{ matrix.targets }}" | tr ',' '\n' | grep '^datafusion:' | sed 's/datafusion://' | tr '\n' ',' | sed 's/,$//')
 
           # Build options string if scale_factor is set
-          opts="--options remote_data_dir=${{ matrix.remote_storage }}"
+          opts="--opt remote_data_dir=${{ matrix.remote_storage }}"
           if [ -n "${{ matrix.scale_factor }}" ]; then
-            opts="--options scale_factor=${{ matrix.scale_factor }} ${opts}"
+            opts="--opt scale_factor=${{ matrix.scale_factor }} ${opts}"
           fi
 
           touch results.json

bench-orchestrator/README.md

@@ -51,7 +51,6 @@ vx-bench run <benchmark> [options]
 - `--exclude-queries`: Queries to skip
 - `--iterations, -i`: Iterations per query (default: 5)
 - `--label, -l`: Label for this run (useful for later reference)
-- `--scale-factor, -s`: Scale factor for TPC benchmarks
 - `--track-memory`: Enable memory usage tracking
 - `--build/--no-build`: Build binaries before running (default: build)
 

bench-orchestrator/bench_orchestrator/cli.py

@@ -4,7 +4,7 @@
 """CLI for benchmark orchestration."""
 
 from datetime import datetime, timedelta
-from typing import Annotated
+from typing import Annotated, List
 
 import pandas as pd
 import typer
@@ -67,23 +67,25 @@ def run(
     exclude_queries: Annotated[str | None, typer.Option("--exclude-queries", help="Queries to skip")] = None,
     iterations: Annotated[int, typer.Option("--iterations", "-i", help="Iterations per query")] = 5,
     label: Annotated[str | None, typer.Option("--label", "-l", help="Label for this run")] = None,
-    scale_factor: Annotated[
-        str | None, typer.Option("--scale-factor", "-s", help="Scale factor for TPC benchmarks")
-    ] = None,
     track_memory: Annotated[bool, typer.Option("--track-memory", help="Track memory usage")] = False,
     build: Annotated[bool, typer.Option("--build/--no-build", help="Build binaries before running")] = True,
     verbose: Annotated[bool, typer.Option("--verbose", "-v", help="Log underlying commands")] = False,
+    options: Annotated[list[str] | None, typer.Option("--opt", help="Engine or benchmark specific options")] = None,
 ) -> None:
     """Run benchmarks with specified configuration."""
     engines = parse_engines(engine)
     formats = parse_formats(format)
     query_list = parse_queries(queries)
     exclude_list = parse_queries(exclude_queries)
-
+    bench_opts = {}
     # Build options dict
-    options: dict[str, str] = {}
-    if scale_factor:
-        options["scale_factor"] = scale_factor
+    if options:
+        for opt in options:
+            for s in opt.split(","):
+                k, v, *rest = s.split("=")
+                if rest:
+                    raise RuntimeError("Options must be key-value pairs separated by =")
+                bench_opts[k] = v
 
     config = RunConfig(
         benchmark=benchmark,
@@ -93,7 +95,7 @@ def run(
         exclude_queries=exclude_list,
         iterations=iterations,
         label=label,
-        options=options,
+        options=bench_opts,
         track_memory=track_memory,
     )
 

bench-orchestrator/bench_orchestrator/runner/executor.py

@@ -72,7 +72,7 @@ def run(
             cmd.append("--track-memory")
         if options:
             for k, v in options.items():
-                cmd.extend(["--options", f"{k}={v}"])
+                cmd.extend(["--opt", f"{k}={v}"])
 
         if self.verbose:
             console.print(f"[dim]$ {' '.join(cmd)}[/dim]")

benchmarks/compress-bench/src/main.rs

@@ -67,7 +67,7 @@ struct Args {
     datasets: Option<String>,
     #[arg(short, long, default_value_t, value_enum)]
     display_format: DisplayFormat,
-    #[arg(short)]
+    #[arg(short, long)]
     output_path: Option<PathBuf>,
     #[arg(long)]
     tracing: bool,

benchmarks/datafusion-bench/src/main.rs

@@ -90,7 +90,7 @@ struct Args {
     #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))]
     formats: Vec<Format>,
 
-    #[arg(long, value_delimiter = ',', value_parser = value_parser!(Opt))]
+    #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))]
     options: Vec<Opt>,
 }
 

benchmarks/duckdb-bench/src/main.rs

@@ -65,7 +65,7 @@ struct Args {
     #[arg(long, value_delimiter = ',', value_parser = value_parser!(Format))]
     formats: Vec<Format>,
 
-    #[arg(long, value_delimiter = ',', value_parser = value_parser!(Opt))]
+    #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))]
     options: Vec<Opt>,
 }
 

benchmarks/lance-bench/src/main.rs

@@ -63,7 +63,7 @@ struct Args {
     #[arg(long, default_value_t = false)]
     track_memory: bool,
 
-    #[arg(long, value_delimiter = ',', value_parser = value_parser!(Opt))]
+    #[arg(long = "opt", value_delimiter = ',', value_parser = value_parser!(Opt))]
     options: Vec<Opt>,
 }