fix[bench]: nightly config that doesn't fail (#3830)

Signed-off-by: Joe Isaacs <[email protected]>

---------

Signed-off-by: Joe Isaacs <[email protected]>

Author: joseph-isaacs

.github/workflows/nightly-bench.yml

@@ -18,37 +18,34 @@ jobs:
     with:
       mode: 'develop'
       machine_type: ${{ matrix.machine_type.instance_name }}
+      # datafusion:vortex uses a lot of memory
       benchmark_matrix: |
         [
           {
             "id": "tpch-nvme",
             "subcommand": "tpch",
             "name": "TPC-H on NVME",
-            "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet",
-            "scale_factor": "--scale-factor ${{ matrix.scale_factor }}"
+            "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex",
+            "scale_factor": "--scale-factor 10"
+          },
+          {
+            "id": "tpch-nvme",
+            "subcommand": "tpch",
+            "name": "TPC-H on NVME",
+            "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
+            "scale_factor": "--scale-factor 100"
           },
           {
             "id": "clickbench-nvme",
             "subcommand": "clickbench",
             "name": "Clickbench on NVME",
             "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
-          },
-          {
-            "id": "tpch-s3",
-            "subcommand": "tpch",
-            "name": "TPC-H on S3",
-            "local_dir": "bench-vortex/data/tpch/${{ matrix.scale_factor }}",
-            "remote_storage": "s3://vortex-bench-dev-eu/develop/tpch/${{ matrix.scale_factor }}/",
-            "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet",
-            "scale_factor": "--scale-factor ${{ matrix.scale_factor }}"
           }
         ]
     strategy:
       # A single run not should kill the others
       fail-fast: false
       matrix:
-        # currently 100 is failing I hope to address this.
-        scale_factor: [ 10, 100 ]
         machine_type:
           - id: x86
             instance_name: c6id.8xlarge

.github/workflows/sql-benchmarks.yml

@@ -89,12 +89,7 @@ jobs:
           RUST_BACKTRACE: full
         run: |
           # Generate data, running each query once to make sure they don't panic.
-          echo "datafusion:parquet"
-          target/release_debug/query_bench ${{ matrix.subcommand }} --targets datafusion:parquet -i1 -d gh-json ${{ matrix.scale_factor }}
-          echo "datafusion:vortex"
-          target/release_debug/query_bench ${{ matrix.subcommand }} --targets datafusion:vortex -i1 -d gh-json ${{ matrix.scale_factor }}
-          echo "duckdb:vortex"
-          target/release_debug/query_bench ${{ matrix.subcommand }} --targets duckdb:vortex -i1 -d gh-json ${{ matrix.scale_factor }}
+          target/release_debug/query_bench ${{ matrix.subcommand }} --targets ${{ matrix.targets }} -i1 -d gh-json ${{ matrix.scale_factor }}
 
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v4

bench-vortex/src/benchmark_driver.rs

@@ -7,7 +7,6 @@ use std::path::PathBuf;
 
 use anyhow::Result;
 use indicatif::ProgressBar;
-use itertools::Itertools;
 use log::warn;
 use vortex_datafusion::metrics::VortexMetricsFinder;
 
@@ -44,9 +43,6 @@ pub fn run_benchmark<B: Benchmark>(benchmark: B, config: DriverConfig) -> Result
         &format!("{}.trace.json", benchmark.dataset_name()),
     )?;
 
-    // Validate arguments
-    validate_args(&config)?;
-
     // Generate data for each target (idempotent)
     for target in &config.targets {
         benchmark.generate_data(target)?;
@@ -107,24 +103,6 @@ pub fn run_benchmark<B: Benchmark>(benchmark: B, config: DriverConfig) -> Result
     )
 }
 
-fn validate_args(config: &DriverConfig) -> Result<()> {
-    let engines = config
-        .targets
-        .iter()
-        .map(|t| t.engine())
-        .unique()
-        .collect_vec();
-
-    if (config.emit_plan || config.export_spans || config.show_metrics || config.threads.is_some())
-        && !engines.contains(&Engine::DataFusion)
-    {
-        vortex_panic!(
-            "--emit-plan, --export-spans, --show-metrics, --threads are only valid if DataFusion is used"
-        );
-    }
-    Ok(())
-}
-
 fn execute_queries<B: Benchmark>(
     queries: &[(usize, String)],
     iterations: usize,

bench-vortex/src/tpch/tpchgen.rs

@@ -311,7 +311,7 @@ struct VortexWriter {
 impl VortexWriter {
     fn new(path: PathBuf, schema: SchemaRef) -> Result<Self> {
         // limit the number of in flight rows.
-        let (sender, receiver) = mpsc::channel(32);
+        let (sender, receiver) = mpsc::channel(2);
         let dtype = DType::from_arrow(schema);
         let file_path = path;
         let write_task = Some(tokio::spawn(async move {