Merge branch 'develop' into ji/use-windows-pool

Author: joseph-isaacs

.github/workflows/bench.yml

@@ -178,20 +178,4 @@ jobs:
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 100"
           },
-          {
-            "id": "gharchive-nvme",
-            "subcommand": "gharchive",
-            "name": "GitHub Archive (NVMe)",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
-          },
-          {
-            "id": "gharchive-s3",
-            "subcommand": "gharchive",
-            "name": "GitHub Archive (S3)",
-            "local_dir": "bench-vortex/data/gharchive",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-            "scale_factor": "--scale-factor 100"
-          },
         ]

.github/workflows/ci.yml

@@ -657,13 +657,10 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-          toolchain: nightly-2025-06-26
           targets: "wasm32-wasip1"
-          components: "rust-src"
       - name: Setup Wasmer
         uses: wasmerio/[email protected]
-        # there is a compiler bug in nightly (but not in nightly-2025-06-26)
-      - run: cargo +nightly-2025-06-26 -Zbuild-std=panic_abort,std build --target wasm32-wasip1
+      - run: cargo build --target wasm32-wasip1
         working-directory: ./wasm-test
       - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm
         working-directory: ./wasm-test

.github/workflows/sql-benchmarks.yml

@@ -85,22 +85,6 @@ on:
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 100"
             },
-            {
-              "id": "gharchive-nvme",
-              "subcommand": "gharchive",
-              "name": "GitHub Archive (NVMe)",
-              "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "--scale-factor 100"
-            },
-            {
-              "id": "gharchive-s3",
-              "subcommand": "gharchive",
-              "name": "GitHub Archive (S3)",
-              "local_dir": "bench-vortex/data/gharchive",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
-              "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
-              "scale_factor": "--scale-factor 100"
-            },
           ]
 
 jobs:

Cargo.lock

@@ -109,7 +109,7 @@ fn compress(
         .map(|f| Target::new(Engine::default(), *f))
         .collect_vec();
 
-    let structlistofints = vec![
+    let structlistofints = [
         StructListOfInts::new(100, 1000, 1),
         StructListOfInts::new(1000, 1000, 1),
         StructListOfInts::new(10000, 1000, 1),

bench-vortex/src/bin/compress.rs

@@ -24,6 +24,7 @@ use {
     crate::bench_run::run_with_setup,
     crate::utils::parquet::convert_utf8view_batch,
     crate::utils::parquet::convert_utf8view_schema,
+    anyhow::Context,
     arrow_array::RecordBatch,
     parking_lot::Mutex,
     std::fs,
@@ -131,9 +132,7 @@ pub fn benchmark_vortex_decompress(
     bench_name: &str,
 ) -> Result<(Duration, CompressionTimingMeasurement)> {
     let mut buf = Vec::new();
-    runtime
-        .block_on(vortex_compress_write(uncompressed, &mut buf))
-        .expect("Failed to compress with vortex for decompression test");
+    runtime.block_on(vortex_compress_write(uncompressed, &mut buf))?;
     let buffer = Bytes::from(buf);
 
     // Run the benchmark and measure time.
@@ -255,7 +254,7 @@ pub fn benchmark_lance_compress(
         .collect::<Result<Vec<_>, _>>()?;
     let converted_schema = convert_utf8view_schema(&schema);
 
-    let temp_dir = tempfile::tempdir().expect("Failed to create temp dir");
+    let temp_dir = tempfile::tempdir().context("Failed to create temp dir")?;
     let iteration_paths: Arc<Mutex<Vec<PathBuf>>> = Arc::new(Mutex::new(Vec::new()));
     let iteration_counter = AtomicU64::new(0);
 
@@ -289,7 +288,7 @@ pub fn benchmark_lance_compress(
     // Calculate size from the last iteration.
     let paths = iteration_paths.lock();
     let lance_compressed_size_val = if let Some(last_path) = paths.last() {
-        calculate_lance_size(last_path).expect("Failed to calculate Lance size")
+        calculate_lance_size(last_path).context("Failed to calculate Lance size")?
     } else {
         0
     };
@@ -320,7 +319,7 @@ pub fn benchmark_lance_decompress(
     // NOTE: Lance requires filesystem access unlike Parquet/Vortex which use in-memory buffers.
     let chunked = uncompressed.as_::<ChunkedVTable>().clone();
     let (batches, schema) = chunked_to_vec_record_batch(chunked);
-    let temp_dir = tempfile::tempdir().expect("Failed to create temp dir");
+    let temp_dir = tempfile::tempdir().context("Failed to create temp dir")?;
 
     // Write the Lance dataset once for all iterations.
     let dataset_path = runtime.block_on(async {

bench-vortex/src/compress/bench.rs

@@ -100,8 +100,7 @@ pub fn make_object_store(
             let s3 = Arc::new(
                 AmazonS3Builder::from_env()
                     .with_bucket_name(bucket_name)
-                    .build()
-                    .unwrap(),
+                    .build()?,
             );
             df.register_object_store(&Url::parse(&format!("s3://{bucket_name}/"))?, s3.clone());
             Ok(s3)
@@ -111,8 +110,7 @@ pub fn make_object_store(
             let gcs = Arc::new(
                 GoogleCloudStorageBuilder::from_env()
                     .with_bucket_name(bucket_name)
-                    .build()
-                    .unwrap(),
+                    .build()?,
             );
             df.register_object_store(&Url::parse(&format!("gs://{bucket_name}/"))?, gcs.clone());
             Ok(gcs)