Merge branch 'develop' into ji/array-session

Author: gatesn

.github/workflows/bench.yml

@@ -124,7 +124,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H SF=1 on S3",
             "local_dir": "bench-vortex/data/tpch/1.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/1.0/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 1.0",
             "build_args": "--features lance"
@@ -142,7 +142,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H SF=10 on S3",
             "local_dir": "bench-vortex/data/tpch/10.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 10.0",
             "build_args": "--features lance"
@@ -174,7 +174,7 @@ jobs:
             "subcommand": "fineweb",
             "name": "FineWeb S3",
             "local_dir": "bench-vortex/data/fineweb",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/fineweb/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 100"
           },
@@ -190,7 +190,7 @@ jobs:
             "subcommand": "gharchive",
             "name": "GitHub Archive (S3)",
             "local_dir": "bench-vortex/data/gharchive",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/gharchive/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
             "scale_factor": "--scale-factor 100"
           },

.github/workflows/nightly-bench.yml

@@ -41,7 +41,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H on S3",
             "local_dir": "bench-vortex/data/tpch/10.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
             "targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex",
             "scale_factor": "--scale-factor 10.0",
             "build_args": "--features lance"
@@ -58,7 +58,7 @@ jobs:
             "subcommand": "tpch",
             "name": "TPC-H on S3",
             "local_dir": "bench-vortex/data/tpch/100.0",
-            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/100.0/",
+            "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/",
             "targets": "datafusion:parquet,duckdb:parquet,duckdb:vortex",
             "scale_factor": "--scale-factor 100.0"
           },

.github/workflows/sql-benchmarks.yml

@@ -35,7 +35,7 @@ on:
               "subcommand": "tpch",
               "name": "TPC-H SF=1 on S3",
               "local_dir": "bench-vortex/data/tpch/1.0",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/1.0/",
+              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 1.0"
             },
@@ -51,7 +51,7 @@ on:
               "subcommand": "tpch",
               "name": "TPC-H SF=10 on S3",
               "local_dir": "bench-vortex/data/tpch/10.0",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/tpch/10.0/",
+              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 10.0"
             },
@@ -81,7 +81,7 @@ on:
               "subcommand": "fineweb",
               "name": "FineWeb S3",
               "local_dir": "bench-vortex/data/fineweb",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/fineweb/",
+              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/fineweb/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 100"
             },
@@ -97,7 +97,7 @@ on:
               "subcommand": "gharchive",
               "name": "GitHub Archive (S3)",
               "local_dir": "bench-vortex/data/gharchive",
-              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/gharchive/",
+              "remote_storage": "s3://vortex-bench-dev-eu/${{github.ref_name}}/${{github.run_id}}/gharchive/",
               "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact",
               "scale_factor": "--scale-factor 100"
             },

Cargo.lock

@@ -9,48 +9,48 @@
 [![Maven - Version](https://img.shields.io/maven-central/v/dev.vortex/vortex-spark)](https://central.sonatype.com/artifact/dev.vortex/vortex-spark)
 [![codecov](https://codecov.io/github/vortex-data/vortex/graph/badge.svg)](https://codecov.io/github/vortex-data/vortex)
 
-🫶 [Join the community on Slack!](https://vortex.dev/slack) | 📚 [Documentation](https://docs.vortex.dev/) | 📊 [Performance Benchmarks](https://bench.vortex.dev)
+[Join the community on Slack!](https://vortex.dev/slack) | [Documentation](https://docs.vortex.dev/) | [Performance Benchmarks](https://bench.vortex.dev)
 
 ## Overview
 
 Vortex is a next-generation columnar file format and toolkit designed for high-performance data processing.
 It is the fastest and most extensible format for building data systems backed by object storage. It provides:
 
-- **⚡️ Blazing Fast Performance**
+- **Blazing Fast Performance**
   - 100x faster random access reads (vs. modern Apache Parquet)
   - 10-20x faster scans
   - 5x faster writes
   - Similar compression ratios
   - Efficient support for wide tables with zero-copy/zero-parse metadata
 
-- **🔧 Extensible Architecture**
+- **Extensible Architecture**
   - Modeled after Apache DataFusion's extensible approach
   - Pluggable encoding system, type system, compression strategy, & layout strategy
   - Zero-copy compatibility with Apache Arrow
 
-- **🗳️ Open Source, Neutral Governance**
+- **Open Source, Neutral Governance**
   - A Linux Foundation (LF AI & Data) Project
   - Apache-2.0 Licensed
 
-- **↔️ Integrations**
+- **Integrations**
   - Arrow, DataFusion, DuckDB, Spark, Pandas, Polars, & more
   - Apache Iceberg (coming soon)
 
 > 🟢 **Development Status**: Library APIs may change from version to version, but we now consider
-> the file format <ins>*stable*</ins>. From release 0.36.0, all future releases of Vortex should
+> the file format <ins>_stable_</ins>. From release 0.36.0, all future releases of Vortex should
 > maintain backwards compatibility of the file format (i.e., be able to read files written by
 > any earlier version >= 0.36.0).
 
 ## Key Features
 
 ### Core Capabilities
 
-- ✨ **Logical Types** - Clean separation between logical schema and physical layout
-- 🔄 **Zero-Copy Arrow Integration** - Seamless conversion to/from Apache Arrow arrays
-- 🧩 **Extensible Encodings** - Pluggable physical layouts with built-in optimizations
-- 📦 **Cascading Compression** - Support for nested encoding schemes
-- 🚀 **High-Performance Computing** - Optimized compute kernels for encoded data
-- 📊 **Rich Statistics** - Lazy-loaded summary statistics for optimization
+- **Logical Types** - Clean separation between logical schema and physical layout
+- **Zero-Copy Arrow Integration** - Seamless conversion to/from Apache Arrow arrays
+- **Extensible Encodings** - Pluggable physical layouts with built-in optimizations
+- **Cascading Compression** - Support for nested encoding schemes
+- **High-Performance Computing** - Optimized compute kernels for encoded data
+- **Rich Statistics** - Lazy-loaded summary statistics for optimization
 
 ### Technical Architecture
 
@@ -152,7 +152,7 @@ If you discovery a security vulnerability, please email <[email protected]>
 Copyright © Vortex a Series of LF Projects, LLC.
 For terms of use, trademark policy, and other project policies please see <https://lfprojects.org>
 
-## Acknowledgments 🏆
+## Acknowledgments
 
 The Vortex project benefits enormously from groundbreaking work from the academic & open-source communities.
 

README.md

@@ -425,10 +425,7 @@ fn run_statpopgen(args: StatPopGenArgs) -> anyhow::Result<()> {
 fn run_fineweb(args: FinewebArgs) -> anyhow::Result<()> {
     setup_logging_and_tracing(args.common.verbose, args.common.tracing)?;
 
-    let data_url = Url::from_directory_path("fineweb".to_data_path())
-        .map_err(|_| anyhow::anyhow!("bad data path"))?;
-
-    let benchmark = Fineweb::new(data_url);
+    let benchmark = Fineweb::with_remote_data_dir(args.common.use_remote_data_dir)?;
 
     let config = DriverConfig {
         targets: args.targets,
@@ -456,10 +453,7 @@ fn run_fineweb(args: FinewebArgs) -> anyhow::Result<()> {
 fn run_gharchive(args: GhArchiveArgs) -> anyhow::Result<()> {
     setup_logging_and_tracing(args.common.verbose, args.common.tracing)?;
 
-    let data_url = Url::from_directory_path("gharchive".to_data_path())
-        .map_err(|_| anyhow::anyhow!("bad data path"))?;
-
-    let benchmark = GithubArchive::new(data_url);
+    let benchmark = GithubArchive::with_remote_data_dir(args.common.use_remote_data_dir)?;
 
     let config = DriverConfig {
         targets: args.targets,

bench-vortex/src/bin/query_bench.rs

@@ -58,6 +58,38 @@ impl Fineweb {
     pub fn new(data_url: Url) -> Self {
         Self { data_url }
     }
+
+    pub fn with_remote_data_dir(use_remote_data_dir: Option<String>) -> anyhow::Result<Self> {
+        let data_url = Self::create_data_url(&use_remote_data_dir)?;
+        Ok(Self { data_url })
+    }
+
+    fn create_data_url(remote_data_dir: &Option<String>) -> anyhow::Result<Url> {
+        match remote_data_dir {
+            None => {
+                let data_dir = crate::IdempotentPath::to_data_path("fineweb");
+                Url::from_directory_path(&data_dir).map_err(|_| {
+                    anyhow::anyhow!("Failed to create URL from directory path: {:?}", &data_dir)
+                })
+            }
+            Some(remote_data_dir) => {
+                if !remote_data_dir.ends_with("/") {
+                    log::warn!(
+                        "Supply a --use-remote-data-dir argument which ends in a slash e.g. s3://vortex-bench-dev-eu/develop/12345/fineweb/"
+                    );
+                }
+                log::info!(
+                    concat!(
+                        "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\n",
+                        "If it does not, you should kill this command, locally generate the files (by running without\n",
+                        "--use-remote-data-dir) and upload data/fineweb/ to some remote location.",
+                    ),
+                    remote_data_dir,
+                );
+                Ok(Url::parse(remote_data_dir)?)
+            }
+        }
+    }
 }
 
 impl Fineweb {
@@ -92,6 +124,17 @@ impl Benchmark for Fineweb {
     }
 
     fn generate_data(&self, target: &Target) -> anyhow::Result<()> {
+        // Skip generation if using remote storage
+        match self.data_url.scheme() {
+            "file" => {
+                // Continue with local generation
+            }
+            _ => {
+                // Remote storage - data should already be uploaded
+                return Ok(());
+            }
+        }
+
         // Before downloading anything, make sure we are using a supported target.
         anyhow::ensure!(
             matches!(

bench-vortex/src/fineweb/mod.rs

@@ -49,6 +49,38 @@ impl GithubArchive {
     pub fn new(data_url: Url) -> Self {
         Self { data_url }
     }
+
+    pub fn with_remote_data_dir(use_remote_data_dir: Option<String>) -> anyhow::Result<Self> {
+        let data_url = Self::create_data_url(&use_remote_data_dir)?;
+        Ok(Self { data_url })
+    }
+
+    fn create_data_url(remote_data_dir: &Option<String>) -> anyhow::Result<Url> {
+        match remote_data_dir {
+            None => {
+                let data_dir = crate::IdempotentPath::to_data_path("gharchive");
+                Url::from_directory_path(&data_dir).map_err(|_| {
+                    anyhow::anyhow!("Failed to create URL from directory path: {:?}", &data_dir)
+                })
+            }
+            Some(remote_data_dir) => {
+                if !remote_data_dir.ends_with("/") {
+                    log::warn!(
+                        "Supply a --use-remote-data-dir argument which ends in a slash e.g. s3://vortex-bench-dev-eu/develop/12345/gharchive/"
+                    );
+                }
+                log::info!(
+                    concat!(
+                        "Assuming data already exists at this remote (e.g. S3, GCS) URL: {}.\n",
+                        "If it does not, you should kill this command, locally generate the files (by running without\n",
+                        "--use-remote-data-dir) and upload data/gharchive/ to some remote location.",
+                    ),
+                    remote_data_dir,
+                );
+                Ok(Url::parse(remote_data_dir)?)
+            }
+        }
+    }
 }
 
 impl GithubArchive {
@@ -95,13 +127,24 @@ impl Benchmark for GithubArchive {
     }
 
     fn generate_data(&self, target: &Target) -> anyhow::Result<()> {
+        // Skip generation if using remote storage
+        match self.data_url.scheme() {
+            "file" => {
+                // Continue with local generation
+            }
+            _ => {
+                // Remote storage - data should already be uploaded
+                return Ok(());
+            }
+        }
+
         // Before downloading anything, make sure we are using a supported target.
         anyhow::ensure!(
             matches!(
                 target.format,
                 Format::Parquet | Format::OnDiskVortex | Format::VortexCompact
             ),
-            "unsupported format for `fineweb` bench: {}",
+            "unsupported format for `gharchive` bench: {}",
             target.format
         );
 

bench-vortex/src/realnest/gharchive.rs

@@ -225,6 +225,14 @@ pub trait ALPFloat: private::Sealed + Float + Display + NativePType {
         encoded.map_each_in_place(move |encoded| Self::decode_single(encoded, exponents))
     }
 
+    fn decode_into(encoded: &[Self::ALPInt], exponents: Exponents, output: &mut [Self]) {
+        assert_eq!(encoded.len(), output.len());
+
+        for i in 0..encoded.len() {
+            output[i] = Self::decode_single(encoded[i], exponents)
+        }
+    }
+
     fn decode_slice_inplace(encoded: &mut [Self::ALPInt], exponents: Exponents) {
         let decoded: &mut [Self] = unsafe { transmute(encoded) };
         decoded.iter_mut().for_each(|v| {