Touch up some benchmarking code and clean up dependencies (#5665)

As I'm struggling through figuring out how to untangle the benchmarks a
bit, these things fell out and I figured they are worth their own PR.
1. Used `cargo machete` to clear up some unused dependencies.
2. Made some lance-only code in benchmarks more explicitly behind a
feature

---------

Signed-off-by: Adam Gutglick <[email protected]>

Author: AdamGS

Cargo.lock

@@ -17,15 +17,15 @@ version = { workspace = true }
 workspace = true
 
 [features]
-lance = ["dep:lance", "dep:lance-encoding"]
+lance = ["dep:lance", "dep:lance-encoding", "dep:arrow-cast"]
 
 [dependencies]
+arrow-cast = { workspace = true, optional = true }
 lance = { version = "0.39.0", optional = true }
 lance-encoding = { version = "0.39.0", optional = true }
 
 anyhow = { workspace = true }
 arrow-array = { workspace = true }
-arrow-cast = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-select = { workspace = true }
 async-trait = { workspace = true }
@@ -57,7 +57,6 @@ opentelemetry-otlp = { workspace = true, features = ["trace"] }
 opentelemetry_sdk = { workspace = true }
 parking_lot = { workspace = true }
 parquet = { workspace = true, features = ["async"] }
-paste = { workspace = true }
 rand = { workspace = true }
 rayon = { workspace = true }
 regex = { workspace = true }

bench-vortex/Cargo.toml

@@ -28,8 +28,8 @@ use crate::metrics::export_plan_spans;
 use crate::query_bench::filter_queries;
 use crate::query_bench::print_memory_usage;
 use crate::query_bench::print_results;
+use crate::utils::file::url_scheme_to_storage;
 use crate::utils::new_tokio_runtime;
-use crate::utils::url_scheme_to_storage;
 use crate::vortex_panic;
 
 /// Mode for EXPLAIN queries

bench-vortex/src/benchmark_driver.rs

@@ -48,8 +48,8 @@ use crate::SESSION;
 use crate::conversions::parquet_to_vortex;
 #[cfg(feature = "lance")]
 use crate::utils;
-use crate::utils::file_utils::idempotent;
-use crate::utils::file_utils::idempotent_async;
+use crate::utils::file::idempotent;
+use crate::utils::file::idempotent_async;
 
 pub static HITS_SCHEMA: LazyLock<Schema> = LazyLock::new(|| {
     use DataType::*;
@@ -248,7 +248,7 @@ pub async fn convert_parquet_to_lance(input_path: &Path) -> anyhow::Result<()> {
 
     // Use the generic converter with no prefix filter (accepts all parquet files)
     // ClickBench also uses Utf8View columns that need conversion for Lance
-    utils::convert_parquet_to_lance(
+    utils::parquet::convert_parquet_to_lance(
         &parquet_dir,
         &lance_dir,
         "hits", // ClickBench uses "hits" as the dataset name

bench-vortex/src/clickbench/clickbench_data.rs

@@ -22,8 +22,8 @@ use vortex::utils::aliases::hash_map::HashMap;
 use {
     super::lance::*,
     crate::bench_run::run_with_setup,
-    crate::utils::convert_utf8view_batch,
-    crate::utils::convert_utf8view_schema,
+    crate::utils::parquet::convert_utf8view_batch,
+    crate::utils::parquet::convert_utf8view_schema,
     arrow_array::RecordBatch,
     parking_lot::Mutex,
     std::fs,

bench-vortex/src/compress/bench.rs

@@ -26,8 +26,8 @@ use lance::dataset::WriteParams;
 use lance_encoding::version::LanceFileVersion;
 use tempfile::TempDir;
 
-use crate::utils::parquet_utils::convert_utf8view_batch;
-use crate::utils::parquet_utils::convert_utf8view_schema;
+use crate::utils::parquet::convert_utf8view_batch;
+use crate::utils::parquet::convert_utf8view_schema;
 
 /// Write pre-converted [`RecordBatch`]es to Lance format.
 pub async fn lance_compress_write_only(

bench-vortex/src/compress/lance.rs

@@ -15,8 +15,8 @@ use reqwest::Client;
 use tokio::fs::File as TokioFile;
 use tokio::io::AsyncWriteExt;
 
-use crate::utils::file_utils::idempotent;
-use crate::utils::file_utils::idempotent_async;
+use crate::utils::file::idempotent;
+use crate::utils::file::idempotent_async;
 
 pub async fn download_data(fname: PathBuf, data_url: &str) -> Result<PathBuf> {
     idempotent_async(&fname, async |path| {

bench-vortex/src/datasets/data_downloads.rs

@@ -12,7 +12,7 @@ use std::sync::LazyLock;
 use clap::ValueEnum;
 use itertools::Itertools;
 use serde::Serialize;
-pub use utils::file_utils::*;
+pub use utils::file::*;
 pub use utils::logging::*;
 use vortex::error::VortexUnwrap;
 use vortex::error::vortex_err;

bench-vortex/src/lib.rs

@@ -44,6 +44,7 @@ use crate::tpch::schema::SUPPLIER;
 use crate::tpch::tpch_queries;
 use crate::tpch::tpchgen;
 use crate::tpch::tpchgen::TpchGenOptions;
+
 #[cfg(feature = "lance")]
 #[rustfmt::skip]
 use crate::{
@@ -378,7 +379,7 @@ pub async fn convert_all_tpch_to_lance(parquet_dir: &Path, lance_dir: &Path) ->
     for table in &tables {
         // Use table_ prefix to avoid matching similar names (e.g., part vs partsupp)
         let file_prefix = format!("{}_", table);
-        utils::convert_parquet_to_lance(
+        utils::parquet::convert_parquet_to_lance(
             parquet_dir,
             lance_dir,
             table,              // Dataset name is the table name

bench-vortex/src/tpch/tpch_benchmark.rs

@@ -42,7 +42,7 @@ use crate::CompactionStrategy;
 use crate::Format;
 use crate::IdempotentPath;
 use crate::SESSION;
-use crate::utils::file_utils::idempotent_async;
+use crate::utils::file::idempotent_async;
 
 type TableFuture<'a> = Pin<Box<dyn Future<Output = Result<()>> + Send + 'a>>;