Add TPC-H and TPC-DS benchmark comparing DuckDB-DuckLake vs DataFusion-DuckLake (#41)

Author: shefeek-jinnah

.gitignore

@@ -4,5 +4,6 @@ target
 /.idea/
 *.iml
 
-# Test data (generated by tests/setup_test_data_v2.sql)
-tests/test_data/
+# Benchmark data and results
+benchmark/data/
+benchmark/results/

Cargo.toml

@@ -1,3 +1,6 @@
+[workspace]
+members = [".", "benchmark"]
+
 [package]
 name = "datafusion-ducklake"
 version = "0.0.4"

benchmark/Cargo.toml

@@ -0,0 +1,51 @@
+[package]
+name = "ducklake-benchmark"
+version = "0.1.0"
+edition = "2024"
+description = "Benchmark comparing DuckDB-DuckLake vs DataFusion-DuckLake performance"
+publish = false
+
+[[bin]]
+name = "ducklake-benchmark"
+path = "src/main.rs"
+
+[[bin]]
+name = "generate-tpch"
+path = "src/bin/generate_tpch.rs"
+
+[[bin]]
+name = "generate-tpcds"
+path = "src/bin/generate_tpcds.rs"
+
+[dependencies]
+# DataFusion-DuckLake: Uncomment ONE of the following options
+
+# Option 1: Local development (default)
+datafusion-ducklake = { path = "..", features = ["metadata-duckdb"] }
+
+# Option 2: Specific git commit
+# datafusion-ducklake = { git = "https://github.com/hotdata-dev/datafusion-ducklake", rev = "COMMIT_HASH", features = ["metadata-duckdb"] }
+
+# Option 3: Git branch
+# datafusion-ducklake = { git = "https://github.com/hotdata-dev/datafusion-ducklake", branch = "main", features = ["metadata-duckdb"] }
+
+# Option 4: Published crates.io version
+# datafusion-ducklake = { version = "0.1.0", features = ["metadata-duckdb"] }
+
+# Core dependencies
+datafusion = "50.1.0"
+duckdb = { version = "1.4.1", features = ["bundled"] }
+tokio = { version = "1", features = ["full"] }
+futures = "0.3"
+
+# CLI and reporting
+clap = { version = "4", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+csv = "1.3"
+chrono = "0.4"
+
+# Utilities
+anyhow = "1"
+regex = "1"
+walkdir = "2"

benchmark/src/bin/generate_tpcds.rs

@@ -0,0 +1,144 @@
+use anyhow::Result;
+use clap::Parser;
+use duckdb::Connection;
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(name = "generate-tpcds")]
+#[command(about = "Generate TPC-DS data in DuckLake format")]
+struct Args {
+    /// Path for the DuckLake catalog database
+    #[arg(short, long, default_value = "benchmark/data/tpcds.ducklake")]
+    catalog: PathBuf,
+
+    /// Path for data files (Parquet storage)
+    #[arg(short, long, default_value = "benchmark/data/tpcds_files")]
+    data_path: PathBuf,
+
+    /// TPC-DS scale factor (1 = 1GB, 10 = 10GB, etc.)
+    #[arg(short, long, default_value = "1")]
+    scale_factor: f64,
+}
+
+/// TPC-DS tables (24 tables)
+const TPCDS_TABLES: &[&str] = &[
+    "call_center",
+    "catalog_page",
+    "catalog_returns",
+    "catalog_sales",
+    "customer",
+    "customer_address",
+    "customer_demographics",
+    "date_dim",
+    "household_demographics",
+    "income_band",
+    "inventory",
+    "item",
+    "promotion",
+    "reason",
+    "ship_mode",
+    "store",
+    "store_returns",
+    "store_sales",
+    "time_dim",
+    "warehouse",
+    "web_page",
+    "web_returns",
+    "web_sales",
+    "web_site",
+];
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    println!("TPC-DS DuckLake Data Generator");
+    println!("==============================");
+    println!("Catalog: {:?}", args.catalog);
+    println!("Data path: {:?}", args.data_path);
+    println!(
+        "Scale factor: {} (~{}GB)",
+        args.scale_factor, args.scale_factor
+    );
+    println!();
+
+    // Ensure directories exist
+    std::fs::create_dir_all(&args.data_path)?;
+    if let Some(parent) = args.catalog.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+
+    // Remove existing catalog if present
+    if args.catalog.exists() {
+        std::fs::remove_file(&args.catalog)?;
+    }
+
+    let conn = Connection::open_in_memory()?;
+
+    // Install and load extensions
+    println!("Installing extensions...");
+    conn.execute_batch(
+        r#"
+        INSTALL tpcds;
+        LOAD tpcds;
+        INSTALL ducklake;
+        LOAD ducklake;
+        "#,
+    )?;
+
+    // Generate TPC-DS data in memory
+    println!("Generating TPC-DS data (SF={})...", args.scale_factor);
+    println!("  This may take a while for large scale factors...");
+    conn.execute_batch(&format!("CALL dsdgen(sf={})", args.scale_factor))?;
+
+    // Create DuckLake catalog
+    println!("Creating DuckLake catalog...");
+    let attach_sql = format!(
+        "ATTACH '{}' AS tpcds_lake (TYPE ducklake, DATA_PATH '{}')",
+        args.catalog.display(),
+        args.data_path.display()
+    );
+    conn.execute(&attach_sql, [])?;
+
+    // Create schema
+    conn.execute("CREATE SCHEMA IF NOT EXISTS tpcds_lake.main", [])?;
+
+    // Copy TPC-DS tables to DuckLake
+    for table in TPCDS_TABLES {
+        print!("  Copying {} to DuckLake... ", table);
+        conn.execute_batch(&format!(
+            "CREATE TABLE tpcds_lake.main.{} AS SELECT * FROM {}",
+            table, table
+        ))?;
+
+        // Get row count
+        let count: i64 = conn.query_row(
+            &format!("SELECT COUNT(*) FROM tpcds_lake.main.{}", table),
+            [],
+            |row| row.get(0),
+        )?;
+        println!("{} rows", count);
+    }
+
+    println!("\nData generation complete!");
+    println!("Catalog saved to: {:?}", args.catalog);
+
+    // Print data size
+    let total_size = dir_size(&args.data_path)?;
+    println!(
+        "\nTotal data size: {:.2} GB",
+        total_size as f64 / 1_000_000_000.0
+    );
+
+    Ok(())
+}
+
+fn dir_size(path: &PathBuf) -> Result<u64> {
+    let mut size = 0;
+    for entry in walkdir::WalkDir::new(path) {
+        let entry = entry?;
+        if entry.file_type().is_file() {
+            size += entry.metadata()?.len();
+        }
+    }
+    Ok(size)
+}

benchmark/src/bin/generate_tpch.rs

@@ -0,0 +1,122 @@
+use anyhow::Result;
+use clap::Parser;
+use duckdb::Connection;
+use std::path::PathBuf;
+
+#[derive(Parser)]
+#[command(name = "generate-tpch")]
+#[command(about = "Generate TPC-H data in DuckLake format")]
+struct Args {
+    /// Path for the DuckLake catalog database
+    #[arg(short, long, default_value = "benchmark/data/tpch.ducklake")]
+    catalog: PathBuf,
+
+    /// Path for data files (Parquet storage)
+    #[arg(short, long, default_value = "benchmark/data/tpch_files")]
+    data_path: PathBuf,
+
+    /// TPC-H scale factor (1 = 1GB, 10 = 10GB, etc.)
+    #[arg(short, long, default_value = "1")]
+    scale_factor: f64,
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    println!("TPC-H DuckLake Data Generator");
+    println!("=============================");
+    println!("Catalog: {:?}", args.catalog);
+    println!("Data path: {:?}", args.data_path);
+    println!(
+        "Scale factor: {} (~{}GB)",
+        args.scale_factor, args.scale_factor
+    );
+    println!();
+
+    // Ensure directories exist
+    std::fs::create_dir_all(&args.data_path)?;
+    if let Some(parent) = args.catalog.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+
+    // Remove existing catalog if present
+    if args.catalog.exists() {
+        std::fs::remove_file(&args.catalog)?;
+    }
+
+    let conn = Connection::open_in_memory()?;
+
+    // Install and load extensions
+    println!("Installing extensions...");
+    conn.execute_batch(
+        r#"
+        INSTALL tpch;
+        LOAD tpch;
+        INSTALL ducklake;
+        LOAD ducklake;
+        "#,
+    )?;
+
+    // Generate TPC-H data in memory
+    println!("Generating TPC-H data (SF={})...", args.scale_factor);
+    conn.execute_batch(&format!("CALL dbgen(sf={})", args.scale_factor))?;
+
+    // Create DuckLake catalog
+    println!("Creating DuckLake catalog...");
+    let attach_sql = format!(
+        "ATTACH '{}' AS tpch_lake (TYPE ducklake, DATA_PATH '{}')",
+        args.catalog.display(),
+        args.data_path.display()
+    );
+    conn.execute(&attach_sql, [])?;
+
+    // Create schema
+    conn.execute("CREATE SCHEMA IF NOT EXISTS tpch_lake.main", [])?;
+
+    // Copy TPC-H tables to DuckLake
+    let tables =
+        ["customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier"];
+
+    for table in &tables {
+        println!("  Copying {} to DuckLake...", table);
+        conn.execute_batch(&format!(
+            "CREATE TABLE tpch_lake.main.{} AS SELECT * FROM {}",
+            table, table
+        ))?;
+    }
+
+    println!("\nData generation complete!");
+    println!("Catalog saved to: {:?}", args.catalog);
+
+    // Print table statistics
+    println!("\nTable Statistics:");
+    println!("-----------------");
+    for table in &tables {
+        let count: i64 = conn.query_row(
+            &format!("SELECT COUNT(*) FROM tpch_lake.main.{}", table),
+            [],
+            |row| row.get(0),
+        )?;
+        println!("  {}: {} rows", table, count);
+    }
+
+    // Print data size
+    let total_size = dir_size(&args.data_path)?;
+    println!(
+        "\nTotal data size: {:.2} MB",
+        total_size as f64 / 1_000_000.0
+    );
+
+    Ok(())
+}
+
+fn dir_size(path: &PathBuf) -> Result<u64> {
+    let mut size = 0;
+    for entry in walkdir::WalkDir::new(path) {
+        let entry = entry?;
+        if entry.file_type().is_file() {
+            size += entry.metadata()?.len();
+        }
+    }
+    Ok(size)
+}