chore: build time vendor duckdb (#3831)

Signed-off-by: Alexander Droste <[email protected]>

Author: 0ax1

.gitmodules

@@ -22,9 +22,9 @@ use std::path::PathBuf;
 /// static lib is not self-contained. This means that it includes symbols which
 /// are not defined as part of the static library.
 fn main() {
-    const DUCKDB_VERSION: &str = "v1.3.1";
+    const DUCKDB_VERSION: &str = "v1.3.2";
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
     let target_dir = manifest_dir.parent().unwrap().join("target");
-    let lib_path = target_dir.join(format!("duckdb-{DUCKDB_VERSION}"));
+    let lib_path = target_dir.join(format!("duckdb-lib-{DUCKDB_VERSION}"));
     println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_path.display());
 }

bench-vortex/build.rs

@@ -2,3 +2,6 @@
 # version pinned as part of the lock file, it can generate different output
 # across machines. rust-bindgen loads libclang dynamically from a system-wide path.
 src/cpp.rs
+
+# Symlinked DuckDB source is ignored. It is downloaded, extracted and symlinked as part of build.rs
+duckdb

vortex-duckdb/.gitignore

@@ -6,13 +6,13 @@
 use std::path::PathBuf;
 use std::{env, fs};
 
-const DUCKDB_VERSION: &str = "v1.3.1";
+const DUCKDB_VERSION: &str = "1.3.2";
 const DUCKDB_BASE_URL: &str = "https://github.com/duckdb/duckdb/releases/download";
 
-fn download_duckdb_archive() -> Result<PathBuf, Box<dyn std::error::Error>> {
+fn download_duckdb_lib_archive() -> Result<PathBuf, Box<dyn std::error::Error>> {
     let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
     let target_dir = manifest_dir.parent().unwrap().join("target");
-    let duckdb_dir = target_dir.join(format!("duckdb-{DUCKDB_VERSION}"));
+    let duckdb_dir = target_dir.join(format!("duckdb-lib-v{DUCKDB_VERSION}"));
 
     let target = env::var("TARGET")?;
     let (platform, arch) = match target.as_str() {
@@ -24,7 +24,7 @@ fn download_duckdb_archive() -> Result<PathBuf, Box<dyn std::error::Error>> {
     };
 
     let archive_name = format!("libduckdb-{platform}-{arch}.zip");
-    let url = format!("{DUCKDB_BASE_URL}/{DUCKDB_VERSION}/{archive_name}");
+    let url = format!("{DUCKDB_BASE_URL}/v{DUCKDB_VERSION}/{archive_name}");
     let archive_path = duckdb_dir.join(&archive_name);
 
     // Create directory if it doesn't exist.
@@ -43,29 +43,95 @@ fn download_duckdb_archive() -> Result<PathBuf, Box<dyn std::error::Error>> {
 }
 
 fn extract_duckdb_libraries(archive_path: PathBuf) -> Result<PathBuf, Box<dyn std::error::Error>> {
-    let duckdb_dir = archive_path
+    let duckdb_lib_dir = archive_path
         .parent()
         .ok_or("Invalid archive path")?
         .to_path_buf();
 
     // Check if already extracted. The archive for Linux only contains a .so library, macOS only .dylib.
-    if duckdb_dir.join("libduckdb.dylib").exists() || duckdb_dir.join("libduckdb.so").exists() {
+    if duckdb_lib_dir.join("libduckdb.dylib").exists()
+        || duckdb_lib_dir.join("libduckdb.so").exists()
+    {
         println!("DuckDB libraries already extracted, skipping extraction");
-        return Ok(duckdb_dir);
+        return Ok(duckdb_lib_dir);
     }
 
     let file = fs::File::open(&archive_path)?;
     let mut archive = zip::ZipArchive::new(file)?;
-    archive.extract(&duckdb_dir)?;
-    println!("Extracting DuckDB libraries to {}", duckdb_dir.display());
+    archive.extract(&duckdb_lib_dir)?;
+    println!(
+        "Extracting DuckDB libraries to {}",
+        duckdb_lib_dir.display()
+    );
 
-    Ok(duckdb_dir)
+    Ok(duckdb_lib_dir)
+}
+
+fn download_duckdb_source_archive() -> Result<PathBuf, Box<dyn std::error::Error>> {
+    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    let target_dir = manifest_dir.parent().unwrap().join("target");
+    let duckdb_source_dir = target_dir.join(format!("duckdb-source-v{DUCKDB_VERSION}"));
+    let archive_name = format!("duckdb-source-v{DUCKDB_VERSION}.zip");
+    let url = format!("https://github.com/duckdb/duckdb/archive/refs/tags/v{DUCKDB_VERSION}.zip");
+    let archive_path = duckdb_source_dir.join(&archive_name);
+
+    // Create directory if it doesn't exist.
+    fs::create_dir_all(&duckdb_source_dir)?;
+
+    if !archive_path.exists() {
+        println!("Downloading DuckDB source code from {url}");
+        let response = reqwest::blocking::get(&url)?;
+        fs::write(&archive_path, &response.bytes()?)?;
+        println!("Downloaded to {}", archive_path.display());
+    } else {
+        println!("Source archive already exists, skipping download");
+    }
+
+    Ok(archive_path)
+}
+
+fn extract_duckdb_source(archive_path: PathBuf) -> Result<PathBuf, Box<dyn std::error::Error>> {
+    let duckdb_source_dir = archive_path
+        .parent()
+        .ok_or("Invalid archive path")?
+        .to_path_buf();
+
+    // Check if the source is already extracted.
+    if duckdb_source_dir
+        .join(format!("duckdb-{DUCKDB_VERSION}/CMakeLists.txt"))
+        .exists()
+    {
+        println!("DuckDB source already extracted, skipping extraction");
+        return Ok(duckdb_source_dir);
+    }
+
+    let file = fs::File::open(&archive_path)?;
+    let mut archive = zip::ZipArchive::new(file)?;
+    archive.extract(&duckdb_source_dir)?;
+    println!(
+        "Extracting DuckDB source to {}",
+        duckdb_source_dir.display()
+    );
+
+    Ok(duckdb_source_dir)
 }
 
 fn main() {
     let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
     let duckdb_repo = crate_dir.join("duckdb");
 
+    // Download and extract prebuilt DuckDB libraries.
+    let zip_lib_path = download_duckdb_lib_archive().unwrap();
+    let extraced_lib_path = extract_duckdb_libraries(zip_lib_path).unwrap();
+
+    // Download, extract and symlink DuckDB source code.
+    let zip_source_path = download_duckdb_source_archive().unwrap();
+    let extracted_source_path = extract_duckdb_source(zip_source_path).unwrap();
+    if duckdb_repo.exists() {
+        fs::remove_file(&duckdb_repo).unwrap();
+    }
+    std::os::unix::fs::symlink(&extracted_source_path, &duckdb_repo).unwrap();
+
     // Generate the _imported_ bindings from our C++ code.
     bindgen::Builder::default()
         .header("cpp/include/duckdb_vx.h")
@@ -81,7 +147,10 @@ fn main() {
         //.generate_cstr(true) // This emits invalid syntax and breaks the Rust formatter
         .clang_arg(format!(
             "-I{}",
-            duckdb_repo.join("src/include").to_str().unwrap()
+            duckdb_repo
+                .join(format!("duckdb-{DUCKDB_VERSION}/src/include"))
+                .to_str()
+                .unwrap()
         ))
         .clang_arg(format!(
             "-I{}",
@@ -97,14 +166,16 @@ fn main() {
         .write_to_file(crate_dir.join("src/cpp.rs"))
         .expect("Couldn't write bindings!");
 
-    // Download and extract prebuilt DuckDB libraries.
-    let zip_path = download_duckdb_archive().unwrap();
-    let lib_path = extract_duckdb_libraries(zip_path).unwrap();
-
     // Link against DuckDB dylib.
-    println!("cargo:rustc-link-search=native={}", lib_path.display());
+    println!(
+        "cargo:rustc-link-search=native={}",
+        extraced_lib_path.display()
+    );
     println!("cargo:rustc-link-lib=dylib=duckdb");
-    println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_path.display());
+    println!(
+        "cargo:rustc-link-arg=-Wl,-rpath,{}",
+        extraced_lib_path.display()
+    );
 
     if env::var("TARGET").unwrap().contains("linux") {
         println!("cargo:rustc-link-lib=stdc++");
@@ -119,7 +190,7 @@ fn main() {
         .flag("-Wno-c++20-designator")
         .flag("-Wno-unused-parameter")
         // We include DuckDB headers from the DuckDB extension submodule.
-        .include(duckdb_repo.join("src/include"))
+        .include(duckdb_repo.join(format!("duckdb-{DUCKDB_VERSION}/src/include")))
         .include("cpp/include")
         .file("cpp/copy_function.cpp")
         .file("cpp/data.cpp")

vortex-duckdb/build.rs

@@ -24,8 +24,7 @@ endif()
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-c++20-designator")
 
 # Include directories
-include_directories(./include)
-include_directories(../duckdb/src/include)
+include_directories(include ../duckdb/src/include)
 
 # Add library with source files
 add_library(vortex

vortex-duckdb/cpp/CMakeLists.txt

@@ -6,7 +6,7 @@
 #include "duckdb_vx.h"
 #include "duckdb_vx/logical_type.h"
 
-#include "../duckdb/src/include/duckdb/common/types.hpp"
+#include "duckdb/common/types.hpp"
 #include <cassert>
 
 duckdb_logical_type duckdb_vx_logical_type_copy(duckdb_logical_type ty) {