add pgvector bundled

Author: nicoloboschi

.github/workflows/ci.yml

@@ -17,8 +17,6 @@ jobs:
         uses: dtolnay/rust-toolchain@stable
 
       - name: Build
-        env:
-          BUNDLE_POSTGRESQL: "true"
         run: cargo build --release
 
       - name: Upload CLI artifact

.github/workflows/release-cli.yml

@@ -76,8 +76,6 @@ jobs:
           echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
 
       - name: Build
-        env:
-          BUNDLE_POSTGRESQL: "true"
         run: cargo build --release --target ${{ matrix.target }}
 
       - name: Prepare artifact (Unix)
@@ -146,9 +144,9 @@ jobs:
           echo "- \`pg0-windows-x86_64.exe\` - Windows x64" >> release_notes.md
           echo "" >> release_notes.md
           echo "### Bundled Components" >> release_notes.md
-          echo "PostgreSQL is bundled directly in the binary - no downloads required on first run!" >> release_notes.md
+          echo "Everything is bundled directly in the binary - no downloads required, works completely offline!" >> release_notes.md
           echo "- PostgreSQL ${PG_VERSION}" >> release_notes.md
-          echo "- pgvector ${PGVECTOR_VERSION} (downloaded on first run)" >> release_notes.md
+          echo "- pgvector ${PGVECTOR_VERSION}" >> release_notes.md
           echo "" >> release_notes.md
           echo "### Installation (macOS/Linux)" >> release_notes.md
           echo "\`\`\`bash" >> release_notes.md

README.md

@@ -396,9 +396,7 @@ pg0 start -c shared_buffers=1GB -c maintenance_work_mem=2GB
 
 ## How It Works
 
-PostgreSQL is **bundled directly** into the pg0 binary - no downloads required on first run! On first start, pg0 extracts PostgreSQL to `~/.pg0/installation/` and initializes the database.
-
-pgvector is downloaded on first run from pre-compiled binaries and cached in the installation directory.
+PostgreSQL and pgvector are **bundled directly** into the pg0 binary - no downloads required, works completely offline! On first start, pg0 extracts PostgreSQL and pgvector to `~/.pg0/installation/` and initializes the database.
 
 Data is stored in `~/.pg0/instances/<name>/data/` (or your custom `--data-dir`) and persists between restarts.
 

build.rs

@@ -5,7 +5,6 @@ use std::path::PathBuf;
 
 fn main() {
     println!("cargo:rerun-if-changed=versions.env");
-    println!("cargo:rerun-if-env-changed=BUNDLE_POSTGRESQL");
 
     // Load versions from versions.env
     let versions_env = fs::read_to_string("versions.env").expect("Failed to read versions.env");
@@ -35,27 +34,11 @@ fn main() {
     println!("cargo:rustc-env=PGVECTOR_COMPILED_TAG={}", pgvector_tag);
     println!("cargo:rustc-env=PGVECTOR_COMPILED_REPO={}", pgvector_repo);
 
-    // Check if we should bundle PostgreSQL
-    let bundle = env::var("BUNDLE_POSTGRESQL")
-        .map(|v| v == "1" || v == "true")
-        .unwrap_or(false);
-
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 
-    if bundle {
-        bundle_postgresql(&pg_version, &out_dir);
-    } else {
-        // Create an empty marker file so include_bytes! doesn't fail
-        let marker = out_dir.join("postgresql_bundle.tar.gz");
-        if !marker.exists() {
-            fs::write(&marker, b"").expect("Failed to create empty bundle marker");
-        }
-        println!(
-            "cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
-            marker.display()
-        );
-        println!("cargo:rustc-env=POSTGRESQL_BUNDLED=false");
-    }
+    // Bundle PostgreSQL and pgvector
+    bundle_postgresql(&pg_version, &out_dir);
+    bundle_pgvector(&pg_version, &pgvector_tag, &pgvector_repo, &out_dir);
 }
 
 fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) {
@@ -115,7 +98,71 @@ fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) {
         "cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
         bundle_path.display()
     );
-    println!("cargo:rustc-env=POSTGRESQL_BUNDLED=true");
+}
+
+fn bundle_pgvector(pg_version: &str, pgvector_tag: &str, pgvector_repo: &str, out_dir: &PathBuf) {
+    let target = env::var("TARGET").unwrap();
+
+    // Map Rust target to pgvector platform name
+    let pgvector_platform = match target.as_str() {
+        "aarch64-apple-darwin" => "aarch64-apple-darwin",
+        "x86_64-apple-darwin" => "x86_64-apple-darwin",
+        "x86_64-unknown-linux-gnu" => "x86_64-unknown-linux-gnu",
+        "x86_64-unknown-linux-musl" => "x86_64-unknown-linux-gnu", // musl uses gnu pgvector
+        "aarch64-unknown-linux-gnu" => "aarch64-unknown-linux-gnu",
+        "aarch64-unknown-linux-musl" => "aarch64-unknown-linux-gnu", // musl uses gnu pgvector
+        "x86_64-pc-windows-msvc" => {
+            eprintln!("Warning: pgvector not available for Windows, skipping bundle");
+            let marker = out_dir.join("pgvector_bundle.tar.gz");
+            fs::write(&marker, b"").expect("Failed to create empty pgvector marker");
+            println!(
+                "cargo:rustc-env=PGVECTOR_BUNDLE_PATH={}",
+                marker.display()
+            );
+            return;
+        }
+        _ => {
+            eprintln!(
+                "Warning: Unknown target {}, pgvector will not be bundled",
+                target
+            );
+            let marker = out_dir.join("pgvector_bundle.tar.gz");
+            fs::write(&marker, b"").expect("Failed to create empty pgvector marker");
+            println!(
+                "cargo:rustc-env=PGVECTOR_BUNDLE_PATH={}",
+                marker.display()
+            );
+            return;
+        }
+    };
+
+    // Get PG major version (e.g., "18" from "18.1.0")
+    let pg_major = pg_version.split('.').next().unwrap_or("18");
+
+    let filename = format!("pgvector-{}-pg{}.tar.gz", pgvector_platform, pg_major);
+    let url = format!(
+        "https://github.com/{}/releases/download/{}/{}",
+        pgvector_repo, pgvector_tag, filename
+    );
+
+    let bundle_path = out_dir.join(&filename);
+
+    // Download if not already cached
+    if !bundle_path.exists() {
+        eprintln!(
+            "Downloading pgvector for {} (PG {})...",
+            pgvector_platform, pg_major
+        );
+        download_file(&url, &bundle_path).expect("Failed to download pgvector bundle");
+        eprintln!("Downloaded to {}", bundle_path.display());
+    } else {
+        eprintln!("Using cached pgvector bundle: {}", bundle_path.display());
+    }
+
+    println!(
+        "cargo:rustc-env=PGVECTOR_BUNDLE_PATH={}",
+        bundle_path.display()
+    );
 }
 
 fn download_file(url: &str, dest: &PathBuf) -> io::Result<()> {

docker-tests/README.md

@@ -47,22 +47,6 @@ chmod +x *.sh
 ./test_alpine_arm64.sh
 ```
 
-### Using GitHub Token (Avoid Rate Limiting)
-
-To avoid GitHub API rate limiting, set a personal access token:
-
-```bash
-# Create a token at https://github.com/settings/tokens
-# No scopes needed for public repos
-
-export GITHUB_TOKEN=ghp_your_token_here
-
-# Now run tests with 5000 req/hour instead of 60
-./test_debian_amd64.sh
-```
-
-The token is automatically passed into the Docker containers.
-
 ## Expected Results
 
 ### Debian/Ubuntu (glibc)
@@ -78,15 +62,11 @@ The token is automatically passed into the Docker containers.
 ## Requirements
 
 - Docker installed and running
-- Internet connection (to download pg0 and PostgreSQL)
-- ~500MB free space for PostgreSQL binaries
+- Internet connection (to download pg0 binary)
+- ~50MB free space (PostgreSQL and pgvector are bundled in the binary)
 
 ## Troubleshooting
 
-### Test hangs during PostgreSQL download
-
-This may be due to GitHub API rate limiting. Wait a few minutes and try again.
-
 ### ARM64 tests are slow
 
 ARM64 tests use emulation on x86_64 hosts, which is slower. This is expected.

docker-tests/test_alpine_amd64.sh

@@ -12,7 +12,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh"
 
 docker run --rm --platform=linux/amd64 \
-  -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
   -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \
   python:3.11-alpine sh -c '
 set -e
@@ -39,7 +38,6 @@ echo ""
 echo "=== Switching to non-root user for pg0 ==="
 su - pguser << EOF
 set -e
-export GITHUB_TOKEN="${GITHUB_TOKEN}"
 
 echo "=== Installing pg0 ==="
 bash /usr/local/bin/install.sh

docker-tests/test_alpine_arm64.sh

@@ -12,7 +12,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh"
 
 docker run --rm --platform=linux/arm64 \
-  -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
   -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \
   python:3.11-alpine sh -c '
 set -e
@@ -39,7 +38,6 @@ echo ""
 echo "=== Switching to non-root user for pg0 ==="
 su - pguser << EOF
 set -e
-export GITHUB_TOKEN="${GITHUB_TOKEN}"
 
 echo "=== Installing pg0 ==="
 bash /usr/local/bin/install.sh

docker-tests/test_debian_amd64.sh

@@ -21,7 +21,6 @@ if [ -n "${PG0_BINARY_PATH:-}" ]; then
 fi
 
 docker run --rm --platform=linux/amd64 \
-  -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
   $BINARY_ENV \
   -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \
   $VOLUME_ARGS \
@@ -54,7 +53,6 @@ echo ""
 echo "=== Switching to non-root user for pg0 ==="
 su - pguser << EOF
 set -e
-export GITHUB_TOKEN="${GITHUB_TOKEN}"
 export PG0_BINARY_URL="${PG0_BINARY_URL}"
 
 echo "=== Installing pg0 ==="

docker-tests/test_debian_arm64.sh

@@ -12,7 +12,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh"
 
 docker run --rm --platform=linux/arm64 \
-  -e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
   -v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \
   python:3.11-slim bash -c '
 set -e
@@ -43,7 +42,6 @@ echo ""
 echo "=== Switching to non-root user for pg0 ==="
 su - pguser << EOF
 set -e
-export GITHUB_TOKEN="${GITHUB_TOKEN}"
 
 echo "=== Installing pg0 ==="
 bash /usr/local/bin/install.sh