Skip to content

Commit a565d5b

Browse files
nicoloboschinicoloboschi
committed
bundle and improve runtime perf
1 parent c5565c8 commit a565d5b

File tree

8 files changed

+537
-40
lines changed

8 files changed

+537
-40
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
9+
jobs:
10+
build:
11+
name: Build CLI
12+
runs-on: macos-latest
13+
steps:
14+
- uses: actions/checkout@v4
15+
16+
- name: Install Rust
17+
uses: dtolnay/rust-toolchain@stable
18+
19+
- name: Build
20+
env:
21+
BUNDLE_POSTGRESQL: "true"
22+
run: cargo build --release
23+
24+
- name: Upload CLI artifact
25+
uses: actions/upload-artifact@v4
26+
with:
27+
name: pg0-macos
28+
path: target/release/pg0
29+
30+
sdk-python:
31+
name: Python SDK Tests
32+
needs: build
33+
runs-on: macos-latest
34+
steps:
35+
- uses: actions/checkout@v4
36+
37+
- name: Download CLI
38+
uses: actions/download-artifact@v4
39+
with:
40+
name: pg0-macos
41+
path: ~/.local/bin
42+
43+
- name: Make CLI executable
44+
run: chmod +x ~/.local/bin/pg0
45+
46+
- name: Set up Python
47+
uses: actions/setup-python@v5
48+
with:
49+
python-version: "3.11"
50+
51+
- name: Install uv
52+
uses: astral-sh/setup-uv@v4
53+
54+
- name: Install dependencies
55+
working-directory: sdk/python
56+
run: uv sync --dev
57+
58+
- name: Run tests
59+
working-directory: sdk/python
60+
run: |
61+
export PATH="$HOME/.local/bin:$PATH"
62+
uv run pytest tests/ -v
63+
64+
sdk-node:
65+
name: Node.js SDK Tests
66+
needs: build
67+
runs-on: macos-latest
68+
steps:
69+
- uses: actions/checkout@v4
70+
71+
- name: Download CLI
72+
uses: actions/download-artifact@v4
73+
with:
74+
name: pg0-macos
75+
path: ~/.local/bin
76+
77+
- name: Make CLI executable
78+
run: chmod +x ~/.local/bin/pg0
79+
80+
- name: Set up Node.js
81+
uses: actions/setup-node@v4
82+
with:
83+
node-version: "20"
84+
85+
- name: Install dependencies
86+
working-directory: sdk/node
87+
run: npm install
88+
89+
- name: Run tests
90+
working-directory: sdk/node
91+
run: |
92+
export PATH="$HOME/.local/bin:$PATH"
93+
npm test
94+
95+
docker-tests:
96+
name: Docker Tests (${{ matrix.platform }})
97+
runs-on: ${{ matrix.runner }}
98+
strategy:
99+
fail-fast: false
100+
matrix:
101+
include:
102+
- platform: debian-amd64
103+
runner: ubuntu-latest
104+
script: docker-tests/test_debian_amd64.sh
105+
- platform: debian-arm64
106+
runner: ubuntu-latest
107+
script: docker-tests/test_debian_arm64.sh
108+
- platform: alpine-amd64
109+
runner: ubuntu-latest
110+
script: docker-tests/test_alpine_amd64.sh
111+
- platform: alpine-arm64
112+
runner: ubuntu-latest
113+
script: docker-tests/test_alpine_arm64.sh
114+
115+
steps:
116+
- uses: actions/checkout@v4
117+
118+
- name: Set up QEMU (for ARM64 emulation)
119+
if: contains(matrix.platform, 'arm64')
120+
uses: docker/setup-qemu-action@v3
121+
with:
122+
platforms: arm64
123+
124+
- name: Run Docker test
125+
run: |
126+
chmod +x ${{ matrix.script }}
127+
bash ${{ matrix.script }}

‎.github/workflows/release-cli.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ jobs:
7676
echo "RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV
7777
7878
- name: Build
79+
env:
80+
BUNDLE_POSTGRESQL: "true"
7981
run: cargo build --release --target ${{ matrix.target }}
8082

8183
- name: Prepare artifact (Unix)
@@ -144,9 +146,9 @@ jobs:
144146
echo "- \`pg0-windows-x86_64.exe\` - Windows x64" >> release_notes.md
145147
echo "" >> release_notes.md
146148
echo "### Bundled Components" >> release_notes.md
147-
echo "On first run, the CLI automatically downloads:" >> release_notes.md
148-
echo "- PostgreSQL ${PG_VERSION} (from theseus-rs)" >> release_notes.md
149-
echo "- pgvector ${PGVECTOR_VERSION}" >> release_notes.md
149+
echo "PostgreSQL is bundled directly in the binary - no downloads required on first run!" >> release_notes.md
150+
echo "- PostgreSQL ${PG_VERSION}" >> release_notes.md
151+
echo "- pgvector ${PGVECTOR_VERSION} (downloaded on first run)" >> release_notes.md
150152
echo "" >> release_notes.md
151153
echo "### Installation (macOS/Linux)" >> release_notes.md
152154
echo "\`\`\`bash" >> release_notes.md

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ dirs = "5"
2020
thiserror = "1"
2121
tracing = "0.1"
2222
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
23+
flate2 = "1"
24+
tar = "0.4"
2325

2426
[profile.release]
2527
strip = true

‎README.md‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,10 +408,67 @@ pg0 start -c shared_buffers=1GB -c maintenance_work_mem=2GB
408408

409409
## How It Works
410410

411-
On first run, pg0 downloads PostgreSQL from [theseus-rs](https://github.com/theseus-rs/postgresql-binaries) and pgvector from pre-compiled binaries. These are cached in `~/.pg0/installation/` for subsequent runs.
411+
PostgreSQL is **bundled directly** into the pg0 binary - no downloads required on first run! On first start, pg0 extracts PostgreSQL to `~/.pg0/installation/` and initializes the database.
412+
413+
pgvector is downloaded on first run from pre-compiled binaries and cached in the installation directory.
412414

413415
Data is stored in `~/.pg0/instances/<name>/data/` (or your custom `--data-dir`) and persists between restarts.
414416

417+
## Troubleshooting
418+
419+
### GitHub API Rate Limit Exceeded
420+
421+
pgvector is downloaded from GitHub on first run. If you see this error:
422+
423+
```
424+
Error: HTTP status client error (403 rate limit exceeded) for url (https://api.github.com/...)
425+
```
426+
427+
This means you've hit GitHub's API rate limit for unauthenticated requests (60 requests/hour).
428+
429+
**Solution:** Set a GitHub token to increase the rate limit to 5,000 requests/hour:
430+
431+
```bash
432+
# Create a token at https://github.com/settings/tokens (no scopes needed)
433+
export GITHUB_TOKEN=ghp_your_token_here
434+
pg0 start
435+
```
436+
437+
Or inline:
438+
439+
```bash
440+
GITHUB_TOKEN=ghp_your_token_here pg0 start
441+
```
442+
443+
The token doesn't need any special permissions - a basic token with no scopes works fine.
444+
445+
### PostgreSQL Cannot Run as Root
446+
447+
PostgreSQL refuses to run as root for security reasons. If you see permission errors in Docker or Linux:
448+
449+
```bash
450+
# Create a non-root user
451+
useradd -m pguser
452+
su - pguser -c "pg0 start"
453+
```
454+
455+
See the [Docker](#docker) section for complete examples.
456+
457+
### Port Already in Use
458+
459+
If port 5432 is already in use, pg0 will automatically find the next available port when starting without an explicit `--port`:
460+
461+
```bash
462+
pg0 start --name second-instance
463+
# Output: Port 5432 is in use, using port 5433 instead.
464+
```
465+
466+
To use a specific port, specify it explicitly:
467+
468+
```bash
469+
pg0 start --port 5433
470+
```
471+
415472
## Build from Source
416473

417474
```bash

‎build.rs‎

Lines changed: 123 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,136 @@
1+
use std::env;
12
use std::fs;
3+
use std::io;
4+
use std::path::PathBuf;
25

36
fn main() {
4-
// Read versions from versions.env
5-
let versions = fs::read_to_string("versions.env").expect("Failed to read versions.env");
7+
println!("cargo:rerun-if-changed=versions.env");
8+
println!("cargo:rerun-if-env-changed=BUNDLE_POSTGRESQL");
9+
10+
// Load versions from versions.env
11+
let versions_env = fs::read_to_string("versions.env").expect("Failed to read versions.env");
12+
let mut pg_version = String::new();
13+
let mut pgvector_version = String::new();
14+
let mut pgvector_tag = String::new();
15+
let mut pgvector_repo = String::new();
616

7-
for line in versions.lines() {
17+
for line in versions_env.lines() {
818
let line = line.trim();
919
if line.is_empty() || line.starts_with('#') {
1020
continue;
1121
}
1222
if let Some((key, value)) = line.split_once('=') {
13-
println!("cargo:rustc-env={}={}", key.trim(), value.trim());
23+
match key.trim() {
24+
"PG_VERSION" => pg_version = value.trim().to_string(),
25+
"PGVECTOR_VERSION" => pgvector_version = value.trim().to_string(),
26+
"PGVECTOR_COMPILED_TAG" => pgvector_tag = value.trim().to_string(),
27+
"PGVECTOR_COMPILED_REPO" => pgvector_repo = value.trim().to_string(),
28+
_ => {}
29+
}
1430
}
1531
}
1632

17-
// Re-run if versions.env changes
18-
println!("cargo:rerun-if-changed=versions.env");
33+
println!("cargo:rustc-env=PG_VERSION={}", pg_version);
34+
println!("cargo:rustc-env=PGVECTOR_VERSION={}", pgvector_version);
35+
println!("cargo:rustc-env=PGVECTOR_COMPILED_TAG={}", pgvector_tag);
36+
println!("cargo:rustc-env=PGVECTOR_COMPILED_REPO={}", pgvector_repo);
37+
38+
// Check if we should bundle PostgreSQL
39+
let bundle = env::var("BUNDLE_POSTGRESQL")
40+
.map(|v| v == "1" || v == "true")
41+
.unwrap_or(false);
42+
43+
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
44+
45+
if bundle {
46+
bundle_postgresql(&pg_version, &out_dir);
47+
} else {
48+
// Create an empty marker file so include_bytes! doesn't fail
49+
let marker = out_dir.join("postgresql_bundle.tar.gz");
50+
if !marker.exists() {
51+
fs::write(&marker, b"").expect("Failed to create empty bundle marker");
52+
}
53+
println!(
54+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
55+
marker.display()
56+
);
57+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=false");
58+
}
59+
}
60+
61+
fn bundle_postgresql(pg_version: &str, out_dir: &PathBuf) {
62+
let target = env::var("TARGET").unwrap();
63+
64+
// Map Rust target to theseus-rs binary name
65+
let pg_target = match target.as_str() {
66+
"aarch64-apple-darwin" => "aarch64-apple-darwin",
67+
"x86_64-apple-darwin" => "x86_64-apple-darwin",
68+
"x86_64-unknown-linux-gnu" => "x86_64-unknown-linux-gnu",
69+
"x86_64-unknown-linux-musl" => "x86_64-unknown-linux-musl",
70+
"aarch64-unknown-linux-gnu" => "aarch64-unknown-linux-gnu",
71+
"aarch64-unknown-linux-musl" => "aarch64-unknown-linux-musl",
72+
"x86_64-pc-windows-msvc" => "x86_64-pc-windows-msvc",
73+
_ => {
74+
eprintln!(
75+
"Warning: Unknown target {}, PostgreSQL will not be bundled",
76+
target
77+
);
78+
let marker = out_dir.join("postgresql_bundle.tar.gz");
79+
fs::write(&marker, b"").expect("Failed to create empty bundle marker");
80+
println!(
81+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
82+
marker.display()
83+
);
84+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=false");
85+
return;
86+
}
87+
};
88+
89+
let ext = if target.contains("windows") {
90+
"zip"
91+
} else {
92+
"tar.gz"
93+
};
94+
let filename = format!("postgresql-{}-{}.{}", pg_version, pg_target, ext);
95+
let url = format!(
96+
"https://github.com/theseus-rs/postgresql-binaries/releases/download/{}/{}",
97+
pg_version, filename
98+
);
99+
100+
let bundle_path = out_dir.join(&filename);
101+
102+
// Download if not already cached
103+
if !bundle_path.exists() {
104+
eprintln!(
105+
"Downloading PostgreSQL {} for {}...",
106+
pg_version, pg_target
107+
);
108+
download_file(&url, &bundle_path).expect("Failed to download PostgreSQL bundle");
109+
eprintln!("Downloaded to {}", bundle_path.display());
110+
} else {
111+
eprintln!("Using cached PostgreSQL bundle: {}", bundle_path.display());
112+
}
113+
114+
println!(
115+
"cargo:rustc-env=POSTGRESQL_BUNDLE_PATH={}",
116+
bundle_path.display()
117+
);
118+
println!("cargo:rustc-env=POSTGRESQL_BUNDLED=true");
119+
}
120+
121+
fn download_file(url: &str, dest: &PathBuf) -> io::Result<()> {
122+
// Use curl for downloading (available on all CI platforms)
123+
let status = std::process::Command::new("curl")
124+
.args(["-fsSL", url, "-o"])
125+
.arg(dest)
126+
.status()?;
127+
128+
if !status.success() {
129+
return Err(io::Error::new(
130+
io::ErrorKind::Other,
131+
format!("Failed to download {}", url),
132+
));
133+
}
134+
135+
Ok(())
19136
}

‎docker-tests/test_debian_amd64.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,20 @@ echo "=================================="
1111
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
1212
INSTALL_SCRIPT="$SCRIPT_DIR/../install.sh"
1313

14+
# Check if PG0_BINARY_PATH is set (local binary to test)
15+
VOLUME_ARGS=""
16+
BINARY_ENV=""
17+
if [ -n "${PG0_BINARY_PATH:-}" ]; then
18+
echo "Using local binary: $PG0_BINARY_PATH"
19+
VOLUME_ARGS="-v $PG0_BINARY_PATH:/tmp/pg0-binary:ro"
20+
BINARY_ENV="-e PG0_BINARY_URL=file:///tmp/pg0-binary"
21+
fi
22+
1423
docker run --rm --platform=linux/amd64 \
1524
-e GITHUB_TOKEN="${GITHUB_TOKEN:-}" \
25+
$BINARY_ENV \
1626
-v "$INSTALL_SCRIPT:/tmp/install.sh:ro" \
27+
$VOLUME_ARGS \
1728
python:3.11-slim bash -c '
1829
set -e
1930
@@ -44,6 +55,7 @@ echo "=== Switching to non-root user for pg0 ==="
4455
su - pguser << EOF
4556
set -e
4657
export GITHUB_TOKEN="${GITHUB_TOKEN}"
58+
export PG0_BINARY_URL="${PG0_BINARY_URL}"
4759
4860
echo "=== Installing pg0 ==="
4961
bash /usr/local/bin/install.sh

0 commit comments

Comments
 (0)