Skip to content

Commit 4c71095

Browse files
mudlermudler
authored
feat: add postgresql engine and tests (#34)
* feat: add postgresql engine and tests Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add pgsql custom image Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * add pgvector Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * CI and docs Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixup tests Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add logging Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Use Makefile in CI Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * tAdd tests for hybrid/semantic search and reset Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 542712a commit 4c71095

34 files changed

+3175
-94
lines changed

‎.github/workflows/image.yml‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070
prefix=
7171
suffix=
7272
73-
- name: Build
73+
- name: Build LocalRecall
7474
uses: docker/build-push-action@v6
7575
with:
7676
builder: ${{ steps.buildx.outputs.name }}
@@ -83,3 +83,29 @@ jobs:
8383
#tags: ${{ steps.prep.outputs.tags }}
8484
tags: ${{ steps.meta.outputs.tags }}
8585
labels: ${{ steps.meta.outputs.labels }}
86+
87+
- name: Extract metadata for PostgreSQL image
88+
id: meta-postgres
89+
uses: docker/metadata-action@2a4836ac76fe8f5d0ee3a0d89aa12a80cc552ad3
90+
with:
91+
images: quay.io/mudler/localrecall
92+
tags: |
93+
type=ref,event=branch,suffix=-postgresql-{{date 'YYYYMMDDHHmmss'}}
94+
type=semver,pattern={{raw}},suffix=-postgresql
95+
type=sha,suffix=-postgresql-{{date 'YYYYMMDDHHmmss'}}
96+
type=ref,event=branch,suffix=-postgresql
97+
flavor: |
98+
latest=auto,suffix=-postgresql
99+
prefix=
100+
suffix=-postgresql
101+
102+
- name: Build PostgreSQL
103+
uses: docker/build-push-action@v6
104+
with:
105+
builder: ${{ steps.buildx.outputs.name }}
106+
context: ./
107+
file: ./Dockerfile.pgsql
108+
platforms: linux/amd64,linux/arm64
109+
push: true
110+
tags: ${{ steps.meta-postgres.outputs.tags }}
111+
labels: ${{ steps.meta-postgres.outputs.labels }}

‎.github/workflows/test.yml‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,44 +8,68 @@ on:
88
branches:
99
- '**'
1010

11-
1211
concurrency:
1312
group: ci-${{ github.head_ref || github.ref }}-${{ github.repository }}
1413
cancel-in-progress: true
1514

16-
1715
jobs:
18-
test:
16+
unit-tests:
1917
runs-on: ubuntu-latest
2018

2119
steps:
2220
- name: Checkout code
23-
uses: actions/checkout@v2
21+
uses: actions/checkout@v4
2422

2523
- name: Set up Go
26-
uses: actions/setup-go@v2
24+
uses: actions/setup-go@v4
2725
with:
2826
go-version: '^1.22'
2927

3028
- name: Install dependencies
3129
run: go mod tidy
32-
- name: Run tests
33-
run: |
34-
make test
35-
e2e:
30+
31+
- name: Run unit tests
32+
run: make test-unit
33+
34+
integration-tests:
35+
runs-on: ubuntu-latest
36+
37+
steps:
38+
- name: Checkout code
39+
uses: actions/checkout@v4
40+
41+
- name: Set up Go
42+
uses: actions/setup-go@v4
43+
with:
44+
go-version: '^1.22'
45+
46+
- name: Install dependencies
47+
run: go mod tidy
48+
49+
- name: Run integration tests
50+
env:
51+
INTEGRATION: "true"
52+
run: make test-integration
53+
54+
e2e-tests:
3655
runs-on: ubuntu-latest
3756

3857
steps:
3958
- name: Checkout code
40-
uses: actions/checkout@v2
59+
uses: actions/checkout@v4
4160

4261
- name: Set up Go
43-
uses: actions/setup-go@v2
62+
uses: actions/setup-go@v4
4463
with:
4564
go-version: '^1.22'
4665

4766
- name: Install dependencies
4867
run: go mod tidy
49-
- name: Run tests
50-
run: |
51-
make test-e2e
68+
69+
- name: Set up Docker Buildx
70+
uses: docker/setup-buildx-action@v3
71+
72+
- name: Run E2E tests
73+
env:
74+
E2E: "true"
75+
run: make test-e2e

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Build the Go binary in a separate stage utilizing Makefile
2-
FROM golang:1.23 AS builder
2+
FROM golang:1.24 AS builder
33

44
WORKDIR /app
55
COPY go.mod go.sum ./

‎Dockerfile.pgsql‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
FROM ubuntu:24.04
2+
3+
# Set environment variables to non-interactive to prevent prompts
4+
ENV DEBIAN_FRONTEND=noninteractive
5+
ENV TZ=UTC
6+
7+
# Install necessary packages
8+
RUN apt-get update && \
9+
apt-get install -y \
10+
git \
11+
tzdata \
12+
gnupg2 \
13+
wget \
14+
curl \
15+
make \
16+
gcc \
17+
pkg-config \
18+
clang \
19+
libssl-dev \
20+
lsb-release \
21+
software-properties-common \
22+
postgresql-common \
23+
jq \
24+
build-essential \
25+
&& rm -rf /var/lib/apt/lists/*
26+
27+
# Add the PostgreSQL 18 repository
28+
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \
29+
sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
30+
apt-get update
31+
32+
33+
# Install PostgreSQL 18, contrib modules, pgvector, and timescaledb
34+
RUN apt-get install -y \
35+
postgresql-18 \
36+
postgresql-contrib-18 \
37+
postgresql-18-pgvector \
38+
postgresql-18-timescaledb \
39+
postgresql-server-dev-18 \
40+
&& rm -rf /var/lib/apt/lists/*
41+
42+
# Ensure PostgreSQL binaries are in the PATH
43+
ENV PATH="/usr/lib/postgresql/18/bin:${PATH}"
44+
45+
# Build and install pg_textsearch extension
46+
RUN git clone https://github.com/timescale/pg_textsearch /tmp/pg_textsearch && \
47+
cd /tmp/pg_textsearch && \
48+
make && \
49+
make install && \
50+
cd / && \
51+
rm -rf /tmp/pg_textsearch
52+
53+
# Install Rust (required for pgvectorscale)
54+
# Install with default target for the build platform
55+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal && \
56+
. $HOME/.cargo/env && \
57+
rustup default stable && \
58+
rustup target add $(rustc -vV | grep host | cut -d' ' -f2)
59+
ENV PATH="/root/.cargo/bin:${PATH}"
60+
ENV CARGO_TARGET_DIR="/tmp/cargo-target"
61+
62+
# Build and install pgvectorscale extension (provides diskann access method)
63+
# Note: This may fail with SIGILL errors in some Docker environments due to Rust toolchain issues.
64+
# If this fails, pgvectorscale can be built separately and installed, or the system will fall back to pgvector.
65+
# For production, use a specific version tag instead of main branch.
66+
RUN cd /tmp && \
67+
git clone --depth 1 https://github.com/timescale/pgvectorscale && \
68+
cd pgvectorscale/pgvectorscale && \
69+
PGRX_VERSION=$(cargo metadata --format-version 1 2>/dev/null | jq -r '.packages[] | select(.name == "pgrx") | .version' 2>/dev/null || \
70+
grep -E 'pgrx\s*=\s*"' Cargo.toml | head -1 | sed -E 's/.*"([^"]+)".*/\1/' || echo "0.11.8") && \
71+
echo "Installing cargo-pgrx version: $PGRX_VERSION" && \
72+
cargo install --locked cargo-pgrx --version "$PGRX_VERSION" && \
73+
cargo pgrx init --pg18 $(which pg_config) && \
74+
cargo pgrx install --release && \
75+
echo "Verifying vectorscale extension installation..." && \
76+
ls -la /usr/lib/postgresql/18/lib/vectorscale*.so 2>/dev/null && \
77+
ls -la /usr/share/postgresql/18/extension/vectorscale.control 2>/dev/null && \
78+
echo "Extension files found successfully" || \
79+
(echo "Warning: Some extension files not found" && \
80+
find /usr/lib/postgresql/18 -name "*vectorscale*" 2>/dev/null && \
81+
find /usr/share/postgresql/18 -name "*vectorscale*" 2>/dev/null) && \
82+
cd / && \
83+
rm -rf /tmp/pgvectorscale && \
84+
rm -rf /root/.cargo
85+
# Create directory for init scripts
86+
RUN mkdir -p /docker-entrypoint-initdb.d && \
87+
chmod 755 /docker-entrypoint-initdb.d
88+
89+
# Copy the initialization script
90+
COPY internal/init-db.sh /docker-entrypoint-initdb.d/01-init-extensions.sh
91+
RUN chmod +x /docker-entrypoint-initdb.d/01-init-extensions.sh
92+
93+
# Create a script to initialize PostgreSQL if needed
94+
COPY internal/postgres-init.sh /usr/local/bin/postgres-init.sh
95+
RUN chmod +x /usr/local/bin/postgres-init.sh
96+
97+
# Set up PostgreSQL data directory
98+
ENV PGDATA=/var/lib/postgresql/data
99+
ENV POSTGRES_DB=localrecall
100+
ENV POSTGRES_USER=localrecall
101+
ENV POSTGRES_PASSWORD=localrecall
102+
RUN mkdir -p "$PGDATA" && \
103+
chown -R postgres:postgres "$PGDATA" && \
104+
chmod 700 "$PGDATA"
105+
106+
# Expose PostgreSQL port
107+
EXPOSE 5432
108+
109+
# Use the standard PostgreSQL entrypoint approach
110+
USER postgres
111+
112+
# Initialize database if it doesn't exist, then start PostgreSQL
113+
CMD ["/usr/local/bin/postgres-init.sh"]

0 commit comments

Comments
 (0)