Skip to content

Commit 7bd365c

Browse files
marcobambinimarcobambini
authored
Merge pull request #1 from sqliteai/add-cross-platform-builds
add cross platform builds
2 parents 9297a55 + 734302a commit 7bd365c

File tree

5 files changed

+358
-2
lines changed

5 files changed

+358
-2
lines changed

‎.github/workflows/main.yml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
name: build, test and release sqlite-vector
2+
on:
3+
push:
4+
workflow_dispatch:
5+
6+
permissions:
7+
contents: write
8+
9+
jobs:
10+
build:
11+
runs-on: ${{ matrix.os }}
12+
name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'isim' && matrix.name != 'ios' && ' + test' || ''}}
13+
timeout-minutes: 20
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
include:
18+
- os: ubuntu-latest
19+
arch: x86_64
20+
name: linux
21+
- os: LinuxARM64
22+
arch: arm64
23+
name: linux
24+
- os: macos-latest
25+
name: macos
26+
- os: windows-latest
27+
arch: x86_64
28+
name: windows
29+
- os: ubuntu-latest
30+
arch: arm64-v8a
31+
name: android
32+
make: PLATFORM=android ARCH=arm64-v8a
33+
- os: ubuntu-latest
34+
arch: x86_64
35+
name: android
36+
make: PLATFORM=android ARCH=x86_64
37+
sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip
38+
- os: macos-latest
39+
name: ios
40+
make: PLATFORM=ios
41+
- os: macos-latest
42+
name: isim
43+
make: PLATFORM=isim
44+
45+
defaults:
46+
run:
47+
shell: bash
48+
49+
steps:
50+
51+
- uses: actions/[email protected]
52+
53+
- name: build sqlite-vector
54+
run: make extension ${{ matrix.make && matrix.make || ''}}
55+
56+
- name: windows install sqlite3
57+
if: matrix.os == 'windows-latest'
58+
run: choco install sqlite -y
59+
60+
- name: macos install sqlite3 without SQLITE_OMIT_LOAD_EXTENSION
61+
if: matrix.name == 'macos'
62+
run: brew link sqlite --force
63+
64+
- name: android setup test environment
65+
if: matrix.name == 'android' && matrix.arch != 'arm64-v8a'
66+
run: |
67+
68+
echo "::group::enable kvm group perms"
69+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
70+
sudo udevadm control --reload-rules
71+
sudo udevadm trigger --name-match=kvm
72+
echo "::endgroup::"
73+
74+
echo "::group::download and build sqlite3 without SQLITE_OMIT_LOAD_EXTENSION"
75+
curl -O ${{ matrix.sqlite-amalgamation-zip }}
76+
unzip sqlite-amalgamation-*.zip
77+
export ${{ matrix.make }}
78+
$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.arch }}-linux-android26-clang sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl
79+
# remove unused folders to save up space
80+
rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* openssl
81+
echo "::endgroup::"
82+
83+
echo "::group::prepare the test script"
84+
make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator"
85+
cat > commands.sh << EOF
86+
mv -f /data/local/tmp/sqlite3 /system/xbin
87+
cd /data/local/tmp
88+
$(make test PLATFORM=$PLATFORM ARCH=$ARCH -n)
89+
EOF
90+
echo "::endgroup::"
91+
92+
- name: android test sqlite-vector
93+
if: matrix.name == 'android' && matrix.arch != 'arm64-v8a'
94+
uses: reactivecircus/[email protected]
95+
with:
96+
api-level: 26
97+
arch: ${{ matrix.arch }}
98+
script: |
99+
adb root
100+
adb remount
101+
adb push ${{ github.workspace }}/. /data/local/tmp/
102+
adb shell "sh /data/local/tmp/commands.sh"
103+
104+
- name: test sqlite-vector
105+
if: matrix.name == 'linux' || matrix.name == 'windows' || matrix.name == 'macos'
106+
run: make test
107+
108+
- uses: actions/[email protected]
109+
if: always()
110+
with:
111+
name: vector-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }}
112+
path: dist/vector.*
113+
if-no-files-found: error
114+
115+
release:
116+
runs-on: ubuntu-latest
117+
name: release
118+
needs: build
119+
if: github.ref == 'refs/heads/main'
120+
121+
env:
122+
GH_TOKEN: ${{ github.token }}
123+
124+
steps:
125+
126+
- uses: actions/[email protected]
127+
128+
- uses: actions/[email protected]
129+
with:
130+
path: artifacts
131+
132+
- name: release tag version from sqlite-vector.h
133+
id: tag
134+
run: |
135+
FILE="src/sqlite-vector.h"
136+
VERSION=$(grep -oP '#define SQLITE_VECTOR_VERSION\s+"\K[^"]+' "$FILE")
137+
if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
138+
LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name')
139+
if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
140+
echo "version=$VERSION" >> $GITHUB_OUTPUT
141+
else
142+
echo "::warning file=src/sqlite-vector.h::To release a new version, please update the SQLITE_VECTOR_VERSION in src/sqlite-vector.h to be different than the latest $LATEST"
143+
fi
144+
exit 0
145+
fi
146+
echo "❌ SQLITE_VECTOR_VERSION not found in sqlite-vector.h"
147+
exit 1
148+
149+
- name: zip artifacts
150+
run: |
151+
for folder in "artifacts"/*; do
152+
if [ -d "$folder" ]; then
153+
name=$(basename "$folder")
154+
if [[ "$name" != "github-pages" ]]; then
155+
zip -jq "${name}-${{ steps.tag.outputs.version }}.zip" "$folder"/*
156+
tar -cJf "${name}-${{ steps.tag.outputs.version }}.tar.xz" -C "$folder" .
157+
tar -czf "${name}-${{ steps.tag.outputs.version }}.tar.gz" -C "$folder" .
158+
fi
159+
fi
160+
done
161+
162+
- uses: softprops/[email protected]
163+
if: steps.tag.outputs.version != ''
164+
with:
165+
generate_release_notes: true
166+
tag_name: ${{ steps.tag.outputs.version }}
167+
files: |
168+
vector-*-${{ steps.tag.outputs.version }}.zip
169+
vector-*-${{ steps.tag.outputs.version }}.tar.xz
170+
vector-*-${{ steps.tag.outputs.version }}.tar.gz
171+
make_latest: true

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,10 @@
11
.DS_Store
2+
*.xcworkspacedata
3+
*.xcuserstate
4+
*.xcbkptlist
5+
*.plist
6+
/build
7+
/dist
8+
*.sqlite
9+
*.a
10+
.vscode

‎Makefile‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# Makefile for SQLite Vector Extension
2+
# Supports compilation for Linux, macOS, Windows, Android and iOS
3+
4+
# customize sqlite3 executable with
5+
# make test SQLITE3=/opt/homebrew/Cellar/sqlite/3.49.1/bin/sqlite3
6+
SQLITE3 ?= sqlite3
7+
8+
# Set default platform if not specified
9+
ifeq ($(OS),Windows_NT)
10+
PLATFORM := windows
11+
HOST := windows
12+
CPUS := $(shell powershell -Command "[Environment]::ProcessorCount")
13+
else
14+
HOST = $(shell uname -s | tr '[:upper:]' '[:lower:]')
15+
ifeq ($(HOST),darwin)
16+
PLATFORM := macos
17+
CPUS := $(shell sysctl -n hw.ncpu)
18+
else
19+
PLATFORM := $(HOST)
20+
CPUS := $(shell nproc)
21+
endif
22+
endif
23+
24+
# Speed up builds by using all available CPU cores
25+
MAKEFLAGS += -j$(CPUS)
26+
27+
# Compiler and flags
28+
CC = gcc
29+
CFLAGS = -Wall -Wextra -Wno-unused-parameter -I$(SRC_DIR) -I$(LIB_DIR)
30+
31+
# Directories
32+
SRC_DIR = src
33+
DIST_DIR = dist
34+
LIB_DIR = libs
35+
VPATH = $(SRC_DIR):$(LIB_DIR)
36+
BUILD_DIR = build
37+
38+
# Files
39+
SRC_FILES = $(wildcard $(SRC_DIR)/*.c)
40+
OBJ_FILES = $(patsubst %.c, $(BUILD_DIR)/%.o, $(notdir $(SRC_FILES)))
41+
42+
# Platform-specific settings
43+
ifeq ($(PLATFORM),windows)
44+
TARGET := $(DIST_DIR)/vector.dll
45+
LDFLAGS += -shared
46+
# Create .def file for Windows
47+
DEF_FILE := $(BUILD_DIR)/vector.def
48+
else ifeq ($(PLATFORM),macos)
49+
TARGET := $(DIST_DIR)/vector.dylib
50+
LDFLAGS += -arch x86_64 -arch arm64 -dynamiclib -undefined dynamic_lookup
51+
CFLAGS += -arch x86_64 -arch arm64
52+
else ifeq ($(PLATFORM),android)
53+
# Set ARCH to find Android NDK's Clang compiler, the user should set the ARCH
54+
ifeq ($(filter %,$(ARCH)),)
55+
$(error "Android ARCH must be set to ARCH=x86_64 or ARCH=arm64-v8a")
56+
endif
57+
# Set ANDROID_NDK path to find android build tools
58+
# e.g. on MacOS: export ANDROID_NDK=/Users/username/Library/Android/sdk/ndk/25.2.9519653
59+
ifeq ($(filter %,$(ANDROID_NDK)),)
60+
$(error "Android NDK must be set")
61+
endif
62+
63+
BIN = $(ANDROID_NDK)/toolchains/llvm/prebuilt/$(HOST)-x86_64/bin
64+
PATH := $(BIN):$(PATH)
65+
66+
ifneq (,$(filter $(ARCH),arm64 arm64-v8a))
67+
override ARCH := aarch64
68+
endif
69+
70+
CC = $(BIN)/$(ARCH)-linux-android26-clang
71+
TARGET := $(DIST_DIR)/vector.so
72+
LDFLAGS += -shared
73+
else ifeq ($(PLATFORM),ios)
74+
TARGET := $(DIST_DIR)/vector.dylib
75+
SDK := -isysroot $(shell xcrun --sdk iphoneos --show-sdk-path) -miphoneos-version-min=11.0
76+
LDFLAGS += -dynamiclib $(SDK)
77+
CFLAGS += -arch arm64 $(SDK)
78+
else ifeq ($(PLATFORM),isim)
79+
TARGET := $(DIST_DIR)/vector.dylib
80+
SDK := -isysroot $(shell xcrun --sdk iphonesimulator --show-sdk-path) -miphonesimulator-version-min=11.0
81+
LDFLAGS += -arch x86_64 -arch arm64 -dynamiclib $(SDK)
82+
CFLAGS += -arch x86_64 -arch arm64 $(SDK)
83+
else # linux
84+
TARGET := $(DIST_DIR)/vector.so
85+
LDFLAGS += -shared
86+
endif
87+
88+
# Windows .def file generation
89+
$(DEF_FILE):
90+
ifeq ($(PLATFORM),windows)
91+
@echo "LIBRARY vector.dll" > $@
92+
@echo "EXPORTS" >> $@
93+
@echo " sqlite3_vector_init" >> $@
94+
endif
95+
96+
# Make sure the build and dist directories exist
97+
$(shell mkdir -p $(BUILD_DIR) $(DIST_DIR))
98+
99+
# Default target
100+
extension: $(TARGET)
101+
all: $(TARGET)
102+
103+
# Loadable library
104+
$(TARGET): $(OBJ_FILES) $(DEF_FILE)
105+
$(CC) $(OBJ_FILES) $(DEF_FILE) -o $@ $(LDFLAGS)
106+
ifeq ($(PLATFORM),windows)
107+
# Generate import library for Windows
108+
dlltool -D $@ -d $(DEF_FILE) -l $(DIST_DIR)/vector.lib
109+
endif
110+
111+
# Object files
112+
$(BUILD_DIR)/%.o: %.c
113+
$(CC) $(CFLAGS) -O3 -fPIC -c $< -o $@
114+
115+
test: $(TARGET)
116+
$(SQLITE3) ":memory:" -cmd ".bail on" ".load ./$<" "SELECT vector_version();"
117+
118+
# Clean up generated files
119+
clean:
120+
rm -rf $(BUILD_DIR)/* $(DIST_DIR)/* *.gcda *.gcno *.gcov *.sqlite
121+
122+
# Help message
123+
help:
124+
@echo "SQLite Vector Extension Makefile"
125+
@echo "Usage:"
126+
@echo " make [PLATFORM=platform] [ARCH=arch] [ANDROID_NDK=\$$ANDROID_HOME/ndk/26.1.10909125] [target]"
127+
@echo ""
128+
@echo "Platforms:"
129+
@echo " linux (default on Linux)"
130+
@echo " macos (default on macOS)"
131+
@echo " windows (default on Windows)"
132+
@echo " android (needs ARCH to be set to x86_64 or arm64-v8a and ANDROID_NDK to be set)"
133+
@echo " ios (only on macOS)"
134+
@echo " isim (only on macOS)"
135+
@echo ""
136+
@echo "Targets:"
137+
@echo " all - Build the extension (default)"
138+
@echo " clean - Remove built files"
139+
@echo " test - Test the extension"
140+
@echo " help - Display this help message"
141+
142+
.PHONY: all clean test extension help

‎src/distance-neon.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,8 @@ static inline float uint8_distance_l2_impl_neon(const void *v1, const void *v2,
155155
uint8x16_t vb = vld1q_u8(b + i);
156156

157157
// compute 8-bit differences widened to signed 16-bit
158-
int16x8_t diff_lo = vsubl_u8(vget_low_u8(va), vget_low_u8(vb));
159-
int16x8_t diff_hi = vsubl_u8(vget_high_u8(va), vget_high_u8(vb));
158+
int16x8_t diff_lo = (int16x8_t)vsubl_u8(vget_low_u8(va), vget_low_u8(vb));
159+
int16x8_t diff_hi = (int16x8_t)vsubl_u8(vget_high_u8(va), vget_high_u8(vb));
160160

161161
// widen to signed 32-bit and square
162162
int32x4_t diff_lo_0 = vmovl_s16(vget_low_s16(diff_lo));

‎src/sqlite-vector.c‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,32 @@
1818
#include <unistd.h>
1919
#include <stdint.h>
2020
#include <stdbool.h>
21+
#include <stddef.h>
22+
23+
#ifdef _WIN32
24+
char *strcasestr(const char *haystack, const char *needle) {
25+
if (!haystack || !needle) return NULL;
26+
if (!*needle) return (char *)haystack;
27+
for (; *haystack; ++haystack) {
28+
const char *h = haystack;
29+
const char *n = needle;
30+
while (*h && *n && tolower((unsigned char)*h) == tolower((unsigned char)*n)) {
31+
++h;
32+
++n;
33+
}
34+
if (!*n) return (char *)haystack;
35+
}
36+
return NULL;
37+
}
38+
#endif
39+
40+
#if defined(_WIN32) || defined(__linux__)
41+
#include <float.h>
42+
#endif
43+
44+
#ifndef SQLITE_CORE
45+
SQLITE_EXTENSION_INIT1
46+
#endif
2147

2248
#define DEBUG_VECTOR_ALWAYS(...) do {printf(__VA_ARGS__ );printf("\n");} while (0)
2349

@@ -842,8 +868,13 @@ static int vector_rebuild_quantization (sqlite3_context *context, const char *ta
842868

843869
// STEP 1
844870
// find global min/max across ALL vectors
871+
#if defined(_WIN32) || defined(__linux__)
872+
float min_val = FLT_MAX;
873+
float max_val = -FLT_MAX;
874+
#else
845875
float min_val = MAXFLOAT;
846876
float max_val = -MAXFLOAT;
877+
#endif
847878

848879
while (1) {
849880
rc = sqlite3_step(vm);
@@ -1814,6 +1845,9 @@ static void vector_backend (sqlite3_context *context, int argc, sqlite3_value **
18141845
// MARK: -
18151846

18161847
SQLITE_VECTOR_API int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) {
1848+
#ifndef SQLITE_CORE
1849+
SQLITE_EXTENSION_INIT2(pApi);
1850+
#endif
18171851
int rc = SQLITE_OK;
18181852

18191853
init_distance_functions(false);

0 commit comments

Comments
 (0)