Skip to content

Commit 7dc6469

Browse files
committed
feat: multiple versions for the vector extension
Build multiple versions of the vector extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17.
1 parent a3d559f commit 7dc6469

File tree

4 files changed

+289
-22
lines changed

4 files changed

+289
-22
lines changed

nix/checks.nix

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,10 @@
309309
;
310310
}
311311
// pkgs.lib.optionalAttrs (system == "x86_64-linux") {
312+
pgvector = import ./ext/tests/pgvector.nix {
313+
inherit self;
314+
inherit pkgs;
315+
};
312316
wrappers = import ./ext/tests/wrappers.nix {
313317
inherit self;
314318
inherit pkgs;

nix/ext/pgvector.nix

Lines changed: 78 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,92 @@
11
{
2+
pkgs,
23
lib,
34
stdenv,
45
fetchFromGitHub,
56
postgresql,
67
}:
8+
let
9+
pname = "vector";
710

8-
stdenv.mkDerivation rec {
9-
pname = "pgvector";
10-
version = "0.8.0";
11+
# Load version configuration from external file
12+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};
1113

12-
buildInputs = [ postgresql ];
14+
# Filter versions compatible with current PostgreSQL version
15+
supportedVersions = lib.filterAttrs (
16+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
17+
) allVersions;
1318

14-
src = fetchFromGitHub {
15-
owner = "pgvector";
16-
repo = pname;
17-
rev = "refs/tags/v${version}";
18-
hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q=";
19-
};
19+
# Derived version information
20+
versions = lib.naturalSort (lib.attrNames supportedVersions);
21+
latestVersion = lib.last versions;
22+
numberOfVersions = builtins.length versions;
23+
packages = builtins.attrValues (
24+
lib.mapAttrs (name: value: build name value.hash) supportedVersions
25+
);
26+
27+
# Build function for individual versions
28+
build =
29+
version: hash:
30+
stdenv.mkDerivation rec {
31+
inherit pname version;
32+
33+
buildInputs = [ postgresql ];
34+
35+
src = fetchFromGitHub {
36+
owner = "pgvector";
37+
repo = "pgvector";
38+
rev = "refs/tags/v${version}";
39+
inherit hash;
40+
};
41+
42+
installPhase = ''
43+
mkdir -p $out/{lib,share/postgresql/extension}
44+
45+
# Install shared library with version suffix
46+
mv ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
47+
48+
# Create version-specific control file
49+
sed -e "/^default_version =/d" \
50+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
51+
${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
52+
53+
# Copy SQL file to install the specific version
54+
cp sql/${pname}.sql $out/share/postgresql/extension/${pname}--${version}.sql
55+
56+
# For the latest version, copy sql upgrade script, default control file and symlink
57+
if [[ "${version}" == "${latestVersion}" ]]; then
58+
cp sql/*.sql $out/share/postgresql/extension
59+
{
60+
echo "default_version = '${latestVersion}'"
61+
cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
62+
} > $out/share/postgresql/extension/${pname}.control
63+
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
64+
fi
2065
21-
installPhase = ''
22-
mkdir -p $out/{lib,share/postgresql/extension}
66+
runHook postInstall
67+
'';
2368

24-
cp *${postgresql.dlSuffix} $out/lib
25-
cp sql/*.sql $out/share/postgresql/extension
26-
cp *.control $out/share/postgresql/extension
27-
'';
69+
meta = with lib; {
70+
description = "Open-source vector similarity search for Postgres";
71+
homepage = "https://github.com/${src.owner}/${src.repo}";
72+
maintainers = with maintainers; [ olirice ];
73+
platforms = postgresql.meta.platforms;
74+
license = licenses.postgresql;
75+
};
76+
};
77+
in
78+
pkgs.buildEnv {
79+
name = pname;
80+
paths = packages;
81+
pathsToLink = [
82+
"/lib"
83+
"/share/postgresql/extension"
84+
];
2885

29-
meta = with lib; {
30-
description = "Open-source vector similarity search for Postgres";
31-
homepage = "https://github.com/${src.owner}/${src.repo}";
32-
maintainers = with maintainers; [ olirice ];
33-
platforms = postgresql.meta.platforms;
34-
license = licenses.postgresql;
86+
passthru = {
87+
inherit versions numberOfVersions;
88+
pname = "${pname}-all";
89+
version =
90+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
3591
};
3692
}

nix/ext/tests/pgvector.nix

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{ self, pkgs }:
2+
let
3+
pname = "vector";
4+
inherit (pkgs) lib;
5+
installedExtension =
6+
postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all";
7+
versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions;
8+
postgresqlWithExtension =
9+
postgresql:
10+
let
11+
majorVersion = lib.versions.major postgresql.version;
12+
pkg = pkgs.buildEnv {
13+
name = "postgresql-${majorVersion}-${pname}";
14+
paths = [
15+
postgresql
16+
postgresql.lib
17+
(installedExtension majorVersion)
18+
];
19+
passthru = {
20+
inherit (postgresql) version psqlSchema;
21+
lib = pkg;
22+
withPackages = _: pkg;
23+
};
24+
nativeBuildInputs = [ pkgs.makeWrapper ];
25+
pathsToLink = [
26+
"/"
27+
"/bin"
28+
"/lib"
29+
];
30+
postBuild = ''
31+
wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
32+
wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
33+
wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
34+
'';
35+
};
36+
in
37+
pkg;
38+
pgsodiumGetKey = lib.getExe (
39+
pkgs.writeShellScriptBin "pgsodium-getkey" ''
40+
echo 0000000000000000000000000000000000000000000000000000000000000000
41+
''
42+
);
43+
in
44+
self.inputs.nixpkgs.lib.nixos.runTest {
45+
name = pname;
46+
hostPkgs = pkgs;
47+
nodes.server =
48+
{ config, ... }:
49+
{
50+
virtualisation = {
51+
forwardPorts = [
52+
{
53+
from = "host";
54+
host.port = 13022;
55+
guest.port = 22;
56+
}
57+
];
58+
};
59+
60+
services.postgresql = {
61+
enable = true;
62+
package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
63+
settings = {
64+
"shared_preload_libraries" = pname;
65+
"pgsodium.getkey_script" = pgsodiumGetKey;
66+
};
67+
};
68+
69+
specialisation.postgresql17.configuration = {
70+
services.postgresql = {
71+
package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17);
72+
};
73+
74+
systemd.services.postgresql-migrate = {
75+
serviceConfig = {
76+
Type = "oneshot";
77+
RemainAfterExit = true;
78+
User = "postgres";
79+
Group = "postgres";
80+
StateDirectory = "postgresql";
81+
WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}";
82+
};
83+
script =
84+
let
85+
oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
86+
newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17;
87+
oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}";
88+
newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}";
89+
in
90+
''
91+
if [[ ! -d ${newDataDir} ]]; then
92+
install -d -m 0700 -o postgres -g postgres "${newDataDir}"
93+
${newPostgresql}/bin/initdb -D "${newDataDir}"
94+
echo "shared_preload_libraries = '${pname}'" >> "${newDataDir}/postgresql.conf"
95+
echo "pgsodium.getkey_script = '${pgsodiumGetKey}'" >> "${newDataDir}/postgresql.conf";
96+
${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \
97+
--old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin"
98+
else
99+
echo "${newDataDir} already exists"
100+
fi
101+
'';
102+
};
103+
104+
systemd.services.postgresql = {
105+
after = [ "postgresql-migrate.service" ];
106+
requires = [ "postgresql-migrate.service" ];
107+
};
108+
};
109+
};
110+
testScript =
111+
{ nodes, ... }:
112+
let
113+
pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
114+
in
115+
''
116+
versions = {
117+
"15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
118+
"17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
119+
}
120+
121+
def run_sql(query):
122+
return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip()
123+
124+
def check_upgrade_path(pg_version):
125+
with subtest("Check ${pname} upgrade path"):
126+
firstVersion = versions[pg_version][0]
127+
server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'")
128+
run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""")
129+
installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""")
130+
assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}"
131+
for version in versions[pg_version][1:]:
132+
run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""")
133+
installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""")
134+
assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}"
135+
136+
start_all()
137+
138+
server.wait_for_unit("multi-user.target")
139+
server.wait_for_unit("postgresql.service")
140+
141+
check_upgrade_path("15")
142+
143+
with subtest("Check ${pname} latest extension version"):
144+
server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'")
145+
server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'")
146+
installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""")
147+
latestVersion = versions["15"][-1]
148+
assert f"${pname},{latestVersion}" in installed_extensions
149+
150+
with subtest("switch to postgresql 17"):
151+
server.succeed(
152+
"${pg17-configuration}/bin/switch-to-configuration test >&2"
153+
)
154+
155+
check_upgrade_path("17")
156+
'';
157+
}

nix/ext/versions.json

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,55 @@
2020
],
2121
"hash": "sha256-G0eQk2bY5CNPMeokN/nb05g03CuiplRf902YXFVQFbs="
2222
}
23+
},
24+
"vector": {
25+
"0.4.0": {
26+
"postgresql": [
27+
"15"
28+
],
29+
"hash": "sha256-bOckX7zvHhgJDDhoAm+VZVIeVIf2hG/3oWZWuTtnZPo="
30+
},
31+
"0.5.1": {
32+
"postgresql": [
33+
"15",
34+
"17"
35+
],
36+
"hash": "sha256-ZNzq+dATZn9LUgeOczsaadr5hwdbt9y/+sAOPIdr77U="
37+
},
38+
"0.6.0": {
39+
"postgresql": [
40+
"15",
41+
"17"
42+
],
43+
"hash": "sha256-hXm+k0BZ9xZP1Tnek14jPoKCPQkA5ovscu9IX2mW7Kc="
44+
},
45+
"0.6.2": {
46+
"postgresql": [
47+
"15",
48+
"17"
49+
],
50+
"hash": "sha256-r+TpFJg6WrMn0L2B7RpmSRvw3XxpHzMRtpFWDCzLvgs="
51+
},
52+
"0.7.0": {
53+
"postgresql": [
54+
"15",
55+
"17"
56+
],
57+
"hash": "sha256-vFn7sNphOYyig6Jl1HILMaC2t9strFQBQ8ywL8Ibx1M="
58+
},
59+
"0.7.4": {
60+
"postgresql": [
61+
"15",
62+
"17"
63+
],
64+
"hash": "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI="
65+
},
66+
"0.8.0": {
67+
"postgresql": [
68+
"15",
69+
"17"
70+
],
71+
"hash": "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q="
72+
}
2373
}
2474
}

0 commit comments

Comments
 (0)