Skip to content

Commit 20dc72e

Browse files
yvan-srakasamrosejfroche
authored
feat: multiple versions for the timescaledb-apache extension (#1749)
* chores: opted-out nix-fast-build * chores: set up nix-github-actions * chores: comment out vestigal testinfra-ami-build * chores: use custom github runners * feat: multiple versions for the timescaledb-apache extension * feat: multiversion timescaledb * fix: refactor a bit to complete the extension * test: remove older version of integration test * fix: make sure integration tests do not run timescaledb tests on pg > 15 * test: revert to use test in nix/tests for timescale * fix: integrtion tests * Move timescaledb nixos test to nix/ext/tests * chore: bump for test * fix: need to also handle timescaledb-loader * chore: bump version for fix * fix: didn't need "loader" * chore: bump version for testing * fix: does need loader * feat: switch both timescaledb and its loader when switching versions Also reuse default tests defined in lib.py to test extension upgrade paths and switching extension versions. * chore: bump to test * fix: must symlink to timescaledb.so on switch * chore: bump for release --------- Co-authored-by: Sam Rose <[email protected]> Co-authored-by: samrose <[email protected]> Co-authored-by: Jean-François Roche <[email protected]>
1 parent b97ce59 commit 20dc72e

File tree

8 files changed

+238
-112
lines changed

8 files changed

+238
-112
lines changed

ansible/vars.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ postgres_major:
99

1010
# Full version strings for each major version
1111
postgres_release:
12-
postgresorioledb-17: "17.5.1.026-orioledb"
13-
postgres17: "17.6.1.005"
14-
postgres15: "15.14.1.005"
12+
postgresorioledb-17: "17.5.1.027-orioledb"
13+
postgres17: "17.6.1.006"
14+
postgres15: "15.14.1.006"
1515

1616
# Non Postgres Extensions
1717
pgbouncer_release: "1.19.0"

nix/ext/tests/lib.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ def check_switch_extension_with_background_worker(
147147
)
148148
last_version = available_versions[-1]
149149
assert ext_version.endswith(
150-
f"{self.extension_name}-{last_version}.so"
150+
f"{last_version}.so"
151151
), f"Expected {self.extension_name} version {last_version}, but found {ext_version}"
152152

153153
# Switch to the first version
@@ -157,13 +157,13 @@ def check_switch_extension_with_background_worker(
157157
# Check that we are using the first version now
158158
ext_version = self.vm.succeed(f"readlink -f {extension_lib_path}").strip()
159159
assert ext_version.endswith(
160-
f"{self.extension_name}-{first_version}.so"
160+
f"{first_version}.so"
161161
), f"Expected {self.extension_name} version {first_version}, but found {ext_version}"
162162

163163
# Switch to the first version
164164
self.vm.succeed(f"switch_{self.extension_name}_version {last_version}")
165165
# Check that we are using the last version now
166166
ext_version = self.vm.succeed(f"readlink -f {extension_lib_path}").strip()
167167
assert ext_version.endswith(
168-
f"{self.extension_name}-{last_version}.so"
168+
f"{last_version}.so"
169169
), f"Expected {self.extension_name} version {last_version}, but found {ext_version}"

nix/ext/tests/timescaledb.nix

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
{ self, pkgs }:
2+
let
3+
pname = "timescaledb";
4+
inherit (pkgs) lib;
5+
installedExtension =
6+
postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all";
7+
versions = (installedExtension "15").versions;
8+
postgresqlWithExtension =
9+
postgresql:
10+
let
11+
majorVersion = lib.versions.major postgresql.version;
12+
pkg = pkgs.buildEnv {
13+
name = "postgresql-${majorVersion}-${pname}";
14+
paths = [
15+
postgresql
16+
postgresql.lib
17+
(installedExtension majorVersion)
18+
];
19+
passthru = {
20+
inherit (postgresql) version psqlSchema;
21+
lib = pkg;
22+
withPackages = _: pkg;
23+
};
24+
nativeBuildInputs = [ pkgs.makeWrapper ];
25+
pathsToLink = [
26+
"/"
27+
"/bin"
28+
"/lib"
29+
];
30+
postBuild = ''
31+
wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib
32+
wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib
33+
wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib
34+
'';
35+
};
36+
in
37+
pkg;
38+
psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15;
39+
in
40+
self.inputs.nixpkgs.lib.nixos.runTest {
41+
name = "timescaledb";
42+
hostPkgs = pkgs;
43+
nodes.server =
44+
{ ... }:
45+
{
46+
services.postgresql = {
47+
enable = true;
48+
package = (postgresqlWithExtension psql_15);
49+
settings = {
50+
shared_preload_libraries = "timescaledb";
51+
};
52+
};
53+
};
54+
testScript =
55+
{ ... }:
56+
''
57+
${builtins.readFile ./lib.py}
58+
59+
start_all()
60+
61+
server.wait_for_unit("multi-user.target")
62+
server.wait_for_unit("postgresql.service")
63+
64+
versions = {
65+
"15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') versions)}],
66+
}
67+
extension_name = "${pname}"
68+
support_upgrade = True
69+
70+
test = PostgresExtensionTest(server, extension_name, versions, support_upgrade)
71+
72+
with subtest("Check upgrade path with postgresql 15"):
73+
test.check_upgrade_path("15")
74+
75+
with subtest("Test switch_${pname}_version"):
76+
test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15")
77+
'';
78+
}

nix/ext/timescaledb-2.9.1.nix

Lines changed: 0 additions & 65 deletions
This file was deleted.

nix/ext/timescaledb.nix

Lines changed: 134 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,53 +6,147 @@
66
postgresql,
77
openssl,
88
libkrb5,
9+
buildEnv,
10+
makeWrapper,
11+
switch-ext-version,
12+
coreutils,
13+
writeShellApplication,
914
}:
1015

11-
stdenv.mkDerivation rec {
12-
pname = "timescaledb-apache";
13-
version = "2.16.1";
16+
let
17+
pname = "timescaledb";
18+
build =
19+
version: hash: _revision:
20+
stdenv.mkDerivation rec {
21+
inherit pname version;
1422

15-
nativeBuildInputs = [ cmake ];
16-
buildInputs = [
17-
postgresql
18-
openssl
19-
libkrb5
20-
];
23+
nativeBuildInputs = [ cmake ];
24+
buildInputs = [
25+
postgresql
26+
openssl
27+
libkrb5
28+
];
29+
30+
src = fetchFromGitHub {
31+
owner = "timescale";
32+
repo = "timescaledb";
33+
rev = version;
34+
inherit hash;
35+
};
36+
37+
cmakeFlags = [
38+
"-DSEND_TELEMETRY_DEFAULT=OFF"
39+
"-DREGRESS_CHECKS=OFF"
40+
"-DTAP_CHECKS=OFF"
41+
"-DAPACHE_ONLY=1"
42+
] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
43+
44+
postPatch = ''
45+
for x in CMakeLists.txt sql/CMakeLists.txt; do
46+
if [ -f "$x" ]; then
47+
substituteInPlace "$x" \
48+
--replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
49+
fi
50+
done
51+
52+
for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
53+
if [ -f "$x" ]; then
54+
substituteInPlace "$x" \
55+
--replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
56+
fi
57+
done
58+
'';
59+
60+
installPhase = ''
61+
# Run cmake install first
62+
cmake --install . --prefix=$out
63+
64+
# TimescaleDB creates two libraries:
65+
# 1. timescaledb.so (loader) -> rename to timescaledb-<version>-loader.so
66+
# 2. timescaledb-<version>.so (actual extension) -> keep as is
67+
68+
# Rename the loader library to be version-specific
69+
if [ -f $out/lib/timescaledb${postgresql.dlSuffix} ]; then
70+
mv $out/lib/timescaledb${postgresql.dlSuffix} $out/lib/timescaledb-loader-${version}${postgresql.dlSuffix}
71+
fi
2172
22-
src = fetchFromGitHub {
23-
owner = "timescale";
24-
repo = "timescaledb";
25-
rev = version;
26-
hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=";
73+
# The versioned library (timescaledb-VERSION.so) is already correctly named
74+
75+
# Create versioned control file with default_version removed and module_pathname pointing to symlink
76+
if [ -f $out/share/postgresql/extension/timescaledb.control ]; then
77+
sed -e "/^default_version =/d" \
78+
-e "s|^module_pathname = .*|module_pathname = '\$libdir/timescaledb'|" \
79+
$out/share/postgresql/extension/timescaledb.control > $out/share/postgresql/extension/timescaledb--${version}.control
80+
rm $out/share/postgresql/extension/timescaledb.control
81+
fi
82+
'';
83+
84+
meta = with lib; {
85+
description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
86+
homepage = "https://www.timescale.com/";
87+
changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
88+
license = licenses.asl20;
89+
inherit (postgresql.meta) platforms;
90+
};
91+
};
92+
93+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb;
94+
supportedVersions = lib.filterAttrs (
95+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
96+
) allVersions;
97+
versions = lib.naturalSort (lib.attrNames supportedVersions);
98+
latestVersion = lib.last versions;
99+
numberOfVersions = builtins.length versions;
100+
packages = builtins.attrValues (
101+
lib.mapAttrs (name: value: build name value.hash (value.revision or name)) supportedVersions
102+
);
103+
switch-timescaledb-loader = writeShellApplication {
104+
name = "switch_timescaledb_loader";
105+
runtimeInputs = [ coreutils ];
106+
text = ''
107+
EXT_LOADER_TO_USE="$EXT_WRAPPER_LIB/$EXT_NAME-loader-$VERSION${postgresql.dlSuffix}"
108+
if [ -f "$EXT_LOADER_TO_USE" ]; then
109+
ln -sfnv "$EXT_LOADER_TO_USE" "$EXT_WRAPPER_LIB/$EXT_NAME${postgresql.dlSuffix}"
110+
fi
111+
'';
27112
};
113+
in
114+
buildEnv {
115+
name = pname;
116+
paths = packages;
117+
nativeBuildInputs = [ makeWrapper ];
118+
postBuild = ''
119+
{
120+
echo "default_version = '${latestVersion}'"
121+
cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
122+
} > $out/share/postgresql/extension/${pname}.control
123+
124+
# Create symlink from the latest versioned loader to timescaledb.so
125+
ln -sfn ${pname}-loader-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
28126
29-
cmakeFlags = [
30-
"-DSEND_TELEMETRY_DEFAULT=OFF"
31-
"-DREGRESS_CHECKS=OFF"
32-
"-DTAP_CHECKS=OFF"
33-
"-DAPACHE_ONLY=1"
34-
] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
35-
36-
# Fix the install phase which tries to install into the pgsql extension dir,
37-
# and cannot be manually overridden. This is rather fragile but works OK.
38-
postPatch = ''
39-
for x in CMakeLists.txt sql/CMakeLists.txt; do
40-
substituteInPlace "$x" \
41-
--replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
42-
done
43-
44-
for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
45-
substituteInPlace "$x" \
46-
--replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
47-
done
127+
# The versioned extension libraries (timescaledb-VERSION.so) are already in place
128+
129+
# checks - we should have loader files and versioned extension files
130+
(set -x
131+
test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" -gt 0
132+
)
133+
makeWrapper ${lib.getExe switch-ext-version} $out/bin/switch_timescaledb_version \
134+
--prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" --prefix EXTRA_STEPS : ${lib.getExe switch-timescaledb-loader}
48135
'';
49136

50-
meta = with lib; {
51-
description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
52-
homepage = "https://www.timescale.com/";
53-
changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
54-
platforms = postgresql.meta.platforms;
55-
license = licenses.asl20;
56-
broken = versionOlder postgresql.version "13";
137+
pathsToLink = [
138+
"/lib"
139+
"/share/postgresql/extension"
140+
];
141+
142+
passthru = {
143+
inherit versions numberOfVersions switch-ext-version;
144+
pname = "${pname}-all";
145+
hasBackgroundWorker = true;
146+
defaultSettings = {
147+
shared_preload_libraries = [ "timescaledb" ];
148+
};
149+
version =
150+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
57151
};
58152
}

nix/ext/versions.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,20 @@
130130
],
131131
"hash": "sha256-Cpi2iASi1QJoED0Qs1dANqg/BNZTsz5S+pw8iYyW03Y="
132132
}
133+
},
134+
"timescaledb": {
135+
"2.9.1": {
136+
"postgresql": [
137+
"15"
138+
],
139+
"hash": "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="
140+
},
141+
"2.16.1": {
142+
"postgresql": [
143+
"15"
144+
],
145+
"hash": "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="
146+
}
133147
},
134148
"vector": {
135149
"0.4.0": {

nix/packages/postgres.nix

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
ourExtensions = [
2020
../ext/rum.nix
2121
../ext/timescaledb.nix
22-
../ext/timescaledb-2.9.1.nix
2322
../ext/pgroonga.nix
2423
../ext/index_advisor.nix
2524
../ext/wal2json.nix

0 commit comments

Comments
 (0)