diff --git a/.gitignore b/.gitignore index 5372bfdeb..a25b52763 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ result* .history .envrc .direnv +.nixos-test-history #IDE diff --git a/ansible/vars.yml b/ansible/vars.yml index cd7ea6a28..949b0870f 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,9 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.5.1.017-orioledb" - postgres17: "17.4.1.074" - postgres15: "15.8.1.131" + postgresorioledb-17: "17.5.1.017-orioledb-pg-tle-1" + postgres17: "17.4.1.074-pg-tle-1" + postgres15: "15.8.1.131-pg-tle-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/nix/checks.nix b/nix/checks.nix index 2dc51797a..d81b134f9 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -309,6 +309,10 @@ ; } // pkgs.lib.optionalAttrs (system == "x86_64-linux") { + pg_tle = import ./ext/tests/pg_tle.nix { + inherit self; + inherit pkgs; + }; wrappers = import ./ext/tests/wrappers.nix { inherit self; inherit pkgs; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index df83d4307..30e0431bf 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -1,45 +1,108 @@ { lib, stdenv, + buildEnv, fetchFromGitHub, postgresql, flex, openssl, libkrb5, }: - -stdenv.mkDerivation rec { +let pname = "pg_tle"; - version = "1.4.0"; + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; - nativeBuildInputs = [ flex ]; - buildInputs = [ - openssl - postgresql - libkrb5 - ]; + nativeBuildInputs = [ flex ]; + buildInputs = [ + openssl + postgresql + libkrb5 + ]; - src = fetchFromGitHub { - owner = "aws"; - repo = pname; - rev = "refs/tags/v${version}"; - hash = "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces="; - }; + src = fetchFromGitHub { + owner = "aws"; + repo = pname; + rev = "refs/tags/v${version}"; + inherit hash; + }; + + makeFlags = [ "FLEX=flex" ]; + + installPhase = '' - makeFlags = [ "FLEX=flex" ]; + mkdir -p $out/{lib,share/postgresql/extension} - installPhase = '' - mkdir -p $out/{lib,share/postgresql/extension} + mv ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} - cp *${postgresql.dlSuffix} $out/lib - cp *.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension + create_sql_files() { + if test -f ${pname}--${version}.sql; then + cp ${pname}--${version}.sql $out/share/postgresql/extension + fi + echo "Creating SQL files for previous versions..." + if [[ "${version}" == "${latestVersion}" ]]; then + cp *.sql $out/share/postgresql/extension + fi + } + + create_control_files() { + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \ + ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix} + fi + } + + create_sql_files + create_control_files + ''; + + meta = with lib; { + description = "Framework for 'Trusted Language Extensions' in PostgreSQL"; + homepage = "https://github.com/aws/${pname}"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); +in +buildEnv { + name = pname; + paths = packages; + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + postBuild = '' + # checks + (set -x + test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${ + toString (numberOfVersions + 1) + }" + ) ''; - meta = with lib; { - description = "Framework for 'Trusted Language Extensions' in PostgreSQL"; - homepage = "https://github.com/aws/${pname}"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/pg_tle.nix b/nix/ext/tests/pg_tle.nix new file mode 100644 index 000000000..3e89808d8 --- /dev/null +++ b/nix/ext/tests/pg_tle.nix @@ -0,0 +1,161 @@ +{ self, pkgs }: +let + pname = "pg_tle"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIo+ulCUfJjnCVgfM4946Ih5Nm8DeZZiayYeABHGPEl7 jfroche" + ]; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + settings = { + shared_preload_libraries = "pg_tle"; + }; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + echo "shared_preload_libraries = 'pg_tle'" >> "${newDataDir}/postgresql.conf" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname};'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["15"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check ${pname} latest extension version"): + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["17"][-1] + assert f"${pname},{latestVersion}" in installed_extensions + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index c77b23be6..4b489e6b6 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -20,5 +20,34 @@ ], "hash": "sha256-G0eQk2bY5CNPMeokN/nb05g03CuiplRf902YXFVQFbs=" } + }, + "pg_tle": { + "1.0.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-yAe37nIgObjs0kZY1A0QBGMP/s45sC1HVRwxVRaXwXs=" + }, + "1.0.4": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-W/7pLy/27VatCdzUh1NZ4K2FRMD1erfHiFV2eY2x2W0=" + }, + "1.3.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-Q4D4+eYPBONd8IKlDVfQa8wI4PJyKHkMPPqONqEWFLE=" + }, + "1.4.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces=" + } } }