diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 058d9087b..7551a14fa 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,23 +10,32 @@ on: permissions: id-token: write - # required by testinfra-ami-build dependent workflows + # required by dependent workflows contents: write packages: write jobs: + nix-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v30 + - id: set-matrix + name: Generate Nix Matrix + run: | + set -Eeu + matrix="$(nix eval --json '.#githubActions.matrix')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + build-run-image: + name: ${{ matrix.name }} (${{ matrix.system }}) + needs: nix-matrix + runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: - include: - - runner: large-linux-x86 - arch: amd64 - - runner: large-linux-arm - arch: arm64 - - runner: macos-latest-xlarge - arch: arm64 - runs-on: ${{ matrix.runner }} + matrix: ${{fromJSON(needs.nix-matrix.outputs.matrix)}} timeout-minutes: 180 steps: - name: Checkout Repo @@ -104,23 +113,12 @@ jobs: sudo rm -rf /tmp/* 2>/dev/null || true echo "=== AFTER CLEANUP ===" df -h - - name: Build psql bundle - run: > - nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" - -- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} - --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')" + - run: nix build -L '.#${{ matrix.attr }}' env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }} - run-testinfra: - needs: build-run-image - if: ${{ success() }} - uses: ./.github/workflows/testinfra-ami-build.yml - secrets: - DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} - run-tests: needs: build-run-image if: ${{ success() }} diff --git a/flake.lock b/flake.lock index 9d2865e1d..b9a8b2f83 100644 --- a/flake.lock +++ b/flake.lock @@ -173,6 +173,26 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1737420293, + "narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nix2container": { "inputs": { "flake-utils": "flake-utils_2", @@ -309,6 +329,7 @@ "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", + "nix-github-actions": "nix-github-actions", "nix2container": "nix2container", "nixpkgs": "nixpkgs_4", "nixpkgs-go124": "nixpkgs-go124", diff --git a/flake.nix b/flake.nix index db14dac9a..f91facd4b 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,8 @@ git-hooks.url = "github:cachix/git-hooks.nix"; git-hooks.inputs.nixpkgs.follows = "nixpkgs"; nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; + nix-github-actions.url = "github:nix-community/nix-github-actions"; + nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = @@ -36,6 +38,7 @@ nix/nixpkgs.nix nix/packages nix/overlays + nix/github-actions.nix ]; }); } diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix deleted file mode 100644 index 0df743671..000000000 --- a/nix/ext/timescaledb-2.9.1.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - lib, - stdenv, - fetchFromGitHub, - cmake, - postgresql, - openssl, - libkrb5, -}: - -stdenv.mkDerivation rec { - pname = "timescaledb-apache"; - version = "2.9.1"; - - nativeBuildInputs = [ cmake ]; - buildInputs = [ - postgresql - openssl - libkrb5 - ]; - - src = fetchFromGitHub { - owner = "timescale"; - repo = "timescaledb"; - rev = version; - hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; - }; - - cmakeFlags = [ - "-DSEND_TELEMETRY_DEFAULT=OFF" - "-DREGRESS_CHECKS=OFF" - "-DTAP_CHECKS=OFF" - "-DAPACHE_ONLY=1" - ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; - - # Fix the install phase which tries to install into the pgsql extension dir, - # and cannot be manually overridden. This is rather fragile but works OK. - postPatch = '' - for x in CMakeLists.txt sql/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" - done - - for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" - done - ''; - - # timescaledb-2.9.1.so already exists in the lib directory - # we have no need for the timescaledb.so or control file - postInstall = '' - rm $out/lib/timescaledb.so - rm $out/share/postgresql/extension/timescaledb.control - ''; - - meta = with lib; { - description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; - homepage = "https://www.timescale.com/"; - changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.asl20; - broken = versionOlder postgresql.version "13"; - }; -} diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix index 6f5681546..c8e224e9e 100644 --- a/nix/ext/timescaledb.nix +++ b/nix/ext/timescaledb.nix @@ -1,4 +1,5 @@ { + pkgs, lib, stdenv, fetchFromGitHub, @@ -8,51 +9,104 @@ libkrb5, }: -stdenv.mkDerivation rec { - pname = "timescaledb-apache"; - version = "2.16.1"; +let + pname = "timescaledb"; + build = + version: hash: _revision: + stdenv.mkDerivation rec { + inherit pname version; - nativeBuildInputs = [ cmake ]; - buildInputs = [ - postgresql - openssl - libkrb5 - ]; + nativeBuildInputs = [ cmake ]; + buildInputs = [ + postgresql + openssl + libkrb5 + ]; - src = fetchFromGitHub { - owner = "timescale"; - repo = "timescaledb"; - rev = version; - hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="; - }; + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + inherit hash; + }; - cmakeFlags = [ - "-DSEND_TELEMETRY_DEFAULT=OFF" - "-DREGRESS_CHECKS=OFF" - "-DTAP_CHECKS=OFF" - "-DAPACHE_ONLY=1" - ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; - - # Fix the install phase which tries to install into the pgsql extension dir, - # and cannot be manually overridden. This is rather fragile but works OK. - postPatch = '' - for x in CMakeLists.txt sql/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" - done - - for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" - done - ''; + cmakeFlags = [ + "-DSEND_TELEMETRY_DEFAULT=OFF" + "-DREGRESS_CHECKS=OFF" + "-DTAP_CHECKS=OFF" + "-DAPACHE_ONLY=1" + ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + if [ -f "$x" ]; then + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + fi + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + if [ -f "$x" ]; then + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + fi + done + ''; - meta = with lib; { - description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; - homepage = "https://www.timescale.com/"; - changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.asl20; - broken = versionOlder postgresql.version "13"; + postInstall = '' + if [ -f $out/lib/timescaledb.so ]; then + mv $out/lib/timescaledb.so $out/lib/timescaledb-${version}.so + fi + if [ -f $out/share/postgresql/extension/timescaledb.control ]; then + mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb--${version}.control + fi + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb; + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash (value.revision or name)) supportedVersions + ); +in +pkgs.buildEnv { + name = pname; + paths = packages; + postBuild = '' + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix} + + # checks + (set -x + test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${ + toString (numberOfVersions + 1) + }" + ) + ''; + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/versions.json b/nix/ext/versions.json index c77b23be6..60809af4c 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -20,5 +20,19 @@ ], "hash": "sha256-G0eQk2bY5CNPMeokN/nb05g03CuiplRf902YXFVQFbs=" } + }, + "timescaledb": { + "2.9.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8=" + }, + "2.16.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=" + } } } diff --git a/nix/github-actions.nix b/nix/github-actions.nix new file mode 100644 index 000000000..752f0b41a --- /dev/null +++ b/nix/github-actions.nix @@ -0,0 +1,18 @@ +{ inputs, ... }: +let + githubPlatforms = { + "x86_64-linux" = "large-linux-x86"; + "aarch64-linux" = "large-linux-arm"; + "aarch64-darwin" = "macos-latest-xlarge"; + }; +in +{ + flake.githubActions = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = inputs.nixpkgs.lib.getAttrs [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + ] inputs.self.checks; + platforms = githubPlatforms; + }; +} diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix index bf8710b4d..a6467ef43 100644 --- a/nix/packages/postgres.nix +++ b/nix/packages/postgres.nix @@ -19,7 +19,6 @@ ourExtensions = [ ../ext/rum.nix ../ext/timescaledb.nix - ../ext/timescaledb-2.9.1.nix ../ext/pgroonga.nix ../ext/index_advisor.nix ../ext/wal2json.nix diff --git a/nix/tests/timescaledb.nix b/nix/tests/timescaledb.nix new file mode 100644 index 000000000..2978055e7 --- /dev/null +++ b/nix/tests/timescaledb.nix @@ -0,0 +1,100 @@ +{ self, pkgs }: +let + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: + self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/timescaledb-all"; + versions = (installedExtension "15").versions; + firstVersion = lib.head versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-timescaledb"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = "timescaledb"; + hostPkgs = pkgs; + nodes.server = + { ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIArkmq6Th79Z4klW6Urgi4phN8yq769/l/10jlE00tU9" + ]; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + settings = { + shared_preload_libraries = "timescaledb"; + }; + }; + + specialisation.postgresql15.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15); + }; + }; + }; + testScript = + { ... }: + '' + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(): + with subtest("Check timescaledb upgrade path"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS timescaledb;'") + run_sql(r"""CREATE EXTENSION timescaledb WITH VERSION \"${firstVersion}\";""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""") + assert installed_version == "${firstVersion}", f"Expected timescaledb version ${firstVersion}, but found {installed_version}" + for version in [${lib.concatStringsSep ", " (map (s: ''"${s}"'') versions)}][1:]: + run_sql(f"""ALTER EXTENSION timescaledb UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""") + assert installed_version == version, f"Expected timescaledb version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path() + ''; +}