diff --git a/Dockerfile-15 b/Dockerfile-15 index 6224767ff..7b1d7099f 100644 --- a/Dockerfile-15 +++ b/Dockerfile-15 @@ -120,14 +120,14 @@ ENV PGDATA=/var/lib/postgresql/data #################### FROM base as walg ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g +WORKDIR /nixpg + +RUN nix profile install .#wal-g-3 && \ + ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g +RUN nix store gc + +WORKDIR / # #################### # # Download gosu for easy step-down from root # #################### @@ -179,7 +179,8 @@ RUN sed -i \ -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ usermod -aG postgres wal-g && \ mkdir -p /etc/postgresql-custom && \ @@ -219,4 +220,7 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE=C.UTF-8 ENV LC_COLLATE=C.UTF-8 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-orioledb-17 b/Dockerfile-orioledb-17 index 5f8fed70e..fe9b902cb 100644 --- a/Dockerfile-orioledb-17 +++ b/Dockerfile-orioledb-17 @@ -37,7 +37,7 @@ ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 ARG supautils_release=2.2.0 -ARG wal_g_release=2.0.1 +ARG wal_g_release=3.0.5 FROM ubuntu:focal as base @@ -118,14 +118,15 @@ ENV PGDATA=/var/lib/postgresql/data #################### FROM base as walg ARG wal_g_release -# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz -RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ - apt-get update && apt-get install -y --no-install-recommends curl && \ - curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ - tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ - rm -rf /tmp/wal-g.tar.gz && \ - mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g +WORKDIR /nixpg + +RUN nix profile install .#wal-g-3 && \ + ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g + +RUN nix store gc + +WORKDIR / # #################### # # Download gosu for easy step-down from root # #################### @@ -178,6 +179,7 @@ RUN sed -i \ echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ usermod -aG postgres wal-g && \ mkdir -p /etc/postgresql-custom && \ @@ -231,4 +233,8 @@ ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE=C.UTF-8 ENV LC_COLLATE=C.UTF-8 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +RUN mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 9907d3ee7..5ba268cf9 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,68 +1,3 @@ -# Downloading dependencies -# - name: wal-g dependencies -# become: yes -# apt: -# pkg: -# - libbrotli-dev -# - liblzo2-dev -# - libsodium-dev -# - cmake - -# install go dependency for WAL-G -# - name: wal-g go dependency -# get_url: -# url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" -# dest: /tmp -# checksum: "{{ golang_version_checksum[platform] }}" -# timeout: 60 - -# - name: unpack go archive -# unarchive: -# remote_src: yes -# src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" -# dest: /usr/local - -# # Download WAL-G -# - name: wal-g - download latest version -# git: -# repo: https://github.com/wal-g/wal-g.git -# dest: /tmp/wal-g -# version: "v{{ wal_g_release }}" -# become: yes - -# - name: wal-g - pg_clean -# make: -# chdir: /tmp/wal-g -# target: pg_clean -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes -# ignore_errors: yes - -# - name: wal-g - deps -# make: -# chdir: /tmp/wal-g -# target: deps -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes -# ignore_errors: yes - -# - name: wal-g - build and install -# community.general.make: -# chdir: /tmp/wal-g -# target: pg_install -# jobs: "{{ parallel_jobs | default(omit) }}" -# params: -# GOBIN: "/usr/local/bin" -# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" -# USE_LIBSODIUM: true -# become: yes - - name: Create wal-g group group: name: wal-g @@ -86,20 +21,36 @@ mode: '0770' when: nixpkg_mode -- name: Install wal-g from nix binary cache +- name: Install wal-g 2 from nix binary cache become: yes shell: | - sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g" + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-2" when: stage2_nix -- name: Create symlink for wal-g from Nix profile to /usr/local/bin +- name: Install wal-g 3 from nix binary cache + become: yes + shell: | + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3" + when: stage2_nix + +- name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin ansible.builtin.file: - src: /home/wal-g/.nix-profile/bin/wal-g - dest: /usr/local/bin/wal-g + src: /home/wal-g/.nix-profile/bin/wal-g-3 + dest: /usr/local/bin/wal-g-v3 state: link force: yes # This will replace existing file/symlink if it exists become: yes # Need sudo to write to /usr/local/bin when: stage2_nix + +- name: Create symlink to make wal-g-v2 the default wal-g + ansible.builtin.file: + src: /usr/local/bin/wal-g-v2 + dest: /usr/local/bin/wal-g + state: link + force: yes + become: yes + when: stage2_nix + - name: Create /etc/wal-g/config.json file: path: /etc/wal-g/config.json diff --git a/ansible/vars.yml b/ansible/vars.yml index 88fe86749..495b65858 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.050-orioledb" - postgres15: "15.8.1.057" + postgresorioledb-17: "17.0.1.051-orioledb" + postgres15: "15.8.1.058" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 48286907e..135031afc 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,12 +10,14 @@ SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') nix build .#checks.$SYSTEM.psql_15 -L --no-link nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link -nix build .#psql_15/bin -o psql_15 +nix build .#psql_15/bin -o psql_15 -L -nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -nix build .#wal-g -o wal-g +nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L +nix build .#wal-g-2 -o wal-g-2 -L +nix build .#wal-g-3 -o wal-g-3 -L # Copy to S3 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-2 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 if [ "$SYSTEM" = "aarch64-linux" ]; then @@ -27,5 +29,4 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_src - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g fi diff --git a/flake.nix b/flake.nix index dadc55253..b36ccda57 100644 --- a/flake.nix +++ b/flake.nix @@ -87,7 +87,7 @@ sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; - wal-g = pkgs.callPackage ./nix/wal-g.nix { }; + inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3; # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by # listing their name. Anytime the version of nixpkgs is upgraded, these @@ -406,7 +406,8 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_orioledb-17 = postgresVersions.psql_orioledb-17; - wal-g = wal-g; + wal-g-2 = wal-g-2; + wal-g-3 = wal-g-3; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_orioledb-17; diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 13f2bbc90..764b1c9e4 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -1,58 +1,83 @@ -{ - lib, - buildGoModule, - fetchFromGitHub, - brotli, - libsodium, - installShellFiles, +{ lib +, buildGoModule +, fetchFromGitHub +, brotli +, libsodium +, installShellFiles +, }: -buildGoModule rec { - pname = "wal-g"; - version = "3.0.5"; +let + walGCommon = { version, vendorHash, sha256, majorVersion }: + buildGoModule rec { + pname = "wal-g-${majorVersion}"; + inherit version; - src = fetchFromGitHub { - owner = "wal-g"; - repo = "wal-g"; - rev = "v${version}"; - sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; + src = fetchFromGitHub { + owner = "wal-g"; + repo = "wal-g"; + rev = "v${version}"; + inherit sha256; + }; + + inherit vendorHash; + + nativeBuildInputs = [ installShellFiles ]; + + buildInputs = [ + brotli + libsodium + ]; + + subPackages = [ "main/pg" ]; + + tags = [ + "brotli" + "libsodium" + ]; + + ldflags = [ + "-s" + "-w" + "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}" + "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}" + ]; + + postInstall = '' + mv $out/bin/pg $out/bin/wal-g-${majorVersion} + + # Create version-specific completions + mkdir -p $out/share/bash-completion/completions + $out/bin/wal-g-${majorVersion} completion bash > $out/share/bash-completion/completions/wal-g-${majorVersion} + + mkdir -p $out/share/zsh/site-functions + $out/bin/wal-g-${majorVersion} completion zsh > $out/share/zsh/site-functions/_wal-g-${majorVersion} + + ''; + + meta = with lib; { + homepage = "https://github.com/wal-g/wal-g"; + license = licenses.asl20; + description = "Archival restoration tool for PostgreSQL"; + mainProgram = "wal-g-${majorVersion}"; + maintainers = [ samrose ]; + }; + }; +in +{ + # wal-g v2.0.1 + wal-g-2 = walGCommon { + version = "2.0.1"; + sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok="; + vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g="; + majorVersion = "2"; }; - vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; - - nativeBuildInputs = [ installShellFiles ]; - - buildInputs = [ - brotli - libsodium - ]; - - subPackages = [ "main/pg" ]; - - tags = [ - "brotli" - "libsodium" - ]; - - ldflags = [ - "-s" - "-w" - "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}" - "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}" - ]; - - postInstall = '' - mv $out/bin/pg $out/bin/wal-g - installShellCompletion --cmd wal-g \ - --bash <($out/bin/wal-g completion bash) \ - --zsh <($out/bin/wal-g completion zsh) - ''; - - meta = with lib; { - homepage = "https://github.com/wal-g/wal-g"; - license = licenses.asl20; - description = "Archival restoration tool for PostgreSQL"; - mainProgram = "wal-g"; - maintainers = [ samrose ]; + # wal-g v3.0.5 + wal-g-3 = walGCommon { + version = "3.0.5"; + sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; + vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; + majorVersion = "3"; }; }