Sam/wal g 2 and 3

Dockerfile-15

@@ -120,14 +120,14 @@ ENV PGDATA=/var/lib/postgresql/data
 ####################
 FROM base as walg
 ARG wal_g_release
-# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
-RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
-    apt-get update && apt-get install -y --no-install-recommends curl && \
-    curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
-    tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
-    rm -rf /tmp/wal-g.tar.gz && \
-    mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
+WORKDIR /nixpg
+
+RUN nix profile install .#wal-g-3 && \
+    ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g
 
+RUN nix store gc
+
+WORKDIR /
 # ####################
 # # Download gosu for easy step-down from root
 # ####################
@@ -179,7 +179,8 @@ RUN sed -i \
     -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
     echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
     #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
-    echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
+    echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \    
+    echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
     usermod -aG postgres wal-g && \
     mkdir -p /etc/postgresql-custom && \
@@ -219,4 +220,7 @@ ENV LC_ALL en_US.UTF-8
 ENV LC_CTYPE=C.UTF-8
 ENV LC_COLLATE=C.UTF-8
 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
+RUN mkdir -p /usr/share/postgresql/extension/ && \
+    ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \
+    chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh
 CMD ["postgres", "-D", "/etc/postgresql"]

Dockerfile-orioledb-17

@@ -37,7 +37,7 @@ ARG pgvector_release=0.4.0
 ARG pg_tle_release=1.3.2
 ARG index_advisor_release=0.2.0
 ARG supautils_release=2.2.0
-ARG wal_g_release=2.0.1
+ARG wal_g_release=3.0.5
 
 FROM ubuntu:focal as base
 
@@ -118,14 +118,15 @@ ENV PGDATA=/var/lib/postgresql/data
 ####################
 FROM base as walg
 ARG wal_g_release
-# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz
-RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \
-    apt-get update && apt-get install -y --no-install-recommends curl && \
-    curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \
-    tar -xvf /tmp/wal-g.tar.gz -C /tmp && \
-    rm -rf /tmp/wal-g.tar.gz && \
-    mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g
 
+WORKDIR /nixpg
+
+RUN nix profile install .#wal-g-3 && \
+    ln -s /nix/var/nix/profiles/default/bin/wal-g-3 /tmp/wal-g
+
+RUN nix store gc
+
+WORKDIR /
 # ####################
 # # Download gosu for easy step-down from root
 # ####################
@@ -178,6 +179,7 @@ RUN sed -i \
     echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \
     #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
     echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
+    echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
     usermod -aG postgres wal-g && \
     mkdir -p /etc/postgresql-custom && \
@@ -231,4 +233,8 @@ ENV LC_ALL en_US.UTF-8
 ENV LC_CTYPE=C.UTF-8
 ENV LC_COLLATE=C.UTF-8
 ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive
+RUN mkdir -p /usr/share/postgresql/extension/ && \
+    ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \
+    chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh
+
 CMD ["postgres", "-D", "/etc/postgresql"]

ansible/tasks/setup-wal-g.yml

@@ -1,68 +1,3 @@
-# Downloading dependencies
-# - name: wal-g dependencies
-#   become: yes
-#   apt:
-#     pkg:
-#       - libbrotli-dev
-#       - liblzo2-dev
-#       - libsodium-dev
-#       - cmake
-
-# install go dependency for WAL-G
-# - name: wal-g go dependency
-#   get_url:
-#     url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-#     dest: /tmp
-#     checksum: "{{ golang_version_checksum[platform] }}"
-#     timeout: 60
-
-# - name: unpack go archive
-#   unarchive:
-#     remote_src: yes
-#     src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-#     dest: /usr/local
-
-# # Download WAL-G
-# - name: wal-g - download latest version
-#   git:
-#     repo: https://github.com/wal-g/wal-g.git
-#     dest: /tmp/wal-g
-#     version: "v{{ wal_g_release }}"
-#   become: yes
-
-# - name: wal-g - pg_clean
-#   make:
-#     chdir: /tmp/wal-g
-#     target: pg_clean
-#     params:
-#       GOBIN: "/usr/local/bin"
-#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-#       USE_LIBSODIUM: true
-#   become: yes
-#   ignore_errors: yes
-
-# - name: wal-g - deps
-#   make:
-#     chdir: /tmp/wal-g
-#     target: deps
-#     params:
-#       GOBIN: "/usr/local/bin"
-#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-#       USE_LIBSODIUM: true
-#   become: yes
-#   ignore_errors: yes
-
-# - name: wal-g - build and install
-#   community.general.make:
-#     chdir: /tmp/wal-g
-#     target: pg_install
-#     jobs: "{{ parallel_jobs | default(omit) }}"
-#     params:
-#       GOBIN: "/usr/local/bin"
-#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-#       USE_LIBSODIUM: true
-#   become: yes
-
 - name: Create wal-g group
   group:
     name: wal-g
@@ -86,20 +21,45 @@
     mode: '0770'
   when: nixpkg_mode
 
-- name: Install wal-g from nix binary cache
+- name: Install wal-g 2 from nix binary cache
   become: yes
   shell: |
-    sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g"
+    sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-2"
+  when: stage2_nix
+
+- name: Install wal-g 3 from nix binary cache
+  become: yes
+  shell: |
+    sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3"
   when: stage2_nix
 
 - name: Create symlink for wal-g from Nix profile to /usr/local/bin
   ansible.builtin.file:
-    src: /home/wal-g/.nix-profile/bin/wal-g
-    dest: /usr/local/bin/wal-g
+    src: /home/wal-g/.nix-profile/bin/wal-g-2
+    dest: /usr/local/bin/wal-g-v2
     state: link
     force: yes  # This will replace existing file/symlink if it exists
   become: yes   # Need sudo to write to /usr/local/bin
   when: stage2_nix
+
+- name: Create symlink for wal-g-3 from Nix profile to /usr/local/bin
+  ansible.builtin.file:
+    src: /home/wal-g/.nix-profile/bin/wal-g-3
+    dest: /usr/local/bin/wal-g-v3
+    state: link
+    force: yes  # This will replace existing file/symlink if it exists
+  become: yes   # Need sudo to write to /usr/local/bin
+  when: stage2_nix
+
+- name: Create symlink to make wal-g-v3 the default wal-g
+  ansible.builtin.file:
+    src: /usr/local/bin/wal-g-v3
+    dest: /usr/local/bin/wal-g
+    state: link
+    force: yes
+  become: yes
+  when: stage2_nix
+
 - name: Create /etc/wal-g/config.json
   file:
     path: /etc/wal-g/config.json

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.050-orioledb"
-  postgres15: "15.8.1.057"
+  postgresorioledb-17: "17.0.1.051-orioledb"
+  postgres15: "15.8.1.058"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

docker/nix/build_nix.sh

@@ -10,12 +10,14 @@ SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"')
 
 nix build .#checks.$SYSTEM.psql_15 -L --no-link
 nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link
-nix build .#psql_15/bin -o psql_15
+nix build .#psql_15/bin -o psql_15 -L
 
-nix build .#psql_orioledb-17/bin -o psql_orioledb_17
-nix build .#wal-g -o wal-g
+nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L
+nix build .#wal-g-2 -o wal-g-2 -L
+nix build .#wal-g-3 -o wal-g-3 -L
 # Copy to S3
-nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-2
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
 if [ "$SYSTEM" = "aarch64-linux" ]; then
@@ -27,5 +29,4 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_15_src
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_orioledb-17_src
-    nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./wal-g
 fi

flake.nix

@@ -87,7 +87,7 @@
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
         supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
         mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
-        wal-g = pkgs.callPackage ./nix/wal-g.nix { };
+        inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3;
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
         # listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -406,7 +406,8 @@
           # PostgreSQL versions.
           psql_15 = postgresVersions.psql_15;
           psql_orioledb-17 = postgresVersions.psql_orioledb-17;
-          wal-g = wal-g;
+          wal-g-2 = wal-g-2;
+          wal-g-3 = wal-g-3;
           sfcgal = sfcgal;
           pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
           inherit postgresql_15 postgresql_orioledb-17;

nix/wal-g.nix

@@ -1,58 +1,83 @@
-{
-  lib,
-  buildGoModule,
-  fetchFromGitHub,
-  brotli,
-  libsodium,
-  installShellFiles,
+{ lib
+, buildGoModule
+, fetchFromGitHub
+, brotli
+, libsodium
+, installShellFiles
+,
 }:
 
-buildGoModule rec {
-  pname = "wal-g";
-  version = "3.0.5";
+let
+  walGCommon = { version, vendorHash, sha256, majorVersion }:
+    buildGoModule rec {
+      pname = "wal-g-${majorVersion}";
+      inherit version;
 
-  src = fetchFromGitHub {
-    owner = "wal-g";
-    repo = "wal-g";
-    rev = "v${version}";
-    sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
+      src = fetchFromGitHub {
+        owner = "wal-g";
+        repo = "wal-g";
+        rev = "v${version}";
+        inherit sha256;
+      };
+
+      inherit vendorHash;
+
+      nativeBuildInputs = [ installShellFiles ];
+
+      buildInputs = [
+        brotli
+        libsodium
+      ];
+
+      subPackages = [ "main/pg" ];
+
+      tags = [
+        "brotli"
+        "libsodium"
+      ];
+
+      ldflags = [
+        "-s"
+        "-w"
+        "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
+        "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
+      ];
+
+      postInstall = ''
+        mv $out/bin/pg $out/bin/wal-g-${majorVersion}
+
+        # Create version-specific completions
+        mkdir -p $out/share/bash-completion/completions
+        $out/bin/wal-g-${majorVersion} completion bash > $out/share/bash-completion/completions/wal-g-${majorVersion}
+
+        mkdir -p $out/share/zsh/site-functions
+        $out/bin/wal-g-${majorVersion} completion zsh > $out/share/zsh/site-functions/_wal-g-${majorVersion}
+
+      '';
+
+      meta = with lib; {
+        homepage = "https://github.com/wal-g/wal-g";
+        license = licenses.asl20;
+        description = "Archival restoration tool for PostgreSQL";
+        mainProgram = "wal-g-${majorVersion}";
+        maintainers = [ samrose ];
+      };
+    };
+in
+{
+  # wal-g v2.0.1
+  wal-g-2 = walGCommon {
+    version = "2.0.1";
+    sha256 = "sha256-5mwA55aAHwEFabGZ6c3pi8NLcYofvoe4bb/cFj7NWok=";
+    vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g=";
+    majorVersion = "2";
   };
 
-  vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
-
-  nativeBuildInputs = [ installShellFiles ];
-
-  buildInputs = [
-    brotli
-    libsodium
-  ];
-
-  subPackages = [ "main/pg" ];
-
-  tags = [
-    "brotli"
-    "libsodium"
-  ];
-
-  ldflags = [
-    "-s"
-    "-w"
-    "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
-    "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
-  ];
-
-  postInstall = ''
-    mv $out/bin/pg $out/bin/wal-g
-    installShellCompletion --cmd wal-g \
-      --bash <($out/bin/wal-g completion bash) \
-      --zsh <($out/bin/wal-g completion zsh)
-  '';
-
-  meta = with lib; {
-    homepage = "https://github.com/wal-g/wal-g";
-    license = licenses.asl20;
-    description = "Archival restoration tool for PostgreSQL";
-    mainProgram = "wal-g";
-    maintainers = [ samrose ];
+  # wal-g v3.0.5
+  wal-g-3 = walGCommon {
+    version = "3.0.5";
+    sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
+    vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
+    majorVersion = "3";
   };
 }