feature: wal-g v 3 (#1456)

* feature: wal-g v 3

* chore: install, symlink

* chore: wal-g user and groups in stage 1

* chore: bump versions to test wal-g

* chore: build wal-g in ci and copy to binary cache on success

* chore: minor correction

* chore: maintainer

* chore: release wal-g 3

Author: samrose

ansible/playbook.yml

@@ -38,7 +38,7 @@
 
     - name: Install WAL-G
       import_tasks: tasks/setup-wal-g.yml
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install Gotrue
       import_tasks: tasks/setup-gotrue.yml

ansible/tasks/setup-wal-g.yml

@@ -1,72 +1,73 @@
 # Downloading dependencies
-- name: wal-g dependencies
-  become: yes
-  apt:
-    pkg:
-      - libbrotli-dev
-      - liblzo2-dev
-      - libsodium-dev
-      - cmake
+# - name: wal-g dependencies
+#   become: yes
+#   apt:
+#     pkg:
+#       - libbrotli-dev
+#       - liblzo2-dev
+#       - libsodium-dev
+#       - cmake
 
 # install go dependency for WAL-G
-- name: wal-g go dependency
-  get_url:
-    url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-    dest: /tmp
-    checksum: "{{ golang_version_checksum[platform] }}"
-    timeout: 60
+# - name: wal-g go dependency
+#   get_url:
+#     url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+#     dest: /tmp
+#     checksum: "{{ golang_version_checksum[platform] }}"
+#     timeout: 60
 
-- name: unpack go archive
-  unarchive:
-    remote_src: yes
-    src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-    dest: /usr/local
+# - name: unpack go archive
+#   unarchive:
+#     remote_src: yes
+#     src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+#     dest: /usr/local
 
-# Download WAL-G
-- name: wal-g - download latest version
-  git:
-    repo: https://github.com/wal-g/wal-g.git
-    dest: /tmp/wal-g
-    version: "v{{ wal_g_release }}"
-  become: yes
+# # Download WAL-G
+# - name: wal-g - download latest version
+#   git:
+#     repo: https://github.com/wal-g/wal-g.git
+#     dest: /tmp/wal-g
+#     version: "v{{ wal_g_release }}"
+#   become: yes
 
-- name: wal-g - pg_clean
-  make:
-    chdir: /tmp/wal-g
-    target: pg_clean
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
-  ignore_errors: yes
+# - name: wal-g - pg_clean
+#   make:
+#     chdir: /tmp/wal-g
+#     target: pg_clean
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
+#   ignore_errors: yes
 
-- name: wal-g - deps
-  make:
-    chdir: /tmp/wal-g
-    target: deps
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
-  ignore_errors: yes
+# - name: wal-g - deps
+#   make:
+#     chdir: /tmp/wal-g
+#     target: deps
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
+#   ignore_errors: yes
 
-- name: wal-g - build and install
-  community.general.make:
-    chdir: /tmp/wal-g
-    target: pg_install
-    jobs: "{{ parallel_jobs | default(omit) }}"
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
+# - name: wal-g - build and install
+#   community.general.make:
+#     chdir: /tmp/wal-g
+#     target: pg_install
+#     jobs: "{{ parallel_jobs | default(omit) }}"
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
 
 - name: Create wal-g group
   group:
     name: wal-g
     state: present
+  when: nixpkg_mode
 
 - name: Create wal-g user
   user:
@@ -75,22 +76,38 @@
     comment: WAL-G user
     group: wal-g
     groups: wal-g, postgres
-
+  when: nixpkg_mode
 - name: Create a config directory owned by wal-g
   file:
     path: /etc/wal-g
     state: directory
     owner: wal-g
     group: wal-g
     mode: '0770'
+  when: nixpkg_mode
 
+- name: Install wal-g from nix binary cache
+  become: yes
+  shell: |
+    sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g"
+  when: stage2_nix
+
+- name: Create symlink for wal-g from Nix profile to /usr/local/bin
+  ansible.builtin.file:
+    src: /home/wal-g/.nix-profile/bin/wal-g
+    dest: /usr/local/bin/wal-g
+    state: link
+    force: yes  # This will replace existing file/symlink if it exists
+  become: yes   # Need sudo to write to /usr/local/bin
+  when: stage2_nix
 - name: Create /etc/wal-g/config.json
   file:
     path: /etc/wal-g/config.json
     state: touch
     owner: wal-g
     group: wal-g
     mode: '0664'
+  when: stage2_nix
 
 - name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf
   template:
@@ -99,6 +116,7 @@
     mode: 0664
     owner: postgres
     group: postgres
+  when: stage2_nix
 
 - name: Add script to be run for restore_command
   template:
@@ -107,24 +125,20 @@
     mode: 0500
     owner: postgres
     group: postgres
+  when: stage2_nix
 
 - name: Add helper script for wal_fetch.sh
   template:
     src: "files/walg_helper_scripts/wal_change_ownership.sh"
     dest: /root/wal_change_ownership.sh
     mode: 0700
     owner: root
+  when: stage2_nix
 
 - name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf
   become: yes
   replace:
     path: /etc/postgresql/postgresql.conf
     regexp: "#include = '/etc/postgresql-custom/wal-g.conf'"
     replace: "include = '/etc/postgresql-custom/wal-g.conf'"
-
-# Clean up Go
-- name: Uninstall Go
-  become: yes
-  file:
-    path: /usr/local/go
-    state: absent
+  when: stage2_nix

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.048-orioledb"
-  postgres15: "15.8.1.055"
+  postgresorioledb-17: "17.0.1.049-orioledb"
+  postgres15: "15.8.1.056"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"
@@ -45,7 +45,6 @@ kong_deb_checksum: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61
 nginx_release: 1.22.0
 nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93
 
-wal_g_release: "2.0.1"
 
 postgres_exporter_release: "0.15.0"
 postgres_exporter_release_checksum:

docker/nix/build_nix.sh

@@ -13,8 +13,9 @@ nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link
 nix build .#psql_15/bin -o psql_15
 
 nix build .#psql_orioledb-17/bin -o psql_orioledb_17
-
+nix build .#wal-g -o wal-g
 # Copy to S3
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
 if [ "$SYSTEM" = "aarch64-linux" ]; then
@@ -26,4 +27,5 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_15_src
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_orioledb-17_src
+    nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./wal-g
 fi

flake.nix

@@ -87,6 +87,7 @@
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
         supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
         mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
+        wal-g = pkgs.callPackage ./nix/wal-g.nix { };
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
         # listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -405,6 +406,7 @@
           # PostgreSQL versions.
           psql_15 = postgresVersions.psql_15;
           psql_orioledb-17 = postgresVersions.psql_orioledb-17;
+          wal-g = wal-g;
           sfcgal = sfcgal;
           pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
           inherit postgresql_15 postgresql_orioledb-17;

nix/wal-g.nix

@@ -0,0 +1,58 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+  brotli,
+  libsodium,
+  installShellFiles,
+}:
+
+buildGoModule rec {
+  pname = "wal-g";
+  version = "3.0.5";
+
+  src = fetchFromGitHub {
+    owner = "wal-g";
+    repo = "wal-g";
+    rev = "v${version}";
+    sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
+  };
+
+  vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
+
+  nativeBuildInputs = [ installShellFiles ];
+
+  buildInputs = [
+    brotli
+    libsodium
+  ];
+
+  subPackages = [ "main/pg" ];
+
+  tags = [
+    "brotli"
+    "libsodium"
+  ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
+    "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
+  ];
+
+  postInstall = ''
+    mv $out/bin/pg $out/bin/wal-g
+    installShellCompletion --cmd wal-g \
+      --bash <($out/bin/wal-g completion bash) \
+      --zsh <($out/bin/wal-g completion zsh)
+  '';
+
+  meta = with lib; {
+    homepage = "https://github.com/wal-g/wal-g";
+    license = licenses.asl20;
+    description = "Archival restoration tool for PostgreSQL";
+    mainProgram = "wal-g";
+    maintainers = [ samrose ];
+  };
+}