From 1b2b4367e18d14044197a85697d7f114c3cbdf84 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Feb 2025 12:22:19 -0500 Subject: [PATCH 1/8] feature: wal-g v 3 --- flake.nix | 2 ++ nix/wal-g.nix | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 nix/wal-g.nix diff --git a/flake.nix b/flake.nix index 3abdbafff..ba7ad70ec 100644 --- a/flake.nix +++ b/flake.nix @@ -87,6 +87,7 @@ sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; + wal-g = pkgs.callPackage ./nix/wal-g.nix { }; # Our list of PostgreSQL extensions which come from upstream Nixpkgs. # These are maintained upstream and can easily be used here just by # listing their name. Anytime the version of nixpkgs is upgraded, these @@ -405,6 +406,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_orioledb-17 = postgresVersions.psql_orioledb-17; + wal-g = wal-g; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_orioledb-17; diff --git a/nix/wal-g.nix b/nix/wal-g.nix new file mode 100644 index 000000000..a2f2e5dd8 --- /dev/null +++ b/nix/wal-g.nix @@ -0,0 +1,58 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, + brotli, + libsodium, + installShellFiles, +}: + +buildGoModule rec { + pname = "wal-g"; + version = "3.0.5"; + + src = fetchFromGitHub { + owner = "wal-g"; + repo = "wal-g"; + rev = "v${version}"; + sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U="; + }; + + vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw="; + + nativeBuildInputs = [ installShellFiles ]; + + buildInputs = [ + brotli + libsodium + ]; + + subPackages = [ "main/pg" ]; + + tags = [ + "brotli" + "libsodium" + ]; + + ldflags = [ + "-s" + "-w" + "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}" + "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}" + ]; + + postInstall = '' + mv $out/bin/pg $out/bin/wal-g + installShellCompletion --cmd wal-g \ + --bash <($out/bin/wal-g completion bash) \ + --zsh <($out/bin/wal-g completion zsh) + ''; + + meta = with lib; { + homepage = "https://github.com/wal-g/wal-g"; + license = licenses.asl20; + description = "Archival restoration tool for PostgreSQL"; + mainProgram = "wal-g"; + maintainers = [ ]; + }; +} \ No newline at end of file From 88a5f97e8f02c29df64131a7e1bdb7679209dd05 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 20 Feb 2025 13:05:50 -0500 Subject: [PATCH 2/8] chore: install, symlink --- ansible/playbook.yml | 2 +- ansible/tasks/setup-wal-g.yml | 142 +++++++++++++++++++--------------- 2 files changed, 79 insertions(+), 65 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 5c5f8da3a..08754b8ce 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -38,7 +38,7 @@ - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml - when: debpkg_mode or nixpkg_mode + when: debpkg_mode or stage2_nix - name: Install Gotrue import_tasks: tasks/setup-gotrue.yml diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index bbc64cdde..2aaae10bf 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,72 +1,73 @@ # Downloading dependencies -- name: wal-g dependencies - become: yes - apt: - pkg: - - libbrotli-dev - - liblzo2-dev - - libsodium-dev - - cmake +# - name: wal-g dependencies +# become: yes +# apt: +# pkg: +# - libbrotli-dev +# - liblzo2-dev +# - libsodium-dev +# - cmake # install go dependency for WAL-G -- name: wal-g go dependency - get_url: - url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /tmp - checksum: "{{ golang_version_checksum[platform] }}" - timeout: 60 +# - name: wal-g go dependency +# get_url: +# url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" +# dest: /tmp +# checksum: "{{ golang_version_checksum[platform] }}" +# timeout: 60 -- name: unpack go archive - unarchive: - remote_src: yes - src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" - dest: /usr/local +# - name: unpack go archive +# unarchive: +# remote_src: yes +# src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" +# dest: /usr/local -# Download WAL-G -- name: wal-g - download latest version - git: - repo: https://github.com/wal-g/wal-g.git - dest: /tmp/wal-g - version: "v{{ wal_g_release }}" - become: yes +# # Download WAL-G +# - name: wal-g - download latest version +# git: +# repo: https://github.com/wal-g/wal-g.git +# dest: /tmp/wal-g +# version: "v{{ wal_g_release }}" +# become: yes -- name: wal-g - pg_clean - make: - chdir: /tmp/wal-g - target: pg_clean - params: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - USE_LIBSODIUM: true - become: yes - ignore_errors: yes +# - name: wal-g - pg_clean +# make: +# chdir: /tmp/wal-g +# target: pg_clean +# params: +# GOBIN: "/usr/local/bin" +# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" +# USE_LIBSODIUM: true +# become: yes +# ignore_errors: yes -- name: wal-g - deps - make: - chdir: /tmp/wal-g - target: deps - params: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - USE_LIBSODIUM: true - become: yes - ignore_errors: yes +# - name: wal-g - deps +# make: +# chdir: /tmp/wal-g +# target: deps +# params: +# GOBIN: "/usr/local/bin" +# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" +# USE_LIBSODIUM: true +# become: yes +# ignore_errors: yes -- name: wal-g - build and install - community.general.make: - chdir: /tmp/wal-g - target: pg_install - jobs: "{{ parallel_jobs | default(omit) }}" - params: - GOBIN: "/usr/local/bin" - PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" - USE_LIBSODIUM: true - become: yes +# - name: wal-g - build and install +# community.general.make: +# chdir: /tmp/wal-g +# target: pg_install +# jobs: "{{ parallel_jobs | default(omit) }}" +# params: +# GOBIN: "/usr/local/bin" +# PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" +# USE_LIBSODIUM: true +# become: yes - name: Create wal-g group group: name: wal-g state: present + when: stage2_nix - name: Create wal-g user user: @@ -75,7 +76,7 @@ comment: WAL-G user group: wal-g groups: wal-g, postgres - + when: stage2_nix - name: Create a config directory owned by wal-g file: path: /etc/wal-g @@ -83,7 +84,22 @@ owner: wal-g group: wal-g mode: '0770' + when: stage2_nix +- name: Install wal-g from nix binary cache + become: yes + shell: | + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" + when: stage2_nix + +- name: Create symlink for wal-g from Nix profile to /usr/local/bin + ansible.builtin.file: + src: /home/wal-g/.nix-profile/bin/wal-g + dest: /usr/local/bin/wal-g + state: link + force: yes # This will replace existing file/symlink if it exists + become: yes # Need sudo to write to /usr/local/bin + when: stage2_nix - name: Create /etc/wal-g/config.json file: path: /etc/wal-g/config.json @@ -91,6 +107,7 @@ owner: wal-g group: wal-g mode: '0664' + when: stage2_nix - name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf template: @@ -99,6 +116,7 @@ mode: 0664 owner: postgres group: postgres + when: stage2_nix - name: Add script to be run for restore_command template: @@ -107,6 +125,7 @@ mode: 0500 owner: postgres group: postgres + when: stage2_nix - name: Add helper script for wal_fetch.sh template: @@ -114,6 +133,7 @@ dest: /root/wal_change_ownership.sh mode: 0700 owner: root + when: stage2_nix - name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf become: yes @@ -121,10 +141,4 @@ path: /etc/postgresql/postgresql.conf regexp: "#include = '/etc/postgresql-custom/wal-g.conf'" replace: "include = '/etc/postgresql-custom/wal-g.conf'" - -# Clean up Go -- name: Uninstall Go - become: yes - file: - path: /usr/local/go - state: absent + when: stage2_nix From f20a214e747ffcbc41f18732144d4f9f63fc8d16 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Feb 2025 14:08:28 -0500 Subject: [PATCH 3/8] chore: wal-g user and groups in stage 1 --- ansible/playbook.yml | 2 +- ansible/tasks/setup-wal-g.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 08754b8ce..5c8a9f70e 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -38,7 +38,7 @@ - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml - when: debpkg_mode or stage2_nix + when: debpkg_mode or nixpkg_mode or stage2_nix - name: Install Gotrue import_tasks: tasks/setup-gotrue.yml diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 2aaae10bf..258ea1ea9 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -67,7 +67,7 @@ group: name: wal-g state: present - when: stage2_nix + when: nixpkg_mode - name: Create wal-g user user: @@ -76,7 +76,7 @@ comment: WAL-G user group: wal-g groups: wal-g, postgres - when: stage2_nix + when: nixpkg_mode - name: Create a config directory owned by wal-g file: path: /etc/wal-g @@ -84,7 +84,7 @@ owner: wal-g group: wal-g mode: '0770' - when: stage2_nix + when: nixpkg_mode - name: Install wal-g from nix binary cache become: yes From c6fe1fde35f6676e6a1b0f00d73aa0ef2c4325e2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 28 Feb 2025 14:26:53 -0500 Subject: [PATCH 4/8] chore: bump versions to test wal-g --- ansible/vars.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 381fb9510..784b15dcc 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.048-orioledb" - postgres15: "15.8.1.055" + postgresorioledb-17: "17.0.1.041-orioledb-walg-1" + postgres15: "15.8.1.047-walg-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" @@ -45,7 +45,6 @@ kong_deb_checksum: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61 nginx_release: 1.22.0 nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93 -wal_g_release: "2.0.1" postgres_exporter_release: "0.15.0" postgres_exporter_release_checksum: From 4dfc2953d21107f8453feef4544443877365aec0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 5 Mar 2025 11:38:08 -0500 Subject: [PATCH 5/8] chore: build wal-g in ci and copy to binary cache on success --- docker/nix/build_nix.sh | 4 +++- nix/wal-g.nix | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index b0eef9863..48286907e 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -13,8 +13,9 @@ nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link nix build .#psql_15/bin -o psql_15 nix build .#psql_orioledb-17/bin -o psql_orioledb_17 - +nix build .#wal-g -o wal-g # Copy to S3 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 if [ "$SYSTEM" = "aarch64-linux" ]; then @@ -26,4 +27,5 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_src + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g fi diff --git a/nix/wal-g.nix b/nix/wal-g.nix index a2f2e5dd8..94798ae0b 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -55,4 +55,4 @@ buildGoModule rec { mainProgram = "wal-g"; maintainers = [ ]; }; -} \ No newline at end of file +} From 4bdd38a434a0a8f52af6d41883c00b8c96f95e3d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 6 Mar 2025 10:51:33 -0500 Subject: [PATCH 6/8] chore: minor correction --- ansible/tasks/setup-wal-g.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 258ea1ea9..9907d3ee7 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -89,7 +89,7 @@ - name: Install wal-g from nix binary cache become: yes shell: | - sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" + sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g" when: stage2_nix - name: Create symlink for wal-g from Nix profile to /usr/local/bin From c4cb9c4abcfcf87f5f858c22075f095afec5f359 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 11 Mar 2025 12:55:16 +0100 Subject: [PATCH 7/8] chore: maintainer --- nix/wal-g.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/wal-g.nix b/nix/wal-g.nix index 94798ae0b..13f2bbc90 100644 --- a/nix/wal-g.nix +++ b/nix/wal-g.nix @@ -53,6 +53,6 @@ buildGoModule rec { license = licenses.asl20; description = "Archival restoration tool for PostgreSQL"; mainProgram = "wal-g"; - maintainers = [ ]; + maintainers = [ samrose ]; }; } From 1994d1ecef486a201e013ab5d3e1674bfb391731 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 18 Mar 2025 16:45:11 -0400 Subject: [PATCH 8/8] chore: release wal-g 3 --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 784b15dcc..d8e963b70 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.041-orioledb-walg-1" - postgres15: "15.8.1.047-walg-1" + postgresorioledb-17: "17.0.1.049-orioledb" + postgres15: "15.8.1.056" # Non Postgres Extensions pgbouncer_release: "1.19.0"