Merge branch 'develop' into feat/reland-vault-wo-pgsodium

Author: soedirgo

README.md

@@ -16,9 +16,8 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 | [hypopg](https://github.com/HypoPG/hypopg/archive/refs/tags/1.4.1.tar.gz) | [1.4.1](https://github.com/HypoPG/hypopg/archive/refs/tags/1.4.1.tar.gz) | Hypothetical Indexes for PostgreSQL |
 | [index_advisor](https://github.com/olirice/index_advisor/archive/v0.2.0.tar.gz) | [0.2.0](https://github.com/olirice/index_advisor/archive/v0.2.0.tar.gz) | Recommend indexes to improve query performance in PostgreSQL |
 | [pg-safeupdate](https://github.com/eradman/pg-safeupdate/archive/1.4.tar.gz) | [1.4](https://github.com/eradman/pg-safeupdate/archive/1.4.tar.gz) | A simple extension to PostgreSQL that requires criteria for UPDATE and DELETE |
-| [pg_backtrace](https://github.com/pashkinelfe/pg_backtrace/archive/d100bac815a7365e199263f5b3741baf71b14c70.tar.gz) | [1.1](https://github.com/pashkinelfe/pg_backtrace/archive/d100bac815a7365e199263f5b3741baf71b14c70.tar.gz) | Updated fork of pg_backtrace |
 | [pg_cron](https://github.com/citusdata/pg_cron/archive/v1.6.4.tar.gz) | [1.6.4](https://github.com/citusdata/pg_cron/archive/v1.6.4.tar.gz) | Run Cron jobs through PostgreSQL |
-| [pg_graphql](https://github.com/supabase/pg_graphql/archive/v1.5.9.tar.gz) | [1.5.9](https://github.com/supabase/pg_graphql/archive/v1.5.9.tar.gz) | GraphQL support for PostreSQL |
+| [pg_graphql](https://github.com/supabase/pg_graphql/archive/v1.5.11.tar.gz) | [1.5.11](https://github.com/supabase/pg_graphql/archive/v1.5.11.tar.gz) | GraphQL support for PostreSQL |
 | [pg_hashids](https://github.com/iCyberon/pg_hashids/archive/cd0e1b31d52b394a0df64079406a14a4f7387cd6.tar.gz) | [cd0e1b31d52b394a0df64079406a14a4f7387cd6](https://github.com/iCyberon/pg_hashids/archive/cd0e1b31d52b394a0df64079406a14a4f7387cd6.tar.gz) | Generate short unique IDs in PostgreSQL |
 | [pg_jsonschema](https://github.com/supabase/pg_jsonschema/archive/v0.3.3.tar.gz) | [0.3.3](https://github.com/supabase/pg_jsonschema/archive/v0.3.3.tar.gz) | JSON Schema Validation for PostgreSQL |
 | [pg_net](https://github.com/supabase/pg_net/archive/refs/tags/v0.14.0.tar.gz) | [0.14.0](https://github.com/supabase/pg_net/archive/refs/tags/v0.14.0.tar.gz) | Async networking for Postgres |
@@ -39,7 +38,7 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 | [plv8](https://github.com/plv8/plv8/archive/v3.1.10.tar.gz) | [3.1.10](https://github.com/plv8/plv8/archive/v3.1.10.tar.gz) | V8 Engine Javascript Procedural Language add-on for PostgreSQL |
 | [postgis](https://download.osgeo.org/postgis/source/postgis-3.3.7.tar.gz) | [3.3.7](https://download.osgeo.org/postgis/source/postgis-3.3.7.tar.gz) | Geographic Objects for PostgreSQL |
 | [rum](https://github.com/postgrespro/rum/archive/1.3.14.tar.gz) | [1.3.14](https://github.com/postgrespro/rum/archive/1.3.14.tar.gz) | Full text search index method for PostgreSQL |
-| [supabase-wrappers](https://github.com/supabase/wrappers/archive/v0.4.4.tar.gz) | [0.4.4](https://github.com/supabase/wrappers/archive/v0.4.4.tar.gz) | Various Foreign Data Wrappers (FDWs) for PostreSQL |
+| [supabase-wrappers](https://github.com/supabase/wrappers/archive/v0.4.5.tar.gz) | [0.4.5](https://github.com/supabase/wrappers/archive/v0.4.5.tar.gz) | Various Foreign Data Wrappers (FDWs) for PostreSQL |
 | [supautils](https://github.com/supabase/supautils/archive/refs/tags/v2.6.0.tar.gz) | [2.6.0](https://github.com/supabase/supautils/archive/refs/tags/v2.6.0.tar.gz) | PostgreSQL extension for enhanced security |
 | [timescaledb-apache](https://github.com/timescale/timescaledb/archive/2.16.1.tar.gz) | [2.16.1](https://github.com/timescale/timescaledb/archive/2.16.1.tar.gz) | Scales PostgreSQL for time-series data via automatic partitioning across time and space |
 | [vault](https://github.com/supabase/vault/archive/refs/tags/v0.2.9.tar.gz) | [0.2.9](https://github.com/supabase/vault/archive/refs/tags/v0.2.9.tar.gz) | Store encrypted secrets in PostgreSQL |
@@ -52,9 +51,8 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 | [index_advisor](https://github.com/olirice/index_advisor/archive/v0.2.0.tar.gz) | [0.2.0](https://github.com/olirice/index_advisor/archive/v0.2.0.tar.gz) | Recommend indexes to improve query performance in PostgreSQL |
 | [orioledb](https://github.com/orioledb/orioledb/archive/beta9.tar.gz) | [orioledb](https://github.com/orioledb/orioledb/archive/beta9.tar.gz) | orioledb |
 | [pg-safeupdate](https://github.com/eradman/pg-safeupdate/archive/1.4.tar.gz) | [1.4](https://github.com/eradman/pg-safeupdate/archive/1.4.tar.gz) | A simple extension to PostgreSQL that requires criteria for UPDATE and DELETE |
-| [pg_backtrace](https://github.com/pashkinelfe/pg_backtrace/archive/d100bac815a7365e199263f5b3741baf71b14c70.tar.gz) | [1.1](https://github.com/pashkinelfe/pg_backtrace/archive/d100bac815a7365e199263f5b3741baf71b14c70.tar.gz) | Updated fork of pg_backtrace |
 | [pg_cron](https://github.com/citusdata/pg_cron/archive/v1.6.4.tar.gz) | [1.6.4](https://github.com/citusdata/pg_cron/archive/v1.6.4.tar.gz) | Run Cron jobs through PostgreSQL |
-| [pg_graphql](https://github.com/supabase/pg_graphql/archive/v1.5.9.tar.gz) | [1.5.9](https://github.com/supabase/pg_graphql/archive/v1.5.9.tar.gz) | GraphQL support for PostreSQL |
+| [pg_graphql](https://github.com/supabase/pg_graphql/archive/v1.5.11.tar.gz) | [1.5.11](https://github.com/supabase/pg_graphql/archive/v1.5.11.tar.gz) | GraphQL support for PostreSQL |
 | [pg_hashids](https://github.com/iCyberon/pg_hashids/archive/cd0e1b31d52b394a0df64079406a14a4f7387cd6.tar.gz) | [cd0e1b31d52b394a0df64079406a14a4f7387cd6](https://github.com/iCyberon/pg_hashids/archive/cd0e1b31d52b394a0df64079406a14a4f7387cd6.tar.gz) | Generate short unique IDs in PostgreSQL |
 | [pg_jsonschema](https://github.com/supabase/pg_jsonschema/archive/v0.3.3.tar.gz) | [0.3.3](https://github.com/supabase/pg_jsonschema/archive/v0.3.3.tar.gz) | JSON Schema Validation for PostgreSQL |
 | [pg_net](https://github.com/supabase/pg_net/archive/refs/tags/v0.14.0.tar.gz) | [0.14.0](https://github.com/supabase/pg_net/archive/refs/tags/v0.14.0.tar.gz) | Async networking for Postgres |
@@ -74,7 +72,7 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 | [plpgsql-check](https://github.com/okbob/plpgsql_check/archive/v2.7.11.tar.gz) | [2.7.11](https://github.com/okbob/plpgsql_check/archive/v2.7.11.tar.gz) | Linter tool for language PL/pgSQL |
 | [postgis](https://download.osgeo.org/postgis/source/postgis-3.3.7.tar.gz) | [3.3.7](https://download.osgeo.org/postgis/source/postgis-3.3.7.tar.gz) | Geographic Objects for PostgreSQL |
 | [rum](https://github.com/postgrespro/rum/archive/1.3.14.tar.gz) | [1.3.14](https://github.com/postgrespro/rum/archive/1.3.14.tar.gz) | Full text search index method for PostgreSQL |
-| [supabase-wrappers](https://github.com/supabase/wrappers/archive/v0.4.4.tar.gz) | [0.4.4](https://github.com/supabase/wrappers/archive/v0.4.4.tar.gz) | Various Foreign Data Wrappers (FDWs) for PostreSQL |
+| [supabase-wrappers](https://github.com/supabase/wrappers/archive/v0.4.5.tar.gz) | [0.4.5](https://github.com/supabase/wrappers/archive/v0.4.5.tar.gz) | Various Foreign Data Wrappers (FDWs) for PostreSQL |
 | [supautils](https://github.com/supabase/supautils/archive/refs/tags/v2.6.0.tar.gz) | [2.6.0](https://github.com/supabase/supautils/archive/refs/tags/v2.6.0.tar.gz) | PostgreSQL extension for enhanced security |
 | [vault](https://github.com/supabase/vault/archive/refs/tags/v0.2.9.tar.gz) | [0.2.9](https://github.com/supabase/vault/archive/refs/tags/v0.2.9.tar.gz) | Store encrypted secrets in PostgreSQL |
 | [wal2json](https://github.com/eulerto/wal2json/archive/wal2json_2_6.tar.gz) | [2_6](https://github.com/eulerto/wal2json/archive/wal2json_2_6.tar.gz) | PostgreSQL JSON output plugin for changeset extraction |

ansible/files/postgresql_config/supautils.conf.j2

@@ -5,7 +5,8 @@ supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.ide
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility
 # omitted because deprecated:                intagg, xml2
 # omitted because doesn't require superuser: pgmq
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+# omitted because protected:                 plpgsql
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
 supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'postgres'

ansible/playbook.yml

@@ -38,7 +38,7 @@
 
     - name: Install WAL-G
       import_tasks: tasks/setup-wal-g.yml
-      when: debpkg_mode or nixpkg_mode
+      when: debpkg_mode or nixpkg_mode or stage2_nix
 
     - name: Install Gotrue
       import_tasks: tasks/setup-gotrue.yml

ansible/tasks/setup-wal-g.yml

@@ -1,72 +1,73 @@
 # Downloading dependencies
-- name: wal-g dependencies
-  become: yes
-  apt:
-    pkg:
-      - libbrotli-dev
-      - liblzo2-dev
-      - libsodium-dev
-      - cmake
+# - name: wal-g dependencies
+#   become: yes
+#   apt:
+#     pkg:
+#       - libbrotli-dev
+#       - liblzo2-dev
+#       - libsodium-dev
+#       - cmake
 
 # install go dependency for WAL-G
-- name: wal-g go dependency
-  get_url:
-    url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-    dest: /tmp
-    checksum: "{{ golang_version_checksum[platform] }}"
-    timeout: 60
+# - name: wal-g go dependency
+#   get_url:
+#     url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+#     dest: /tmp
+#     checksum: "{{ golang_version_checksum[platform] }}"
+#     timeout: 60
 
-- name: unpack go archive
-  unarchive:
-    remote_src: yes
-    src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
-    dest: /usr/local
+# - name: unpack go archive
+#   unarchive:
+#     remote_src: yes
+#     src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz"
+#     dest: /usr/local
 
-# Download WAL-G
-- name: wal-g - download latest version
-  git:
-    repo: https://github.com/wal-g/wal-g.git
-    dest: /tmp/wal-g
-    version: "v{{ wal_g_release }}"
-  become: yes
+# # Download WAL-G
+# - name: wal-g - download latest version
+#   git:
+#     repo: https://github.com/wal-g/wal-g.git
+#     dest: /tmp/wal-g
+#     version: "v{{ wal_g_release }}"
+#   become: yes
 
-- name: wal-g - pg_clean
-  make:
-    chdir: /tmp/wal-g
-    target: pg_clean
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
-  ignore_errors: yes
+# - name: wal-g - pg_clean
+#   make:
+#     chdir: /tmp/wal-g
+#     target: pg_clean
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
+#   ignore_errors: yes
 
-- name: wal-g - deps
-  make:
-    chdir: /tmp/wal-g
-    target: deps
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
-  ignore_errors: yes
+# - name: wal-g - deps
+#   make:
+#     chdir: /tmp/wal-g
+#     target: deps
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
+#   ignore_errors: yes
 
-- name: wal-g - build and install
-  community.general.make:
-    chdir: /tmp/wal-g
-    target: pg_install
-    jobs: "{{ parallel_jobs | default(omit) }}"
-    params:
-      GOBIN: "/usr/local/bin"
-      PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
-      USE_LIBSODIUM: true
-  become: yes
+# - name: wal-g - build and install
+#   community.general.make:
+#     chdir: /tmp/wal-g
+#     target: pg_install
+#     jobs: "{{ parallel_jobs | default(omit) }}"
+#     params:
+#       GOBIN: "/usr/local/bin"
+#       PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin"
+#       USE_LIBSODIUM: true
+#   become: yes
 
 - name: Create wal-g group
   group:
     name: wal-g
     state: present
+  when: nixpkg_mode
 
 - name: Create wal-g user
   user:
@@ -75,22 +76,38 @@
     comment: WAL-G user
     group: wal-g
     groups: wal-g, postgres
-
+  when: nixpkg_mode
 - name: Create a config directory owned by wal-g
   file:
     path: /etc/wal-g
     state: directory
     owner: wal-g
     group: wal-g
     mode: '0770'
+  when: nixpkg_mode
 
+- name: Install wal-g from nix binary cache
+  become: yes
+  shell: |
+    sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g"
+  when: stage2_nix
+
+- name: Create symlink for wal-g from Nix profile to /usr/local/bin
+  ansible.builtin.file:
+    src: /home/wal-g/.nix-profile/bin/wal-g
+    dest: /usr/local/bin/wal-g
+    state: link
+    force: yes  # This will replace existing file/symlink if it exists
+  become: yes   # Need sudo to write to /usr/local/bin
+  when: stage2_nix
 - name: Create /etc/wal-g/config.json
   file:
     path: /etc/wal-g/config.json
     state: touch
     owner: wal-g
     group: wal-g
     mode: '0664'
+  when: stage2_nix
 
 - name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf
   template:
@@ -99,6 +116,7 @@
     mode: 0664
     owner: postgres
     group: postgres
+  when: stage2_nix
 
 - name: Add script to be run for restore_command
   template:
@@ -107,24 +125,20 @@
     mode: 0500
     owner: postgres
     group: postgres
+  when: stage2_nix
 
 - name: Add helper script for wal_fetch.sh
   template:
     src: "files/walg_helper_scripts/wal_change_ownership.sh"
     dest: /root/wal_change_ownership.sh
     mode: 0700
     owner: root
+  when: stage2_nix
 
 - name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf
   become: yes
   replace:
     path: /etc/postgresql/postgresql.conf
     regexp: "#include = '/etc/postgresql-custom/wal-g.conf'"
     replace: "include = '/etc/postgresql-custom/wal-g.conf'"
-
-# Clean up Go
-- name: Uninstall Go
-  become: yes
-  file:
-    path: /usr/local/go
-    state: absent
+  when: stage2_nix

ansible/vars.yml

@@ -45,7 +45,6 @@ kong_deb_checksum: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61
 nginx_release: 1.22.0
 nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93
 
-wal_g_release: "2.0.1"
 
 postgres_exporter_release: "0.15.0"
 postgres_exporter_release_checksum:

docker/nix/build_nix.sh

@@ -13,8 +13,9 @@ nix build .#checks.$SYSTEM.psql_orioledb-17 -L --no-link
 nix build .#psql_15/bin -o psql_15
 
 nix build .#psql_orioledb-17/bin -o psql_orioledb_17
-
+nix build .#wal-g -o wal-g
 # Copy to S3
+nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15
 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17
 if [ "$SYSTEM" = "aarch64-linux" ]; then
@@ -26,4 +27,5 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_15_src
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug
     nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./postgresql_orioledb-17_src
+    nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key  ./wal-g
 fi

flake.nix

@@ -87,6 +87,7 @@
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
         supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
         mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
+        wal-g = pkgs.callPackage ./nix/wal-g.nix { };
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
         # listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -405,6 +406,7 @@
           # PostgreSQL versions.
           psql_15 = postgresVersions.psql_15;
           psql_orioledb-17 = postgresVersions.psql_orioledb-17;
+          wal-g = wal-g;
           sfcgal = sfcgal;
           pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
           inherit postgresql_15 postgresql_orioledb-17;

nix/wal-g.nix

@@ -0,0 +1,58 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+  brotli,
+  libsodium,
+  installShellFiles,
+}:
+
+buildGoModule rec {
+  pname = "wal-g";
+  version = "3.0.5";
+
+  src = fetchFromGitHub {
+    owner = "wal-g";
+    repo = "wal-g";
+    rev = "v${version}";
+    sha256 = "sha256-wVr0L2ZXMuEo6tc2ajNzPinVQ8ZVzNOSoaHZ4oFsA+U=";
+  };
+
+  vendorHash = "sha256-YDLAmRfDl9TgbabXj/1rxVQ052NZDg3IagXVTe5i9dw=";
+
+  nativeBuildInputs = [ installShellFiles ];
+
+  buildInputs = [
+    brotli
+    libsodium
+  ];
+
+  subPackages = [ "main/pg" ];
+
+  tags = [
+    "brotli"
+    "libsodium"
+  ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X github.com/wal-g/wal-g/cmd/pg.walgVersion=${version}"
+    "-X github.com/wal-g/wal-g/cmd/pg.gitRevision=${src.rev}"
+  ];
+
+  postInstall = ''
+    mv $out/bin/pg $out/bin/wal-g
+    installShellCompletion --cmd wal-g \
+      --bash <($out/bin/wal-g completion bash) \
+      --zsh <($out/bin/wal-g completion zsh)
+  '';
+
+  meta = with lib; {
+    homepage = "https://github.com/wal-g/wal-g";
+    license = licenses.asl20;
+    description = "Archival restoration tool for PostgreSQL";
+    mainProgram = "wal-g";
+    maintainers = [ samrose ];
+  };
+}