feat: reland vault w/o pgsodium

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -152,6 +152,43 @@ EOF
         run_sql -c "update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';"
     fi
 
+    # Patch to handle upgrading to pgsodium-less Vault
+    REENCRYPT_VAULT_SECRETS_QUERY=$(cat <<EOF
+    DO \$\$
+    BEGIN
+      IF EXISTS (SELECT FROM pg_available_extension_versions WHERE name = 'supabase_vault' AND version = '0.3.0')
+        AND EXISTS (SELECT FROM pg_extension WHERE extname = 'supabase_vault')
+      THEN
+        IF (SELECT extversion FROM pg_extension WHERE extname = 'supabase_vault') != '0.2.8' THEN
+          GRANT USAGE ON SCHEMA vault TO postgres WITH GRANT OPTION;
+          GRANT SELECT, DELETE ON vault.secrets, vault.decrypted_secrets TO postgres WITH GRANT OPTION;
+          GRANT EXECUTE ON FUNCTION vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt TO postgres WITH GRANT OPTION;
+        END IF;
+        -- Do an explicit IF EXISTS check to avoid referencing pgsodium objects if the project already migrated away from using pgsodium.
+        IF EXISTS (SELECT FROM vault.secrets WHERE key_id IS NOT NULL) THEN
+          UPDATE vault.secrets s
+          SET
+            secret = encode(
+              vault._crypto_aead_det_encrypt(
+                message := pgsodium.crypto_aead_det_decrypt(decode(s.secret, 'base64'), convert_to(s.id || s.description || s.created_at || s.updated_at, 'utf8'), s.key_id, s.nonce),
+                additional := convert_to(s.id::text, 'utf8'),
+                key_id := 0,
+                context := 'pgsodium'::bytea,
+                nonce := s.nonce
+              ),
+              'base64'
+            ),
+            key_id = NULL
+          WHERE
+            key_id IS NOT NULL;
+        END IF;
+      END IF;
+    END
+    \$\$;
+EOF
+    )
+    run_sql -c "$REENCRYPT_VAULT_SECRETS_QUERY"
+
     run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres"
 }
 

ansible/files/postgresql_config/postgresql.conf.j2

@@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english'
 #local_preload_libraries = ''
 #session_preload_libraries = ''
 
-shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter'	# (change requires restart)
+shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault'	# (change requires restart)
 jit_provider = 'llvmjit'		# JIT library to use
 
 # - Other Defaults -

ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql

@@ -0,0 +1,3 @@
+grant usage on schema vault to postgres with grant option;
+grant select, delete on vault.secrets, vault.decrypted_secrets to postgres with grant option;
+grant execute on function vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt to postgres with grant option;

ansible/tasks/test-image.yml

@@ -11,17 +11,34 @@
 #     cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
 #   when: debpkg_mode or stage2_nix
 
-- name: Temporarily disable PG Sodium references in config
+- name: Temporarily disable PG Sodium and Supabase Vault references in config
   become: yes
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      sed -i.bak 
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
   when: debpkg_mode or stage2_nix
 
+- name: Verify pgsodium and vault removal from config
+  become: yes
+  become_user: postgres
+  shell:
+    cmd: |
+      FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf)
+      if [ ! -z "$FOUND" ]; then
+        echo "Found unremoved references:"
+        echo "$FOUND"
+        exit 1
+      fi
+  register: verify_result
+  failed_when: verify_result.rc != 0
+  when: debpkg_mode or stage2_nix
+
 - name: Start Postgres Database to load all extensions.
   become: yes
   become_user: postgres

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.048-orioledb"
-  postgres15: "15.8.1.055"
+  postgresorioledb-17: "17.0.1.049-orioledb"
+  postgres15: "15.8.1.056"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"

ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -17,7 +17,6 @@ BEGIN
         extension_array := ARRAY[
             'plpgsql',
             'pg_stat_statements',
-            'pgsodium',
             'pgtap',
             'pg_graphql',
             'pgcrypto',
@@ -30,7 +29,6 @@ BEGIN
         extension_array := ARRAY[
             'plpgsql',
             'pg_stat_statements',
-            'pgsodium',
             'pgtap',
             'pg_graphql',
             'pgcrypto',
@@ -44,7 +42,7 @@ BEGIN
     PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false);
 END $$;
 
-SELECT plan(8);
+SELECT no_plan();
 
 SELECT extensions_are(
     string_to_array(current_setting('myapp.extensions'), ',')::text[]
@@ -56,9 +54,5 @@ SELECT has_schema('pg_catalog');
 SELECT has_schema('information_schema');
 SELECT has_schema('public');
 
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']);
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']);
-SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']);
-
 SELECT * FROM finish();
-ROLLBACK;
+ROLLBACK;

flake.nix

@@ -604,42 +604,49 @@
             sqlTests = ./nix/tests/smoke;
             pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
             pg_regress = basePackages.pg_regress;
-            getkey-script = pkgs.writeScriptBin "pgsodium-getkey" ''
-              #!${pkgs.bash}/bin/bash
-              set -euo pipefail
-
-              TMPDIR_BASE=$(mktemp -d)
-
-              if [[ "$(uname)" == "Darwin" ]]; then
-                KEY_DIR="/private/tmp/pgsodium"
-              else
-                KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}"
-              fi
-              KEY_FILE="$KEY_DIR/pgsodium.key"
-
-              if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
-                echo "Error: Could not create key directory $KEY_DIR" >&2
-                exit 1
-              fi
-              chmod 1777 "$KEY_DIR"
-
-              if [[ ! -f "$KEY_FILE" ]]; then
-                if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
-                  if ! (openssl rand -hex 32 > "$KEY_FILE"); then
-                    echo "00000000000000000000000000000000" > "$KEY_FILE"
-                    echo "Warning: Using fallback key" >&2
+            getkey-script = pkgs.stdenv.mkDerivation {
+              name = "pgsodium-getkey";
+              buildCommand = ''
+                mkdir -p $out/bin
+                cat > $out/bin/pgsodium-getkey << 'EOF'
+                #!${pkgs.bash}/bin/bash
+                set -euo pipefail
+
+                TMPDIR_BASE=$(mktemp -d)
+
+                if [[ "$(uname)" == "Darwin" ]]; then
+                  KEY_DIR="/private/tmp/pgsodium"
+                else
+                  KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}"
+                fi
+                KEY_FILE="$KEY_DIR/pgsodium.key"
+
+                if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
+                  echo "Error: Could not create key directory $KEY_DIR" >&2
+                  exit 1
+                fi
+                chmod 1777 "$KEY_DIR"
+
+                if [[ ! -f "$KEY_FILE" ]]; then
+                  if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
+                    if ! (openssl rand -hex 32 > "$KEY_FILE"); then
+                      echo "00000000000000000000000000000000" > "$KEY_FILE"
+                      echo "Warning: Using fallback key" >&2
+                    fi
                   fi
+                  chmod 644 "$KEY_FILE"
                 fi
-                chmod 644 "$KEY_FILE"
-              fi
-
-              if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then
-                cat "$KEY_FILE"
-              else
-                echo "Error: Cannot read key file $KEY_FILE" >&2
-                exit 1
-              fi
-            '';
+
+                if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then
+                  cat "$KEY_FILE"
+                else
+                  echo "Error: Cannot read key file $KEY_FILE" >&2
+                  exit 1
+                fi
+                EOF
+                chmod +x $out/bin/pgsodium-getkey
+              '';
+            };
 
             # Use the shared setup but with a test-specific name
             start-postgres-server-bin = makePostgresDevSetup {
@@ -708,6 +715,8 @@
               echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf
               echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf
               echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf
+              echo "Checking shared_preload_libraries setting:"
+              grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf
               # Remove timescaledb if running orioledb-17 check
               echo "I AM ${pgpkg.version}===================================================="
               if [[ "${pgpkg.version}" == *"17"* ]]; then
@@ -806,6 +815,7 @@
                 --user=supabase_admin \
                 ${builtins.concatStringsSep " " sortedTestList}; then
                 echo "pg_regress tests failed"
+                cat $out/regression_output/regression.diffs
                 exit 1
               fi
 

migrations/schema-15.sql

@@ -44,27 +44,6 @@ CREATE SCHEMA graphql_public;
 CREATE SCHEMA pgbouncer;
 
 
---
--- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: -
---
-
-CREATE SCHEMA pgsodium;
-
-
---
--- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: -
---
-
-CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium;
-
-
---
--- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: -
---
-
-COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.';
-
-
 --
 -- Name: realtime; Type: SCHEMA; Schema: -; Owner: -
 --
@@ -582,28 +561,6 @@ END
 $$;
 
 
---
--- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: -
---
-
-CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger
-    LANGUAGE plpgsql
-    AS $$
-		BEGIN
-		        new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE
-			CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode(
-			  pgsodium.crypto_aead_det_encrypt(
-				pg_catalog.convert_to(new.secret, 'utf8'),
-				pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'),
-				new.key_id::uuid,
-				new.nonce
-			  ),
-				'base64') END END;
-		RETURN new;
-		END;
-		$$;
-
-
 SET default_tablespace = '';
 
 SET default_table_access_method = heap;
@@ -790,30 +747,6 @@ CREATE TABLE storage.objects (
 );
 
 
---
--- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: -
---
-
-CREATE VIEW vault.decrypted_secrets AS
- SELECT secrets.id,
-    secrets.name,
-    secrets.description,
-    secrets.secret,
-        CASE
-            WHEN (secrets.secret IS NULL) THEN NULL::text
-            ELSE
-            CASE
-                WHEN (secrets.key_id IS NULL) THEN NULL::text
-                ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name)
-            END
-        END AS decrypted_secret,
-    secrets.key_id,
-    secrets.nonce,
-    secrets.created_at,
-    secrets.updated_at
-   FROM vault.secrets;
-
-
 --
 -- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: -
 --