feat: vault sans pgsodium

.github/workflows/manual-docker-release.yml

@@ -0,0 +1,250 @@
+name: Manual Docker Artifacts Release 
+
+on:
+  workflow_dispatch:
+    inputs:
+      postgresVersion:
+        description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78'
+        required: false
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix_config: ${{ steps.set-matrix.outputs.matrix_config }}
+    steps:
+      - uses: DeterminateSystems/nix-installer-action@main
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Generate build matrix
+        id: set-matrix
+        run: |
+          nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major)
+          let matrix = ($versions | each { |ver|
+            let version = ($ver | str trim)
+            let dockerfile = $"Dockerfile-($version)"
+            if ($dockerfile | path exists) {
+              {
+                version: $version,
+                dockerfile: $dockerfile
+              }
+            } else {
+              null
+            }
+          } | compact)
+
+          let matrix_config = {
+            include: $matrix
+          }
+
+          $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT'
+  build:
+    needs: prepare
+    strategy:
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }}
+    runs-on: ubuntu-latest
+    outputs:
+      build_args: ${{ steps.args.outputs.result }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - name: Set PostgreSQL version environment variable
+        run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV
+
+      - id: args
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            open ansible/vars.yml 
+            | items { |key value| {name: $key, item: $value} } 
+            | where { |it| ($it.item | describe) == "string" } 
+            | each { |it| $"($it.name)=($it.item)" } 
+            | str join "\n" 
+            | save --append $env.GITHUB_OUTPUT
+          '    
+  build_release_image:
+    needs: [prepare, build]
+    strategy:
+      matrix:
+        postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }}
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v3
+        with:
+          endpoint: builders
+      - uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Get image tag
+        id: image
+        run: |
+          if [[ "${{ matrix.arch }}" == "arm64" ]]; then
+            pg_version=$(sudo nix run nixpkgs#nushell -- -c '
+              let version = "${{ matrix.postgres.version }}"
+              let release_key = if ($version | str contains "orioledb") {
+                $"postgresorioledb-17"
+              } else {
+                $"postgres($version)"
+              }
+              let base_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim)
+              let final_version = if "${{ inputs.postgresVersion }}" != "" {
+                "${{ inputs.postgresVersion }}"
+              } else {
+                $base_version
+              }
+              $final_version | str trim
+            ')
+            echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
+          else
+            pg_version=$(nix run nixpkgs#nushell -- -c '
+              let version = "${{ matrix.postgres.version }}"
+              let release_key = if ($version | str contains "orioledb") {
+                $"postgresorioledb-17"
+              } else {
+                $"postgres($version)"
+              }
+              let base_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim)
+              let final_version = if "${{ inputs.postgresVersion }}" != "" {
+                "${{ inputs.postgresVersion }}"
+              } else {
+                $base_version
+              }
+              $final_version | str trim
+            ')
+            echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT
+          fi
+      - id: build
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          build-args: |
+            ${{ needs.build.outputs.build_args }}
+          target: production
+          tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }}
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
+          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }}
+          file: ${{ matrix.postgres.dockerfile }}
+  merge_manifest:
+    needs: [prepare, build, build_release_image]
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Get image tag
+        id: get_version
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            let version = "${{ matrix.version }}"
+            let release_key = if ($version | str contains "orioledb") {
+              $"postgresorioledb-17"
+            } else {
+              $"postgres($version)"
+            }
+            let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim)
+            $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT
+          '
+      - name: Output version
+        id: output_version
+        run: |
+          echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT
+      - name: Collect versions
+        id: collect_versions
+        run: |
+          echo "${{ steps.output_version.outputs.result }}" >> results.txt  # Append results
+      - name: Upload Results Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: merge_results-${{ matrix.version }}
+          path: results.txt
+          if-no-files-found: warn
+      - name: Merge multi-arch manifests
+        run: |
+          docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \
+          ${{ steps.get_version.outputs.pg_version }}_amd64 \
+          ${{ steps.get_version.outputs.pg_version }}_arm64
+  combine_results:
+    needs: [prepare, merge_manifest]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - name: Debug Input from Prepare
+        run: |
+          echo "Raw matrix_config output:"
+          echo "${{ needs.prepare.outputs.matrix_config }}"
+      - name: Get Versions from Matrix Config
+        id: get_versions
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            # Parse the matrix configuration directly
+            let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json)
+
+            # Get versions directly from include array
+            let versions = ($matrix_config.include | get version)
+
+            echo "Versions: $versions"
+
+            # Convert the versions to a comma-separated string
+            let versions_str = ($versions | str join ",")
+            $"versions=$versions_str" | save --append $env.GITHUB_ENV
+          '
+      - name: Download Results Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: merge_results-*
+      - name: Combine Results
+        id: combine
+        run: |
+          nix run nixpkgs#nushell -- -c '
+            # Get all results files and process them in one go
+            let files = (ls **/results.txt | get name)
+            echo $"Found files: ($files)"
+
+            let matrix = {
+              include: (
+                $files
+                | each { |file| open $file }          # Open each file
+                | each { |content| $content | lines }  # Split into lines
+                | flatten                             # Flatten the nested lists
+                | where { |line| $line != "" }        # Filter empty lines
+                | each { |line| 
+                    # Extract just the version part after the last colon
+                    let version = ($line | parse "supabase/postgres:{version}" | get version.0)
+                    {version: $version}
+                }
+              )
+            }
+
+            let json_output = ($matrix | to json -r)  # -r for raw output
+            echo $"Debug output: ($json_output)"
+
+            $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT
+          '
+      - name: Debug Combined Results
+        run: |
+          echo "Combined Results: '${{ steps.combine.outputs.matrix }}'"
+    outputs:
+      matrix: ${{ steps.combine.outputs.matrix }}
+  publish:
+      needs: combine_results
+      strategy:
+        matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }}
+      uses: ./.github/workflows/mirror.yml
+      with:
+        version: ${{ inputs.postgresVersion != '' && inputs.postgresVersion || matrix.version }}
+      secrets: inherit

.github/workflows/test.yml

@@ -76,7 +76,7 @@ jobs:
           echo "EOF" >> $GITHUB_OUTPUT
       - name: verify schema.sql is committed
         run: |
-          nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }}
+          nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }}
           if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
             echo "Detected changes in schema.sql:"
             git diff migrations/schema-${{ env.PGMAJOR }}.sql

ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh

@@ -150,6 +150,43 @@ EOF
 
         run_sql -c "$PATCH_PGMQ_QUERY"
         run_sql -c "update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';"
+
+        # Patch to handle upgrading to pgsodium-less Vault
+        REENCRYPT_VAULT_SECRETS_QUERY=$(cat <<EOF
+        DO \$\$
+        BEGIN
+          IF EXISTS (SELECT FROM pg_available_extension_versions WHERE name = 'supabase_vault' AND version = '0.3.0')
+            AND EXISTS (SELECT FROM pg_extension WHERE extname = 'supabase_vault')
+          THEN
+            IF (SELECT extversion FROM pg_extension WHERE extname = 'supabase_vault') != '0.2.8' THEN
+              GRANT USAGE ON SCHEMA vault TO postgres WITH GRANT OPTION;
+              GRANT SELECT, DELETE ON vault.secrets, vault.decrypted_secrets TO postgres WITH GRANT OPTION;
+              GRANT EXECUTE ON FUNCTION vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt TO postgres WITH GRANT OPTION;
+            END IF;
+            -- Do an explicit IF EXISTS check to avoid referencing pgsodium objects if the project already migrated away from using pgsodium.
+            IF EXISTS (SELECT FROM vault.secrets WHERE key_id IS NOT NULL) THEN
+              UPDATE vault.secrets s
+              SET
+                secret = encode(
+                  vault._crypto_aead_det_encrypt(
+                    message := pgsodium.crypto_aead_det_decrypt(decode(s.secret, 'base64'), convert_to(s.id || s.description || s.created_at || s.updated_at, 'utf8'), s.key_id, s.nonce),
+                    additional := convert_to(s.id::text, 'utf8'),
+                    key_id := 0,
+                    context := 'pgsodium'::bytea,
+                    nonce := s.nonce
+                  ),
+                  'base64'
+                ),
+                key_id = NULL
+              WHERE
+                key_id IS NOT NULL;
+            END IF;
+          END IF;
+        END
+        \$\$;
+EOF
+        )
+        run_sql -c "$REENCRYPT_VAULT_SECRETS_QUERY"
     fi
 
     run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres"

ansible/files/postgresql_config/postgresql.conf.j2

@@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english'
 #local_preload_libraries = ''
 #session_preload_libraries = ''
 
-shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter'	# (change requires restart)
+shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault'	# (change requires restart)
 jit_provider = 'llvmjit'		# JIT library to use
 
 # - Other Defaults -

ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql

@@ -0,0 +1,3 @@
+grant usage on schema vault to postgres with grant option;
+grant select, delete on vault.secrets, vault.decrypted_secrets to postgres with grant option;
+grant execute on function vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt to postgres with grant option;

ansible/tasks/test-image.yml

@@ -11,17 +11,34 @@
 #     cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf
 #   when: debpkg_mode or stage2_nix
 
-- name: Temporarily disable PG Sodium references in config
+- name: Temporarily disable PG Sodium and Supabase Vault references in config
   become: yes
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      sed -i.bak 
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
   when: debpkg_mode or stage2_nix
 
+- name: Verify pgsodium and vault removal from config
+  become: yes
+  become_user: postgres
+  shell:
+    cmd: |
+      FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf)
+      if [ ! -z "$FOUND" ]; then
+        echo "Found unremoved references:"
+        echo "$FOUND"
+        exit 1
+      fi
+  register: verify_result
+  failed_when: verify_result.rc != 0
+  when: debpkg_mode or stage2_nix
+
 - name: Start Postgres Database to load all extensions.
   become: yes
   become_user: postgres

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.34-orioledb"
-  postgres15: "15.8.1.038"
+  postgresorioledb-17: "17.0.1.035-orioledb"
+  postgres15: "15.8.1.039"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"