diff --git a/.github/workflows/manual-docker-release.yml b/.github/workflows/manual-docker-release.yml new file mode 100644 index 000000000..b9b66b305 --- /dev/null +++ b/.github/workflows/manual-docker-release.yml @@ -0,0 +1,250 @@ +name: Manual Docker Artifacts Release + +on: + workflow_dispatch: + inputs: + postgresVersion: + description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78' + required: false + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} + steps: + - uses: DeterminateSystems/nix-installer-action@main + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Generate build matrix + id: set-matrix + run: | + nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) + let matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let dockerfile = $"Dockerfile-($version)" + if ($dockerfile | path exists) { + { + version: $version, + dockerfile: $dockerfile + } + } else { + null + } + } | compact) + + let matrix_config = { + include: $matrix + } + + $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' + build: + needs: prepare + strategy: + matrix: ${{ fromJson(needs.prepare.outputs.matrix_config) }} + runs-on: ubuntu-latest + outputs: + build_args: ${{ steps.args.outputs.result }} + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV + + - id: args + run: | + nix run nixpkgs#nushell -- -c ' + open ansible/vars.yml + | items { |key value| {name: $key, item: $value} } + | where { |it| ($it.item | describe) == "string" } + | each { |it| $"($it.name)=($it.item)" } + | str join "\n" + | save --append $env.GITHUB_OUTPUT + ' + build_release_image: + needs: [prepare, build] + strategy: + matrix: + postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'arm-runner' }} + timeout-minutes: 180 + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - run: docker context create builders + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: image + run: | + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + pg_version=$(sudo nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let base_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let final_version = if "${{ inputs.postgresVersion }}" != "" { + "${{ inputs.postgresVersion }}" + } else { + $base_version + } + $final_version | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + else + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let base_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let final_version = if "${{ inputs.postgresVersion }}" != "" { + "${{ inputs.postgresVersion }}" + } else { + $base_version + } + $final_version | str trim + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + fi + - id: build + uses: docker/build-push-action@v5 + with: + push: true + build-args: | + ${{ needs.build.outputs.build_args }} + target: production + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + file: ${{ matrix.postgres.dockerfile }} + merge_manifest: + needs: [prepare, build, build_release_image] + strategy: + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: get_version + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let release_key = if ($version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($version)" + } + let pg_version = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' + - name: Output version + id: output_version + run: | + echo "result=${{ steps.get_version.outputs.pg_version }}" >> $GITHUB_OUTPUT + - name: Collect versions + id: collect_versions + run: | + echo "${{ steps.output_version.outputs.result }}" >> results.txt # Append results + - name: Upload Results Artifact + uses: actions/upload-artifact@v4 + with: + name: merge_results-${{ matrix.version }} + path: results.txt + if-no-files-found: warn + - name: Merge multi-arch manifests + run: | + docker buildx imagetools create -t ${{ steps.get_version.outputs.pg_version }} \ + ${{ steps.get_version.outputs.pg_version }}_amd64 \ + ${{ steps.get_version.outputs.pg_version }}_arm64 + combine_results: + needs: [prepare, merge_manifest] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + + - name: Debug Input from Prepare + run: | + echo "Raw matrix_config output:" + echo "${{ needs.prepare.outputs.matrix_config }}" + - name: Get Versions from Matrix Config + id: get_versions + run: | + nix run nixpkgs#nushell -- -c ' + # Parse the matrix configuration directly + let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) + + # Get versions directly from include array + let versions = ($matrix_config.include | get version) + + echo "Versions: $versions" + + # Convert the versions to a comma-separated string + let versions_str = ($versions | str join ",") + $"versions=$versions_str" | save --append $env.GITHUB_ENV + ' + - name: Download Results Artifacts + uses: actions/download-artifact@v4 + with: + pattern: merge_results-* + - name: Combine Results + id: combine + run: | + nix run nixpkgs#nushell -- -c ' + # Get all results files and process them in one go + let files = (ls **/results.txt | get name) + echo $"Found files: ($files)" + + let matrix = { + include: ( + $files + | each { |file| open $file } # Open each file + | each { |content| $content | lines } # Split into lines + | flatten # Flatten the nested lists + | where { |line| $line != "" } # Filter empty lines + | each { |line| + # Extract just the version part after the last colon + let version = ($line | parse "supabase/postgres:{version}" | get version.0) + {version: $version} + } + ) + } + + let json_output = ($matrix | to json -r) # -r for raw output + echo $"Debug output: ($json_output)" + + $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT + ' + - name: Debug Combined Results + run: | + echo "Combined Results: '${{ steps.combine.outputs.matrix }}'" + outputs: + matrix: ${{ steps.combine.outputs.matrix }} + publish: + needs: combine_results + strategy: + matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }} + uses: ./.github/workflows/mirror.yml + with: + version: ${{ inputs.postgresVersion != '' && inputs.postgresVersion || matrix.version }} + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6912b38f..6dc194684 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | - nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} + nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index 515c490f6..c2367116d 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -150,6 +150,43 @@ EOF run_sql -c "$PATCH_PGMQ_QUERY" run_sql -c "update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';" + + # Patch to handle upgrading to pgsodium-less Vault + REENCRYPT_VAULT_SECRETS_QUERY=$(cat < - sed -i.bak - -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' + sed -i.bak + -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' + -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' + -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/' -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/' /etc/postgresql/postgresql.conf when: debpkg_mode or stage2_nix +- name: Verify pgsodium and vault removal from config + become: yes + become_user: postgres + shell: + cmd: | + FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf) + if [ ! -z "$FOUND" ]; then + echo "Found unremoved references:" + echo "$FOUND" + exit 1 + fi + register: verify_result + failed_when: verify_result.rc != 0 + when: debpkg_mode or stage2_nix + - name: Start Postgres Database to load all extensions. become: yes become_user: postgres diff --git a/ansible/vars.yml b/ansible/vars.yml index fe8ec2d35..206cb139f 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.34-orioledb" - postgres15: "15.8.1.038" + postgresorioledb-17: "17.0.1.035-orioledb" + postgres15: "15.8.1.039" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/ebssurrogate/files/unit-tests/unit-test-01.sql b/ebssurrogate/files/unit-tests/unit-test-01.sql index f3d47459f..c466af12e 100644 --- a/ebssurrogate/files/unit-tests/unit-test-01.sql +++ b/ebssurrogate/files/unit-tests/unit-test-01.sql @@ -17,7 +17,6 @@ BEGIN extension_array := ARRAY[ 'plpgsql', 'pg_stat_statements', - 'pgsodium', 'pgtap', 'pg_graphql', 'pgcrypto', @@ -30,7 +29,6 @@ BEGIN extension_array := ARRAY[ 'plpgsql', 'pg_stat_statements', - 'pgsodium', 'pgtap', 'pg_graphql', 'pgcrypto', @@ -44,7 +42,7 @@ BEGIN PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false); END $$; -SELECT plan(8); +SELECT no_plan(); SELECT extensions_are( string_to_array(current_setting('myapp.extensions'), ',')::text[] @@ -56,9 +54,5 @@ SELECT has_schema('pg_catalog'); SELECT has_schema('information_schema'); SELECT has_schema('public'); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']); - SELECT * FROM finish(); -ROLLBACK; \ No newline at end of file +ROLLBACK; diff --git a/flake.nix b/flake.nix index c0ecc45f0..8bbd79525 100644 --- a/flake.nix +++ b/flake.nix @@ -571,42 +571,49 @@ sqlTests = ./nix/tests/smoke; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; pg_regress = basePackages.pg_regress; - getkey-script = pkgs.writeScriptBin "pgsodium-getkey" '' - #!${pkgs.bash}/bin/bash - set -euo pipefail - - TMPDIR_BASE=$(mktemp -d) - - if [[ "$(uname)" == "Darwin" ]]; then - KEY_DIR="/private/tmp/pgsodium" - else - KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" - fi - KEY_FILE="$KEY_DIR/pgsodium.key" - - if ! mkdir -p "$KEY_DIR" 2>/dev/null; then - echo "Error: Could not create key directory $KEY_DIR" >&2 - exit 1 - fi - chmod 1777 "$KEY_DIR" - - if [[ ! -f "$KEY_FILE" ]]; then - if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then - if ! (openssl rand -hex 32 > "$KEY_FILE"); then - echo "00000000000000000000000000000000" > "$KEY_FILE" - echo "Warning: Using fallback key" >&2 + getkey-script = pkgs.stdenv.mkDerivation { + name = "pgsodium-getkey"; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/pgsodium-getkey << 'EOF' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + TMPDIR_BASE=$(mktemp -d) + + if [[ "$(uname)" == "Darwin" ]]; then + KEY_DIR="/private/tmp/pgsodium" + else + KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" + fi + KEY_FILE="$KEY_DIR/pgsodium.key" + + if ! mkdir -p "$KEY_DIR" 2>/dev/null; then + echo "Error: Could not create key directory $KEY_DIR" >&2 + exit 1 + fi + chmod 1777 "$KEY_DIR" + + if [[ ! -f "$KEY_FILE" ]]; then + if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then + if ! (openssl rand -hex 32 > "$KEY_FILE"); then + echo "00000000000000000000000000000000" > "$KEY_FILE" + echo "Warning: Using fallback key" >&2 + fi fi + chmod 644 "$KEY_FILE" fi - chmod 644 "$KEY_FILE" - fi - - if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then - cat "$KEY_FILE" - else - echo "Error: Cannot read key file $KEY_FILE" >&2 - exit 1 - fi - ''; + + if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then + cat "$KEY_FILE" + else + echo "Error: Cannot read key file $KEY_FILE" >&2 + exit 1 + fi + EOF + chmod +x $out/bin/pgsodium-getkey + ''; + }; # Use the shared setup but with a test-specific name start-postgres-server-bin = makePostgresDevSetup { @@ -675,6 +682,8 @@ echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + echo "Checking shared_preload_libraries setting:" + grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf # Remove timescaledb if running orioledb-17 check echo "I AM ${pgpkg.version}====================================================" if [[ "${pgpkg.version}" == *"17"* ]]; then diff --git a/migrations/db/migrations/20221207154255_create_pgsodium_and_vault.sql b/migrations/db/migrations/20221207154255_create_pgsodium_and_vault.sql deleted file mode 100644 index 3d3867ab5..000000000 --- a/migrations/db/migrations/20221207154255_create_pgsodium_and_vault.sql +++ /dev/null @@ -1,40 +0,0 @@ --- migrate:up - -DO $$ -DECLARE - pgsodium_exists boolean; - vault_exists boolean; -BEGIN - pgsodium_exists = ( - select count(*) = 1 - from pg_available_extensions - where name = 'pgsodium' - and default_version in ('3.1.6', '3.1.7', '3.1.8', '3.1.9') - ); - - vault_exists = ( - select count(*) = 1 - from pg_available_extensions - where name = 'supabase_vault' - ); - - IF pgsodium_exists - THEN - create extension if not exists pgsodium; - - grant pgsodium_keyiduser to postgres with admin option; - grant pgsodium_keyholder to postgres with admin option; - grant pgsodium_keymaker to postgres with admin option; - - grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role; - grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role; - grant execute on function pgsodium.crypto_aead_det_keygen to service_role; - - IF vault_exists - THEN - create extension if not exists supabase_vault; - END IF; - END IF; -END $$; - --- migrate:down diff --git a/migrations/db/migrations/20221207154255_create_vault.sql b/migrations/db/migrations/20221207154255_create_vault.sql new file mode 100644 index 000000000..d4f714182 --- /dev/null +++ b/migrations/db/migrations/20221207154255_create_vault.sql @@ -0,0 +1,17 @@ +-- migrate:up + +DO $$ +BEGIN + IF EXISTS (select from pg_available_extensions where name = 'supabase_vault') + THEN + create extension if not exists supabase_vault; + + -- for some reason extension custom scripts aren't run during AMI build, so + -- we manually run it here + grant usage on schema vault to postgres with grant option; + grant select, delete on vault.secrets, vault.decrypted_secrets to postgres with grant option; + grant execute on function vault.create_secret, vault.update_secret, vault._crypto_aead_det_decrypt to postgres with grant option; + END IF; +END $$; + +-- migrate:down diff --git a/migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql b/migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql index 4df82e3f4..013a074fd 100644 --- a/migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql +++ b/migrations/db/migrations/20230529180330_alter_api_roles_for_inherit.sql @@ -4,7 +4,5 @@ ALTER ROLE authenticated inherit; ALTER ROLE anon inherit; ALTER ROLE service_role inherit; -GRANT pgsodium_keyholder to service_role; - -- migrate:down diff --git a/migrations/schema-15.sql b/migrations/schema-15.sql index 1bff8b9d8..cb031f797 100644 --- a/migrations/schema-15.sql +++ b/migrations/schema-15.sql @@ -44,27 +44,6 @@ CREATE SCHEMA graphql_public; CREATE SCHEMA pgbouncer; --- --- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA pgsodium; - - --- --- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; - - --- --- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; - - -- -- Name: realtime; Type: SCHEMA; Schema: -; Owner: - -- @@ -574,28 +553,6 @@ END $$; --- --- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - --- - -CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE - CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( - pgsodium.crypto_aead_det_encrypt( - pg_catalog.convert_to(new.secret, 'utf8'), - pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), - new.key_id::uuid, - new.nonce - ), - 'base64') END END; - RETURN new; - END; - $$; - - SET default_tablespace = ''; SET default_table_access_method = heap; @@ -782,30 +739,6 @@ CREATE TABLE storage.objects ( ); --- --- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - --- - -CREATE VIEW vault.decrypted_secrets AS - SELECT secrets.id, - secrets.name, - secrets.description, - secrets.secret, - CASE - WHEN (secrets.secret IS NULL) THEN NULL::text - ELSE - CASE - WHEN (secrets.key_id IS NULL) THEN NULL::text - ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name) - END - END AS decrypted_secret, - secrets.key_id, - secrets.nonce, - secrets.created_at, - secrets.updated_at - FROM vault.secrets; - - -- -- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - -- diff --git a/migrations/schema-orioledb-17.sql b/migrations/schema-orioledb-17.sql index 531970c37..7026b99e3 100644 --- a/migrations/schema-orioledb-17.sql +++ b/migrations/schema-orioledb-17.sql @@ -45,27 +45,6 @@ CREATE SCHEMA graphql_public; CREATE SCHEMA pgbouncer; --- --- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA pgsodium; - - --- --- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; - - --- --- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; - - -- -- Name: realtime; Type: SCHEMA; Schema: -; Owner: - -- @@ -589,28 +568,6 @@ END $$; --- --- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - --- - -CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE - CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( - pgsodium.crypto_aead_det_encrypt( - pg_catalog.convert_to(new.secret, 'utf8'), - pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), - new.key_id::uuid, - new.nonce - ), - 'base64') END END; - RETURN new; - END; - $$; - - SET default_tablespace = ''; SET default_table_access_method = orioledb; @@ -797,30 +754,6 @@ CREATE TABLE storage.objects ( ); --- --- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - --- - -CREATE VIEW vault.decrypted_secrets AS - SELECT id, - name, - description, - secret, - CASE - WHEN (secret IS NULL) THEN NULL::text - ELSE - CASE - WHEN (key_id IS NULL) THEN NULL::text - ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secret, 'base64'::text), convert_to(((((id)::text || description) || (created_at)::text) || (updated_at)::text), 'utf8'::name), key_id, nonce), 'utf8'::name) - END - END AS decrypted_secret, - key_id, - nonce, - created_at, - updated_at - FROM vault.secrets; - - -- -- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - -- diff --git a/migrations/tests/database/privs.sql b/migrations/tests/database/privs.sql index d164e0cb1..ea4f1318a 100644 --- a/migrations/tests/database/privs.sql +++ b/migrations/tests/database/privs.sql @@ -2,10 +2,6 @@ SELECT database_privs_are( 'postgres', 'postgres', ARRAY['CONNECT', 'TEMPORARY', 'CREATE'] ); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); -SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']); - -- Verify public schema privileges SELECT schema_privs_are('public', 'postgres', array['CREATE', 'USAGE']); SELECT schema_privs_are('public', 'anon', array['USAGE']); diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index c822fcd51..2cbd7e7a9 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -1,23 +1,24 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }: stdenv.mkDerivation rec { pname = "vault"; - version = "0.2.9"; + version = "0.3.1"; - buildInputs = [ postgresql ]; + buildInputs = [ libsodium postgresql ]; src = fetchFromGitHub { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-kXTngBW4K6FkZM8HvJG2Jha6OQqbejhnk7tchxy031I="; + hash = "sha256-MC87bqgtynnDhmNZAu96jvfCpsGDCPB0g5TZfRQHd30="; }; installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp sql/*.sql $out/share/postgresql/extension - cp *.control $out/share/postgresql/extension + install -D *${postgresql.dlSuffix} $out/lib + install -D -t $out/share/postgresql/extension sql/*.sql + install -D -t $out/share/postgresql/extension *.control ''; meta = with lib; { diff --git a/nix/tests/expected/z_15_ext_interface.out b/nix/tests/expected/z_15_ext_interface.out index 9914fa3b9..2fedc4366 100644 --- a/nix/tests/expected/z_15_ext_interface.out +++ b/nix/tests/expected/z_15_ext_interface.out @@ -4750,6 +4750,9 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text + supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer, text | SETOF record @@ -5226,7 +5229,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5055 rows) +(5058 rows) /* @@ -6034,6 +6037,15 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid + supabase_vault | vault | decrypted_secrets | created_at + supabase_vault | vault | decrypted_secrets | decrypted_secret + supabase_vault | vault | decrypted_secrets | description + supabase_vault | vault | decrypted_secrets | id + supabase_vault | vault | decrypted_secrets | key_id + supabase_vault | vault | decrypted_secrets | name + supabase_vault | vault | decrypted_secrets | nonce + supabase_vault | vault | decrypted_secrets | secret + supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -6357,5 +6369,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1097 rows) +(1106 rows) diff --git a/nix/tests/expected/z_17_ext_interface.out b/nix/tests/expected/z_17_ext_interface.out index 37f417f81..a0177327a 100644 --- a/nix/tests/expected/z_17_ext_interface.out +++ b/nix/tests/expected/z_17_ext_interface.out @@ -4707,6 +4707,9 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text + supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer | SETOF record @@ -4906,7 +4909,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4747 rows) +(4750 rows) /* @@ -5321,6 +5324,15 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid + supabase_vault | vault | decrypted_secrets | created_at + supabase_vault | vault | decrypted_secrets | decrypted_secret + supabase_vault | vault | decrypted_secrets | description + supabase_vault | vault | decrypted_secrets | id + supabase_vault | vault | decrypted_secrets | key_id + supabase_vault | vault | decrypted_secrets | name + supabase_vault | vault | decrypted_secrets | nonce + supabase_vault | vault | decrypted_secrets | secret + supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -5338,5 +5350,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(398 rows) +(407 rows) diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index ef860afcb..483a1a8e2 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace, supabase_vault' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use @@ -795,6 +795,7 @@ jit_provider = 'llvmjit' # JIT library to use # Add settings for extensions here pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' +vault.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' auto_explain.log_min_duration = 10s cron.database_name = 'postgres' diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in index 8c489839b..1197228af 100644 --- a/nix/tools/dbmate-tool.sh.in +++ b/nix/tools/dbmate-tool.sh.in @@ -15,39 +15,46 @@ CURRENT_SYSTEM="@CURRENT_SYSTEM@" ANSIBLE_VARS="@ANSIBLE_VARS@" PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ + +# Start PostgreSQL using nix +start_postgres() { + DATDIR=$(mktemp -d) + echo "Starting PostgreSQL in directory: $DATDIR" # Create the DATDIR if it doesn't exist + nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize --datdir "$DATDIR" + echo "PostgreSQL started." +} + # Cleanup function cleanup() { echo "Cleaning up..." - # Kill postgres processes first + # Check if PostgreSQL processes exist if pgrep -f "postgres" >/dev/null; then - pkill -TERM postgres || true - sleep 2 - fi - - # Then kill overmind - if [ -S "./.overmind.sock" ]; then - overmind kill || true - sleep 2 + echo "Stopping PostgreSQL gracefully..." + + # Use pg_ctl to stop PostgreSQL + pg_ctl -D "$DATDIR" stop + + # Wait a bit for graceful shutdown + sleep 5 + + # Check if processes are still running + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be stopped gracefully." + fi + else + echo "PostgreSQL is not running, skipping stop." fi - # Kill tmux sessions explicitly - pkill -f "tmux.*overmind.*postgresql" || true - tmux ls 2>/dev/null | grep 'overmind' | cut -d: -f1 | xargs -I{} tmux kill-session -t {} || true - - # Force kill any stragglers - pkill -9 -f "(postgres|tmux.*overmind.*postgresql)" || true - - rm -f .overmind.sock Procfile - - # Final verification - if ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep >/dev/null; then - ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep - return 1 + # Always exit successfully, log any remaining processes + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be cleaned up:" + pgrep -f "postgres" + else + echo "Cleanup completed successfully" fi } -# Set up trap for cleanup on script exit # Function to display help print_help() { @@ -57,7 +64,7 @@ print_help() { echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" echo " -p, --port PORT Specify the port number to use (default: 5435)" echo " -h, --help Show this help message" - echo + echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" echo "Description:" echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." @@ -66,9 +73,9 @@ print_help() { echo " nix run .#dbmate-tool" echo " nix run .#dbmate-tool -- --version 15" echo " nix run .#dbmate-tool -- --version 16 --port 5433" + echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" } - # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -125,7 +132,7 @@ wait_for_postgres() { local max_attempts=30 # Increased significantly local attempt=1 - # Give overmind a moment to actually start the process + # Give PostgreSQL a moment to actually start the process sleep 2 while [ $attempt -le $max_attempts ]; do @@ -142,7 +149,6 @@ wait_for_postgres() { done echo "PostgreSQL failed to start after $max_attempts attempts" - overmind echo postgres return 1 } @@ -175,26 +181,7 @@ trim_schema() { ;; esac } -overmind_start() { - cat > Procfile << EOF -postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations -EOF - overmind start -D - echo "Waiting for overmind socket..." - max_wait=5 - count=0 - while [ $count -lt $max_wait ]; do - if [ -S "./.overmind.sock" ]; then - # Found the socket, give it a moment to be ready - sleep 5 - echo "Socket file found and ready" - break - fi - echo "Waiting for socket file (attempt $count/$max_wait)" - sleep 1 - count=$((count + 1)) - done -} + perform_dump() { local max_attempts=3 local attempt=1 @@ -214,21 +201,18 @@ perform_dump() { echo "All dump attempts failed" return 1 } + migrate_version() { echo "PSQL_VERSION: $PSQL_VERSION" - overmind kill || true - rm -f .overmind.sock Procfile || true + #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" - # Start overmind - overmind_start - echo "Waiting for overmind socket..." - - + # Start PostgreSQL + start_postgres echo "Waiting for PostgreSQL to be ready..." - #Wait for PostgreSQL to be ready to accept connections + # Wait for PostgreSQL to be ready to accept connections if ! wait_for_postgres; then echo "Failed to connect to PostgreSQL server" exit 1 @@ -255,11 +239,11 @@ EOSQL "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" - #set db url to run dbmate + # Set db url to run dbmate export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - #export path so dbmate can find correct psql and pg_dump + # Export path so dbmate can find correct psql and pg_dump export PATH="$PSQLBIN:$PATH" - # run init scripts + # Run init scripts if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then echo "Error: Initial migration failed" exit 1 diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 75c5f8de7..0586e010b 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -56,12 +56,15 @@ start_postgres() { } stop_postgres() { - pg_ctl stop -D "$DATDIR" -m fast + if [ "$DAEMONIZE" = true ]; then + echo "PostgreSQL is running in daemon mode. Please stop it using pg_ctl." + else + pg_ctl stop -D "$DATDIR" -m fast + fi } trap 'stop_postgres' SIGINT SIGTERM -# Parse arguments # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -104,6 +107,15 @@ while [[ "$#" -gt 0 ]]; do print_help exit 0 ;; + --datdir) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + DATDIR="$2" + shift 2 + else + echo "Error: --datadir requires a directory path" + exit 1 + fi + ;; *) if [[ "$1" =~ ^- ]]; then echo "Unknown option: $1" @@ -161,7 +173,9 @@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ MECAB_LIB=@MECAB_LIB@ # Setup directories and locale settings -DATDIR=$(mktemp -d) +if [[ -z "$DATDIR" ]]; then + DATDIR=$(mktemp -d) +fi LOCALE_ARCHIVE=@LOCALES@ CURRENT_SYSTEM=@CURRENT_SYSTEM@ @@ -209,6 +223,8 @@ sed -e "1i\\ include = '$DATDIR/supautils.conf'" \ -e "\$a\\ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ +-e "\$a\\ +vault.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \ -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \ @@ -329,6 +345,7 @@ EOSQL fi fi echo "Shutting down PostgreSQL..." + stop_postgres # Step 4: Restart PostgreSQL in the foreground (with log output visible) or as a daemon