From 04e7dad1e8d71c83ba2a7b896b68b1e70452cb27 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Tue, 18 Feb 2025 11:16:38 +0800 Subject: [PATCH 1/3] Revert "feat: vault sans pgsodium" This reverts commit 7207d7af7dfe10f8d3271f4a8749d0fc45eea107. --- .github/workflows/test.yml | 2 +- .../pg_upgrade_scripts/complete.sh | 37 ------- .../postgresql_config/postgresql.conf.j2 | 2 +- .../supabase_vault/after-create.sql | 3 - ansible/files/sodium_extension.sql | 6 ++ ansible/tasks/test-image.yml | 23 +--- ansible/vars.yml | 4 +- .../files/unit-tests/unit-test-01.sql | 10 +- flake.nix | 77 ++++++-------- nix/ext/vault.nix | 13 ++- nix/tests/expected/z_15_ext_interface.out | 16 +-- nix/tests/expected/z_17_ext_interface.out | 16 +-- nix/tests/postgresql.conf.in | 3 +- nix/tools/dbmate-tool.sh.in | 100 ++++++++++-------- nix/tools/run-server.sh.in | 23 +--- 15 files changed, 127 insertions(+), 208 deletions(-) delete mode 100644 ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql create mode 100644 ansible/files/sodium_extension.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6dc194684..b6912b38f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | - nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} + nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index c2367116d..515c490f6 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -150,43 +150,6 @@ EOF run_sql -c "$PATCH_PGMQ_QUERY" run_sql -c "update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq';" - - # Patch to handle upgrading to pgsodium-less Vault - REENCRYPT_VAULT_SECRETS_QUERY=$(cat < - sed -i.bak - -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' - -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' - -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/' + sed -i.bak + -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/' /etc/postgresql/postgresql.conf when: debpkg_mode or stage2_nix -- name: Verify pgsodium and vault removal from config - become: yes - become_user: postgres - shell: - cmd: | - FOUND=$(grep -E "shared_preload_libraries.*pgsodium|shared_preload_libraries.*supabase_vault|^pgsodium\.getkey_script" /etc/postgresql/postgresql.conf) - if [ ! -z "$FOUND" ]; then - echo "Found unremoved references:" - echo "$FOUND" - exit 1 - fi - register: verify_result - failed_when: verify_result.rc != 0 - when: debpkg_mode or stage2_nix - - name: Start Postgres Database to load all extensions. become: yes become_user: postgres diff --git a/ansible/vars.yml b/ansible/vars.yml index 206cb139f..fe8ec2d35 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.035-orioledb" - postgres15: "15.8.1.039" + postgresorioledb-17: "17.0.1.34-orioledb" + postgres15: "15.8.1.038" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/ebssurrogate/files/unit-tests/unit-test-01.sql b/ebssurrogate/files/unit-tests/unit-test-01.sql index c466af12e..f3d47459f 100644 --- a/ebssurrogate/files/unit-tests/unit-test-01.sql +++ b/ebssurrogate/files/unit-tests/unit-test-01.sql @@ -17,6 +17,7 @@ BEGIN extension_array := ARRAY[ 'plpgsql', 'pg_stat_statements', + 'pgsodium', 'pgtap', 'pg_graphql', 'pgcrypto', @@ -29,6 +30,7 @@ BEGIN extension_array := ARRAY[ 'plpgsql', 'pg_stat_statements', + 'pgsodium', 'pgtap', 'pg_graphql', 'pgcrypto', @@ -42,7 +44,7 @@ BEGIN PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false); END $$; -SELECT no_plan(); +SELECT plan(8); SELECT extensions_are( string_to_array(current_setting('myapp.extensions'), ',')::text[] @@ -54,5 +56,9 @@ SELECT has_schema('pg_catalog'); SELECT has_schema('information_schema'); SELECT has_schema('public'); +SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); +SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); +SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']); + SELECT * FROM finish(); -ROLLBACK; +ROLLBACK; \ No newline at end of file diff --git a/flake.nix b/flake.nix index 8bbd79525..c0ecc45f0 100644 --- a/flake.nix +++ b/flake.nix @@ -571,49 +571,42 @@ sqlTests = ./nix/tests/smoke; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; pg_regress = basePackages.pg_regress; - getkey-script = pkgs.stdenv.mkDerivation { - name = "pgsodium-getkey"; - buildCommand = '' - mkdir -p $out/bin - cat > $out/bin/pgsodium-getkey << 'EOF' - #!${pkgs.bash}/bin/bash - set -euo pipefail - - TMPDIR_BASE=$(mktemp -d) - - if [[ "$(uname)" == "Darwin" ]]; then - KEY_DIR="/private/tmp/pgsodium" - else - KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" - fi - KEY_FILE="$KEY_DIR/pgsodium.key" - - if ! mkdir -p "$KEY_DIR" 2>/dev/null; then - echo "Error: Could not create key directory $KEY_DIR" >&2 - exit 1 - fi - chmod 1777 "$KEY_DIR" - - if [[ ! -f "$KEY_FILE" ]]; then - if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then - if ! (openssl rand -hex 32 > "$KEY_FILE"); then - echo "00000000000000000000000000000000" > "$KEY_FILE" - echo "Warning: Using fallback key" >&2 - fi + getkey-script = pkgs.writeScriptBin "pgsodium-getkey" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + TMPDIR_BASE=$(mktemp -d) + + if [[ "$(uname)" == "Darwin" ]]; then + KEY_DIR="/private/tmp/pgsodium" + else + KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" + fi + KEY_FILE="$KEY_DIR/pgsodium.key" + + if ! mkdir -p "$KEY_DIR" 2>/dev/null; then + echo "Error: Could not create key directory $KEY_DIR" >&2 + exit 1 + fi + chmod 1777 "$KEY_DIR" + + if [[ ! -f "$KEY_FILE" ]]; then + if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then + if ! (openssl rand -hex 32 > "$KEY_FILE"); then + echo "00000000000000000000000000000000" > "$KEY_FILE" + echo "Warning: Using fallback key" >&2 fi - chmod 644 "$KEY_FILE" - fi - - if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then - cat "$KEY_FILE" - else - echo "Error: Cannot read key file $KEY_FILE" >&2 - exit 1 fi - EOF - chmod +x $out/bin/pgsodium-getkey - ''; - }; + chmod 644 "$KEY_FILE" + fi + + if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then + cat "$KEY_FILE" + else + echo "Error: Cannot read key file $KEY_FILE" >&2 + exit 1 + fi + ''; # Use the shared setup but with a test-specific name start-postgres-server-bin = makePostgresDevSetup { @@ -682,8 +675,6 @@ echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf - echo "Checking shared_preload_libraries setting:" - grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf # Remove timescaledb if running orioledb-17 check echo "I AM ${pgpkg.version}====================================================" if [[ "${pgpkg.version}" == *"17"* ]]; then diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index 2cbd7e7a9..c822fcd51 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -1,24 +1,23 @@ -{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }: +{ lib, stdenv, fetchFromGitHub, postgresql }: stdenv.mkDerivation rec { pname = "vault"; - version = "0.3.1"; + version = "0.2.9"; - buildInputs = [ libsodium postgresql ]; + buildInputs = [ postgresql ]; src = fetchFromGitHub { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-MC87bqgtynnDhmNZAu96jvfCpsGDCPB0g5TZfRQHd30="; + hash = "sha256-kXTngBW4K6FkZM8HvJG2Jha6OQqbejhnk7tchxy031I="; }; installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - install -D *${postgresql.dlSuffix} $out/lib - install -D -t $out/share/postgresql/extension sql/*.sql - install -D -t $out/share/postgresql/extension *.control + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension ''; meta = with lib; { diff --git a/nix/tests/expected/z_15_ext_interface.out b/nix/tests/expected/z_15_ext_interface.out index 2fedc4366..9914fa3b9 100644 --- a/nix/tests/expected/z_15_ext_interface.out +++ b/nix/tests/expected/z_15_ext_interface.out @@ -4750,9 +4750,6 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text - supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea - supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea - supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer, text | SETOF record @@ -5229,7 +5226,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5058 rows) +(5055 rows) /* @@ -6037,15 +6034,6 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid - supabase_vault | vault | decrypted_secrets | created_at - supabase_vault | vault | decrypted_secrets | decrypted_secret - supabase_vault | vault | decrypted_secrets | description - supabase_vault | vault | decrypted_secrets | id - supabase_vault | vault | decrypted_secrets | key_id - supabase_vault | vault | decrypted_secrets | name - supabase_vault | vault | decrypted_secrets | nonce - supabase_vault | vault | decrypted_secrets | secret - supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -6369,5 +6357,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1106 rows) +(1097 rows) diff --git a/nix/tests/expected/z_17_ext_interface.out b/nix/tests/expected/z_17_ext_interface.out index a0177327a..37f417f81 100644 --- a/nix/tests/expected/z_17_ext_interface.out +++ b/nix/tests/expected/z_17_ext_interface.out @@ -4707,9 +4707,6 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text - supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea - supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea - supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer | SETOF record @@ -4909,7 +4906,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4750 rows) +(4747 rows) /* @@ -5324,15 +5321,6 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid - supabase_vault | vault | decrypted_secrets | created_at - supabase_vault | vault | decrypted_secrets | decrypted_secret - supabase_vault | vault | decrypted_secrets | description - supabase_vault | vault | decrypted_secrets | id - supabase_vault | vault | decrypted_secrets | key_id - supabase_vault | vault | decrypted_secrets | name - supabase_vault | vault | decrypted_secrets | nonce - supabase_vault | vault | decrypted_secrets | secret - supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -5350,5 +5338,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(407 rows) +(398 rows) diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 483a1a8e2..ef860afcb 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace, supabase_vault' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use @@ -795,7 +795,6 @@ jit_provider = 'llvmjit' # JIT library to use # Add settings for extensions here pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' -vault.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' auto_explain.log_min_duration = 10s cron.database_name = 'postgres' diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in index 1197228af..8c489839b 100644 --- a/nix/tools/dbmate-tool.sh.in +++ b/nix/tools/dbmate-tool.sh.in @@ -15,46 +15,39 @@ CURRENT_SYSTEM="@CURRENT_SYSTEM@" ANSIBLE_VARS="@ANSIBLE_VARS@" PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ - -# Start PostgreSQL using nix -start_postgres() { - DATDIR=$(mktemp -d) - echo "Starting PostgreSQL in directory: $DATDIR" # Create the DATDIR if it doesn't exist - nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize --datdir "$DATDIR" - echo "PostgreSQL started." -} - # Cleanup function cleanup() { echo "Cleaning up..." - # Check if PostgreSQL processes exist + # Kill postgres processes first if pgrep -f "postgres" >/dev/null; then - echo "Stopping PostgreSQL gracefully..." - - # Use pg_ctl to stop PostgreSQL - pg_ctl -D "$DATDIR" stop - - # Wait a bit for graceful shutdown - sleep 5 - - # Check if processes are still running - if pgrep -f "postgres" >/dev/null; then - echo "Warning: Some PostgreSQL processes could not be stopped gracefully." - fi - else - echo "PostgreSQL is not running, skipping stop." + pkill -TERM postgres || true + sleep 2 fi - # Always exit successfully, log any remaining processes - if pgrep -f "postgres" >/dev/null; then - echo "Warning: Some PostgreSQL processes could not be cleaned up:" - pgrep -f "postgres" - else - echo "Cleanup completed successfully" + # Then kill overmind + if [ -S "./.overmind.sock" ]; then + overmind kill || true + sleep 2 + fi + + # Kill tmux sessions explicitly + pkill -f "tmux.*overmind.*postgresql" || true + tmux ls 2>/dev/null | grep 'overmind' | cut -d: -f1 | xargs -I{} tmux kill-session -t {} || true + + # Force kill any stragglers + pkill -9 -f "(postgres|tmux.*overmind.*postgresql)" || true + + rm -f .overmind.sock Procfile + + # Final verification + if ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep >/dev/null; then + ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep + return 1 fi } +# Set up trap for cleanup on script exit # Function to display help print_help() { @@ -64,7 +57,7 @@ print_help() { echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" echo " -p, --port PORT Specify the port number to use (default: 5435)" echo " -h, --help Show this help message" - echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" + echo echo "Description:" echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." @@ -73,9 +66,9 @@ print_help() { echo " nix run .#dbmate-tool" echo " nix run .#dbmate-tool -- --version 15" echo " nix run .#dbmate-tool -- --version 16 --port 5433" - echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" } + # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -132,7 +125,7 @@ wait_for_postgres() { local max_attempts=30 # Increased significantly local attempt=1 - # Give PostgreSQL a moment to actually start the process + # Give overmind a moment to actually start the process sleep 2 while [ $attempt -le $max_attempts ]; do @@ -149,6 +142,7 @@ wait_for_postgres() { done echo "PostgreSQL failed to start after $max_attempts attempts" + overmind echo postgres return 1 } @@ -181,7 +175,26 @@ trim_schema() { ;; esac } - +overmind_start() { + cat > Procfile << EOF +postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations +EOF + overmind start -D + echo "Waiting for overmind socket..." + max_wait=5 + count=0 + while [ $count -lt $max_wait ]; do + if [ -S "./.overmind.sock" ]; then + # Found the socket, give it a moment to be ready + sleep 5 + echo "Socket file found and ready" + break + fi + echo "Waiting for socket file (attempt $count/$max_wait)" + sleep 1 + count=$((count + 1)) + done +} perform_dump() { local max_attempts=3 local attempt=1 @@ -201,18 +214,21 @@ perform_dump() { echo "All dump attempts failed" return 1 } - migrate_version() { echo "PSQL_VERSION: $PSQL_VERSION" - #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting + overmind kill || true + rm -f .overmind.sock Procfile || true PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" - # Start PostgreSQL - start_postgres + # Start overmind + overmind_start + echo "Waiting for overmind socket..." + + echo "Waiting for PostgreSQL to be ready..." - # Wait for PostgreSQL to be ready to accept connections + #Wait for PostgreSQL to be ready to accept connections if ! wait_for_postgres; then echo "Failed to connect to PostgreSQL server" exit 1 @@ -239,11 +255,11 @@ EOSQL "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" - # Set db url to run dbmate + #set db url to run dbmate export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - # Export path so dbmate can find correct psql and pg_dump + #export path so dbmate can find correct psql and pg_dump export PATH="$PSQLBIN:$PATH" - # Run init scripts + # run init scripts if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then echo "Error: Initial migration failed" exit 1 diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 0586e010b..75c5f8de7 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -56,15 +56,12 @@ start_postgres() { } stop_postgres() { - if [ "$DAEMONIZE" = true ]; then - echo "PostgreSQL is running in daemon mode. Please stop it using pg_ctl." - else - pg_ctl stop -D "$DATDIR" -m fast - fi + pg_ctl stop -D "$DATDIR" -m fast } trap 'stop_postgres' SIGINT SIGTERM +# Parse arguments # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -107,15 +104,6 @@ while [[ "$#" -gt 0 ]]; do print_help exit 0 ;; - --datdir) - if [[ -n "$2" && ! "$2" =~ ^- ]]; then - DATDIR="$2" - shift 2 - else - echo "Error: --datadir requires a directory path" - exit 1 - fi - ;; *) if [[ "$1" =~ ^- ]]; then echo "Unknown option: $1" @@ -173,9 +161,7 @@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ MECAB_LIB=@MECAB_LIB@ # Setup directories and locale settings -if [[ -z "$DATDIR" ]]; then - DATDIR=$(mktemp -d) -fi +DATDIR=$(mktemp -d) LOCALE_ARCHIVE=@LOCALES@ CURRENT_SYSTEM=@CURRENT_SYSTEM@ @@ -223,8 +209,6 @@ sed -e "1i\\ include = '$DATDIR/supautils.conf'" \ -e "\$a\\ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ --e "\$a\\ -vault.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \ -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \ @@ -345,7 +329,6 @@ EOSQL fi fi echo "Shutting down PostgreSQL..." - stop_postgres # Step 4: Restart PostgreSQL in the foreground (with log output visible) or as a daemon From a6f354dbe71b99dc6b870ebd5a5c87148d5aa89a Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Tue, 18 Feb 2025 11:21:27 +0800 Subject: [PATCH 2/3] fix: pgsodium mask_role migration --- ansible/vars.yml | 4 +-- .../20250218031949_pgsodium_mask_role.sql | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 migrations/db/migrations/20250218031949_pgsodium_mask_role.sql diff --git a/ansible/vars.yml b/ansible/vars.yml index fe8ec2d35..6d3d096b1 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,8 +8,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.34-orioledb" - postgres15: "15.8.1.038" + postgresorioledb-17: "17.0.1.036-orioledb" + postgres15: "15.8.1.040" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/migrations/db/migrations/20250218031949_pgsodium_mask_role.sql b/migrations/db/migrations/20250218031949_pgsodium_mask_role.sql new file mode 100644 index 000000000..c4a3eadf4 --- /dev/null +++ b/migrations/db/migrations/20250218031949_pgsodium_mask_role.sql @@ -0,0 +1,25 @@ +-- migrate:up +CREATE OR REPLACE FUNCTION pgsodium.mask_role(masked_role regrole, source_name text, view_name text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path TO '' +AS $function$ +BEGIN + EXECUTE format( + 'GRANT SELECT ON pgsodium.key TO %s', + masked_role); + + EXECUTE format( + 'GRANT pgsodium_keyiduser, pgsodium_keyholder TO %s', + masked_role); + + EXECUTE format( + 'GRANT ALL ON %I TO %s', + view_name, + masked_role); + RETURN; +END +$function$; + +-- migrate:down From dc884d5bd9d12e5b78cde4ca8d776fce784b017c Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 17 Feb 2025 23:46:29 -0500 Subject: [PATCH 3/3] chore: restore flake-url arg (#1450) * chore: restore flake-url arg * chore: restore dbmate and run server --- .github/workflows/test.yml | 2 +- nix/tools/dbmate-tool.sh.in | 100 +++++++++++++++--------------------- nix/tools/run-server.sh.in | 23 +++++++-- 3 files changed, 63 insertions(+), 62 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6912b38f..6dc194684 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | - nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} + nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in index 8c489839b..1197228af 100644 --- a/nix/tools/dbmate-tool.sh.in +++ b/nix/tools/dbmate-tool.sh.in @@ -15,39 +15,46 @@ CURRENT_SYSTEM="@CURRENT_SYSTEM@" ANSIBLE_VARS="@ANSIBLE_VARS@" PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ + +# Start PostgreSQL using nix +start_postgres() { + DATDIR=$(mktemp -d) + echo "Starting PostgreSQL in directory: $DATDIR" # Create the DATDIR if it doesn't exist + nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize --datdir "$DATDIR" + echo "PostgreSQL started." +} + # Cleanup function cleanup() { echo "Cleaning up..." - # Kill postgres processes first + # Check if PostgreSQL processes exist if pgrep -f "postgres" >/dev/null; then - pkill -TERM postgres || true - sleep 2 - fi - - # Then kill overmind - if [ -S "./.overmind.sock" ]; then - overmind kill || true - sleep 2 + echo "Stopping PostgreSQL gracefully..." + + # Use pg_ctl to stop PostgreSQL + pg_ctl -D "$DATDIR" stop + + # Wait a bit for graceful shutdown + sleep 5 + + # Check if processes are still running + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be stopped gracefully." + fi + else + echo "PostgreSQL is not running, skipping stop." fi - # Kill tmux sessions explicitly - pkill -f "tmux.*overmind.*postgresql" || true - tmux ls 2>/dev/null | grep 'overmind' | cut -d: -f1 | xargs -I{} tmux kill-session -t {} || true - - # Force kill any stragglers - pkill -9 -f "(postgres|tmux.*overmind.*postgresql)" || true - - rm -f .overmind.sock Procfile - - # Final verification - if ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep >/dev/null; then - ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep - return 1 + # Always exit successfully, log any remaining processes + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be cleaned up:" + pgrep -f "postgres" + else + echo "Cleanup completed successfully" fi } -# Set up trap for cleanup on script exit # Function to display help print_help() { @@ -57,7 +64,7 @@ print_help() { echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" echo " -p, --port PORT Specify the port number to use (default: 5435)" echo " -h, --help Show this help message" - echo + echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" echo "Description:" echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." @@ -66,9 +73,9 @@ print_help() { echo " nix run .#dbmate-tool" echo " nix run .#dbmate-tool -- --version 15" echo " nix run .#dbmate-tool -- --version 16 --port 5433" + echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" } - # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -125,7 +132,7 @@ wait_for_postgres() { local max_attempts=30 # Increased significantly local attempt=1 - # Give overmind a moment to actually start the process + # Give PostgreSQL a moment to actually start the process sleep 2 while [ $attempt -le $max_attempts ]; do @@ -142,7 +149,6 @@ wait_for_postgres() { done echo "PostgreSQL failed to start after $max_attempts attempts" - overmind echo postgres return 1 } @@ -175,26 +181,7 @@ trim_schema() { ;; esac } -overmind_start() { - cat > Procfile << EOF -postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations -EOF - overmind start -D - echo "Waiting for overmind socket..." - max_wait=5 - count=0 - while [ $count -lt $max_wait ]; do - if [ -S "./.overmind.sock" ]; then - # Found the socket, give it a moment to be ready - sleep 5 - echo "Socket file found and ready" - break - fi - echo "Waiting for socket file (attempt $count/$max_wait)" - sleep 1 - count=$((count + 1)) - done -} + perform_dump() { local max_attempts=3 local attempt=1 @@ -214,21 +201,18 @@ perform_dump() { echo "All dump attempts failed" return 1 } + migrate_version() { echo "PSQL_VERSION: $PSQL_VERSION" - overmind kill || true - rm -f .overmind.sock Procfile || true + #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" - # Start overmind - overmind_start - echo "Waiting for overmind socket..." - - + # Start PostgreSQL + start_postgres echo "Waiting for PostgreSQL to be ready..." - #Wait for PostgreSQL to be ready to accept connections + # Wait for PostgreSQL to be ready to accept connections if ! wait_for_postgres; then echo "Failed to connect to PostgreSQL server" exit 1 @@ -255,11 +239,11 @@ EOSQL "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" - #set db url to run dbmate + # Set db url to run dbmate export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - #export path so dbmate can find correct psql and pg_dump + # Export path so dbmate can find correct psql and pg_dump export PATH="$PSQLBIN:$PATH" - # run init scripts + # Run init scripts if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then echo "Error: Initial migration failed" exit 1 diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 75c5f8de7..0586e010b 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -56,12 +56,15 @@ start_postgres() { } stop_postgres() { - pg_ctl stop -D "$DATDIR" -m fast + if [ "$DAEMONIZE" = true ]; then + echo "PostgreSQL is running in daemon mode. Please stop it using pg_ctl." + else + pg_ctl stop -D "$DATDIR" -m fast + fi } trap 'stop_postgres' SIGINT SIGTERM -# Parse arguments # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -104,6 +107,15 @@ while [[ "$#" -gt 0 ]]; do print_help exit 0 ;; + --datdir) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + DATDIR="$2" + shift 2 + else + echo "Error: --datadir requires a directory path" + exit 1 + fi + ;; *) if [[ "$1" =~ ^- ]]; then echo "Unknown option: $1" @@ -161,7 +173,9 @@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ MECAB_LIB=@MECAB_LIB@ # Setup directories and locale settings -DATDIR=$(mktemp -d) +if [[ -z "$DATDIR" ]]; then + DATDIR=$(mktemp -d) +fi LOCALE_ARCHIVE=@LOCALES@ CURRENT_SYSTEM=@CURRENT_SYSTEM@ @@ -209,6 +223,8 @@ sed -e "1i\\ include = '$DATDIR/supautils.conf'" \ -e "\$a\\ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ +-e "\$a\\ +vault.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \ -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \ @@ -329,6 +345,7 @@ EOSQL fi fi echo "Shutting down PostgreSQL..." + stop_postgres # Step 4: Restart PostgreSQL in the foreground (with log output visible) or as a daemon