diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b6912b38f..6dc194684 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -76,7 +76,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | - nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} + nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/.gitignore b/.gitignore index 74fd9f7b6..005d3ece6 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ result* .idea/ .vscode/ -migrations/db/schema*.sql +db/schema.sql diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index 1604d94f9..88393eadf 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/flake.nix b/flake.nix index c0ecc45f0..8bbd79525 100644 --- a/flake.nix +++ b/flake.nix @@ -571,42 +571,49 @@ sqlTests = ./nix/tests/smoke; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; pg_regress = basePackages.pg_regress; - getkey-script = pkgs.writeScriptBin "pgsodium-getkey" '' - #!${pkgs.bash}/bin/bash - set -euo pipefail - - TMPDIR_BASE=$(mktemp -d) - - if [[ "$(uname)" == "Darwin" ]]; then - KEY_DIR="/private/tmp/pgsodium" - else - KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" - fi - KEY_FILE="$KEY_DIR/pgsodium.key" - - if ! mkdir -p "$KEY_DIR" 2>/dev/null; then - echo "Error: Could not create key directory $KEY_DIR" >&2 - exit 1 - fi - chmod 1777 "$KEY_DIR" - - if [[ ! -f "$KEY_FILE" ]]; then - if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then - if ! (openssl rand -hex 32 > "$KEY_FILE"); then - echo "00000000000000000000000000000000" > "$KEY_FILE" - echo "Warning: Using fallback key" >&2 + getkey-script = pkgs.stdenv.mkDerivation { + name = "pgsodium-getkey"; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/pgsodium-getkey << 'EOF' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + TMPDIR_BASE=$(mktemp -d) + + if [[ "$(uname)" == "Darwin" ]]; then + KEY_DIR="/private/tmp/pgsodium" + else + KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" + fi + KEY_FILE="$KEY_DIR/pgsodium.key" + + if ! mkdir -p "$KEY_DIR" 2>/dev/null; then + echo "Error: Could not create key directory $KEY_DIR" >&2 + exit 1 + fi + chmod 1777 "$KEY_DIR" + + if [[ ! -f "$KEY_FILE" ]]; then + if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then + if ! (openssl rand -hex 32 > "$KEY_FILE"); then + echo "00000000000000000000000000000000" > "$KEY_FILE" + echo "Warning: Using fallback key" >&2 + fi fi + chmod 644 "$KEY_FILE" fi - chmod 644 "$KEY_FILE" - fi - - if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then - cat "$KEY_FILE" - else - echo "Error: Cannot read key file $KEY_FILE" >&2 - exit 1 - fi - ''; + + if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then + cat "$KEY_FILE" + else + echo "Error: Cannot read key file $KEY_FILE" >&2 + exit 1 + fi + EOF + chmod +x $out/bin/pgsodium-getkey + ''; + }; # Use the shared setup but with a test-specific name start-postgres-server-bin = makePostgresDevSetup { @@ -675,6 +682,8 @@ echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + echo "Checking shared_preload_libraries setting:" + grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf # Remove timescaledb if running orioledb-17 check echo "I AM ${pgpkg.version}====================================================" if [[ "${pgpkg.version}" == *"17"* ]]; then diff --git a/migrations/schema-15.sql b/migrations/schema-15.sql index 1bff8b9d8..cb031f797 100644 --- a/migrations/schema-15.sql +++ b/migrations/schema-15.sql @@ -44,27 +44,6 @@ CREATE SCHEMA graphql_public; CREATE SCHEMA pgbouncer; --- --- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA pgsodium; - - --- --- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; - - --- --- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; - - -- -- Name: realtime; Type: SCHEMA; Schema: -; Owner: - -- @@ -574,28 +553,6 @@ END $$; --- --- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - --- - -CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE - CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( - pgsodium.crypto_aead_det_encrypt( - pg_catalog.convert_to(new.secret, 'utf8'), - pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), - new.key_id::uuid, - new.nonce - ), - 'base64') END END; - RETURN new; - END; - $$; - - SET default_tablespace = ''; SET default_table_access_method = heap; @@ -782,30 +739,6 @@ CREATE TABLE storage.objects ( ); --- --- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - --- - -CREATE VIEW vault.decrypted_secrets AS - SELECT secrets.id, - secrets.name, - secrets.description, - secrets.secret, - CASE - WHEN (secrets.secret IS NULL) THEN NULL::text - ELSE - CASE - WHEN (secrets.key_id IS NULL) THEN NULL::text - ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name) - END - END AS decrypted_secret, - secrets.key_id, - secrets.nonce, - secrets.created_at, - secrets.updated_at - FROM vault.secrets; - - -- -- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - -- diff --git a/migrations/schema-orioledb-17.sql b/migrations/schema-orioledb-17.sql index 531970c37..7026b99e3 100644 --- a/migrations/schema-orioledb-17.sql +++ b/migrations/schema-orioledb-17.sql @@ -45,27 +45,6 @@ CREATE SCHEMA graphql_public; CREATE SCHEMA pgbouncer; --- --- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - --- - -CREATE SCHEMA pgsodium; - - --- --- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - --- - -CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; - - --- --- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - --- - -COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; - - -- -- Name: realtime; Type: SCHEMA; Schema: -; Owner: - -- @@ -589,28 +568,6 @@ END $$; --- --- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - --- - -CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE - CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( - pgsodium.crypto_aead_det_encrypt( - pg_catalog.convert_to(new.secret, 'utf8'), - pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), - new.key_id::uuid, - new.nonce - ), - 'base64') END END; - RETURN new; - END; - $$; - - SET default_tablespace = ''; SET default_table_access_method = orioledb; @@ -797,30 +754,6 @@ CREATE TABLE storage.objects ( ); --- --- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - --- - -CREATE VIEW vault.decrypted_secrets AS - SELECT id, - name, - description, - secret, - CASE - WHEN (secret IS NULL) THEN NULL::text - ELSE - CASE - WHEN (key_id IS NULL) THEN NULL::text - ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secret, 'base64'::text), convert_to(((((id)::text || description) || (created_at)::text) || (updated_at)::text), 'utf8'::name), key_id, nonce), 'utf8'::name) - END - END AS decrypted_secret, - key_id, - nonce, - created_at, - updated_at - FROM vault.secrets; - - -- -- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - -- diff --git a/nix/tests/expected/z_15_ext_interface.out b/nix/tests/expected/z_15_ext_interface.out index 9914fa3b9..2fedc4366 100644 --- a/nix/tests/expected/z_15_ext_interface.out +++ b/nix/tests/expected/z_15_ext_interface.out @@ -4750,6 +4750,9 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text + supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer, text | SETOF record @@ -5226,7 +5229,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5055 rows) +(5058 rows) /* @@ -6034,6 +6037,15 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid + supabase_vault | vault | decrypted_secrets | created_at + supabase_vault | vault | decrypted_secrets | decrypted_secret + supabase_vault | vault | decrypted_secrets | description + supabase_vault | vault | decrypted_secrets | id + supabase_vault | vault | decrypted_secrets | key_id + supabase_vault | vault | decrypted_secrets | name + supabase_vault | vault | decrypted_secrets | nonce + supabase_vault | vault | decrypted_secrets | secret + supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -6357,5 +6369,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1097 rows) +(1106 rows) diff --git a/nix/tests/expected/z_17_ext_interface.out b/nix/tests/expected/z_17_ext_interface.out index 37f417f81..a0177327a 100644 --- a/nix/tests/expected/z_17_ext_interface.out +++ b/nix/tests/expected/z_17_ext_interface.out @@ -4707,6 +4707,9 @@ order by sslinfo | public | ssl_issuer_dn | | text sslinfo | public | ssl_issuer_field | text | text sslinfo | public | ssl_version | | text + supabase_vault | vault | _crypto_aead_det_decrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_encrypt | message bytea, additional bytea, key_id bigint, context bytea, nonce bytea | bytea + supabase_vault | vault | _crypto_aead_det_noncegen | | bytea supabase_vault | vault | create_secret | new_secret text, new_name text, new_description text, new_key_id uuid | uuid supabase_vault | vault | update_secret | secret_id uuid, new_secret text, new_name text, new_description text, new_key_id uuid | void tablefunc | public | connectby | text, text, text, text, integer | SETOF record @@ -4906,7 +4909,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(4747 rows) +(4750 rows) /* @@ -5321,6 +5324,15 @@ order by postgis_topology | topology | topology | name postgis_topology | topology | topology | precision postgis_topology | topology | topology | srid + supabase_vault | vault | decrypted_secrets | created_at + supabase_vault | vault | decrypted_secrets | decrypted_secret + supabase_vault | vault | decrypted_secrets | description + supabase_vault | vault | decrypted_secrets | id + supabase_vault | vault | decrypted_secrets | key_id + supabase_vault | vault | decrypted_secrets | name + supabase_vault | vault | decrypted_secrets | nonce + supabase_vault | vault | decrypted_secrets | secret + supabase_vault | vault | decrypted_secrets | updated_at supabase_vault | vault | secrets | created_at supabase_vault | vault | secrets | description supabase_vault | vault | secrets | id @@ -5338,5 +5350,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(398 rows) +(407 rows) diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 3ff7aea4f..483a1a8e2 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -795,6 +795,7 @@ jit_provider = 'llvmjit' # JIT library to use # Add settings for extensions here pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' +vault.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' auto_explain.log_min_duration = 10s cron.database_name = 'postgres' diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in index 8c489839b..621912bec 100644 --- a/nix/tools/dbmate-tool.sh.in +++ b/nix/tools/dbmate-tool.sh.in @@ -15,39 +15,49 @@ CURRENT_SYSTEM="@CURRENT_SYSTEM@" ANSIBLE_VARS="@ANSIBLE_VARS@" PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ + +# Start PostgreSQL using nix +start_postgres() { + echo "Starting PostgreSQL..." + nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations --daemonize + echo "PostgreSQL started." +} + # Cleanup function cleanup() { echo "Cleaning up..." - # Kill postgres processes first + # Check if PostgreSQL processes exist if pgrep -f "postgres" >/dev/null; then - pkill -TERM postgres || true - sleep 2 + echo "Stopping PostgreSQL gracefully..." + + # Use a more specific signal handling approach + pkill -15 -f "postgres" # Send SIGTERM + + # Wait a bit for graceful shutdown + sleep 5 + + # If processes are still running, force kill + if pgrep -f "postgres" >/dev/null; then + echo "Forcing PostgreSQL processes to stop..." + pkill -9 -f "postgres" # Send SIGKILL if SIGTERM didn't work + fi + else + echo "PostgreSQL is not running, skipping stop." fi - # Then kill overmind - if [ -S "./.overmind.sock" ]; then - overmind kill || true - sleep 2 + # Always exit successfully, log any remaining processes + if pgrep -f "postgres" >/dev/null; then + echo "Warning: Some PostgreSQL processes could not be cleaned up:" + pgrep -f "postgres" + else + echo "Cleanup completed successfully" fi - # Kill tmux sessions explicitly - pkill -f "tmux.*overmind.*postgresql" || true - tmux ls 2>/dev/null | grep 'overmind' | cut -d: -f1 | xargs -I{} tmux kill-session -t {} || true - - # Force kill any stragglers - pkill -9 -f "(postgres|tmux.*overmind.*postgresql)" || true - - rm -f .overmind.sock Procfile - - # Final verification - if ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep >/dev/null; then - ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep - return 1 - fi + # Explicitly exit with 0 to prevent workflow failure + exit 0 } -# Set up trap for cleanup on script exit # Function to display help print_help() { @@ -57,7 +67,7 @@ print_help() { echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" echo " -p, --port PORT Specify the port number to use (default: 5435)" echo " -h, --help Show this help message" - echo + echo " -f, --flake-url URL Specify the flake URL to use (default: github:supabase/postgres)" echo "Description:" echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." @@ -66,9 +76,9 @@ print_help() { echo " nix run .#dbmate-tool" echo " nix run .#dbmate-tool -- --version 15" echo " nix run .#dbmate-tool -- --version 16 --port 5433" + echo " nix run .#dbmate-tool -- --version 16 --port 5433 --flake-url github:supabase/postgres/" } - # Parse arguments while [[ "$#" -gt 0 ]]; do case "$1" in @@ -125,7 +135,7 @@ wait_for_postgres() { local max_attempts=30 # Increased significantly local attempt=1 - # Give overmind a moment to actually start the process + # Give PostgreSQL a moment to actually start the process sleep 2 while [ $attempt -le $max_attempts ]; do @@ -142,7 +152,6 @@ wait_for_postgres() { done echo "PostgreSQL failed to start after $max_attempts attempts" - overmind echo postgres return 1 } @@ -175,26 +184,7 @@ trim_schema() { ;; esac } -overmind_start() { - cat > Procfile << EOF -postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" -- "$PSQL_VERSION" --skip-migrations -EOF - overmind start -D - echo "Waiting for overmind socket..." - max_wait=5 - count=0 - while [ $count -lt $max_wait ]; do - if [ -S "./.overmind.sock" ]; then - # Found the socket, give it a moment to be ready - sleep 5 - echo "Socket file found and ready" - break - fi - echo "Waiting for socket file (attempt $count/$max_wait)" - sleep 1 - count=$((count + 1)) - done -} + perform_dump() { local max_attempts=3 local attempt=1 @@ -214,21 +204,18 @@ perform_dump() { echo "All dump attempts failed" return 1 } + migrate_version() { echo "PSQL_VERSION: $PSQL_VERSION" - overmind kill || true - rm -f .overmind.sock Procfile || true + #pkill -f "postgres" || true # Ensure PostgreSQL is stopped before starting PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" - # Start overmind - overmind_start - echo "Waiting for overmind socket..." - - + # Start PostgreSQL + start_postgres echo "Waiting for PostgreSQL to be ready..." - #Wait for PostgreSQL to be ready to accept connections + # Wait for PostgreSQL to be ready to accept connections if ! wait_for_postgres; then echo "Failed to connect to PostgreSQL server" exit 1 @@ -255,11 +242,11 @@ EOSQL "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" - #set db url to run dbmate + # Set db url to run dbmate export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" - #export path so dbmate can find correct psql and pg_dump + # Export path so dbmate can find correct psql and pg_dump export PATH="$PSQLBIN:$PATH" - # run init scripts + # Run init scripts if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then echo "Error: Initial migration failed" exit 1 diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 172f62db7..22b1fb557 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -56,7 +56,11 @@ start_postgres() { } stop_postgres() { - pg_ctl stop -D "$DATDIR" -m fast + if [ "$DAEMONIZE" = true ]; then + echo "PostgreSQL is running in daemon mode. Please stop it using pg_ctl." + else + pg_ctl stop -D "$DATDIR" -m fast + fi } trap 'stop_postgres' SIGINT SIGTERM @@ -331,6 +335,7 @@ EOSQL fi fi echo "Shutting down PostgreSQL..." + stop_postgres # Step 4: Restart PostgreSQL in the foreground (with log output visible) or as a daemon