diff --git a/flake.nix b/flake.nix index a2ae1b973..7f9a446cc 100644 --- a/flake.nix +++ b/flake.nix @@ -300,19 +300,24 @@ name = "pg_ident.conf"; path = ./ansible/files/postgresql_config/pg_ident.conf.j2; }; + postgresqlExtensionCustomScriptsPath = builtins.path { + name = "extension-custom-scripts"; + path = ./ansible/files/postgresql_extension_custom_scripts; + }; getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; localeArchive = if pkgs.stdenv.isDarwin then "${pkgs.darwin.locale}/share/locale" else "${pkgs.glibcLocales}/lib/locale/locale-archive"; in pkgs.runCommand "start-postgres-server" { } '' - mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql + mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts cp ${supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } cp ${pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } cp ${loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } cp ${readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } cp ${pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } cp ${pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } + cp -r ${postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; } echo "Copy operation completed" chmod 644 $out/etc/postgresql-custom/supautils.conf chmod 644 $out/etc/postgresql/postgresql.conf @@ -329,32 +334,23 @@ --subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" \ --subst-var-by 'PG_HBA' "$out/etc/postgresql/pg_hba.conf" \ --subst-var-by 'PG_IDENT' "$out/etc/postgresql/pg_ident.conf" \ - --subst-var-by 'LOCALES' '${localeArchive}' + --subst-var-by 'LOCALES' '${localeArchive}' \ + --subst-var-by 'EXTENSION_CUSTOM_SCRIPTS_DIR' "$out/extension-custom-scripts" chmod +x $out/bin/start-postgres-server ''; - # Start a version of the client. - start-client = pkgs.runCommand "start-postgres-client" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' - chmod +x $out/bin/start-postgres-client - ''; - # Start a version of the client and runs migrations script on server. - start-client-and-migrate = + start-client = let migrationsDir = ./migrations/db; postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql; pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; statExtensionSql = ./ansible/files/stat_extension.sql; in - pkgs.runCommand "start-postgres-client-migrate" { } '' + pkgs.runCommand "start-postgres-client" { } '' mkdir -p $out/bin - substitute ${./nix/tools/run-client-migrate.sh.in} $out/bin/start-postgres-client-migrate \ + substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ @@ -362,7 +358,7 @@ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/start-postgres-client-migrate + chmod +x $out/bin/start-postgres-client ''; # Migrate between two data directories. @@ -517,7 +513,6 @@ { start-server = mkApp "start-server" "start-postgres-server"; start-client = mkApp "start-client" "start-postgres-client"; - start-client-and-migrate = mkApp "start-client-and-migrate" "start-postgres-client-migrate"; start-replica = mkApp "start-replica" "start-postgres-replica"; migration-test = mkApp "migrate-tool" "migrate-postgres"; sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; diff --git a/nix/tools/run-client-migrate.sh.in b/nix/tools/run-client-migrate.sh.in deleted file mode 100644 index 070168ef3..000000000 --- a/nix/tools/run-client-migrate.sh.in +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# shellcheck shell=bash - -[ ! -z "$DEBUG" ] && set -x - -# first argument should be '15' or '16' for the version -if [ "$1" == "15" ]; then - echo "Starting client for PSQL 15" - PSQL15=@PSQL15_BINDIR@ - BINDIR="$PSQL15" -elif [ "$1" == "16" ]; then - echo "Starting client for PSQL 16" - PSQL16=@PSQL16_BINDIR@ - BINDIR="$PSQL16" -elif [ "$1" == "orioledb-16" ]; then - echo "Starting client for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ - BINDIR="$PSQLORIOLEDB16" -else - echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" - exit 1 -fi -#vars for migration.sh -export PATH=$BINDIR/bin:$PATH -export POSTGRES_DB=postgres -export POSTGRES_HOST=localhost -export POSTGRES_PORT=@PGSQL_DEFAULT_PORT@ -PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" -PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ -MIGRATIONS_DIR=@MIGRATIONS_DIR@ -POSTGRESQL_SCHEMA_SQL=@POSTGRESQL_SCHEMA_SQL@ -PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ -STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ -psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL - create role postgres superuser login password '$PGPASSWORD'; - alter database postgres owner to postgres; -EOSQL -psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" -psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" -for sql in "$MIGRATIONS_DIR"/init-scripts/*.sql; do - echo "$0: running $sql" - psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -f "$sql" postgres -done -psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'" -# run migrations as super user - postgres user demoted in post-setup -for sql in "$MIGRATIONS_DIR"/migrations/*.sql; do - echo "$0: running $sql" - psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$sql" postgres -done -psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$POSTGRESQL_SCHEMA_SQL" postgres -# TODO Do we need to reset stats when running migrations locally? -#psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -c 'SELECT extensions.pg_stat_statements_reset(); SELECT pg_stat_reset();' postgres || true - -exec psql -U postgres -p "$PORTNO" -h localhost postgres diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in index a941213aa..18a8f21bb 100644 --- a/nix/tools/run-client.sh.in +++ b/nix/tools/run-client.sh.in @@ -20,9 +20,35 @@ else echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" exit 1 fi - +#vars for migration.sh export PATH=$BINDIR/bin:$PATH - +export POSTGRES_DB=postgres +export POSTGRES_HOST=localhost +export POSTGRES_PORT=@PGSQL_DEFAULT_PORT@ PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" +PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ +MIGRATIONS_DIR=@MIGRATIONS_DIR@ +POSTGRESQL_SCHEMA_SQL=@POSTGRESQL_SCHEMA_SQL@ +PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ +STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ +psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL + create role postgres superuser login password '$PGPASSWORD'; + alter database postgres owner to postgres; +EOSQL +for sql in "$MIGRATIONS_DIR"/init-scripts/*.sql; do + echo "$0: running $sql" + psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -f "$sql" postgres +done +psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'" +psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" +psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" +# run migrations as super user - postgres user demoted in post-setup +for sql in "$MIGRATIONS_DIR"/migrations/*.sql; do + echo "$0: running $sql" + psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$sql" postgres +done +psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -f "$POSTGRESQL_SCHEMA_SQL" postgres +# TODO Do we need to reset stats when running migrations locally? +#psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -p "$PORTNO" -h localhost -c 'SELECT extensions.pg_stat_statements_reset(); SELECT pg_stat_reset();' postgres || true exec psql -U postgres -p "$PORTNO" -h localhost postgres diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 85e358fcf..836407cfe 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -1,8 +1,6 @@ #!/usr/bin/env bash # shellcheck shell=bash - [ ! -z "$DEBUG" ] && set -x - # first argument should be '15' or '16' for the version if [ "$1" == "15" ]; then echo "Starting server for PSQL 15" @@ -20,9 +18,7 @@ else echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" exit 1 fi - export PATH=$BINDIR/bin:$PATH - PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ PSQL_CONF_FILE=@PSQL_CONF_FILE@ PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ @@ -32,6 +28,7 @@ LOGGING_CONFIG_FILE=@LOGGING_CONF_FILE@ READREPL_CONFIG_FILE=@READREPL_CONF_FILE@ PG_HBA_FILE=@PG_HBA@ PG_IDENT_FILE=@PG_IDENT@ +EXTENSION_CUSTOM_SCRIPTS=@EXTENSION_CUSTOM_SCRIPTS_DIR@ DATDIR=$(mktemp -d) LOCALE_ARCHIVE=@LOCALES@ export LOCALE_ARCHIVE @@ -41,22 +38,26 @@ export LC_ALL=en_US.UTF-8 export LANG=en_US.UTF-8 export LC_CTYPE=en_US.UTF-8 mkdir -p "$DATDIR" - echo "NOTE: using port $PORTNO for server" echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" echo "NOTE: you are free to re-use this data directory at will" - initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" - echo "NOTE: patching postgresql.conf files" - -sed -e "\$a\\ - include = '$SUPAUTILS_CONFIG_FILE' \\ - pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ - -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ - -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$PG_HBA_FILE'|" \ - -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$PG_IDENT_FILE'|" \ - -e "s|include = '/etc/postgresql/logging.conf'|#&|" \ - -e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$READREPL_CONFIG_FILE'|" \ - "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" -postgres -p "$PORTNO" -D "$DATDIR" -k /tmp +cp "$PG_HBA_FILE" "$DATDIR/pg_hba.conf" +cp "$PG_IDENT_FILE" "$DATDIR/pg_ident.conf" +cp "$READREPL_CONFIG_FILE" "$DATDIR/read-replica.conf" +cp -r "$EXTENSION_CUSTOM_SCRIPTS" "$DATDIR" +sed "s|supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'|supautils.privileged_extensions_custom_scripts_path = '$DATDIR/extension-custom-scripts'|" "$SUPAUTILS_CONFIG_FILE" > "$DATDIR/supautils.conf" +sed -e "1i\\ +include = '$DATDIR/supautils.conf'" \ +-e "\$a\\ +pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ +-e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ +-e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \ +-e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \ +-e "s|include = '/etc/postgresql/logging.conf'|#&|" \ +-e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$DATDIR/read-replica.conf'|" \ +-e "\$a\\ +session_preload_libraries = 'supautils'" \ +"$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" +postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp \ No newline at end of file