Skip to content

Commit dee6c48

Browse files
samrosesamrose
committed
feat: migration with pg_prove tests
1 parent ab4efe6 commit dee6c48

File tree

2 files changed

+96
-32
lines changed

2 files changed

+96
-32
lines changed

‎flake.nix‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@
6262

6363
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
6464
pg_regress = pkgs.callPackage ./nix/ext/pg_regress.nix { };
65+
pg_prove = pkgs.runCommand "pg_prove"
66+
{
67+
nativeBuildInputs = [ pkgs.makeWrapper ];
68+
} ''
69+
mkdir -p $out/bin
70+
for x in pg_prove pg_tapgen; do
71+
makeWrapper "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}/bin/$x" "$out/bin/$x" \
72+
--set LOCALE_ARCHIVE "${pkgs.glibcLocales}/lib/locale/locale-archive"
73+
done
74+
'';
6575

6676
# Our list of PostgreSQL extensions which come from upstream Nixpkgs.
6777
# These are maintained upstream and can easily be used here just by
@@ -328,17 +338,23 @@
328338
configFile = ./nix/tests/postgresql.conf.in;
329339
getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
330340
primingScript = ./nix/tests/prime.sql;
331-
migrationData = ./nix/tests/migrations/data.sql;
341+
migrationsDir = ./migrations;
342+
pgupgradeTests = ./tests;
343+
pgProve = pg_prove;
332344
in
333345
pkgs.runCommand "migrate-postgres" { } ''
334-
mkdir -p $out/bin
346+
mkdir -p $out/bin $out/migrations $out/tests
347+
cp -r ${migrationsDir}/* $out/migrations
348+
cp -r ${pgupgradeTests}/* $out/tests
349+
335350
substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
336351
--subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
337352
--subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
338353
--subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
339354
--subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
340-
--subst-var-by 'MIGRATION_DATA' '${migrationData}'
341-
355+
--subst-var-by 'MIGRATIONS_DIR' "$out/migrations" \
356+
--subst-var-by 'PGUPGRADE_TESTS' "$out/tests" \
357+
--subst-var-by 'PG_PROVE' "${pgProve}"
342358
chmod +x $out/bin/migrate-postgres
343359
'';
344360

‎nix/tools/migrate-tool.sh.in‎

Lines changed: 76 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -53,71 +53,119 @@ echo "Upgrade method: $UPGRADE_METHOD"
5353
PORTNO="${2:-@PGSQL_DEFAULT_PORT@}"
5454
DATDIR=$(mktemp -d)
5555
NEWDAT=$(mktemp -d)
56+
PGUSER=${PGUSER:-postgres}
57+
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
58+
PG_PROVE=@PG_PROVE@
59+
PGUPGRADE_TESTS=@PGUPGRADE_TESTS@
5660
mkdir -p "$DATDIR" "$NEWDAT"
5761

58-
echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed"
59-
echo "NOTE: you are free to re-use this data directory at will"
62+
echo "using temporary directory $DATDIR for PSQL $1 data, which will not be removed"
63+
echo "you are free to re-use this data directory at will"
6064
echo
6165

62-
$OLDVER/bin/initdb -D "$DATDIR" --locale=C
63-
$NEWVER/bin/initdb -D "$NEWDAT" --locale=C
66+
echo "PGUSER IS $PGUSER"
67+
68+
$OLDVER/bin/initdb -U "$PGUSER" -D "$DATDIR" --locale=C
69+
$NEWVER/bin/initdb -U "$PGUSER" -D "$NEWDAT" --locale=C
6470

6571
# NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script
6672
PSQL_CONF_FILE=@PSQL_CONF_FILE@
6773
PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@
68-
echo "NOTE: patching postgresql.conf files"
74+
echo "patching postgresql.conf files"
6975
for x in "$DATDIR" "$NEWDAT"; do
7076
sed \
7177
"s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \
7278
$PSQL_CONF_FILE > "$x/postgresql.conf"
7379
done
7480

75-
echo "NOTE: Starting first server (v${1}) to load data into the system"
81+
echo "Starting first server (v${1}) to load data into the system"
7682
$OLDVER/bin/pg_ctl start -D "$DATDIR"
7783

7884
PRIMING_SCRIPT=@PRIMING_SCRIPT@
79-
MIGRATION_DATA=@MIGRATION_DATA@
85+
MIGRATIONS_DIR=@MIGRATIONS_DIR@
86+
87+
echo "MIGRATIONS_DIR IS $MIGRATIONS_DIR"
8088

81-
$OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT"
82-
$OLDVER/bin/psql -h localhost -d postgres -Xf "$MIGRATION_DATA"
89+
# $OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT"
90+
for sql in "$MIGRATIONS_DIR"/db/init-scripts/*.sql; do
91+
echo "$0: running $sql"
92+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -f "$sql"
93+
done
94+
95+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
96+
# run migrations as super user - postgres user demoted in post-setup
97+
for sql in "$MIGRATIONS_DIR"/db/migrations/*.sql; do
98+
echo "$0: running $sql"
99+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -f "$sql"
100+
done
83101

84102
if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then
85-
echo "NOTE: Stopping old server (v${1}) to prepare for migration"
103+
# echo "Ensuring proper permissions on $DATDIR"
104+
# chmod 700 "$DATDIR"
105+
106+
echo "Stopping old server"
107+
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
108+
109+
echo "Starting old server"
110+
$OLDVER/bin/pg_ctl start -D "$DATDIR"
111+
112+
echo "Ensuring $PGUSER is a superuser in the old database"
113+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "ALTER USER $PGUSER WITH SUPERUSER;" || true
114+
115+
echo "Running pre-migration checks"
116+
"$PG_PROVE"/bin/pg_prove --psql="$OLDVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 "$MIGRATIONS_DIR"/tests/test.sql
117+
118+
echo "Running fixtures"
119+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/97-enable-extensions.sql"
120+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/98-data-fixtures.sql"
121+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/99-fixtures.sql"
122+
123+
echo "Stopping old server"
86124
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
87125

88-
echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
126+
echo "Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
89127

90128
export PGDATAOLD="$DATDIR"
91129
export PGDATANEW="$NEWDAT"
92130
export PGBINOLD="$OLDVER/bin"
93131
export PGBINNEW="$NEWVER/bin"
94132

95-
if ! $NEWVER/bin/pg_upgrade --check; then
133+
if ! $NEWVER/bin/pg_upgrade -U "$PGUSER" --check; then
96134
echo "ERROR: pg_upgrade check failed"
97135
exit 1
98136
fi
99137

100-
echo "NOTE: pg_upgrade check passed, proceeding with migration"
101-
$NEWVER/bin/pg_upgrade
138+
echo "pg_upgrade check passed, proceeding with migration"
139+
$NEWVER/bin/pg_upgrade -U "$PGUSER"
102140
rm -f delete_old_cluster.sh # we don't need this
141+
echo "Migration complete, running post-migration checks"
142+
$NEWVER/bin/pg_ctl start -D "$NEWDAT"
143+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
144+
"$PGUPGRADE_TESTS/pg_upgrade/tests/01-schema.sql"
145+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
146+
"$PGUPGRADE_TESTS/pg_upgrade/tests/02-data.sql"
147+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
148+
"$PGUPGRADE_TESTS/pg_upgrade/tests/03-settings.sql"
149+
$NEWVER/bin/pg_ctl stop -D "$NEWDAT"
103150
exit 0
104151
fi
105152

106-
if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
107-
SQLDAT="$DATDIR/dump.sql"
108-
echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)"
109-
$NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
110153

111-
echo "NOTE: Stopping old server (v${1}) to prepare for migration"
112-
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
154+
# if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
155+
# SQLDAT="$DATDIR/dump.sql"
156+
# echo "Exporting data via pg_dumpall ($SQLDAT)"
157+
# $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
113158

114-
echo "NOTE: Starting second server (v${2}) to load data into the system"
115-
$NEWVER/bin/pg_ctl start -D "$NEWDAT"
159+
# echo "Stopping old server (v${1}) to prepare for migration"
160+
# $OLDVER/bin/pg_ctl stop -D "$DATDIR"
116161

117-
echo "NOTE: Loading data into new server (v${2}) via 'cat | psql'"
118-
cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
162+
# echo "Starting second server (v${2}) to load data into the system"
163+
# $NEWVER/bin/pg_ctl start -D "$NEWDAT"
119164

120-
printf "\n\n\n\n"
121-
echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT"
122-
$NEWVER/bin/pg_ctl stop -D "$NEWDAT"
123-
fi
165+
# echo "Loading data into new server (v${2}) via 'cat | psql'"
166+
# cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
167+
168+
# printf "\n\n\n\n"
169+
# echo "Done, check logs. Stopping the server; new database is located at $NEWDAT"
170+
# $NEWVER/bin/pg_ctl stop -D "$NEWDAT"
171+
# fi

0 commit comments

Comments
 (0)