Skip to content

Commit 61ed1f0

Browse files
samrosesamrose
committed
feat: migration testing of 2 git versions
1 parent 18bf547 commit 61ed1f0

File tree

2 files changed

+141
-86
lines changed

2 files changed

+141
-86
lines changed

‎flake.nix‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,16 @@
6262

6363
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
6464
pg_regress = pkgs.callPackage ./nix/ext/pg_regress.nix { };
65-
65+
pg_prove = pkgs.runCommand "pg_prove"
66+
{
67+
nativeBuildInputs = [ pkgs.makeWrapper ];
68+
} ''
69+
mkdir -p $out/bin
70+
for x in pg_prove pg_tapgen; do
71+
makeWrapper "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}/bin/$x" "$out/bin/$x" \
72+
${pkgs.lib.optionalString pkgs.stdenv.isLinux "--set LOCALE_ARCHIVE \"${pkgs.glibcLocales}/lib/locale/locale-archive\""}
73+
done
74+
'';
6675
# Our list of PostgreSQL extensions which come from upstream Nixpkgs.
6776
# These are maintained upstream and can easily be used here just by
6877
# listing their name. Anytime the version of nixpkgs is upgraded, these
@@ -367,23 +376,27 @@
367376

368377
# Migrate between two data directories.
369378
migrate-tool =
370-
let
371-
configFile = ./nix/tests/postgresql.conf.in;
372-
getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
373-
primingScript = ./nix/tests/prime.sql;
374-
migrationData = ./nix/tests/migrations/data.sql;
375-
in
376-
pkgs.runCommand "migrate-postgres" { } ''
377-
mkdir -p $out/bin
378-
substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
379-
--subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
380-
--subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
381-
--subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
382-
--subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
383-
--subst-var-by 'MIGRATION_DATA' '${migrationData}'
384-
385-
chmod +x $out/bin/migrate-postgres
386-
'';
379+
let
380+
configFile = ./nix/tests/postgresql.conf.in;
381+
getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
382+
primingScript = ./nix/tests/prime.sql;
383+
migrationsDir = ./migrations;
384+
pgupgradeTests = ./tests;
385+
pgProve = pg_prove;
386+
in
387+
pkgs.runCommand "migrate-postgres" { } ''
388+
mkdir -p $out/bin $out/migrations $out/tests
389+
cp -r ${migrationsDir}/* $out/migrations
390+
cp -r ${pgupgradeTests}/* $out/tests
391+
substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
392+
--subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
393+
--subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
394+
--subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
395+
--subst-var-by 'MIGRATIONS_DIR' "$out/migrations" \
396+
--subst-var-by 'PGUPGRADE_TESTS' "$out/tests" \
397+
--subst-var-by 'PG_PROVE' "${pgProve}"
398+
chmod +x $out/bin/migrate-postgres
399+
'';
387400

388401
start-replica = pkgs.runCommand "start-postgres-replica" { } ''
389402
mkdir -p $out/bin

‎nix/tools/migrate-tool.sh.in‎

Lines changed: 110 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -1,123 +1,165 @@
11
#!/usr/bin/env bash
22

3-
[ ! -z "$DEBUG" ] && set -x
4-
5-
# first argument is the old version; a path 15 or 16
6-
if [[ $1 == /nix/store* ]]; then
7-
if [ ! -L "$1/receipt.json" ] || [ ! -e "$1/receipt.json" ]; then
8-
echo "ERROR: $1 does not look like a valid Postgres install"
3+
[ ! -z "$DEBUG" ] && set -eoux pipefail
4+
5+
# Function to build flake and return the output path
6+
build_flake() {
7+
local flake_url="$1"
8+
local temp_dir=$(mktemp -d)
9+
if ! nix build "$flake_url" -o "$temp_dir/result"; then
10+
echo "ERROR: Failed to build flake $flake_url"
911
exit 1
1012
fi
11-
OLDVER="$1"
12-
elif [ "$1" == "15" ]; then
13-
PSQL15=@PSQL15_BINDIR@
14-
OLDVER="$PSQL15"
15-
elif [ "$1" == "16" ]; then
16-
PSQL16=@PSQL16_BINDIR@
17-
OLDVER="$PSQL16"
18-
else
19-
echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path"
20-
exit 1
21-
fi
13+
echo "$temp_dir/result"
14+
}
2215

23-
# second argument is the new version; 15 or 16
24-
if [[ $2 == /nix/store* ]]; then
25-
if [ ! -L "$2/receipt.json" ] || [ ! -e "$2/receipt.json" ]; then
26-
echo "ERROR: $1 does not look like a valid Postgres install"
27-
exit 1
28-
fi
29-
NEWVER="$2"
30-
elif [ "$2" == "15" ]; then
31-
PSQL15=@PSQL15_BINDIR@
32-
NEWVER="$PSQL15"
33-
elif [ "$2" == "16" ]; then
34-
PSQL16=@PSQL16_BINDIR@
35-
NEWVER="$PSQL16"
36-
echo "NEWVER IS $NEWVER"
37-
else
38-
echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path"
39-
exit 1
40-
fi
16+
# First argument is the old version flake URL
17+
OLDVER=$(build_flake "$1")
18+
19+
# Second argument is the new version flake URL
20+
NEWVER=$(build_flake "$2")
4121

42-
# thid argument is the upgrade method: either pg_dumpall or pg_ugprade
22+
# Third argument is the upgrade method: either pg_dumpall or pg_upgrade
4323
if [ "$3" != "pg_dumpall" ] && [ "$3" != "pg_upgrade" ]; then
4424
echo "Please provide a valid upgrade method (pg_dumpall or pg_upgrade)"
4525
exit 1
4626
fi
4727
UPGRADE_METHOD="$3"
4828

49-
echo "Old server build: PSQL $1"
50-
echo "New server build: PSQL $2"
29+
echo "Old server build: $OLDVER"
30+
echo "New server build: $NEWVER"
5131
echo "Upgrade method: $UPGRADE_METHOD"
5232

53-
PORTNO="${2:-@PGSQL_DEFAULT_PORT@}"
33+
PORTNO="@PGSQL_DEFAULT_PORT@"
5434
DATDIR=$(mktemp -d)
5535
NEWDAT=$(mktemp -d)
36+
PGUSER=${PGUSER:-postgres}
37+
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
38+
PG_PROVE=@PG_PROVE@
39+
PGUPGRADE_TESTS=@PGUPGRADE_TESTS@
5640
mkdir -p "$DATDIR" "$NEWDAT"
5741

58-
echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed"
59-
echo "NOTE: you are free to re-use this data directory at will"
42+
echo "using temporary directory $DATDIR for old data, which will not be removed"
43+
echo "you are free to re-use this data directory at will"
6044
echo
6145

62-
$OLDVER/bin/initdb -D "$DATDIR" --locale=C
63-
$NEWVER/bin/initdb -D "$NEWDAT" --locale=C
46+
echo "PGUSER IS $PGUSER"
47+
48+
$OLDVER/bin/initdb -U "$PGUSER" -D "$DATDIR" --locale=C
49+
$NEWVER/bin/initdb -U "$PGUSER" -D "$NEWDAT" --locale=C
6450

6551
# NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script
6652
PSQL_CONF_FILE=@PSQL_CONF_FILE@
6753
PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@
68-
echo "NOTE: patching postgresql.conf files"
54+
echo "patching postgresql.conf files"
6955
for x in "$DATDIR" "$NEWDAT"; do
7056
sed \
7157
"s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \
7258
$PSQL_CONF_FILE > "$x/postgresql.conf"
7359
done
7460

75-
echo "NOTE: Starting first server (v${1}) to load data into the system"
61+
echo "Starting first server to load data into the system"
7662
$OLDVER/bin/pg_ctl start -D "$DATDIR"
7763

7864
PRIMING_SCRIPT=@PRIMING_SCRIPT@
79-
MIGRATION_DATA=@MIGRATION_DATA@
65+
MIGRATIONS_DIR=@MIGRATIONS_DIR@
66+
67+
echo "MIGRATIONS_DIR IS $MIGRATIONS_DIR"
68+
69+
for sql in "$MIGRATIONS_DIR"/db/init-scripts/*.sql; do
70+
echo "$0: running $sql"
71+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -f "$sql"
72+
done
8073

81-
$OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT"
82-
$OLDVER/bin/psql -h localhost -d postgres -Xf "$MIGRATION_DATA"
74+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
75+
# run migrations as super user - postgres user demoted in post-setup
76+
for sql in "$MIGRATIONS_DIR"/db/migrations/*.sql; do
77+
echo "$0: running $sql"
78+
$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -f "$sql"
79+
done
8380

8481
if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then
85-
echo "NOTE: Stopping old server (v${1}) to prepare for migration"
82+
echo "Stopping old server"
8683
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
8784

88-
echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
85+
echo "Starting old server"
86+
$OLDVER/bin/pg_ctl start -D "$DATDIR"
87+
88+
echo "Ensuring $PGUSER is a superuser in the old database"
89+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "ALTER USER $PGUSER WITH SUPERUSER;" || true
90+
91+
echo "Running pre-migration checks"
92+
"$PG_PROVE"/bin/pg_prove --psql="$OLDVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 "$MIGRATIONS_DIR"/tests/test.sql
93+
94+
echo "Running fixtures"
95+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/97-enable-extensions.sql"
96+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/98-data-fixtures.sql"
97+
"$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/99-fixtures.sql"
98+
99+
echo "Stopping old server"
100+
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
101+
102+
echo "Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
89103

90104
export PGDATAOLD="$DATDIR"
91105
export PGDATANEW="$NEWDAT"
92106
export PGBINOLD="$OLDVER/bin"
93107
export PGBINNEW="$NEWVER/bin"
94108

95-
if ! $NEWVER/bin/pg_upgrade --check; then
109+
# Create a temporary directory for pg_upgrade to work in
110+
UPGRADE_WORKDIR=$(mktemp -d)
111+
echo "Using temporary directory for pg_upgrade: $UPGRADE_WORKDIR"
112+
113+
# Change to the temporary directory before running pg_upgrade
114+
pushd "$UPGRADE_WORKDIR"
115+
116+
if ! $NEWVER/bin/pg_upgrade -U "$PGUSER" --check; then
96117
echo "ERROR: pg_upgrade check failed"
118+
popd
97119
exit 1
98120
fi
99121

100-
echo "NOTE: pg_upgrade check passed, proceeding with migration"
101-
$NEWVER/bin/pg_upgrade
102-
rm -f delete_old_cluster.sh # we don't need this
122+
echo "pg_upgrade check passed, proceeding with migration"
123+
$NEWVER/bin/pg_upgrade -U "$PGUSER"
124+
125+
# Change back to the original directory
126+
popd
127+
echo "Migration complete, running post-migration checks"
128+
echo "Migration complete, running post-migration checks"
129+
echo "NEWDAT IS $NEWDAT"
130+
$NEWVER/bin/pg_ctl start -D "$NEWDAT"
131+
echo "Turning off JIT"
132+
cat << EOF > "$NEWDAT"/jit_off.sql
133+
ALTER SYSTEM SET jit = off;
134+
SELECT pg_reload_conf();
135+
EOF
136+
"$NEWVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$NEWDAT"/jit_off.sql
137+
138+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
139+
"$PGUPGRADE_TESTS/pg_upgrade/tests/01-schema.sql"
140+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
141+
"$PGUPGRADE_TESTS/pg_upgrade/tests/02-data.sql"
142+
"$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
143+
"$PGUPGRADE_TESTS/pg_upgrade/tests/03-settings.sql"
144+
$NEWVER/bin/pg_ctl stop -D "$NEWDAT"
103145
exit 0
104146
fi
105147

106-
if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
107-
SQLDAT="$DATDIR/dump.sql"
108-
echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)"
109-
$NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
148+
# if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
149+
# SQLDAT="$DATDIR/dump.sql"
150+
# echo "Exporting data via pg_dumpall ($SQLDAT)"
151+
# $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
110152

111-
echo "NOTE: Stopping old server (v${1}) to prepare for migration"
112-
$OLDVER/bin/pg_ctl stop -D "$DATDIR"
153+
# echo "Stopping old server (v${1}) to prepare for migration"
154+
# $OLDVER/bin/pg_ctl stop -D "$DATDIR"
113155

114-
echo "NOTE: Starting second server (v${2}) to load data into the system"
115-
$NEWVER/bin/pg_ctl start -D "$NEWDAT"
156+
# echo "Starting second server (v${2}) to load data into the system"
157+
# $NEWVER/bin/pg_ctl start -D "$NEWDAT"
116158

117-
echo "NOTE: Loading data into new server (v${2}) via 'cat | psql'"
118-
cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
159+
# echo "Loading data into new server (v${2}) via 'cat | psql'"
160+
# cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
119161

120-
printf "\n\n\n\n"
121-
echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT"
122-
$NEWVER/bin/pg_ctl stop -D "$NEWDAT"
123-
fi
162+
# printf "\n\n\n\n"
163+
# echo "Done, check logs. Stopping the server; new database is located at $NEWDAT"
164+
# $NEWVER/bin/pg_ctl stop -D "$NEWDAT"
165+
# fi

0 commit comments

Comments
 (0)