Sam/migration testing

flake.nix

@@ -62,6 +62,16 @@
 
         sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
         pg_regress = pkgs.callPackage ./nix/ext/pg_regress.nix { };
+        pg_prove = pkgs.runCommand "pg_prove"
+          {
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+          } ''
+          mkdir -p $out/bin
+          for x in pg_prove pg_tapgen; do
+            makeWrapper "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}/bin/$x" "$out/bin/$x" \
+              ${pkgs.lib.optionalString pkgs.stdenv.isLinux "--set LOCALE_ARCHIVE \"${pkgs.glibcLocales}/lib/locale/locale-archive\""}
+          done
+        '';
 
         # Our list of PostgreSQL extensions which come from upstream Nixpkgs.
         # These are maintained upstream and can easily be used here just by
@@ -323,25 +333,28 @@
             '';
 
           # Migrate between two data directories.
-          migrate-tool =
-            let
-              configFile = ./nix/tests/postgresql.conf.in;
-              getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
-              primingScript = ./nix/tests/prime.sql;
-              migrationData = ./nix/tests/migrations/data.sql;
-            in
-            pkgs.runCommand "migrate-postgres" { } ''
-              mkdir -p $out/bin
-              substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
-                --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
-                --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
-                --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
-                --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
-                --subst-var-by 'MIGRATION_DATA' '${migrationData}'
-
-              chmod +x $out/bin/migrate-postgres
-            '';
-
+        migrate-tool =
+          let
+            configFile = ./nix/tests/postgresql.conf.in;
+            getkeyScript = ./nix/tests/util/pgsodium_getkey.sh;
+            primingScript = ./nix/tests/prime.sql;
+            migrationsDir = ./migrations;
+            pgupgradeTests = ./tests;
+            pgProve = pg_prove;
+          in
+          pkgs.runCommand "migrate-postgres" { } ''
+            mkdir -p $out/bin $out/migrations $out/tests
+            cp -r ${migrationsDir}/* $out/migrations
+            cp -r ${pgupgradeTests}/* $out/tests
+            substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \
+              --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \
+              --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
+              --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \
+              --subst-var-by 'MIGRATIONS_DIR' "$out/migrations"  \
+              --subst-var-by 'PGUPGRADE_TESTS' "$out/tests" \
+              --subst-var-by 'PG_PROVE' "${pgProve}"           
+            chmod +x $out/bin/migrate-postgres
+          '';
           start-replica = pkgs.runCommand "start-postgres-replica" { } ''
             mkdir -p $out/bin
             substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \

nix/ext/postgis.nix

@@ -61,6 +61,14 @@ stdenv.mkDerivation rec {
     ln -s ${postgresql}/bin/postgres $out/bin/postgres
   '';
 
+  installPhase = ''
+    mkdir -p $out/{bin,lib,share/postgresql/extension}
+    cp bin/*      $out/bin
+    cp *.so      $out/lib
+    cp *.sql     $out/share/postgresql/extension
+    cp *.control $out/share/postgresql/extension
+  '';
+
   # create aliases for all commands adding version information
   postInstall = ''
     # Teardown the illusory postgres used for building; see postConfigure.

nix/tools/migrate-tool.sh.in

@@ -1,123 +1,173 @@
 #!/usr/bin/env bash
 
-[ ! -z "$DEBUG" ] && set -x
-
-# first argument is the old version; a path 15 or 16
-if [[ $1 == /nix/store* ]]; then
-    if [ ! -L "$1/receipt.json" ] || [ ! -e "$1/receipt.json" ]; then
-        echo "ERROR: $1 does not look like a valid Postgres install"
+[ ! -z "$DEBUG" ] && set -eoux pipefail 
+
+# Function to build flake and return the output path
+build_flake() {
+    local flake_url="$1"
+    local temp_dir=$(mktemp -d)
+    if ! nix build "$flake_url" -o "$temp_dir/result"; then
+        echo "ERROR: Failed to build flake $flake_url"
         exit 1
     fi
-    OLDVER="$1"
-elif [ "$1" == "15" ]; then
-    PSQL15=@PSQL15_BINDIR@
-    OLDVER="$PSQL15"
-elif [ "$1" == "16" ]; then
-    PSQL16=@PSQL16_BINDIR@
-    OLDVER="$PSQL16"
-else
-    echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path"
-    exit 1
-fi
+    echo "$temp_dir/result"
+}
 
-# second argument is the new version; 15 or 16
-if [[ $2 == /nix/store* ]]; then
-    if [ ! -L "$2/receipt.json" ] || [ ! -e "$2/receipt.json" ]; then
-        echo "ERROR: $1 does not look like a valid Postgres install"
-        exit 1
-    fi
-    NEWVER="$2"
-elif [ "$2" == "15" ]; then
-    PSQL15=@PSQL15_BINDIR@
-    NEWVER="$PSQL15"
-elif [ "$2" == "16" ]; then
-    PSQL16=@PSQL16_BINDIR@
-    NEWVER="$PSQL16"
-    echo "NEWVER IS $NEWVER"
-else
-    echo "Please provide a valid Postgres version (15 or 16), or a /nix/store path"
-    exit 1
-fi
+# First argument is the old version flake URL
+OLDVER=$(build_flake "$1")
+
+# Second argument is the new version flake URL
+NEWVER=$(build_flake "$2")
 
-# thid argument is the upgrade method: either pg_dumpall or pg_ugprade
+# Third argument is the upgrade method: either pg_dumpall or pg_upgrade
 if [ "$3" != "pg_dumpall" ] && [ "$3" != "pg_upgrade" ]; then
     echo "Please provide a valid upgrade method (pg_dumpall or pg_upgrade)"
     exit 1
 fi
 UPGRADE_METHOD="$3"
 
-echo "Old server build: PSQL $1"
-echo "New server build: PSQL $2"
+echo "Old server build: $OLDVER"
+echo "New server build: $NEWVER"
 echo "Upgrade method: $UPGRADE_METHOD"
 
-PORTNO="${2:-@PGSQL_DEFAULT_PORT@}"
+PORTNO="@PGSQL_DEFAULT_PORT@"
 DATDIR=$(mktemp -d)
 NEWDAT=$(mktemp -d)
+PGUSER=${PGUSER:-postgres}
+POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+PG_PROVE=@PG_PROVE@
+PGUPGRADE_TESTS=@PGUPGRADE_TESTS@
 mkdir -p "$DATDIR" "$NEWDAT"
 
-echo "NOTE: using temporary directory $DATDIR for PSQL $1 data, which will not be removed"
-echo "NOTE: you are free to re-use this data directory at will"
+echo "using temporary directory $DATDIR for old data, which will not be removed"
+echo "you are free to re-use this data directory at will"
 echo
 
-$OLDVER/bin/initdb -D "$DATDIR" --locale=C
-$NEWVER/bin/initdb -D "$NEWDAT" --locale=C
+echo "PGUSER IS $PGUSER"
+
+$OLDVER/bin/initdb -U "$PGUSER" -D "$DATDIR" --locale=C
+$NEWVER/bin/initdb -U "$PGUSER" -D "$NEWDAT" --locale=C
 
 # NOTE (aseipp): we need to patch postgresql.conf to have the right pgsodium_getkey script
 PSQL_CONF_FILE=@PSQL_CONF_FILE@
 PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@
-echo "NOTE: patching postgresql.conf files"
+echo "patching postgresql.conf files"
 for x in "$DATDIR" "$NEWDAT"; do
   sed \
     "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \
     $PSQL_CONF_FILE > "$x/postgresql.conf"
 done
 
-echo "NOTE: Starting first server (v${1}) to load data into the system"
+echo "Starting first server to load data into the system"
 $OLDVER/bin/pg_ctl start -D "$DATDIR"
 
 PRIMING_SCRIPT=@PRIMING_SCRIPT@
-MIGRATION_DATA=@MIGRATION_DATA@
+MIGRATIONS_DIR=@MIGRATIONS_DIR@
+
+echo "MIGRATIONS_DIR IS $MIGRATIONS_DIR"
+
+for sql in "$MIGRATIONS_DIR"/db/init-scripts/*.sql; do
+    echo "$0: running $sql"
+    $OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -f "$sql"
+done
 
-$OLDVER/bin/psql -h localhost -d postgres -Xf "$PRIMING_SCRIPT"
-$OLDVER/bin/psql -h localhost -d postgres -Xf "$MIGRATION_DATA"
+$OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
+# run migrations as super user - postgres user demoted in post-setup
+for sql in "$MIGRATIONS_DIR"/db/migrations/*.sql; do
+    echo "$0: running $sql"
+    $OLDVER/bin/psql -h localhost -d postgres -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -f "$sql"
+done
 
 if [ "$UPGRADE_METHOD" == "pg_upgrade" ]; then
-  echo "NOTE: Stopping old server (v${1}) to prepare for migration"
+  echo "Stopping old server"
   $OLDVER/bin/pg_ctl stop -D "$DATDIR"
 
-  echo "NOTE: Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
+  echo "Starting old server"
+  $OLDVER/bin/pg_ctl start -D "$DATDIR" 
+
+  echo "Ensuring $PGUSER is a superuser in the old database"
+  "$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "ALTER USER $PGUSER WITH SUPERUSER;" || true
+
+  echo "Running pre-migration checks"
+  "$PG_PROVE"/bin/pg_prove --psql="$OLDVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 "$MIGRATIONS_DIR"/tests/test.sql
+
+  echo "Running fixtures"
+  "$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/97-enable-extensions.sql"
+  "$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/98-data-fixtures.sql"
+  "$OLDVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$PGUPGRADE_TESTS/pg_upgrade/tests/99-fixtures.sql"
+
+  echo "Stopping old server"
+  $OLDVER/bin/pg_ctl stop -D "$DATDIR"
+
+  echo "Migrating old data $DATDIR to $NEWDAT using pg_upgrade"
 
   export PGDATAOLD="$DATDIR"
   export PGDATANEW="$NEWDAT"
   export PGBINOLD="$OLDVER/bin"
   export PGBINNEW="$NEWVER/bin"
 
-  if ! $NEWVER/bin/pg_upgrade --check; then
+  # Create a temporary directory for pg_upgrade to work in
+  UPGRADE_WORKDIR=$(mktemp -d)
+  echo "Using temporary directory for pg_upgrade: $UPGRADE_WORKDIR"
+
+  # Change to the temporary directory before running pg_upgrade
+  pushd "$UPGRADE_WORKDIR"
+
+  if ! $NEWVER/bin/pg_upgrade -U "$PGUSER" --check; then
       echo "ERROR: pg_upgrade check failed"
+      popd
       exit 1
   fi
 
-  echo "NOTE: pg_upgrade check passed, proceeding with migration"
-  $NEWVER/bin/pg_upgrade
-  rm -f delete_old_cluster.sh # we don't need this
+  echo "pg_upgrade check passed, proceeding with migration"
+  $NEWVER/bin/pg_upgrade -U "$PGUSER"
+
+  # Change back to the original directory
+  popd
+  echo "Migration complete, running post-migration checks"
+  echo "NEWDAT IS $NEWDAT"
+  $NEWVER/bin/pg_ctl start -D "$NEWDAT"
+  echo "Turning off JIT"
+  cat << EOF > "$NEWDAT"/jit_off.sql
+ALTER SYSTEM SET jit = off;
+SELECT pg_reload_conf();
+EOF
+  "$NEWVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -f "$NEWDAT"/jit_off.sql  
+
+  echo "Setting password encryption method to scram-sha-256"
+  "$NEWVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "ALTER SYSTEM SET password_encryption = 'scram-sha-256';"
+  "$NEWVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "SELECT pg_reload_conf();"
+
+  echo "Running post-migration data checks"
+  "$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
+    "$PGUPGRADE_TESTS/pg_upgrade/tests/01-schema.sql"
+  "$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
+    "$PGUPGRADE_TESTS/pg_upgrade/tests/02-data.sql"
+  "$PG_PROVE"/bin/pg_prove --psql="$NEWVER"/bin/psql -h localhost -U supabase_admin -d postgres -p 5432 \
+    "$PGUPGRADE_TESTS/pg_upgrade/tests/03-settings.sql"
+
+  # echo "Dumping database statistics for debugging"
+  # "$NEWVER"/bin/psql -h localhost -U supabase_admin -p 5432 -d postgres -c "SELECT schemaname, relname, n_live_tup FROM pg_stat_user_tables ORDER BY n_live_tup DESC;"
+
+  $NEWVER/bin/pg_ctl stop -D "$NEWDAT"
   exit 0
 fi
 
-if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
-    SQLDAT="$DATDIR/dump.sql"
-    echo "NOTE: Exporting data via pg_dumpall ($SQLDAT)"
-    $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
+# if [ "$UPGRADE_METHOD" == "pg_dumpall" ]; then
+#     SQLDAT="$DATDIR/dump.sql"
+#     echo "Exporting data via pg_dumpall ($SQLDAT)"
+#     $NEWVER/bin/pg_dumpall -h localhost > "$SQLDAT"
 
-    echo "NOTE: Stopping old server (v${1}) to prepare for migration"
-    $OLDVER/bin/pg_ctl stop -D "$DATDIR"
+#     echo "Stopping old server (v${1}) to prepare for migration"
+#     $OLDVER/bin/pg_ctl stop -D "$DATDIR"
 
-    echo "NOTE: Starting second server (v${2}) to load data into the system"
-    $NEWVER/bin/pg_ctl start -D "$NEWDAT"
+#     echo "Starting second server (v${2}) to load data into the system"
+#     $NEWVER/bin/pg_ctl start -D "$NEWDAT"
 
-    echo "NOTE: Loading data into new server (v${2}) via 'cat | psql'"
-    cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
+#     echo "Loading data into new server (v${2}) via 'cat | psql'"
+#     cat "$SQLDAT" | $NEWVER/bin/psql -h localhost -d postgres
 
-    printf "\n\n\n\n"
-    echo "NOTE: Done, check logs. Stopping the server; new database is located at $NEWDAT"
-    $NEWVER/bin/pg_ctl stop -D "$NEWDAT"
-fi
+#     printf "\n\n\n\n"
+#     echo "Done, check logs. Stopping the server; new database is located at $NEWDAT"
+#     $NEWVER/bin/pg_ctl stop -D "$NEWDAT"
+# fi