chore: cleanup of test and dbmage migration tool

Author: samrose

.github/workflows/test.yml

@@ -72,87 +72,6 @@ jobs:
           echo "$ARGS" >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
-      - run: docker context create builders
-      - uses: docker/setup-buildx-action@v3
-        with:
-          endpoint: builders
-      - uses: docker/build-push-action@v5
-        with:
-          load: true
-          context: .
-          file: Dockerfile-${{ env.PGMAJOR }}
-          target: production
-          build-args: |
-            ${{ steps.args.outputs.result }}
-          tags: supabase/postgres:${{ steps.settings.outputs.postgres-version }},supabase_postgres
-          cache-from: |
-            type=gha,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-            type=gha,scope=${{ github.base_ref }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-          cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }}
-
-      - name: Start Postgres
-        run: |
-          docker run --rm --pull=never \
-          -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \
-          -p ${{ env.POSTGRES_PORT }}:5432 \
-          --name supabase_postgres \
-          -d supabase/postgres:${{ steps.settings.outputs.postgres-version }}
-
-      - name: Install psql
-        run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-          sudo apt update
-          sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }}
-
-      - name: Install pg_prove
-        run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP
-        env:
-          SHELL: /bin/bash
-
-      - name: Wait for healthy database
-        run: |
-          count=0
-          until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do
-              exit=$?
-              count=$((count + 1))
-              if [ $count -ge "$retries" ]; then
-                  echo "Retry $count/$retries exited $exit, no more retries left."
-                  docker stop -t 2 "$container"
-                  return $exit
-              fi
-              sleep 1;
-          done;
-          echo "$container container is healthy"
-        env:
-          retries: 20
-          container: supabase_postgres
-
-      - name: Run tests
-        run: pg_prove migrations/tests/test.sql
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Check migrations are idempotent
-        run: |
-          for sql in ./migrations/db/migrations/*.sql; do
-            echo "$0: running $sql"
-            psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql"
-          done
-        env:
-          PGHOST: localhost
-          PGPORT: ${{ env.POSTGRES_PORT }}
-          PGDATABASE: postgres
-          PGUSER: supabase_admin
-          PGPASSWORD: ${{ env.POSTGRES_PASSWORD }}
-
-      - name: Update Dockerfile.dbmate version
-        run: |
-          sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate
       #TODO PR Convert to develop branch flakeurl
       - name: verify schema.sql is committed
         run: |

flake.nix

@@ -678,10 +678,11 @@
             start-server = mkApp "start-server" "start-postgres-server";
             start-client = mkApp "start-client" "start-postgres-client";
             start-replica = mkApp "start-replica" "start-postgres-replica";
-            migration-test = mkApp "migrate-tool" "migrate-postgres";
+            migrate-postgres = mkApp "migrate-tool" "migrate-postgres";
             sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions";
             pg-restore = mkApp "pg-restore" "pg-restore";
             dbmate-tool = mkApp "dbmate-tool" "dbmate-tool";
+            migration-unit-tests = mkApp "migration-unit-tests" "migration-unit-tests";
           };
 
         # 'devShells.default' lists the set of packages that are included in the

nix/tools/dbmate-tool.sh.in

@@ -7,13 +7,13 @@
 PSQL_VERSION="ALL"
 MIGRATION_FILE=""
 PORTNO="@PGSQL_DEFAULT_PORT@"
-PGSQL_SUPERUSER=@PGSQL_SUPERUSER@
+PGSQL_SUPERUSER="@PGSQL_SUPERUSER@"
 PGPASSWORD="${PGPASSWORD:-postgres}"
 PGSQL_USER="postgres"
 FLAKE_URL="github:supabase/postgres"
 MIGRATIONS_DIR="@MIGRATIONS_DIR@"
-CURRENT_SYSTEM=@CURRENT_SYSTEM@
-ANSIBLE_VARS=@ANSIBLE_VARS@
+CURRENT_SYSTEM="@CURRENT_SYSTEM@"
+ANSIBLE_VARS="@ANSIBLE_VARS@"
 # Cleanup function
 cleanup() {
     echo "Cleaning up..."
@@ -174,8 +174,9 @@ EOF
     done
 }
 migrate_version() {
+    echo "PSQL_VERSION: $PSQL_VERSION"
     overmind kill || true
-    rm -f .overmind.sock Procfile || true
+    rm -f .overmind.sock Procfile  || true
     PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"')
     echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN"
 
@@ -243,6 +244,14 @@ EOSQL
 
     # If we get here, all commands succeeded
     echo "PostgreSQL migration completed successfully"
+    echo "Check migrations are idempotent"
+    for sql in ./migrations/db/migrations/*.sql; do
+        echo "$0: running $sql"
+        "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -f "$sql" || {
+            echo "Failed to execute $sql"
+            exit 1
+        }
+    done
 }
 
 if [ "$PSQL_VERSION" == "all" ]; then

nix/tools/run-server.sh.in

@@ -2,15 +2,6 @@
 # shellcheck shell=bash
 [ ! -z "$DEBUG" ] && set -x
 # first argument should be '15' or '16' for the version
-cleanup() {
-    echo "Cleaning up temporary directory $DATDIR"
-    if [ -d "$DATDIR" ]; then
-        # Change permissions to allow removal
-        chmod -R 700 "$DATDIR" 2>/dev/null || true
-        rm -rf "$DATDIR" 2>/dev/null || sudo rm -rf "$DATDIR"
-    fi
-}
-trap cleanup EXIT
 if [ "$1" == "15" ]; then
     echo "Starting server for PSQL 15"
     PSQL15=@PSQL15_BINDIR@
@@ -47,7 +38,6 @@ export LANGUAGE=en_US.UTF-8
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 export LC_CTYPE=en_US.UTF-8
-mkdir -p "$DATDIR"
 echo "NOTE: using port $PORTNO for server"
 echo "NOTE: using temporary directory $DATDIR for data, which will not be removed"
 echo "NOTE: you are free to re-use this data directory at will"
@@ -59,6 +49,9 @@ cp "$READREPL_CONFIG_FILE" "$DATDIR/read-replica.conf"
 cp -r "$EXTENSION_CUSTOM_SCRIPTS" "$DATDIR"
 sed "s|supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'|supautils.privileged_extensions_custom_scripts_path = '$DATDIR/extension-custom-scripts'|" "$SUPAUTILS_CONFIG_FILE" > "$DATDIR/supautils.conf"
 sed -e "1i\\
+dynamic_shared_memory_type = mmap\\
+shared_buffers = '64MB'\\
+max_connections = 20\\
 include = '$DATDIR/supautils.conf'" \
 -e "\$a\\
 pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \