diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1457eb100..d22b16f16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,10 @@ jobs: uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - name: Set PostgreSQL versions id: set-versions @@ -30,9 +34,9 @@ jobs: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - - runner: [self-hosted, X64] + - runner: ubuntu-22.04 arch: amd64 - - runner: arm-runner + - runner: ubuntu-22.04 arch: arm64 runs-on: ${{ matrix.runner }} timeout-minutes: 180 @@ -40,9 +44,14 @@ jobs: POSTGRES_PORT: 5478 POSTGRES_PASSWORD: password steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main + with: + extra-conf: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV @@ -54,7 +63,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file @@ -67,96 +76,16 @@ jobs: - name: Generate args id: args run: | - ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) + ARGS=$(nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - run: docker context create builders - - uses: docker/setup-buildx-action@v3 - with: - endpoint: builders - - uses: docker/build-push-action@v5 - with: - load: true - context: . - file: Dockerfile-${{ env.PGMAJOR }} - target: production - build-args: | - ${{ steps.args.outputs.result }} - tags: supabase/postgres:${{ steps.settings.outputs.postgres-version }},supabase_postgres - cache-from: | - type=gha,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }} - type=gha,scope=${{ github.base_ref }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }} - - - name: Start Postgres - run: | - docker run --rm --pull=never \ - -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \ - -p ${{ env.POSTGRES_PORT }}:5432 \ - --name supabase_postgres \ - -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} - - - name: Install psql - run: | - sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - - sudo apt update - sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - - - name: Install pg_prove - run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP - env: - SHELL: /bin/bash - - - name: Wait for healthy database - run: | - count=0 - until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do - exit=$? - count=$((count + 1)) - if [ $count -ge "$retries" ]; then - echo "Retry $count/$retries exited $exit, no more retries left." - docker stop -t 2 "$container" - return $exit - fi - sleep 1; - done; - echo "$container container is healthy" - env: - retries: 20 - container: supabase_postgres - - - name: Run tests - run: pg_prove migrations/tests/test.sql - env: - PGHOST: localhost - PGPORT: ${{ env.POSTGRES_PORT }} - PGDATABASE: postgres - PGUSER: supabase_admin - PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - - - name: Check migrations are idempotent - run: | - for sql in ./migrations/db/migrations/*.sql; do - echo "$0: running $sql" - psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql" - done - env: - PGHOST: localhost - PGPORT: ${{ env.POSTGRES_PORT }} - PGDATABASE: postgres - PGUSER: supabase_admin - PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - - - name: Update Dockerfile.dbmate version - run: | - sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - + #TODO PR Convert to develop branch flakeurl - name: verify schema.sql is committed run: | - docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit + GIT_SHA=${{github.sha}} + nix run github:supabase/postgres/${GIT_SHA}#dbmate-tool -- --version ${{ env.PGMAJOR }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/.gitignore b/.gitignore index a6950d2ff..ae375b82a 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,5 @@ result* #IDE .idea/ .vscode/ + +db \ No newline at end of file diff --git a/flake.nix b/flake.nix index f1cff1abd..ac639a336 100644 --- a/flake.nix +++ b/flake.nix @@ -524,7 +524,7 @@ chmod +x $out/bin/pg-restore ''; sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' - mkdir -p $out/bin + mkdir -p $out/bin substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ @@ -533,8 +533,42 @@ --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix' chmod +x $out/bin/sync-exts-versions ''; + dbmate-tool = + let + migrationsDir = ./migrations/db; + ansibleVars = ./ansible/vars.yml; + pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + statExtensionSql = ./ansible/files/stat_extension.sql; + in + pkgs.runCommand "dbmate-tool" { + buildInputs = with pkgs; [ + overmind + dbmate + nix + jq + yq + ]; + nativeBuildInputs = with pkgs; [ + makeWrapper + ]; + } '' + mkdir -p $out/bin $out/migrations + cp -r ${migrationsDir}/* $out + substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ + --subst-var-by 'MIGRATIONS_DIR' $out \ + --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ + --subst-var-by 'CURRENT_SYSTEM' '${system}' \ + --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ + --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' + chmod +x $out/bin/dbmate-tool + wrapProgram $out/bin/dbmate-tool \ + --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]} + ''; }; + # Create a testing harness for a PostgreSQL package. This is used for # 'nix flake check', and works with any PostgreSQL package you hand it. makeCheckHarness = pgpkg: @@ -648,9 +682,11 @@ start-server = mkApp "start-server" "start-postgres-server"; start-client = mkApp "start-client" "start-postgres-client"; start-replica = mkApp "start-replica" "start-postgres-replica"; - migration-test = mkApp "migrate-tool" "migrate-postgres"; + migrate-postgres = mkApp "migrate-tool" "migrate-postgres"; sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; pg-restore = mkApp "pg-restore" "pg-restore"; + dbmate-tool = mkApp "dbmate-tool" "dbmate-tool"; + migration-unit-tests = mkApp "migration-unit-tests" "migration-unit-tests"; }; # 'devShells.default' lists the set of packages that are included in the @@ -689,6 +725,7 @@ basePackages.start-replica basePackages.migrate-tool basePackages.sync-exts-versions + dbmate ]; shellHook = '' export HISTFILE=.history diff --git a/migrations/README.md b/migrations/README.md index bfd7308d4..19d2bf4b3 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,3 +1,36 @@ +# Usage + +from the root of the `supabase/postgres` project, you can run the following commands: + + +```shell +Usage: nix run .#dbmate-tool -- [options] + +Options: + -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all) + -p, --port PORT Specify the port number to use (default: 5435) + -h, --help Show this help message + +Description: + Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions. + NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file. + +Examples: + nix run .#dbmate-tool + nix run .#dbmate-tool -- --version 15 + nix run .#dbmate-tool -- --version 16 --port 5433 + +``` + +This can also be run from a github "flake url" for example: + +```shell +nix run github:supabase/postgres#dbmate-tool -- --version 15 + +or + +nix run github:supabase/postgres/mybranch#dbmate-tool -- --version 15 +``` # supabase/migrations `supabase/migrations` is a consolidation of SQL migrations from: @@ -9,6 +42,8 @@ aiming to provide a single source of truth for migrations on the platform that can be depended upon by those components. For more information on goals see [the RFC](https://www.notion.so/supabase/Centralize-SQL-Migrations-cd3847ae027d4f2bba9defb2cc82f69a) + + ## How it was Created Migrations were pulled (in order) from: @@ -20,10 +55,12 @@ For compatibility with hosted projects, we include [migrate.sh](migrate.sh) that 1. Run all `db/init-scripts` with `postgres` superuser role. 2. Run all `db/migrations` with `supabase_admin` superuser role. -3. Finalize role passwords with `/etc/postgres.schema.sql` if present. +3. Finalize role passwords with `/etc/postgresql.schema.sql` if present. Additionally, [supabase/postgres](https://github.com/supabase/postgres/blob/develop/ansible/playbook-docker.yml#L9) image contains several migration scripts to configure default extensions. These are run first by docker entrypoint and included in ami by ansible. + + ## Guidelines - Migrations are append only. Never edit existing migrations once they are on master. diff --git a/nix/tools/dbmate-tool.sh.in b/nix/tools/dbmate-tool.sh.in new file mode 100644 index 000000000..b7aa0d26e --- /dev/null +++ b/nix/tools/dbmate-tool.sh.in @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +[ ! -z "$DEBUG" ] && set -x + +# Default values +PSQL_VERSION="ALL" +PORTNO="@PGSQL_DEFAULT_PORT@" +PGSQL_SUPERUSER="@PGSQL_SUPERUSER@" +PGPASSWORD="${PGPASSWORD:-postgres}" +PGSQL_USER="postgres" +FLAKE_URL="github:supabase/postgres" +MIGRATIONS_DIR="@MIGRATIONS_DIR@" +CURRENT_SYSTEM="@CURRENT_SYSTEM@" +ANSIBLE_VARS="@ANSIBLE_VARS@" +PGBOUNCER_AUTH_SCHEMA_SQL=@PGBOUNCER_AUTH_SCHEMA_SQL@ +STAT_EXTENSION_SQL=@STAT_EXTENSION_SQL@ +# Cleanup function +cleanup() { + echo "Cleaning up..." + + # Kill overmind processes first + if [ -S "./.overmind.sock" ]; then + overmind kill || true + sleep 2 + fi + + # Kill any remaining postgres processes + echo "Killing any remaining postgres processes..." + pkill -9 postgres || true + pkill -9 -f "tmux.*overmind.*postgresql" || true + + # Extra cleanup for tmux sessions + tmux ls 2>/dev/null | grep 'overmind' | cut -d: -f1 | xargs -I{} tmux kill-session -t {} || true + + # Remove socket and Procfile + rm -f .overmind.sock Procfile + + # Verify cleanup + remaining=$(ps aux | grep -E "(postgres|overmind|tmux.*postgresql)" | grep -v grep || true) + if [ ! -z "$remaining" ]; then + echo "Warning: Some processes might still be running:" + echo "$remaining" + fi +} + +# Set up trap for cleanup on script exit + +# Function to display help +print_help() { + echo "Usage: nix run .#dbmate-tool -- [options]" + echo + echo "Options:" + echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)" + echo " -p, --port PORT Specify the port number to use (default: 5435)" + echo " -h, --help Show this help message" + echo + echo "Description:" + echo " Runs 'dbmate up' against a locally running the version of database you specify. Or 'all' to run against all versions." + echo " NOTE: To create a migration, you must run 'nix develop' and then 'dbmate new ' to create a new migration file." + echo + echo "Examples:" + echo " nix run .#dbmate-tool" + echo " nix run .#dbmate-tool -- --version 15" + echo " nix run .#dbmate-tool -- --version 16 --port 5433" +} + + +# Parse arguments +while [[ "$#" -gt 0 ]]; do + case "$1" in + -v|--version) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PSQL_VERSION="$2" + shift 2 + else + echo "Error: --version requires an argument (15, 16, or orioledb-17)" + exit 1 + fi + ;; + -u|--user) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PGSQL_USER="$2" + shift 2 + else + echo "Error: --user requires an argument" + exit 1 + fi + ;; + -f|--flake-url) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + FLAKE_URL="$2" + shift 2 + else + echo "Error: --flake-url requires an argument" + exit 1 + fi + ;; + -p|--port) + if [[ -n "$2" && ! "$2" =~ ^- ]]; then + PORTNO="$2" + shift 2 + else + echo "Error: --port requires an argument" + exit 1 + fi + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo "Unknown option: $1" + print_help + exit 1 + ;; + esac +done + +# Function to wait for PostgreSQL to be ready +wait_for_postgres() { + local max_attempts=30 # Increased significantly + local attempt=1 + + # Give overmind a moment to actually start the process + sleep 2 + + while [ $attempt -le $max_attempts ]; do + "${PSQLBIN}/pg_isready" -h localhost -p "$PORTNO" -U "$PGSQL_SUPERUSER" -d postgres + local status=$? + + if [ $status -eq 0 ]; then + echo "PostgreSQL is ready!" + return 0 + fi + echo "Waiting for PostgreSQL to start (attempt $attempt/$max_attempts)..." + sleep 2 + attempt=$((attempt + 1)) + done + + echo "PostgreSQL failed to start after $max_attempts attempts" + overmind echo postgres + return 1 +} + +trim_schema() { + case "$CURRENT_SYSTEM" in + "x86_64-darwin"|"aarch64-darwin") + sed -i '' '/INSERT INTO public.schema_migrations/,$d' "./db/schema.sql" + echo "Matched: $CURRENT_SYSTEM" + ;; + *) + sed -i '/INSERT INTO public.schema_migrations/,$d' "./db/schema.sql" + ;; + esac +} +overmind_start() { + cat > Procfile << EOF +postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" "$PSQL_VERSION" +EOF + overmind start -D + echo "Waiting for overmind socket..." + max_wait=5 + count=0 + while [ $count -lt $max_wait ]; do + if [ -S "./.overmind.sock" ]; then + # Found the socket, give it a moment to be ready + sleep 2 + echo "Socket file found and ready" + break + fi + echo "Waiting for socket file (attempt $count/$max_wait)" + sleep 1 + count=$((count + 1)) + done +} +migrate_version() { + echo "PSQL_VERSION: $PSQL_VERSION" + overmind kill || true + rm -f .overmind.sock Procfile || true + PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"') + echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN" + + # Start overmind + overmind_start + echo "Waiting for overmind socket..." + + + echo "Waiting for PostgreSQL to be ready..." + + #Wait for PostgreSQL to be ready to accept connections + if ! wait_for_postgres; then + echo "Failed to connect to PostgreSQL server" + exit 1 + fi + + echo "PostgreSQL server is ready" + + # Configure PostgreSQL roles and permissions + if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL +create role postgres superuser login password '$PGPASSWORD'; +alter database postgres owner to postgres; +EOSQL + then + echo "Failed to configure PostgreSQL roles and permissions" + exit 1 + fi + "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$PGBOUNCER_AUTH_SCHEMA_SQL" + "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -d postgres -f "$STAT_EXTENSION_SQL" + + #set db url to run dbmate + export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" + #export path so dbmate can find correct psql and pg_dump + export PATH="$PSQLBIN:$PATH" + # run init scripts + if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then + echo "Error: Initial migration failed" + exit 1 + fi + + # Password update command + if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"; then + echo "Error: Failed to update supabase_admin password" + exit 1 + fi + + # Set up database URL + export DATABASE_URL="postgres://$PGSQL_SUPERUSER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable" + # Run migrations + if ! dbmate --migrations-dir "$MIGRATIONS_DIR/migrations" up; then + echo "Error: Final migration failed" + exit 1 + fi + + echo "Running dbmate dump with $PSQLBIN" + dbmate dump + + echo "CURRENT_SYSTEM: $CURRENT_SYSTEM" + if [ -f "./db/schema.sql" ]; then + trim_schema + cp "./db/schema.sql" "./migrations/schema-$PSQL_VERSION.sql" + echo "Schema file moved to ./migrations/schema-$PSQL_VERSION.sql" + echo "PSQLBIN is $PSQLBIN" + else + echo "Warning: schema.sql file not found in ./db directory" + exit 1 + fi + + # If we get here, all commands succeeded + echo "PostgreSQL migration completed successfully" + echo "Check migrations are idempotent" + for sql in ./migrations/db/migrations/*.sql; do + echo "$0: running $sql" + "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -f "$sql" || { + echo "Failed to execute $sql" + exit 1 + } + done +} + +if [ "$PSQL_VERSION" == "all" ]; then + VERSIONS=$(yq '.postgres_major[]' "$ANSIBLE_VARS" | tr -d '"') + echo "$VERSIONS" | while read -r version; do + PSQL_VERSION="$version" + echo "Migrating to PostgreSQL version $PSQL_VERSION" + migrate_version + cleanup + done +else + echo "Migrating to PostgreSQL version $PSQL_VERSION" + migrate_version + cleanup +fi diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 977a437fb..2b079bbf1 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -38,7 +38,6 @@ export LANGUAGE=en_US.UTF-8 export LC_ALL=en_US.UTF-8 export LANG=en_US.UTF-8 export LC_CTYPE=en_US.UTF-8 -mkdir -p "$DATDIR" echo "NOTE: using port $PORTNO for server" echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" echo "NOTE: you are free to re-use this data directory at will"