Skip to content

Commit 2089e13

Browse files
committed
feat: test against schema.sql per mjr vrs + tool to generate
1 parent 5c6e0aa commit 2089e13

File tree

6 files changed

+125
-157
lines changed

6 files changed

+125
-157
lines changed

.github/workflows/test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ jobs:
156156
157157
- name: verify schema.sql is committed
158158
run: |
159-
docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit
159+
sudo nix run .#dbmate-tool -- --version ${{ env.PGMAJOR }}
160160
if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then
161161
echo "Detected changes in schema.sql:"
162162
git diff migrations/schema-${{ env.PGMAJOR }}.sql

flake.nix

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -536,13 +536,15 @@
536536
dbmate-tool =
537537
let
538538
migrationsDir = ./migrations/db;
539+
ansibleVars = ./ansible/vars.yml;
539540
in
540541
pkgs.runCommand "dbmate-tool" {
541542
buildInputs = with pkgs; [
542543
overmind
543544
dbmate
544545
nix
545546
jq
547+
yq
546548
];
547549
nativeBuildInputs = with pkgs; [
548550
makeWrapper
@@ -553,10 +555,12 @@
553555
substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \
554556
--subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
555557
--subst-var-by 'MIGRATIONS_DIR' $out \
556-
--subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}'
558+
--subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
559+
--subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \
560+
--subst-var-by 'CURRENT_SYSTEM' '${system}'
557561
chmod +x $out/bin/dbmate-tool
558562
wrapProgram $out/bin/dbmate-tool \
559-
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq]}
563+
--prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]}
560564
'';
561565
};
562566

migrations/schema-15.sql

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,6 @@ CREATE SCHEMA graphql;
3737
CREATE SCHEMA graphql_public;
3838

3939

40-
--
41-
-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: -
42-
--
43-
44-
CREATE SCHEMA pgbouncer;
45-
46-
4740
--
4841
-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: -
4942
--
@@ -100,20 +93,6 @@ CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql;
10093
COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support';
10194

10295

103-
--
104-
-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: -
105-
--
106-
107-
CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions;
108-
109-
110-
--
111-
-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: -
112-
--
113-
114-
COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed';
115-
116-
11796
--
11897
-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: -
11998
--
@@ -490,23 +469,6 @@ $_$;
490469
COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql';
491470

492471

493-
--
494-
-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: -
495-
--
496-
497-
CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text)
498-
LANGUAGE plpgsql SECURITY DEFINER
499-
AS $$
500-
BEGIN
501-
RAISE WARNING 'PgBouncer auth request: %', p_usename;
502-
503-
RETURN QUERY
504-
SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow
505-
WHERE usename = p_usename;
506-
END;
507-
$$;
508-
509-
510472
--
511473
-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: -
512474
--

migrations/schema-16.sql

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,6 @@ CREATE SCHEMA graphql;
3737
CREATE SCHEMA graphql_public;
3838

3939

40-
--
41-
-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: -
42-
--
43-
44-
CREATE SCHEMA pgbouncer;
45-
46-
4740
--
4841
-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: -
4942
--
@@ -100,20 +93,6 @@ CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql;
10093
COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support';
10194

10295

103-
--
104-
-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: -
105-
--
106-
107-
CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions;
108-
109-
110-
--
111-
-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: -
112-
--
113-
114-
COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed';
115-
116-
11796
--
11897
-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: -
11998
--
@@ -490,23 +469,6 @@ $_$;
490469
COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql';
491470

492471

493-
--
494-
-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: -
495-
--
496-
497-
CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text)
498-
LANGUAGE plpgsql SECURITY DEFINER
499-
AS $$
500-
BEGIN
501-
RAISE WARNING 'PgBouncer auth request: %', p_usename;
502-
503-
RETURN QUERY
504-
SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow
505-
WHERE usename = p_usename;
506-
END;
507-
$$;
508-
509-
510472
--
511473
-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: -
512474
--

nix/tools/dbmate-tool.sh.in

Lines changed: 109 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@
44
[ ! -z "$DEBUG" ] && set -x
55

66
# Default values
7-
PSQL_VERSION="15"
7+
PSQL_VERSION="ALL"
88
MIGRATION_FILE=""
99
PORTNO="@PGSQL_DEFAULT_PORT@"
1010
PGSQL_SUPERUSER=@PGSQL_SUPERUSER@
1111
PGPASSWORD="${PGPASSWORD:-postgres}"
1212
PGSQL_USER="postgres"
1313
FLAKE_URL="github:supabase/postgres"
1414
MIGRATIONS_DIR="@MIGRATIONS_DIR@"
15-
15+
CURRENT_SYSTEM=@CURRENT_SYSTEM@
16+
ANSIBLE_VARS=@ANSIBLE_VARS@
1617
# Cleanup function
1718
cleanup() {
1819
echo "Cleaning up..."
@@ -43,14 +44,13 @@ cleanup() {
4344
}
4445

4546
# Set up trap for cleanup on script exit
46-
trap cleanup EXIT
4747

4848
# Function to display help
4949
print_help() {
5050
echo "Usage: nix run .#dbmate-tool -- [options]"
5151
echo
5252
echo "Options:"
53-
echo " -v, --version [15|16|orioledb-17] Specify the PostgreSQL version to use (required defaults to --all)"
53+
echo " -v, --version [15|16|orioledb-17|all] Specify the PostgreSQL version to use (required defaults to --version all)"
5454
echo " -p, --port PORT Specify the port number to use (default: 5435)"
5555
echo " -h, --help Show this help message"
5656
echo
@@ -116,8 +116,6 @@ while [[ "$#" -gt 0 ]]; do
116116
esac
117117
done
118118

119-
PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"')
120-
echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN"
121119
# Function to wait for PostgreSQL to be ready
122120
wait_for_postgres() {
123121
local max_attempts=30 # Increased significantly
@@ -144,89 +142,122 @@ wait_for_postgres() {
144142
return 1
145143
}
146144

147-
# Create Procfile
148-
cat > Procfile << EOF
145+
trim_schema() {
146+
case "$CURRENT_SYSTEM" in
147+
"x86_64-darwin"|"aarch64-darwin")
148+
sed -i '' '/INSERT INTO public.schema_migrations/,$d' "./db/schema.sql"
149+
echo "Matched: $CURRENT_SYSTEM"
150+
;;
151+
*)
152+
sed -i '/INSERT INTO public.schema_migrations/,$d' "./db/schema.sql"
153+
;;
154+
esac
155+
}
156+
migrate_version() {
157+
overmind kill || true
158+
rm -f .overmind.sock Procfile || true
159+
PSQLBIN=$(nix build --no-link "$FLAKE_URL#psql_$PSQL_VERSION/bin" --json | jq -r '.[].outputs.out + "/bin"')
160+
echo "Using PostgreSQL version $PSQL_VERSION from $PSQLBIN"
161+
162+
# Create Procfile
163+
cat > Procfile << EOF
149164
postgres_${PSQL_VERSION}: exec nix run "$FLAKE_URL#start-server" "$PSQL_VERSION"
150165
EOF
151166

152-
echo "Starting PostgreSQL server with: exec nix run $FLAKE_URL#start-server $PSQL_VERSION"
153-
cat Procfile
167+
echo "Starting PostgreSQL server with: exec nix run $FLAKE_URL#start-server $PSQL_VERSION"
168+
cat Procfile
154169

155-
# Start services with Overmind
156-
overmind start -D
170+
# Start services with Overmind
171+
overmind start -D
157172

158173

159-
echo "Waiting for overmind socket..."
160-
max_wait=5
161-
count=0
162-
while [ $count -lt $max_wait ]; do
163-
if [ -S "./.overmind.sock" ]; then
164-
# Found the socket, give it a moment to be ready
165-
sleep 2
166-
echo "Socket file found and ready"
167-
break
168-
fi
169-
echo "Waiting for socket file (attempt $count/$max_wait)"
170-
sleep 1
171-
count=$((count + 1))
172-
done
174+
echo "Waiting for overmind socket..."
175+
max_wait=5
176+
count=0
177+
while [ $count -lt $max_wait ]; do
178+
if [ -S "./.overmind.sock" ]; then
179+
# Found the socket, give it a moment to be ready
180+
sleep 2
181+
echo "Socket file found and ready"
182+
break
183+
fi
184+
echo "Waiting for socket file (attempt $count/$max_wait)"
185+
sleep 1
186+
count=$((count + 1))
187+
done
173188

174189

175-
echo "Waiting for PostgreSQL to be ready..."
190+
echo "Waiting for PostgreSQL to be ready..."
176191

177-
#Wait for PostgreSQL to be ready to accept connections
178-
if ! wait_for_postgres; then
179-
echo "Failed to connect to PostgreSQL server"
180-
exit 1
181-
fi
192+
#Wait for PostgreSQL to be ready to accept connections
193+
if ! wait_for_postgres; then
194+
echo "Failed to connect to PostgreSQL server"
195+
exit 1
196+
fi
182197

183-
echo "PostgreSQL server is ready"
198+
echo "PostgreSQL server is ready"
184199

185-
# Configure PostgreSQL roles and permissions
186-
if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL
187-
create role postgres superuser login password '$PGPASSWORD';
188-
alter database postgres owner to postgres;
200+
# Configure PostgreSQL roles and permissions
201+
if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres <<-EOSQL
202+
create role postgres superuser login password '$PGPASSWORD';
203+
alter database postgres owner to postgres;
189204
EOSQL
190-
then
191-
echo "Failed to configure PostgreSQL roles and permissions"
192-
exit 1
193-
fi
194-
#set db url to run dbmate
195-
export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable"
196-
197-
ls -la "$MIGRATIONS_DIR"
198-
pwd
199-
# First dbmate command
200-
if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then
201-
echo "Error: Initial migration failed"
202-
exit 1
203-
fi
204-
205-
# Password update command
206-
if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"; then
207-
echo "Error: Failed to update supabase_admin password"
208-
exit 1
209-
fi
210-
211-
# Set up database URL
212-
export DATABASE_URL="postgres://$PGSQL_SUPERUSER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable"
213-
214-
# Second dbmate command
215-
if ! dbmate --migrations-dir "$MIGRATIONS_DIR/migrations" up; then
216-
echo "Error: Final migration failed"
217-
exit 1
218-
fi
219-
220-
if [ -f "./db/schema.sql" ]; then
221-
mv "./db/schema.sql" "./db/schema-$PSQL_VERSION.sql"
222-
rm -rf ./db
223-
echo "Schema file moved to ./migrations/schema-$PSQL_VERSION.sql"
224-
else
225-
echo "Warning: schema.sql file not found in ./db directory"
226-
exit 1
227-
fi
205+
then
206+
echo "Failed to configure PostgreSQL roles and permissions"
207+
exit 1
208+
fi
209+
#set db url to run dbmate
210+
export DATABASE_URL="postgres://$PGSQL_USER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable"
211+
#export path so dbmate can find correct psql and pg_dump
212+
export PATH="$PSQLBIN:$PATH"
213+
# run init scripts
214+
if ! dbmate --migrations-dir "$MIGRATIONS_DIR/init-scripts" up; then
215+
echo "Error: Initial migration failed"
216+
exit 1
217+
fi
218+
219+
# Password update command
220+
if ! "${PSQLBIN}/psql" -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -p "$PORTNO" -h localhost -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"; then
221+
echo "Error: Failed to update supabase_admin password"
222+
exit 1
223+
fi
224+
225+
# Set up database URL
226+
export DATABASE_URL="postgres://$PGSQL_SUPERUSER:$PGPASSWORD@localhost:$PORTNO/postgres?sslmode=disable"
227+
# Run migrations
228+
if ! dbmate --migrations-dir "$MIGRATIONS_DIR/migrations" up; then
229+
echo "Error: Final migration failed"
230+
exit 1
231+
fi
228232

229-
"${PSQLBIN}/pg_dump" -U "$PGSQL_SUPERUSER" -p "$PORTNO" -h localhost -d postgres -s -f "./pg-dump-schema-$PSQL_VERSION.sql"
233+
echo "Running dbmate dump with $PSQLBIN"
234+
dbmate dump
230235

231-
# If we get here, all commands succeeded
232-
echo "PostgreSQL migration completed successfully"
236+
echo "CURRENT_SYSTEM: $CURRENT_SYSTEM"
237+
if [ -f "./db/schema.sql" ]; then
238+
trim_schema
239+
cp "./db/schema.sql" "./migrations/schema-$PSQL_VERSION.sql"
240+
echo "Schema file moved to ./migrations/schema-$PSQL_VERSION.sql"
241+
echo "PSQLBIN is $PSQLBIN"
242+
else
243+
echo "Warning: schema.sql file not found in ./db directory"
244+
exit 1
245+
fi
246+
247+
# If we get here, all commands succeeded
248+
echo "PostgreSQL migration completed successfully"
249+
}
250+
251+
if [ "$PSQL_VERSION" == "all" ]; then
252+
VERSIONS=$(yq '.postgres_major[]' "$ANSIBLE_VARS" | tr -d '"')
253+
echo "$VERSIONS" | while read -r version; do
254+
PSQL_VERSION="$version"
255+
echo "Migrating to PostgreSQL version $PSQL_VERSION"
256+
migrate_version
257+
cleanup
258+
done
259+
else
260+
echo "Migrating to PostgreSQL version $PSQL_VERSION"
261+
migrate_version
262+
cleanup
263+
fi

0 commit comments

Comments
 (0)