Skip to content

Commit a247689

Browse files
samrosesamrose
committed
chore: refactoring to create reusable makePostgresDevSetup
1 parent 924ce92 commit a247689

File tree

2 files changed

+197
-186
lines changed

2 files changed

+197
-186
lines changed

‎flake.nix‎

Lines changed: 195 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,116 @@
260260
recurseForDerivations = true;
261261
};
262262

263+
makePostgresDevSetup = { pkgs, name, extraSubstitutions ? {} }:
264+
let
265+
paths = {
266+
migrationsDir = builtins.path {
267+
name = "migrations";
268+
path = ./migrations/db;
269+
};
270+
postgresqlSchemaSql = builtins.path {
271+
name = "postgresql-schema";
272+
path = ./nix/tools/postgresql_schema.sql;
273+
};
274+
pgbouncerAuthSchemaSql = builtins.path {
275+
name = "pgbouncer-auth-schema";
276+
path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
277+
};
278+
statExtensionSql = builtins.path {
279+
name = "stat-extension";
280+
path = ./ansible/files/stat_extension.sql;
281+
};
282+
pgconfigFile = builtins.path {
283+
name = "postgresql.conf";
284+
path = ./ansible/files/postgresql_config/postgresql.conf.j2;
285+
};
286+
supautilsConfigFile = builtins.path {
287+
name = "supautils.conf";
288+
path = ./ansible/files/postgresql_config/supautils.conf.j2;
289+
};
290+
loggingConfigFile = builtins.path {
291+
name = "logging.conf";
292+
path = ./ansible/files/postgresql_config/postgresql-csvlog.conf;
293+
};
294+
readReplicaConfigFile = builtins.path {
295+
name = "readreplica.conf";
296+
path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2;
297+
};
298+
pgHbaConfigFile = builtins.path {
299+
name = "pg_hba.conf";
300+
path = ./ansible/files/postgresql_config/pg_hba.conf.j2;
301+
};
302+
pgIdentConfigFile = builtins.path {
303+
name = "pg_ident.conf";
304+
path = ./ansible/files/postgresql_config/pg_ident.conf.j2;
305+
};
306+
postgresqlExtensionCustomScriptsPath = builtins.path {
307+
name = "extension-custom-scripts";
308+
path = ./ansible/files/postgresql_extension_custom_scripts;
309+
};
310+
getkeyScript = builtins.path {
311+
name = "pgsodium_getkey.sh";
312+
path = ./nix/tests/util/pgsodium_getkey.sh;
313+
};
314+
};
315+
316+
localeArchive = if pkgs.stdenv.isDarwin
317+
then "${pkgs.darwin.locale}/share/locale"
318+
else "${pkgs.glibcLocales}/lib/locale/locale-archive";
319+
320+
substitutions = {
321+
SHELL_PATH = "${pkgs.bash}/bin/bash";
322+
PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}";
323+
PGSQL_SUPERUSER = "${pgsqlSuperuser}";
324+
PSQL15_BINDIR = "${basePackages.psql_15.bin}";
325+
PSQL_CONF_FILE = "${paths.pgconfigFile}";
326+
PSQL16_BINDIR = "${basePackages.psql_16.bin}";
327+
PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}";
328+
PGSODIUM_GETKEY = "${paths.getkeyScript}";
329+
READREPL_CONF_FILE = "${paths.readReplicaConfigFile}";
330+
LOGGING_CONF_FILE = "${paths.loggingConfigFile}";
331+
SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}";
332+
PG_HBA = "${paths.pgHbaConfigFile}";
333+
PG_IDENT = "${paths.pgIdentConfigFile}";
334+
LOCALES = "${localeArchive}";
335+
EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}";
336+
MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so";
337+
GROONGA_DIR = "${supabase-groonga}";
338+
MIGRATIONS_DIR = "${paths.migrationsDir}";
339+
POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}";
340+
PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}";
341+
STAT_EXTENSION_SQL = "${paths.statExtensionSql}";
342+
CURRENT_SYSTEM = "${system}";
343+
} // extraSubstitutions; # Merge in any extra substitutions
344+
in pkgs.runCommand name {
345+
inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql;
346+
} ''
347+
set -x
348+
mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts
349+
350+
# Copy config files with error handling
351+
cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
352+
cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
353+
cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
354+
cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
355+
cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
356+
cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
357+
cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }
358+
359+
echo "Copy operation completed"
360+
chmod 644 $out/etc/postgresql-custom/supautils.conf
361+
chmod 644 $out/etc/postgresql/postgresql.conf
362+
chmod 644 $out/etc/postgresql-custom/logging.conf
363+
chmod 644 $out/etc/postgresql/pg_hba.conf
364+
365+
substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \
366+
${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs
367+
(name: value: "--subst-var-by '${name}' '${value}'")
368+
substitutions
369+
))}
370+
chmod +x $out/bin/start-postgres-server
371+
'';
372+
263373
# The base set of packages that we export from this Nix Flake, that can
264374
# be used with 'nix build'. Don't use the names listed below; check the
265375
# name in 'nix flake show' in order to make sure exactly what name you
@@ -381,103 +491,10 @@
381491
supabase_groonga = supabase-groonga;
382492
pg_regress = makePgRegress activeVersion;
383493
# Start a version of the server.
384-
start-server =
385-
let
386-
migrationsDir = builtins.path {
387-
name = "migrations";
388-
path = ./migrations/db;
389-
};
390-
postgresqlSchemaSql = builtins.path {
391-
name = "postgresql-schema";
392-
path = ./nix/tools/postgresql_schema.sql;
393-
};
394-
pgbouncerAuthSchemaSql = builtins.path {
395-
name = "pgbouncer-auth-schema";
396-
path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql;
397-
};
398-
statExtensionSql = builtins.path {
399-
name = "stat-extension";
400-
path = ./ansible/files/stat_extension.sql;
401-
};
402-
pgconfigFile = builtins.path {
403-
name = "postgresql.conf";
404-
path = ./ansible/files/postgresql_config/postgresql.conf.j2;
405-
};
406-
supautilsConfigFile = builtins.path {
407-
name = "supautils.conf";
408-
path = ./ansible/files/postgresql_config/supautils.conf.j2;
409-
};
410-
loggingConfigFile = builtins.path {
411-
name = "logging.conf";
412-
path = ./ansible/files/postgresql_config/postgresql-csvlog.conf;
413-
};
414-
readReplicaConfigFile = builtins.path {
415-
name = "readreplica.conf";
416-
path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2;
417-
};
418-
pgHbaConfigFile = builtins.path {
419-
name = "pg_hba.conf";
420-
path = ./ansible/files/postgresql_config/pg_hba.conf.j2;
421-
};
422-
pgIdentConfigFile = builtins.path {
423-
name = "pg_ident.conf";
424-
path = ./ansible/files/postgresql_config/pg_ident.conf.j2;
425-
};
426-
postgresqlExtensionCustomScriptsPath = builtins.path {
427-
name = "extension-custom-scripts";
428-
path = ./ansible/files/postgresql_extension_custom_scripts;
429-
};
430-
getkeyScript = builtins.path {
431-
name = "pgsodium_getkey.sh";
432-
path = ./nix/tests/util/pgsodium_getkey.sh;
433-
};
434-
localeArchive = if pkgs.stdenv.isDarwin
435-
then "${pkgs.darwin.locale}/share/locale"
436-
else "${pkgs.glibcLocales}/lib/locale/locale-archive";
437-
in
438-
pkgs.runCommand "start-postgres-server" {
439-
inherit migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql;
440-
} ''
441-
set -x
442-
mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts
443-
cp ${supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
444-
cp ${pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
445-
cp ${loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
446-
cp ${readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
447-
cp ${pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
448-
cp ${pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
449-
cp -r ${postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }
450-
echo "Copy operation completed"
451-
chmod 644 $out/etc/postgresql-custom/supautils.conf
452-
chmod 644 $out/etc/postgresql/postgresql.conf
453-
chmod 644 $out/etc/postgresql-custom/logging.conf
454-
chmod 644 $out/etc/postgresql/pg_hba.conf
455-
substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \
456-
--subst-var-by 'SHELL_PATH' '${pkgs.bash}/bin/bash' \
457-
--subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \
458-
--subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \
459-
--subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \
460-
--subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \
461-
--subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \
462-
--subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \
463-
--subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \
464-
--subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \
465-
--subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \
466-
--subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" \
467-
--subst-var-by 'PG_HBA' "$out/etc/postgresql/pg_hba.conf" \
468-
--subst-var-by 'PG_IDENT' "$out/etc/postgresql/pg_ident.conf" \
469-
--subst-var-by 'LOCALES' '${localeArchive}' \
470-
--subst-var-by 'EXTENSION_CUSTOM_SCRIPTS_DIR' "$out/extension-custom-scripts" \
471-
--subst-var-by 'MECAB_LIB' '${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so' \
472-
--subst-var-by 'GROONGA_DIR' '${supabase-groonga}' \
473-
--subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \
474-
--subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \
475-
--subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \
476-
--subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' \
477-
--subst-var-by 'CURRENT_SYSTEM' '${system}'
478-
479-
chmod +x $out/bin/start-postgres-server
480-
'';
494+
start-server = makePostgresDevSetup {
495+
inherit pkgs;
496+
name = "start-postgres-server";
497+
};
481498

482499
# Start a version of the client and runs migrations script on server.
483500
start-client =
@@ -596,7 +613,47 @@
596613
sqlTests = ./nix/tests/smoke;
597614
pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
598615
pg_regress = basePackages.pg_regress;
599-
start-postgres-server-bin = basePackages.start-server;
616+
getkey-script = pkgs.writeScriptBin "pgsodium-getkey" ''
617+
#!${pkgs.bash}/bin/bash
618+
set -euo pipefail
619+
620+
# Default to a path in /tmp if no argument is provided
621+
KEY_FILE=''${1:-"/tmp/pgsodium.key"}
622+
KEY_DIR=$(dirname "$KEY_FILE")
623+
624+
# Ensure key directory exists with explicit error handling
625+
if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
626+
echo "Warning: Could not create key directory $KEY_DIR, using /tmp" >&2
627+
KEY_FILE="/tmp/pgsodium.key"
628+
fi
629+
630+
# Generate key if it doesn't exist
631+
if [[ ! -f "$KEY_FILE" ]]; then
632+
# Try dd first, fallback to openssl
633+
if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
634+
if ! (openssl rand -hex 32 > "$KEY_FILE"); then
635+
echo "00000000000000000000000000000000" > "$KEY_FILE"
636+
echo "Warning: Using fallback key" >&2
637+
fi
638+
fi
639+
fi
640+
641+
if [[ -f "$KEY_FILE" ]]; then
642+
cat "$KEY_FILE"
643+
else
644+
echo "00000000000000000000000000000000"
645+
echo "Warning: Using default key" >&2
646+
fi
647+
'';
648+
# Use the shared setup but with a test-specific name
649+
start-postgres-server-bin = makePostgresDevSetup {
650+
inherit pkgs;
651+
name = "start-postgres-server-test";
652+
extraSubstitutions = {
653+
PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey";
654+
};
655+
};
656+
600657
getVersionArg = pkg:
601658
let
602659
name = pkg.version;
@@ -610,67 +667,48 @@
610667
{
611668
nativeBuildInputs = with pkgs; [
612669
coreutils bash pgpkg pg_prove pg_regress procps
613-
start-postgres-server-bin which
670+
start-postgres-server-bin which getkey-script
614671
];
615672
} ''
616-
set -e
617-
echo "Contents of nativeBuildInputs PATH:"
618-
echo $PATH
619-
echo "Checking start-postgres-server:"
620-
which start-postgres-server
621-
ls -l $(which start-postgres-server)
622-
file $(which start-postgres-server)
623-
# echo "Creating the getKeyScript..."
624-
# cat > getKeyScript.sh <<'EOF'
625-
# #!/usr/bin/env bash
626-
# set -euo pipefail
627-
628-
# if [[ ! -f "$KEY_FILE" ]]; then
629-
# head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"
630-
# fi
631-
# cat "$KEY_FILE"
632-
# EOF
633-
634-
# Make the script executable
635-
${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
636-
637-
for i in {1..60}; do
638-
if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
639-
echo "PostgreSQL is ready"
640-
break
641-
fi
642-
sleep 1
643-
if [ $i -eq 60 ]; then
644-
echo "PostgreSQL failed to start"
645-
exit 1
646-
fi
647-
done
648-
649-
650-
if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
651-
echo "Error executing SQL file"
652-
exit 1
653-
fi
654-
655-
mkdir -p $out/regression_output
656-
if ! pg_regress \
657-
--use-existing \
658-
--dbname=postgres \
659-
--inputdir=${./nix/tests} \
660-
--outputdir=$out/regression_output \
661-
--host=localhost \
662-
--port=5435 \
663-
--user=supabase_admin \
664-
$(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort ); then
665-
echo "pg_regress tests failed"
666-
exit 1
667-
fi
668-
669-
# Copy logs to output
670-
for logfile in $(find /tmp -name postgresql.log -type f); do
671-
cp "$logfile" $out/postgresql.log
672-
done
673-
exit 0
673+
set -e
674+
${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
675+
676+
for i in {1..60}; do
677+
if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
678+
echo "PostgreSQL is ready"
679+
break
680+
fi
681+
sleep 1
682+
if [ $i -eq 60 ]; then
683+
echo "PostgreSQL failed to start"
684+
exit 1
685+
fi
686+
done
687+
688+
if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
689+
echo "Error executing SQL file"
690+
exit 1
691+
fi
692+
693+
mkdir -p $out/regression_output
694+
if ! pg_regress \
695+
--use-existing \
696+
--dbname=postgres \
697+
--inputdir=${./nix/tests} \
698+
--outputdir=$out/regression_output \
699+
--host=localhost \
700+
--port=5435 \
701+
--user=supabase_admin \
702+
$(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort ); then
703+
echo "pg_regress tests failed"
704+
exit 1
705+
fi
706+
707+
# Copy logs to output
708+
for logfile in $(find /tmp -name postgresql.log -type f); do
709+
cp "$logfile" $out/postgresql.log
710+
done
711+
exit 0
674712
'';
675713
in
676714
rec {

0 commit comments

Comments
 (0)