Skip to content

Commit 3d34484

Browse files
samrosesamrose
committed
chore: getting pg tap tests and pg_regress tests to run for pg_15 and oriole 17
1 parent bd401ea commit 3d34484

File tree

1 file changed

+215
-100
lines changed

1 file changed

+215
-100
lines changed

‎flake.nix‎

Lines changed: 215 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -603,108 +603,223 @@
603603
# Create a testing harness for a PostgreSQL package. This is used for
604604
# 'nix flake check', and works with any PostgreSQL package you hand it.
605605

606-
makeCheckHarness = pgpkg:
606+
makeCheckHarness = pgpkg:
607+
let
608+
sqlTests = ./nix/tests/smoke;
609+
pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
610+
pg_regress = basePackages.pg_regress;
611+
getkey-script = pkgs.writeScriptBin "pgsodium-getkey" ''
612+
#!${pkgs.bash}/bin/bash
613+
set -euo pipefail
614+
615+
TMPDIR_BASE=$(mktemp -d)
616+
617+
if [[ "$(uname)" == "Darwin" ]]; then
618+
KEY_DIR="/private/tmp/pgsodium"
619+
else
620+
KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}"
621+
fi
622+
KEY_FILE="$KEY_DIR/pgsodium.key"
623+
624+
if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
625+
echo "Error: Could not create key directory $KEY_DIR" >&2
626+
exit 1
627+
fi
628+
chmod 1777 "$KEY_DIR"
629+
630+
if [[ ! -f "$KEY_FILE" ]]; then
631+
if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
632+
if ! (openssl rand -hex 32 > "$KEY_FILE"); then
633+
echo "00000000000000000000000000000000" > "$KEY_FILE"
634+
echo "Warning: Using fallback key" >&2
635+
fi
636+
fi
637+
chmod 644 "$KEY_FILE"
638+
fi
639+
640+
if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then
641+
cat "$KEY_FILE"
642+
else
643+
echo "Error: Cannot read key file $KEY_FILE" >&2
644+
exit 1
645+
fi
646+
'';
647+
648+
# Use the shared setup but with a test-specific name
649+
start-postgres-server-bin = makePostgresDevSetup {
650+
inherit pkgs;
651+
name = "start-postgres-server-test";
652+
extraSubstitutions = {
653+
PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey";
654+
};
655+
};
656+
657+
getVersionArg = pkg:
658+
let
659+
name = pkg.version;
660+
in
661+
if builtins.match "15.*" name != null then "15"
662+
else if builtins.match "16.*" name != null then "16"
663+
else if builtins.match "17.*" name != null then "orioledb-17"
664+
else throw "Unsupported PostgreSQL version: ${name}";
665+
666+
# Helper function to filter SQL files based on version
667+
filterTestFiles = version: dir:
668+
let
669+
files = builtins.readDir dir;
670+
isValidFile = name:
607671
let
608-
sqlTests = ./nix/tests/smoke;
609-
pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP;
610-
pg_regress = basePackages.pg_regress;
611-
getkey-script = pkgs.writeScriptBin "pgsodium-getkey" ''
612-
#!${pkgs.bash}/bin/bash
613-
set -euo pipefail
614-
615-
# Default to a path in /tmp if no argument is provided
616-
KEY_FILE=''${1:-"/tmp/pgsodium.key"}
617-
KEY_DIR=$(dirname "$KEY_FILE")
618-
619-
# Ensure key directory exists with explicit error handling
620-
if ! mkdir -p "$KEY_DIR" 2>/dev/null; then
621-
echo "Warning: Could not create key directory $KEY_DIR, using /tmp" >&2
622-
KEY_FILE="/tmp/pgsodium.key"
623-
fi
624-
625-
# Generate key if it doesn't exist
626-
if [[ ! -f "$KEY_FILE" ]]; then
627-
# Try dd first, fallback to openssl
628-
if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then
629-
if ! (openssl rand -hex 32 > "$KEY_FILE"); then
630-
echo "00000000000000000000000000000000" > "$KEY_FILE"
631-
echo "Warning: Using fallback key" >&2
632-
fi
633-
fi
634-
fi
635-
636-
if [[ -f "$KEY_FILE" ]]; then
637-
cat "$KEY_FILE"
638-
else
639-
echo "00000000000000000000000000000000"
640-
echo "Warning: Using default key" >&2
641-
fi
642-
'';
643-
# Use the shared setup but with a test-specific name
644-
start-postgres-server-bin = makePostgresDevSetup {
645-
inherit pkgs;
646-
name = "start-postgres-server-test";
647-
extraSubstitutions = {
648-
PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey";
649-
};
650-
};
651-
652-
getVersionArg = pkg:
653-
let
654-
name = pkg.version;
655-
in
656-
if builtins.match "15.*" name != null then "15"
657-
else if builtins.match "16.*" name != null then "16"
658-
else if builtins.match "17.*" name != null then "orioledb-17"
659-
else throw "Unsupported PostgreSQL version: ${name}";
672+
isVersionSpecific = builtins.match "z_([0-9]+)_.*" name != null;
673+
matchesVersion =
674+
if isVersionSpecific
675+
then builtins.match ("z_" + version + "_.*") name != null
676+
else true;
660677
in
661-
pkgs.runCommand "postgres-${pgpkg.version}-check-harness"
662-
{
663-
nativeBuildInputs = with pkgs; [
664-
coreutils bash perl pgpkg pg_prove pg_regress procps
665-
start-postgres-server-bin which getkey-script
666-
];
667-
} ''
668-
set -e
669-
${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
670-
671-
for i in {1..60}; do
672-
if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
673-
echo "PostgreSQL is ready"
674-
break
675-
fi
676-
sleep 1
677-
if [ $i -eq 60 ]; then
678-
echo "PostgreSQL failed to start"
679-
exit 1
680-
fi
681-
done
682-
683-
if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
684-
echo "Error executing SQL file"
685-
exit 1
686-
fi
687-
688-
mkdir -p $out/regression_output
689-
if ! pg_regress \
690-
--use-existing \
691-
--dbname=postgres \
692-
--inputdir=${./nix/tests} \
693-
--outputdir=$out/regression_output \
694-
--host=localhost \
695-
--port=5435 \
696-
--user=supabase_admin \
697-
$(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort ); then
698-
echo "pg_regress tests failed"
699-
exit 1
700-
fi
701-
702-
# Copy logs to output
703-
for logfile in $(find /tmp -name postgresql.log -type f); do
704-
cp "$logfile" $out/postgresql.log
705-
done
706-
exit 0
707-
'';
678+
pkgs.lib.hasSuffix ".sql" name && matchesVersion;
679+
in
680+
pkgs.lib.filterAttrs (name: _: isValidFile name) files;
681+
682+
# Get the major version for filtering
683+
majorVersion =
684+
if builtins.match ".*17.*" pgpkg.version != null
685+
then "17"
686+
else "15";
687+
688+
# Filter SQL test files
689+
filteredSqlTests = filterTestFiles majorVersion ./nix/tests/sql;
690+
691+
# Convert filtered tests to a sorted list of basenames (without extension)
692+
testList = pkgs.lib.mapAttrsToList (name: _:
693+
builtins.substring 0 (pkgs.lib.stringLength name - 4) name
694+
) filteredSqlTests;
695+
sortedTestList = builtins.sort (a: b: a < b) testList;
696+
697+
in
698+
pkgs.runCommand "postgres-${pgpkg.version}-check-harness"
699+
{
700+
nativeBuildInputs = with pkgs; [
701+
coreutils bash perl pgpkg pg_prove pg_regress procps
702+
start-postgres-server-bin which getkey-script supabase-groonga
703+
];
704+
} ''
705+
set -e
706+
707+
#First we need to create a generic pg cluster for pgtap tests and run those
708+
export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins
709+
PGTAP_CLUSTER=$(mktemp -d)
710+
initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER"
711+
substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \
712+
--subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey"
713+
echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf
714+
echo "port = 5435" >> "$PGTAP_CLUSTER"/postgresql.conf
715+
echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf
716+
# Remove timescaledb if running orioledb-17 check
717+
echo "I AM ${pgpkg.version}===================================================="
718+
if [[ "${pgpkg.version}" == *"17"* ]]; then
719+
perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf"
720+
fi
721+
#NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole
722+
echo "PGTAP_CLUSTER directory contents:"
723+
ls -la "$PGTAP_CLUSTER"
724+
725+
# Check if postgresql.conf exists
726+
if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then
727+
echo "postgresql.conf is missing!"
728+
exit 1
729+
fi
730+
731+
# PostgreSQL startup
732+
if [[ "$(uname)" == "Darwin" ]]; then
733+
pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p 5435 -d 5" start 2>&1
734+
else
735+
mkdir -p "$PGTAP_CLUSTER/sockets"
736+
pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p 5435 -d 5" start 2>&1
737+
fi || {
738+
echo "pg_ctl failed to start PostgreSQL"
739+
echo "Contents of postgresql.log:"
740+
cat "$PGTAP_CLUSTER"/postgresql.log
741+
exit 1
742+
}
743+
for i in {1..60}; do
744+
if pg_isready -h localhost -p 5435; then
745+
echo "PostgreSQL is ready"
746+
break
747+
fi
748+
sleep 1
749+
if [ $i -eq 60 ]; then
750+
echo "PostgreSQL is not ready after 60 seconds"
751+
echo "PostgreSQL status:"
752+
pg_ctl -D "$PGTAP_CLUSTER" status
753+
echo "PostgreSQL log content:"
754+
cat "$PGTAP_CLUSTER"/postgresql.log
755+
exit 1
756+
fi
757+
done
758+
createdb -p 5435 -h localhost --username=supabase_admin testing
759+
if ! psql -p 5435 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
760+
echo "Error executing SQL file. PostgreSQL log content:"
761+
cat "$PGTAP_CLUSTER"/postgresql.log
762+
pg_ctl -D "$PGTAP_CLUSTER" stop
763+
exit 1
764+
fi
765+
SORTED_DIR=$(mktemp -d)
766+
for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do
767+
psql -p 5435 -h localhost --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true
768+
done
769+
rm -rf "$SORTED_DIR"
770+
pg_ctl -D "$PGTAP_CLUSTER" stop
771+
rm -rf $PGTAP_CLUSTER
772+
773+
# End of pgtap tests
774+
# from here on out we are running pg_regress tests, we use a different cluster for this
775+
# which is start by the start-postgres-server-bin script
776+
# start-postgres-server-bin script closely matches our AMI setup, configurations and migrations
777+
778+
# Ensure pgsodium key directory exists with proper permissions
779+
if [[ "$(uname)" == "Darwin" ]]; then
780+
mkdir -p /private/tmp/pgsodium
781+
chmod 1777 /private/tmp/pgsodium
782+
fi
783+
unset GRN_PLUGINS_DIR
784+
${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize
785+
786+
for i in {1..60}; do
787+
if pg_isready -h localhost -p 5435 -U supabase_admin -q; then
788+
echo "PostgreSQL is ready"
789+
break
790+
fi
791+
sleep 1
792+
if [ $i -eq 60 ]; then
793+
echo "PostgreSQL failed to start"
794+
exit 1
795+
fi
796+
done
797+
798+
if ! psql -p 5435 -h localhost --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then
799+
echo "Error executing SQL file"
800+
exit 1
801+
fi
802+
803+
mkdir -p $out/regression_output
804+
if ! pg_regress \
805+
--use-existing \
806+
--dbname=postgres \
807+
--inputdir=${./nix/tests} \
808+
--outputdir=$out/regression_output \
809+
--host=localhost \
810+
--port=5435 \
811+
--user=supabase_admin \
812+
${builtins.concatStringsSep " " sortedTestList}; then
813+
echo "pg_regress tests failed"
814+
exit 1
815+
fi
816+
817+
# Copy logs to output
818+
for logfile in $(find /tmp -name postgresql.log -type f); do
819+
cp "$logfile" $out/postgresql.log
820+
done
821+
exit 0
822+
'';
708823
in
709824
rec {
710825
# The list of all packages that can be built with 'nix build'. The list

0 commit comments

Comments
 (0)