From 4b58a58712db9ae4a4feaea64d263147710909a5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 14 Aug 2024 21:15:29 +0000 Subject: [PATCH 1/5] test: include supautils conf --- flake.nix | 21 ++++++++++++++++----- nix/tests/postgresql.conf.in | 2 +- nix/tools/run-server.sh.in | 4 +++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index 34e757212..07690dd7c 100644 --- a/flake.nix +++ b/flake.nix @@ -276,18 +276,29 @@ # Start a version of the server. start-server = let - configFile = ./nix/tests/postgresql.conf.in; + pgconfigFile = ./nix/tests/postgresql.conf.in; + supautilsConfigFile = builtins.path { + name = "supautils.conf"; + path = ./ansible/files/postgresql_config/supautils.conf.j2; + }; getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; in pkgs.runCommand "start-postgres-server" { } '' - mkdir -p $out/bin + mkdir -p $out/bin $out/etc/postgresql-custom + echo "Copying from: ${supautilsConfigFile}" + echo "Copying to: $out/etc/postgresql-custom/supautils.conf" + cp ${supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } + echo "Copy operation completed" + chmod 644 $out/etc/postgresql-custom/supautils.conf + cat $out/etc/postgresql-custom/supautils.conf substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' - + --subst-var-by 'PSQL_CONF_FILE' '${pgconfigFile}' \ + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ + --subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" + chmod +x $out/bin/start-postgres-server ''; diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..f3583e00d 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -793,7 +793,7 @@ jit_provider = 'llvmjit' # JIT library to use #------------------------------------------------------------------------------ # Add settings for extensions here - +include = '@SUPAUTILS_CONFIG_FILE@' pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' auto_explain.log_min_duration = 10s diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index b620a0f15..676df3c3c 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -28,6 +28,7 @@ PSQL_CONF_FILE=@PSQL_CONF_FILE@ PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" PLJAVA_LIBJVM_LOCATION=@LIBJVM_LOCATION@ +SUPAUTILS_CONFIG_FILE=@SUPAUTILS_CONF_FILE@ DATDIR=$(mktemp -d) mkdir -p "$DATDIR" @@ -42,6 +43,7 @@ echo "NOTE: patching postgresql.conf files" echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ + -e "s#@SUPAUTILS_CONFIG_FILE@#$SUPAUTILS_CONFIG_FILE#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" - +cat $DATDIR/postgresql.conf exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 5046f0f7ee115b8fb11d45028f6973e89d2be1d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 19:50:51 +0000 Subject: [PATCH 2/5] feat: local dev config matching AMI release --- flake.nix | 44 ++++++++++++++++++++++++++++++++------ logfile | 5 +++++ nix/tools/run-server.sh.in | 33 +++++++++++++++++++--------- 3 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 logfile diff --git a/flake.nix b/flake.nix index 07690dd7c..8892b1ce9 100644 --- a/flake.nix +++ b/flake.nix @@ -276,28 +276,58 @@ # Start a version of the server. start-server = let - pgconfigFile = ./nix/tests/postgresql.conf.in; + pgconfigFile = builtins.path { + name = "postgresql.conf"; + path = ./ansible/files/postgresql_config/postgresql.conf.j2; + }; supautilsConfigFile = builtins.path { name = "supautils.conf"; path = ./ansible/files/postgresql_config/supautils.conf.j2; }; + loggingConfigFile = builtins.path { + name = "logging.conf"; + path = ./ansible/files/postgresql_config/postgresql-csvlog.conf; + }; + readReplicaConfigFile = builtins.path { + name = "readreplica.conf"; + path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2; + }; + pgHbaConfigFile = builtins.path { + name = "pg_hba.conf"; + path = ./ansible/files/postgresql_config/pg_hba.conf.j2; + }; + pgIdentConfigFile = builtins.path { + name = "pg_ident.conf"; + path = ./ansible/files/postgresql_config/pg_ident.conf.j2; + }; getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; + localeArchive = "${pkgs.glibcLocales}/lib/locale/locale-archive"; in pkgs.runCommand "start-postgres-server" { } '' - mkdir -p $out/bin $out/etc/postgresql-custom - echo "Copying from: ${supautilsConfigFile}" - echo "Copying to: $out/etc/postgresql-custom/supautils.conf" + mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql cp ${supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } + cp ${pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } + cp ${loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } + cp ${readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } + cp ${pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } + cp ${pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } echo "Copy operation completed" chmod 644 $out/etc/postgresql-custom/supautils.conf - cat $out/etc/postgresql-custom/supautils.conf + chmod 644 $out/etc/postgresql/postgresql.conf + chmod 644 $out/etc/postgresql-custom/logging.conf + chmod 644 $out/etc/postgresql/pg_hba.conf substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL_CONF_FILE' '${pgconfigFile}' \ + --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" + --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ + --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ + --subst-var-by 'SUPAUTILS_CONF_FILE' "$out/etc/postgresql-custom/supautils.conf" \ + --subst-var-by 'PG_HBA' "$out/etc/postgresql/pg_hba.conf" \ + --subst-var-by 'PG_IDENT' "$out/etc/postgresql/pg_ident.conf" \ + --subst-var-by 'LOCALES' '${localeArchive}' chmod +x $out/bin/start-postgres-server ''; diff --git a/logfile b/logfile new file mode 100644 index 000000000..7156c2ad3 --- /dev/null +++ b/logfile @@ -0,0 +1,5 @@ + 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_messages": "en_US.UTF-8" + 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_monetary": "en_US.UTF-8" + 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_numeric": "en_US.UTF-8" + 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_time": "en_US.UTF-8" + 2024-08-15 19:02:30.234 UTC [168132] FATAL: configuration file "/tmp/tmp.ARsCfJV3Wf/postgresql.conf" contains errors diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 676df3c3c..85e358fcf 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -27,23 +27,36 @@ PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ PSQL_CONF_FILE=@PSQL_CONF_FILE@ PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" -PLJAVA_LIBJVM_LOCATION=@LIBJVM_LOCATION@ SUPAUTILS_CONFIG_FILE=@SUPAUTILS_CONF_FILE@ +LOGGING_CONFIG_FILE=@LOGGING_CONF_FILE@ +READREPL_CONFIG_FILE=@READREPL_CONF_FILE@ +PG_HBA_FILE=@PG_HBA@ +PG_IDENT_FILE=@PG_IDENT@ DATDIR=$(mktemp -d) +LOCALE_ARCHIVE=@LOCALES@ +export LOCALE_ARCHIVE +export LANG=en_US.UTF-8 +export LANGUAGE=en_US.UTF-8 +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 +export LC_CTYPE=en_US.UTF-8 mkdir -p "$DATDIR" echo "NOTE: using port $PORTNO for server" echo "NOTE: using temporary directory $DATDIR for data, which will not be removed" echo "NOTE: you are free to re-use this data directory at will" -echo -initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C +initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" echo "NOTE: patching postgresql.conf files" -echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" -sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ - -e "s#@SUPAUTILS_CONFIG_FILE@#$SUPAUTILS_CONFIG_FILE#g" \ - $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" -cat $DATDIR/postgresql.conf -exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp + +sed -e "\$a\\ + include = '$SUPAUTILS_CONFIG_FILE' \\ + pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ + -e "s|data_directory = '/var/lib/postgresql/data'|data_directory = '$DATDIR'|" \ + -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$PG_HBA_FILE'|" \ + -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$PG_IDENT_FILE'|" \ + -e "s|include = '/etc/postgresql/logging.conf'|#&|" \ + -e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$READREPL_CONFIG_FILE'|" \ + "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" +postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 9481589a28c4a93056b315efc0cbc83f8117d695 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 20:10:28 +0000 Subject: [PATCH 3/5] fix: restore old testing conf --- nix/tests/postgresql.conf.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index f3583e00d..d2d667b9a 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -793,7 +793,7 @@ jit_provider = 'llvmjit' # JIT library to use #------------------------------------------------------------------------------ # Add settings for extensions here -include = '@SUPAUTILS_CONFIG_FILE@' + pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' auto_explain.log_min_duration = 10s From fd88452c622e2854bfe27ae9dab6e946f81bf7be Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 15 Aug 2024 16:20:30 -0400 Subject: [PATCH 4/5] feat: make sure macos needed support included --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8892b1ce9..79e032fb1 100644 --- a/flake.nix +++ b/flake.nix @@ -301,7 +301,9 @@ path = ./ansible/files/postgresql_config/pg_ident.conf.j2; }; getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; - localeArchive = "${pkgs.glibcLocales}/lib/locale/locale-archive"; + localeArchive = if pkgs.stdenv.isDarwin + then "${pkgs.darwin.locale}/share/locale" + else "${pkgs.glibcLocales}/lib/locale/locale-archive"; in pkgs.runCommand "start-postgres-server" { } '' mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql From d674b7255480921757208777653a553033a34d9a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 20 Aug 2024 08:36:16 -0400 Subject: [PATCH 5/5] chore: rm logfile --- logfile | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 logfile diff --git a/logfile b/logfile deleted file mode 100644 index 7156c2ad3..000000000 --- a/logfile +++ /dev/null @@ -1,5 +0,0 @@ - 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_messages": "en_US.UTF-8" - 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_monetary": "en_US.UTF-8" - 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_numeric": "en_US.UTF-8" - 2024-08-15 19:02:30.233 UTC [168132] LOG: invalid value for parameter "lc_time": "en_US.UTF-8" - 2024-08-15 19:02:30.234 UTC [168132] FATAL: configuration file "/tmp/tmp.ARsCfJV3Wf/postgresql.conf" contains errors