From cdc64524f25374081f2541428e9c89a4429ab6cd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:28:53 -0400 Subject: [PATCH 01/47] feat: build and flake check of pg 16.3 with exts/wrappers --- flake.nix | 35 ++++++++-- nix/ext/hypopg.nix | 2 +- nix/ext/pg_backtrace.nix | 2 +- nix/ext/pg_cron.nix | 2 +- nix/ext/pg_hashids.nix | 2 +- nix/ext/pg_net.nix | 2 +- nix/ext/pg_plan_filter.nix | 2 +- nix/ext/pg_stat_monitor.nix | 2 +- nix/ext/pg_tle.nix | 2 +- nix/ext/pgrouting.nix | 33 +++++++++- nix/ext/pgsodium.nix | 2 +- nix/ext/pgsql-http.nix | 2 +- nix/ext/pgvector.nix | 2 +- nix/ext/plv8.nix | 76 ++-------------------- nix/ext/supautils.nix | 2 +- nix/postgresql/16.nix | 4 ++ nix/postgresql/default.nix | 1 + nix/tests/sql/extensions_sql_interface.sql | 12 ++++ 18 files changed, 96 insertions(+), 89 deletions(-) create mode 100644 nix/postgresql/16.nix diff --git a/flake.nix b/flake.nix index 82409b185..3bb6d1a45 100644 --- a/flake.nix +++ b/flake.nix @@ -290,8 +290,7 @@ # Define the available PostgreSQL versions postgresVersions = { psql_15 = makePostgres "15"; - # Uncomment the line below to enable PostgreSQL 16 - # psql_16 = makePostgres "16"; + psql_16 = makePostgres "16"; # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; }; @@ -307,15 +306,17 @@ postgresql = postgresqlPackage; }; postgresql_15 = getPostgresqlPackage "15"; + postgresql_16 = getPostgresqlPackage "16"; in postgresVersions //{ supabase-groonga = supabase-groonga; # PostgreSQL versions. - psql_15 = makePostgres "15"; + psql_15 = postgresVersions.psql_15; + psql_16 = postgresVersions.psql_16; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15; + inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; @@ -339,6 +340,28 @@ platforms = platforms.all; }; }; + postgresql_16_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-16-src"; + version = postgresql_16.version; + + src = postgresql_16.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; @@ -397,6 +420,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -425,6 +449,7 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ @@ -569,7 +594,7 @@ # flake check'. This is run in the CI system, as well. checks = { psql_15 = makeCheckHarness basePackages.psql_15.bin; - #psql_16 = makeCheckHarness basePackages.psql_16.bin; + psql_16 = makeCheckHarness basePackages.psql_16.bin; #psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; }; diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index 4fc00a82f..300a44900 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_backtrace.nix b/nix/ext/pg_backtrace.nix index b0169124c..47ede88e2 100644 --- a/nix/ext/pg_backtrace.nix +++ b/nix/ext/pg_backtrace.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 5c546c797..d51254aa2 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 41c3ba664..50bff95da 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..c0946f425 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 2d7d224ec..0ed527251 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 8784067d2..c3e91fe2b 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 6b1c7b1ca..0221d8768 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index 36edf3404..5587566cd 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -14,9 +14,38 @@ stdenv.mkDerivation rec { hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; }; + #disable compile time warnings for incompatible pointer types only on macos and pg16 + NIX_CFLAGS_COMPILE = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") + "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; + + cmakeFlags = [ + "-DPOSTGRESQL_VERSION=${postgresql.version}" + ] ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ + "-DCMAKE_MACOSX_RPATH=ON" + "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" + "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" + ]; + + preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + export DLSUFFIX=.dylib + export CMAKE_SHARED_LIBRARY_SUFFIX=.dylib + export CMAKE_SHARED_MODULE_SUFFIX=.dylib + export MACOSX_RPATH=ON + ''; + + postBuild = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + shopt -s nullglob + for file in lib/libpgrouting-*.so; do + if [ -f "$file" ]; then + mv "$file" "''${file%.so}.dylib" + fi + done + shopt -u nullglob + ''; + installPhase = '' - install -D lib/*.so -t $out/lib - install -D sql/pgrouting--${version}.sql -t $out/share/postgresql/extension + install -D lib/*${postgresql.dlSuffix} -t $out/lib + install -D sql/pgrouting--*.sql -t $out/share/postgresql/extension install -D sql/common/pgrouting.control -t $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index e3b0da228..4e184faa9 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 2edaa9d53..de8fa68c6 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 476f7606c..aefeeea3a 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..785f5e601 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -105,81 +105,17 @@ stdenv.mkDerivation (finalAttrs: { rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ''; - passthru = { - tests = - let - postgresqlWithSelf = postgresql.withPackages (_: [ - finalAttrs.finalPackage - ]); - in { - smoke = runCommand "plv8-smoke-test" {} '' - export PATH=${lib.makeBinPath [ - postgresqlWithSelf - coreutils - gnugrep - ]} - db="$PWD/testdb" - initdb "$db" - postgres -k "$db" -D "$db" & - pid="$!" - - for i in $(seq 1 100); do - if psql -h "$db" -d postgres -c "" 2>/dev/null; then - break - elif ! kill -0 "$pid"; then - exit 1 - else - sleep 0.1 - fi - done - - psql -h "$db" -d postgres -c 'CREATE EXTENSION plv8; DO $$ plv8.elog(NOTICE, plv8.version); $$ LANGUAGE plv8;' 2> "$out" - grep -q "${finalAttrs.version}" "$out" - kill -0 "$pid" - ''; - - regression = stdenv.mkDerivation { - name = "plv8-regression"; - inherit (finalAttrs) src patches nativeBuildInputs buildInputs dontConfigure; - - buildPhase = '' - runHook preBuild - - # The regression tests need to be run in the order specified in the Makefile. - echo -e "include Makefile\nprint_regress_files:\n\t@echo \$(REGRESS)" > Makefile.regress - REGRESS_TESTS=$(make -f Makefile.regress print_regress_files) - - ${postgresql}/lib/pgxs/src/test/regress/pg_regress \ - --bindir='${postgresqlWithSelf}/bin' \ - --temp-instance=regress-instance \ - --dbname=contrib_regression \ - $REGRESS_TESTS - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - touch "$out" - - runHook postInstall - ''; - }; - }; - }; - meta = with lib; { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; @@ -187,4 +123,4 @@ stdenv.mkDerivation (finalAttrs: { platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; license = licenses.postgresql; }; -}) \ No newline at end of file +}) diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..ee1d691b2 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/lib - install -D supautils.so -t $out/lib + install -D *${postgresql.dlSuffix} -t $out/lib ''; meta = with lib; { diff --git a/nix/postgresql/16.nix b/nix/postgresql/16.nix new file mode 100644 index 000000000..c964fc580 --- /dev/null +++ b/nix/postgresql/16.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "16.3"; + hash = "sha256-Mxlj1dPcTK9CFqBJ+kC2bWvLjHMGFYWUEblRh2TmBYU="; +} diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 6ee0452dd..5f92262b2 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -3,6 +3,7 @@ let #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; + postgresql_16 = ./16.nix; }; mkAttributes = jitSupport: diff --git a/nix/tests/sql/extensions_sql_interface.sql b/nix/tests/sql/extensions_sql_interface.sql index 947a4a612..ad0f63ed8 100644 --- a/nix/tests/sql/extensions_sql_interface.sql +++ b/nix/tests/sql/extensions_sql_interface.sql @@ -63,6 +63,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, From 91ec362374e3ec73ada1f7215daa2a0c2c0de236 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:44:54 -0500 Subject: [PATCH 02/47] pg_partman test 15/16 compat --- nix/tests/expected/pg_partman.out | 5 ++++- nix/tests/sql/pg_partman.sql | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out index 527d58b12..25aa80d2e 100644 --- a/nix/tests/expected/pg_partman.out +++ b/nix/tests/expected/pg_partman.out @@ -28,7 +28,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; table_name | table_type -----------------------------+------------ time_taptest_table | BASE TABLE diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql index 621e1a7c9..d1fb9f4ba 100644 --- a/nix/tests/sql/pg_partman.sql +++ b/nix/tests/sql/pg_partman.sql @@ -35,7 +35,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; select public.create_parent( From 0321dfa85e67b274f06a3dd705aa6407d8f8cace Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:46:33 -0500 Subject: [PATCH 03/47] merge sql interface test --- nix/tests/expected/extensions_sql_interface.out | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 5714fbcec..07c0d6650 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -146,6 +146,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, @@ -1076,7 +1088,6 @@ order by moddatetime | public | moddatetime | | trigger old_snapshot | public | pg_old_snapshot_time_mapping | OUT array_offset integer, OUT end_timestamp timestamp with time zone, OUT newest_xmin xid | SETOF record pageinspect | public | brin_metapage_info | page bytea, OUT magic text, OUT version integer, OUT pagesperrange integer, OUT lastrevmappage bigint | record - pageinspect | public | brin_page_items | page bytea, index_oid regclass, OUT itemoffset integer, OUT blknum bigint, OUT attnum integer, OUT allnulls boolean, OUT hasnulls boolean, OUT placeholder boolean, OUT value text | SETOF record pageinspect | public | brin_page_type | page bytea | text pageinspect | public | brin_revmap_data | page bytea, OUT pages tid | SETOF tid pageinspect | public | bt_metap | relname text, OUT magic integer, OUT version integer, OUT root bigint, OUT level bigint, OUT fastroot bigint, OUT fastlevel bigint, OUT last_cleanup_num_delpages bigint, OUT last_cleanup_num_tuples double precision, OUT allequalimage boolean | record @@ -1300,9 +1311,7 @@ order by pg_visibility | public | pg_visibility_map_summary | regclass, OUT all_visible bigint, OUT all_frozen bigint | record pg_walinspect | public | pg_get_wal_record_info | in_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | record pg_walinspect | public | pg_get_wal_records_info | start_lsn pg_lsn, end_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record - pg_walinspect | public | pg_get_wal_records_info_till_end_of_wal | start_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record pg_walinspect | public | pg_get_wal_stats | start_lsn pg_lsn, end_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record - pg_walinspect | public | pg_get_wal_stats_till_end_of_wal | start_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record pgaudit | public | pgaudit_ddl_command_end | | event_trigger pgaudit | public | pgaudit_sql_drop | | event_trigger pgcrypto | public | armor | bytea | text @@ -5225,7 +5234,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5072 rows) +(5069 rows) /* From aa5dd9a271eb4c79bf97e77e3e92d628900c281e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:57:53 -0400 Subject: [PATCH 04/47] tests: build test and cache both versions --- docker/nix/build_nix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index a8a8639e2..3aba72c2c 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -6,5 +6,7 @@ if [ -d "/workspace" ]; then cd /workspace fi nix build .#psql_15/bin -o psql_15 +nix build .#psql_16/bin -o psql_16 nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From 65fdc5d97a00ac46a025b990f149333f51e5a052 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 18:31:02 -0400 Subject: [PATCH 05/47] chore: run checks individually --- docker/nix/build_nix.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3aba72c2c..3cf8505c0 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,8 +5,9 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From f10881b512b5e0497f35e7d3ed59169098f1daca Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 10 Oct 2024 13:01:02 -0400 Subject: [PATCH 06/47] pg 15 and 16 packer/ansible/ghactions (#1268) * fix: reformat ec2 cleanup commands (#1267) Co-authored-by: Sam Rose * feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * feat: realease 15 and 16 to staging * chore: update versions * chore: make yq available * chore: run yq from nix * chore: more setup for staging AMI * fix: yq usage * chore: shell vars * fix: When --init none is used, only users who can elevate to sudo privileges can run Nix * fix: no -i * fix: quote correction * fix: newline extra quotes * fix: no need for pg major version on packer * fix: postgresql_major * fix: ql * fix: no ansible args in stage to invocation * fix: unique val * fix: adjustments to build scripts * chore: env var handling * fix: bump to build * chore: set up more required vars * chore: bump var * feat: pg 16 debug symbols * feat: matrix pg versions build on testinfra * feat: matrix on Test Database * chore: running nix in the right context * feat: just use existing Dockerfile + pg version * chore: refer to var * fix: read name without including quotes * chore: try format function * fix: strip quotes from version number * chore: env var * fix: pg client * fix * fix: try to use psql from our own corresponding pkg * fix: try psql from ppa * fix: dbmate per pg version * build dbmate and then install client * fix: account for architecture * chore: limit changes detection migrations/schema.sql * missing docker compose call * ore: drop tests while investigating * test: try on pg15 only * chore: schema needs update * chore: now run on all versions in matrix * test: trying a version of schema per major pg version as there are type diffs --------- Co-authored-by: Sam Rose Co-authored-by: Oliver Rice --- .github/workflows/ami-release-nix.yml | 65 +- .github/workflows/test.yml | 65 +- .github/workflows/testinfra-nix.yml | 45 +- Dockerfile-15 | 223 +++++ Dockerfile-16 | 223 +++++ ansible/tasks/stage2-setup-postgres.yml | 14 +- ansible/vars.yml | 14 +- common-nix.vars.pkr.hcl | 1 - flake.nix | 1 + migrations/Dockerfile.dbmate | 23 + migrations/docker-compose.yaml | 4 +- migrations/schema-15.sql | 1064 +++++++++++++++++++++++ migrations/schema-16.sql | 1064 +++++++++++++++++++++++ migrations/schema.sql | 2 +- scripts/nix-provision.sh | 7 + stage2-nix-psql.pkr.hcl | 8 +- 16 files changed, 2780 insertions(+), 43 deletions(-) create mode 100644 Dockerfile-15 create mode 100644 Dockerfile-16 delete mode 100644 common-nix.vars.pkr.hcl create mode 100644 migrations/Dockerfile.dbmate create mode 100644 migrations/schema-15.sql create mode 100644 migrations/schema-16.sql diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 194c8de21..643f26fc7 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -8,12 +8,31 @@ on: paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' + - 'ansible/vars.yml' workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 @@ -31,42 +50,55 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} - # Update `ci.yaml` too if changing constraints. run: | - SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl) + SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 fi - # extensions are build in nix prior to this step - # so we can just use the binaries from the nix store - # for postgres, extensions and wrappers + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" + VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Create nix flake revision tarball run: | GIT_SHA=${{github.sha}} - MAJOR_VERSION=$(echo "${{ steps.process_release_version.outputs.version }}" | cut -d. -f1) + MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" @@ -84,17 +116,13 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 staging run: | aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - #Our self hosted github runner already has permissions to publish images - #but they're limited to only that; - #so if we want s3 access we'll need to config credentials with the below steps - # (which overwrites existing perms) after the ami build - - name: configure aws credentials - prod uses: aws-actions/configure-aws-credentials@v4 with: @@ -107,6 +135,7 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 prod @@ -130,12 +159,12 @@ jobs: SLACK_MESSAGE: 'Building Postgres AMI failed' SLACK_FOOTER: '' - - name: Cleanup resources on build cancellation + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2be061941..1457eb100 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,27 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: [self-hosted, X64] arch: amd64 @@ -23,14 +41,36 @@ jobs: POSTGRES_PASSWORD: password steps: - uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Strip quotes from pg major and set env var + run: | + stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') + echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + - id: settings # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + - name: Generate args + id: args + run: | + ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) + echo "result<> $GITHUB_OUTPUT + echo "$ARGS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -40,7 +80,7 @@ jobs: with: load: true context: . - file: "Dockerfile-156" + file: Dockerfile-${{ env.PGMAJOR }} target: production build-args: | ${{ steps.args.outputs.result }} @@ -57,10 +97,13 @@ jobs: -p ${{ env.POSTGRES_PORT }}:5432 \ --name supabase_postgres \ -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} + - name: Install psql run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt update - sudo apt install -y --no-install-recommends postgresql-client + sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP @@ -107,11 +150,15 @@ jobs: PGUSER: supabase_admin PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + - name: Update Dockerfile.dbmate version + run: | + sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate + - name: verify schema.sql is committed run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then - echo "Detected uncommitted changes after build. See status below:" - git diff + if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then + echo "Detected changes in schema.sql:" + git diff migrations/schema-${{ env.PGMAJOR }}.sql exit 1 fi diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 3835a9a00..4a51c159c 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -5,17 +5,35 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + test-ami-nix: + needs: prepare strategy: fail-fast: false matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 ubuntu_release: focal ubuntu_version: 20.04 mcpu: neoverse-n1 - runs-on: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} timeout-minutes: 150 permissions: contents: write @@ -40,18 +58,29 @@ jobs: - name: Generate random string id: random run: echo "random_string=$(openssl rand -hex 8)" >> $GITHUB_OUTPUT + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 @@ -65,12 +94,12 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} - - - name: Cleanup resources on build cancellation + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids + + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true - name: Cleanup AMIs if: always() @@ -91,4 +120,4 @@ jobs: # Deregister AMIs deregister_ami_by_name "$STAGE1_AMI_NAME" - deregister_ami_by_name "$STAGE2_AMI_NAME" \ No newline at end of file + deregister_ami_by_name "$STAGE2_AMI_NAME" diff --git a/Dockerfile-15 b/Dockerfile-15 new file mode 100644 index 000000000..ce83237f2 --- /dev/null +++ b/Dockerfile-15 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_15/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-16 b/Dockerfile-16 new file mode 100644 index 000000000..db5772cee --- /dev/null +++ b/Dockerfile-16 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_16/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 20ad069e2..e22cbe67b 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,7 +6,7 @@ - name: Install Postgres from nix binary cache become: yes shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#psql_15/bin" + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" #TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix @@ -22,6 +22,18 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supabase-groonga" when: stage2_nix +- name: Install debug symbols for postgres version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" + when: stage2_nix + +- name: Install source files for postgresql version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" + when: stage2_nix + - name: Set ownership and permissions for /etc/ssl/private become: yes file: diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..f78b56c97 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,9 +2,17 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -postgresql_major: "15" -postgresql_release: "15.1" -postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +# postgresql_major: "15" +# postgresql_release: "15.1" +# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +postgres_major: + - "15" + - "16" + +# Full version strings for each major version +postgres_release: + postgres15: "15.8.1.003-staging-5" + postgres16: "16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl deleted file mode 100644 index 8419992a8..000000000 --- a/common-nix.vars.pkr.hcl +++ /dev/null @@ -1 +0,0 @@ -postgres-version = "15.8.1.003" diff --git a/flake.nix b/flake.nix index 3bb6d1a45..40202d2f0 100644 --- a/flake.nix +++ b/flake.nix @@ -318,6 +318,7 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; + postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; diff --git a/migrations/Dockerfile.dbmate b/migrations/Dockerfile.dbmate new file mode 100644 index 000000000..0af8193f4 --- /dev/null +++ b/migrations/Dockerfile.dbmate @@ -0,0 +1,23 @@ +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y curl wget gnupg2 lsb-release + +RUN ARCH=$(dpkg --print-architecture); \ + case ${ARCH} in \ + amd64) DBMATE_ARCH="linux-amd64" ;; \ + arm64) DBMATE_ARCH="linux-arm64" ;; \ + *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; \ + esac && \ + curl -fsSL -o /usr/local/bin/dbmate \ + https://github.com/amacneil/dbmate/releases/latest/download/dbmate-${DBMATE_ARCH} && \ + chmod +x /usr/local/bin/dbmate + +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN apt-get update && apt-get install -y postgresql-client-%VERSION% + +ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" + +RUN dbmate --version + +ENTRYPOINT ["dbmate"] \ No newline at end of file diff --git a/migrations/docker-compose.yaml b/migrations/docker-compose.yaml index 2609d6008..1b3e8b143 100644 --- a/migrations/docker-compose.yaml +++ b/migrations/docker-compose.yaml @@ -33,7 +33,9 @@ services: command: pg_prove /tests/test.sql dbmate: - image: amacneil/dbmate:1.16.2 + build: + context: . + dockerfile: Dockerfile.dbmate depends_on: db: condition: service_healthy diff --git a/migrations/schema-15.sql b/migrations/schema-15.sql new file mode 100644 index 000000000..1bff8b9d8 --- /dev/null +++ b/migrations/schema-15.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT secrets.id, + secrets.name, + secrets.description, + secrets.secret, + CASE + WHEN (secrets.secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (secrets.key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name) + END + END AS decrypted_secret, + secrets.key_id, + secrets.nonce, + secrets.created_at, + secrets.updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- + diff --git a/migrations/schema-16.sql b/migrations/schema-16.sql new file mode 100644 index 000000000..49873a109 --- /dev/null +++ b/migrations/schema-16.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT id, + name, + description, + secret, + CASE + WHEN (secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secret, 'base64'::text), convert_to(((((id)::text || description) || (created_at)::text) || (updated_at)::text), 'utf8'::name), key_id, nonce), 'utf8'::name) + END + END AS decrypted_secret, + key_id, + nonce, + created_at, + updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- + diff --git a/migrations/schema.sql b/migrations/schema.sql index f68d131e8..1bff8b9d8 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -737,7 +737,7 @@ COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure sch -- CREATE TABLE public.schema_migrations ( - version character varying(255) NOT NULL + version character varying(128) NOT NULL ); diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..c4776b3a8 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -25,16 +25,23 @@ function install_nix() { function execute_stage2_playbook { + echo "POSTGRES_MAJOR_VERSION: ${POSTGRES_MAJOR_VERSION}" + echo "GIT_SHA: ${GIT_SHA}" sudo tee /etc/ansible/ansible.cfg < Date: Thu, 10 Oct 2024 13:10:37 -0400 Subject: [PATCH 07/47] chore: rm dead code --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 40202d2f0..1e4fa8132 100644 --- a/flake.nix +++ b/flake.nix @@ -141,8 +141,6 @@ #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; - #we will add supported versions to this list in the future - supportedVersions = [ "15" ]; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL From 3454b624004a1f2a3f8823941768af3412357c87 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 12:04:50 -0400 Subject: [PATCH 08/47] chore: resolve versions from ansible/vars.yml instead of packer file create a matrix build for 15 16 (or how ever many versions there are) --- ...ublish-nix-pgupgrade-bin-flake-version.yml | 36 ++++++++++++---- .../publish-nix-pgupgrade-scripts.yml | 41 +++++++++++++++---- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 5b985f4be..ebfa27fe3 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -11,8 +11,28 @@ permissions: id-token: write jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + publish-staging: + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo @@ -21,10 +41,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" @@ -58,6 +76,10 @@ jobs: publish-prod: runs-on: ubuntu-latest if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + needs: prepare + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo @@ -66,10 +88,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index eb5f7a755..59de9a788 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -7,7 +7,6 @@ on: - release/* paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' - - 'common-nix.vars.pkr.hcl' workflow_dispatch: inputs: postgresVersion: @@ -18,20 +17,40 @@ permissions: id-token: write jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + publish-staging: + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts @@ -61,9 +80,15 @@ jobs: SLACK_FOOTER: '' publish-prod: + needs: prepare runs-on: ubuntu-latest if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} + + steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -71,10 +96,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts From d576b0174bb7d7dacf9761b901b5e668904baaea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 13:15:47 -0400 Subject: [PATCH 09/47] feat: WIP orioledb 16 --- flake.nix | 6 +++--- nix/ext/orioledb.nix | 6 +++--- nix/overlays/psql_16-oriole.nix | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 1e4fa8132..74b98807b 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ # pull them from the overlays/ directory automatically, but we don't # want to have an arbitrary order, since it might matter. being # explicit is better. - (import ./nix/overlays/cargo-pgrx.nix) + (import ./nix/overlays/cargo-pgrx-0-11-3.nix) (import ./nix/overlays/psql_16-oriole.nix) ]; @@ -289,7 +289,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + psql_orioledb_16 = makeOrioleDbPostgres "16_29" postgresql_orioledb_16; }; # Find the active PostgreSQL version @@ -311,7 +311,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + psql_orioledb_16 = postgresVersions.psql_orioledb_16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 4d8c51bbd..a0ffc119c 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -7,11 +7,11 @@ stdenv.mkDerivation rec { owner = "orioledb"; repo = "orioledb"; rev = "main"; - sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY="; + sha256 = "sha256-VWjb2JHYad0VZkId70m8UOhRTJRGY4nkEuC7m5ae7w4="; }; - version = "patches16_23"; + version = "patches16_29"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=29"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index d55af10a2..00b1d4571 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_23"; + version = "16_29"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz"; - sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_29.tar.gz"; + sha256 = "sha256-JiC4upjta7OWOfnuxgo/KL41tWcsBUyykOJakn6XjvA="; }; buildInputs = old.buildInputs ++ [ prev.bison From ce435309f714634b8b7a895fbe569a3414f89128 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 13:24:50 -0400 Subject: [PATCH 10/47] feat: intro into CI building psql16 bundle oriole version --- docker/nix/build_nix.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3cf8505c0..dfd769382 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -7,7 +7,10 @@ if [ -d "/workspace" ]; then fi nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link +#no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 +nix build .#psql_orioledb_16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 \ No newline at end of file From b6fdd1172e7c9dfda5e4b0a8094955e4ff60e1a9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 16 Oct 2024 09:50:15 -0400 Subject: [PATCH 11/47] test: working through build compat for oriole --- nix/ext/orioledb.nix | 8 ++++---- nix/overlays/psql_16-oriole.nix | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index a0ffc119c..266f552d8 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,12 +6,12 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "main"; - sha256 = "sha256-VWjb2JHYad0VZkId70m8UOhRTJRGY4nkEuC7m5ae7w4="; + rev = "0dafcb1bc799e9af393094c122c1c3c630797222"; + sha256 = "sha256-dsfDqUXkMeAkUI5l9+J09tsRZOGJVsqcKEVo5YAzMjU="; }; - version = "patches16_29"; + version = "patches16_30"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=29"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=30"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index 00b1d4571..d0eaa606c 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_29"; + version = "16_30"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_29.tar.gz"; - sha256 = "sha256-JiC4upjta7OWOfnuxgo/KL41tWcsBUyykOJakn6XjvA="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_30.tar.gz"; + sha256 = "sha256-VITYbOZYKgbtstdfG2wl7+wV+tl/DDYB34BkIM/Xnvs="; }; buildInputs = old.buildInputs ++ [ prev.bison From 859cbca7c60643f78b6aaaec1fdbd1d47fe1eb83 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 10:00:32 -0400 Subject: [PATCH 12/47] working orioledb build --- flake.nix | 4 +++- nix/ext/orioledb.nix | 10 +++++----- nix/overlays/psql_16-oriole.nix | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 74b98807b..8875c726e 100644 --- a/flake.nix +++ b/flake.nix @@ -134,7 +134,9 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option - orioledbExtension = ourExtensions ++ [ ./nix/ext/orioledb.nix ]; + #we're not using timescaledb in the orioledb version of supabase extensions + orioleFilteredExtensions = builtins.filter (x: x != ./nix/ext/timescaledb.nix) ourExtensions; + orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 266f552d8..ff7c12a96 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,17 +6,17 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "0dafcb1bc799e9af393094c122c1c3c630797222"; - sha256 = "sha256-dsfDqUXkMeAkUI5l9+J09tsRZOGJVsqcKEVo5YAzMjU="; + rev = "bd8e32d0ebaafd0ea3ec3074233b65167f3b6fb7"; + sha256 = "sha256-bzH1SgPZ6q90HpqRsECY2XQPghEcd2Hg4X55G43unNo="; }; - version = "patches16_30"; + version = "patches16_31"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=30"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=31"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index d0eaa606c..20051315d 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_30"; + version = "16_31"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_30.tar.gz"; - sha256 = "sha256-VITYbOZYKgbtstdfG2wl7+wV+tl/DDYB34BkIM/Xnvs="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; + sha256 = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; }; buildInputs = old.buildInputs ++ [ prev.bison From a4b4998dc50103ba82c713ac4587ff239a936eee Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 10:04:31 -0400 Subject: [PATCH 13/47] fix: update value --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8875c726e..e76672f4d 100644 --- a/flake.nix +++ b/flake.nix @@ -291,7 +291,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_orioledb_16 = makeOrioleDbPostgres "16_29" postgresql_orioledb_16; + psql_orioledb_16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; }; # Find the active PostgreSQL version From 8c13a58e2b77232f7d4d0769f414de0e96a7377b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 13:24:10 -0400 Subject: [PATCH 14/47] chore: cut staging only release for oriole16 --- .github/workflows/ami-release-nix.yml | 5 +++-- ansible/vars.yml | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 643f26fc7..f7005a4b0 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,8 +3,9 @@ name: Release AMI Nix on: push: branches: - - develop - - release/* + #- develop + #- release/* + - sam/oriole16 paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' diff --git a/ansible/vars.yml b/ansible/vars.yml index f78b56c97..30cbf2dab 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,17 +2,16 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -# postgresql_major: "15" -# postgresql_release: "15.1" -# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c postgres_major: - "15" - "16" + - "oriole-16" # Full version strings for each major version postgres_release: postgres15: "15.8.1.003-staging-5" postgres16: "16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 07117e3828e3cbc2ab62552f27139d75179186ba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 13:31:47 -0400 Subject: [PATCH 15/47] narrow down to just oriole staging ami here --- ansible/vars.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 30cbf2dab..1f161f82e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -3,14 +3,14 @@ ebssurrogate_mode: true async_mode: true postgres_major: - - "15" - - "16" + #- "15" + #- "16" - "oriole-16" # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-5" - postgres16: "16.3.1.000-staging-5" + #postgres15: "15.8.1.003-staging-5" + #postgres16: "16.3.1.000-staging-5" postgresoriole-16: "oriole-16.3.1.000-staging-1" # Non Postgres Extensions From 2ea1e907f5c4ae72396a251a87ca65586fd36e7c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:06:06 -0400 Subject: [PATCH 16/47] fix: make sure name lines up with gh action convention --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9c26efbb0..108483071 100644 --- a/flake.nix +++ b/flake.nix @@ -291,7 +291,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_orioledb_16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; + psql_oriole-16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; }; # Find the active PostgreSQL version @@ -313,7 +313,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - psql_orioledb_16 = postgresVersions.psql_orioledb_16; + psql_oriole-16 = postgresVersions.psql_oriole-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; From c7f68df1eabefd8cb70900fb59920cd9e5ac848a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:11:26 -0400 Subject: [PATCH 17/47] fix: get correct naming convention in here too for oriole bundle --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 822648bd7..78c843cd1 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,7 +10,7 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_orioledb_16/bin -o psql_orioledb_16 +nix build .#psql_oriole-16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 From 870f591aacaf089d7395a57e485749f1c92b157a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:17:16 -0400 Subject: [PATCH 18/47] chore: bump to cut release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 1f161f82e..fe6bbfdc7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-1" + postgresoriole-16: "oriole-16.3.1.000-staging-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" From d483a0786bee7266ac975863695613912246bee2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 15:24:18 -0400 Subject: [PATCH 19/47] fix: make sure src and debug builds available for oriole too --- flake.nix | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 108483071..27f46a11a 100644 --- a/flake.nix +++ b/flake.nix @@ -307,6 +307,7 @@ }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; + postgresql_oriole-16 = postgresql_orioledb_16; in postgresVersions //{ supabase-groonga = supabase-groonga; @@ -316,9 +317,10 @@ psql_oriole-16 = postgresVersions.psql_oriole-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16; + inherit postgresql_15 postgresql_16 postgresql_oriole-16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; + postgresql_oriole-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb_16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -363,6 +365,28 @@ platforms = platforms.all; }; }; + postgresql_oriole-16_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-16-src"; + version = postgresql_oriole-16.version; + + src = postgresql_16.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; From 5e69b003408e84ccbf3d5d6aa63c5c055f3ca353 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 15:28:36 -0400 Subject: [PATCH 20/47] chore: bump to staging release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index fe6bbfdc7..413704ad4 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-2" + postgresoriole-16: "oriole-16.3.1.000-staging-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From b62e770796de2ed009e863fd8ef520f1a23d3975 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 16:27:44 -0400 Subject: [PATCH 21/47] fix: when oriole16 rm timescale from supautils conf --- ansible/tasks/stage2-setup-postgres.yml | 16 ++++++++++++++++ ansible/vars.yml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index e22cbe67b..e6a2616f5 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -3,6 +3,22 @@ # shell: | # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task + +- name: Check psql_version and modify supautils.conf if necessary + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Remove 'timescaledb,' from supautils.conf + ansible.builtin.lineinfile: + path: /etc/postgresql-custom/supautils.conf + regexp: 'timescaledb,' + line: '' + state: absent + when: is_psql_oriole_16 and stage2_nix + become: yes + - name: Install Postgres from nix binary cache become: yes shell: | diff --git a/ansible/vars.yml b/ansible/vars.yml index 413704ad4..0b870393d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-3" + postgresoriole-16: "oriole-16.3.1.000-staging-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 889dc70f07da836291004e029af5bbda8fe6443f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 21:03:40 -0400 Subject: [PATCH 22/47] fix: also remove from postgresql conf --- ansible/tasks/stage2-setup-postgres.yml | 10 +++++++++- ansible/vars.yml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index e6a2616f5..a9c167499 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,7 +10,15 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from supautils.conf + - name: Remove 'timescaledb,' from supautils.conf if oriole-16 build + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf if oriole-16 build + regexp: 'timescaledb,' + line: '' + state: absent + when: is_psql_oriole_16 and stage2_nix + become: yes + - name: Remove 'timescaledb,' from postgresql.conf ansible.builtin.lineinfile: path: /etc/postgresql-custom/supautils.conf regexp: 'timescaledb,' diff --git a/ansible/vars.yml b/ansible/vars.yml index 0b870393d..5c9331f17 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-4" + postgresoriole-16: "oriole-16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" From a67d8e1de65eee5f4719219831690c50cf8ba0a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 08:39:12 -0400 Subject: [PATCH 23/47] fix: find all instances and remove in this condition for tiemscaledb --- ansible/tasks/stage2-setup-postgres.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index a9c167499..5b4ee2a28 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,20 +10,14 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from supautils.conf if oriole-16 build - ansible.builtin.lineinfile: - path: /etc/postgresql/postgresql.conf if oriole-16 build - regexp: 'timescaledb,' - line: '' - state: absent + - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build + ansible.builtin.command: + cmd: sed -i 's/timescaledb,//g' /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Remove 'timescaledb,' from postgresql.conf - ansible.builtin.lineinfile: - path: /etc/postgresql-custom/supautils.conf - regexp: 'timescaledb,' - line: '' - state: absent + - name: Remove 'timescaledb,' from supautils.conf + ansible.builtin.command: + cmd: sed -i 's/timescaledb,//g' /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes @@ -31,7 +25,6 @@ become: yes shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" -#TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix - name: Install pg_prove from nix binary cache From 67382a646447b26546657b0aa187b826e7bba6ac Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 09:28:33 -0400 Subject: [PATCH 24/47] fix: maybe broke pgsodium with too many spaces --- ansible/tasks/stage2-setup-postgres.yml | 4 ++-- ansible/vars.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 5b4ee2a28..485a068f6 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -12,12 +12,12 @@ - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/timescaledb,//g' /etc/postgresql/postgresql.conf + cmd: sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - name: Remove 'timescaledb,' from supautils.conf ansible.builtin.command: - cmd: sed -i 's/timescaledb,//g' /etc/postgresql-custom/supautils.conf + cmd: sed -i 's/ timescaledb,//g' /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 5c9331f17..2360c2fc7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-6" # Non Postgres Extensions pgbouncer_release: "1.19.0" From c08b661756562529bbdb1eea08321cde35b5ea5e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 10:24:58 -0400 Subject: [PATCH 25/47] fix: create a symlink for missing path --- .github/workflows/test.yml | 3 ++- ansible/tasks/stage2-setup-postgres.yml | 9 +++++++++ ansible/vars.yml | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1457eb100..c58df7d15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,8 @@ on: push: branches: - develop - pull_request: + #TODO PR uncomment this line + #pull_request: workflow_dispatch: jobs: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 485a068f6..400af65ac 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -256,6 +256,15 @@ line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh' when: stage2_nix +#TODO PR may need to remove this. +- name: Create symbolic link for pgsodium_getkey script + file: + src: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" + dest: "/usr/lib/postgresql/share/postgresql/extension/pgsodium_getkey" + state: link + become: yes + when: stage2_nix + - name: Append GRN_PLUGINS_DIR to /etc/environment.d/postgresql.env ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env diff --git a/ansible/vars.yml b/ansible/vars.yml index 2360c2fc7..e924d826e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,10 +8,11 @@ postgres_major: - "oriole-16" # Full version strings for each major version +# TOD PR uncomment these lines postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-6" + postgresoriole-16: "oriole-16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 59b44b6364f0cd0191e7ce35a34cd0e748e9a258 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 11:49:53 -0400 Subject: [PATCH 26/47] fix: try a more universal way to deactivate pgsodium at this stage --- ansible/tasks/test-image.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index d6e8223f7..669bfec79 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -4,11 +4,22 @@ - libtap-parser-sourcehandler-pgtap-perl when: debpkg_mode +# - name: Temporarily disable PG Sodium references in config +# become: yes +# become_user: postgres +# shell: +# cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf +# when: debpkg_mode or stage2_nix + - name: Temporarily disable PG Sodium references in config become: yes become_user: postgres shell: - cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + cmd: > + sed -i.bak + -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' + -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/' + /etc/postgresql/postgresql.conf when: debpkg_mode or stage2_nix - name: Start Postgres Database to load all extensions. From f9613716fa8b64a3815ec31f4529ef908042c8f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 12:24:20 -0400 Subject: [PATCH 27/47] chore: when oriole16 rm timescale plv8 postgis pgvector pgrouting --- ansible/tasks/stage2-setup-postgres.yml | 14 +++++++++----- flake.nix | 10 +++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 400af65ac..0b4e3963d 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,17 +10,21 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build + - name: Remove specified extensions from postgresql.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf + cmd: > + sed -i 's/ timescaledb,//g' + /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Remove 'timescaledb,' from supautils.conf + + - name: Remove specified extensions from supautils.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/ timescaledb,//g' /etc/postgresql-custom/supautils.conf + cmd: > + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' + /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Install Postgres from nix binary cache become: yes shell: | diff --git a/flake.nix b/flake.nix index 27f46a11a..3c5cb0fc1 100644 --- a/flake.nix +++ b/flake.nix @@ -135,7 +135,15 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option #we're not using timescaledb in the orioledb version of supabase extensions - orioleFilteredExtensions = builtins.filter (x: x != ./nix/ext/timescaledb.nix) ourExtensions; + orioleFilteredExtensions = builtins.filter ( + x: + x != ./nix/ext/timescaledb.nix && + x != ./nix/ext/pgvector.nix && + x != ./nix/ext/plv8.nix && + x != ./nix/ext/postgis.nix && + x != ./nix/ext/pgrouting.nix + ) ourExtensions; + orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql From a8f1a6be717a8f3412c30140143cc5e4ba54308c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 12:34:36 -0400 Subject: [PATCH 28/47] fix: more handling of oriole16 differences --- ansible/tasks/test-image.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 669bfec79..da77fb166 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -63,6 +63,21 @@ failed_when: retval.rc != 0 when: debpkg_mode or stage2_nix +- name: Check psql_version and modify migrations if orioe-16 + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Remove specified extensions from SQL file + ansible.builtin.command: + cmd: > + sed -i '/\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\)/d' + /tmp/migrations/tests/test.sql + when: is_psql_oriole_16 and stage2_nix + become: yes + + - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval From c35fa68409acc61017ca0c11c529e7699b4e646e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 13:35:31 -0400 Subject: [PATCH 29/47] chore more oriole16 handling --- ansible/tasks/stage2-setup-postgres.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 0b4e3963d..dbc61e5fd 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -4,7 +4,7 @@ # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task -- name: Check psql_version and modify supautils.conf if necessary +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary block: - name: Check if psql_version is psql_oriole-16 set_fact: @@ -25,6 +25,7 @@ /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes + - name: Install Postgres from nix binary cache become: yes shell: | @@ -219,13 +220,19 @@ recurse: yes when: stage2_nix -- name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir - shell: > - sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ - sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" - become: yes - when: stage2_nix +- name: Check psql_version and run postgis linking if not oriole-16 + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir + shell: > + sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ + sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" + become: yes + when: stage2_nix and not is_psql_oriole_16 - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: From d1e06d473e2e205e62bf539d771ca58085afe3c1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 14:25:28 -0400 Subject: [PATCH 30/47] fix: refine sed regex --- ansible/tasks/test-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index da77fb166..c253be3ed 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -72,12 +72,12 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\)/d' - /tmp/migrations/tests/test.sql - when: is_psql_oriole_16 and stage2_nix + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/test.sql + when: is_psql_oriole_16 become: yes + - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval From 304aa68b4856cd1fdb341ef9251d885bb5c2594d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 15:03:42 -0400 Subject: [PATCH 31/47] fix: also cond rm actual files --- ansible/tasks/test-image.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index c253be3ed..65c03871a 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -76,6 +76,25 @@ when: is_psql_oriole_16 become: yes + - name: Remove specified extension files from extensions directory + ansible.builtin.find: + paths: /tmp/migrations/tests/extensions + patterns: + - '*timescaledb*.sql' + - '*postgis*.sql' + - '*pgrouting*.sql' + - '*plv8*.sql' + - '*pgvector*.sql' + register: files_to_remove + when: is_psql_oriole_16 + + - name: Delete matched extension files + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ files_to_remove.files }}" + when: is_psql_oriole_16 + become: yes - name: Run migrations tests From 55b97630abef9a8d807dbed02bb3b8f8277b1925 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 16:05:02 -0400 Subject: [PATCH 32/47] fix: correct dir --- ansible/tasks/test-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 65c03871a..f342d2e00 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -72,7 +72,7 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/test.sql + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql when: is_psql_oriole_16 become: yes From 292327efcc35f5563e41cd00328d68b956ff36b2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 16:48:41 -0400 Subject: [PATCH 33/47] chore: bump to release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e924d826e..56fc36357 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-7" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 32163be55fc44bc5cbbdc8f829232232a9fc9fb0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 21 Oct 2024 15:44:09 -0400 Subject: [PATCH 34/47] feat: parse and handle oropledb-16 in start-server --- flake.nix | 1 + nix/tools/run-server.sh.in | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 3c5cb0fc1..940e7564c 100644 --- a/flake.nix +++ b/flake.nix @@ -454,6 +454,7 @@ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 977a437fb..1a634b8d6 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -1,7 +1,7 @@ #!/usr/bin/env bash # shellcheck shell=bash [ ! -z "$DEBUG" ] && set -x -# first argument should be '15' or '16' for the version + if [ "$1" == "15" ]; then echo "Starting server for PSQL 15" PSQL15=@PSQL15_BINDIR@ @@ -61,5 +61,17 @@ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "\$a\\ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" + +remove_config_items() { + if [ "$1" == "orioledb-16" ]; then + # Remove items from supautils.conf + sed -i '/supautils.privileged_extensions/d' "$DATDIR/supautils.conf" + + # Remove items from postgresql.conf + sed -i 's/ timescaledb,//g' "$DATDIR/postgresql.conf" + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + fi +} +remove_config_items "$1" export GRN_PLUGINS_DIR=$GROONGA/lib/groonga/plugins postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp From e1c199c3a0e8a4d41c320fc6a1f715aa81f9c93d Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 21 Oct 2024 15:51:11 -0400 Subject: [PATCH 35/47] feat: start-client for oriole16 --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 940e7564c..af25775ad 100644 --- a/flake.nix +++ b/flake.nix @@ -484,6 +484,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ From 6a5648bb03ac96c1b420a930cd371187d8a17108 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 22 Oct 2024 08:56:28 -0400 Subject: [PATCH 36/47] wip for orioledb push to build on linux --- flake.nix | 104 ++++++++++++++++++++++++-------- nix/overlays/psql_16-oriole.nix | 6 +- 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/flake.nix b/flake.nix index af25775ad..71e671c67 100644 --- a/flake.nix +++ b/flake.nix @@ -116,7 +116,7 @@ ./nix/ext/pg_cron.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix - ./nix/ext/pg_net.nix + #./nix/ext/pg_net.nix ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix @@ -141,13 +141,14 @@ x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && - x != ./nix/ext/pgrouting.nix + x != ./nix/ext/pgrouting.nix && + x != ./nix/ext/pg_net.nix ) ourExtensions; orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql - postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb; #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; @@ -242,26 +243,81 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; - makeOrioleDbPostgresBin = version: patchedPostgres: - let - postgresql = patchedPostgres; - upstreamExts = map - (ext: { - name = postgresql.pkgs."${ext}".pname; - version = postgresql.pkgs."${ext}".version; - }) - orioledbPsqlExtensions; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); - - pgbin = postgresql.withPackages (ps: - (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) - ); - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - }; - + # makeOrioleDbPostgresBin = version: patchedPostgres: + # let + # postgresql = patchedPostgres; + # upstreamExts = map + # (ext: { + # name = postgresql.pkgs."${ext}".pname; + # version = postgresql.pkgs."${ext}".version; + # }) + # orioledbPsqlExtensions; + # ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); + + # pgbin = postgresql.withPackages (ps: + # (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) + # ); + # in + # pkgs.symlinkJoin { + # inherit (pgbin) name version; + # paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + # }; +makeOrioleDbPostgresBin = version: patchedPostgres: + let + postgresql = patchedPostgres; + upstreamExts = map + (ext: { + name = patchedPostgres.pkgs."${ext}".pname; + version = patchedPostgres.pkgs."${ext}".version; + }) + orioledbPsqlExtensions; + ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); + + # Create the scope and new self/super for package resolution + scope = { + inherit (patchedPostgres) jitSupport; + inherit (oriole_pkgs.llvmPackages) llvm; + postgresql = patchedPostgres; + inherit (postgresql) stdenv'; + }; + newSelf = self // scope; + newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; + + # Get the packages using the passthru pattern + selectedPackages = + (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ + (makeOurOrioleDbPostgresPkgs version postgresql); + + # Build the environment with the selected packages + pgbin = oriole_pkgs.buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = [ postgresql ] ++ selectedPackages; + buildInputs = [ oriole_pkgs.makeWrapper ]; + passthru = postgresql.passthru // { + inherit postgresql; + args = selectedPackages; + }; + + postBuild = '' + mkdir -p $out/bin + for bin in ${postgresql}/bin/*; do + ln -sf $bin $out/bin/ + done + + rm -f $out/bin/postgres + makeWrapper ${postgresql}/bin/postgres $out/bin/postgres \ + --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ + --prefix PATH : $out/bin \ + --prefix LD_LIBRARY_PATH : $out/lib + ''; + }// { + version = postgresql.version; + }; + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + }; # Create an attribute set, containing all the relevant packages for a # PostgreSQL install, wrapped up with a bow on top. There are three @@ -299,7 +355,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_oriole-16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; + psql_oriole-16 = makeOrioleDbPostgres "16" postgresql_orioledb_16; }; # Find the active PostgreSQL version diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index 20051315d..309129f1f 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,6 +1,6 @@ final: prev: { - postgresql_16 = prev.postgresql_16.overrideAttrs (old: { - pname = "postgresql_16"; + pg_orioledb = prev.postgresql_16.overrideAttrs (old: { + pname = "postgresql_orioledb"; version = "16_31"; src = prev.fetchurl { url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; @@ -17,5 +17,5 @@ final: prev: { prev.perl ]; }); - postgresql_orioledb_16 = final.postgresql_16; + postgresql_orioledb = final.pg_orioledb; } From 049010ae6d3f2345bed72ddd5af92b059e825b3b Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 22 Oct 2024 11:12:49 -0400 Subject: [PATCH 37/47] fix: running server --- flake.nix | 59 ++++++++++++++++++++++++++++++-------- nix/tools/run-server.sh.in | 5 +--- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 71e671c67..11d5425ea 100644 --- a/flake.nix +++ b/flake.nix @@ -116,7 +116,7 @@ ./nix/ext/pg_cron.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix - #./nix/ext/pg_net.nix + ./nix/ext/pg_net.nix ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix @@ -273,7 +273,6 @@ makeOrioleDbPostgresBin = version: patchedPostgres: orioledbPsqlExtensions; ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); - # Create the scope and new self/super for package resolution scope = { inherit (patchedPostgres) jitSupport; inherit (oriole_pkgs.llvmPackages) llvm; @@ -283,12 +282,10 @@ makeOrioleDbPostgresBin = version: patchedPostgres: newSelf = self // scope; newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; - # Get the packages using the passthru pattern selectedPackages = (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql); - # Build the environment with the selected packages pgbin = oriole_pkgs.buildEnv { name = "postgresql-and-plugins-${postgresql.version}"; paths = [ postgresql ] ++ selectedPackages; @@ -299,18 +296,56 @@ makeOrioleDbPostgresBin = version: patchedPostgres: }; postBuild = '' + # Ensure all necessary directories exist mkdir -p $out/bin + mkdir -p $out/lib/postgresql + mkdir -p $out/share/postgresql/extension + + # Create proper links for all binaries with environment for bin in ${postgresql}/bin/*; do - ln -sf $bin $out/bin/ + binary_name=$(basename $bin) + rm -f "$out/bin/$binary_name" + makeWrapper $bin $out/bin/$binary_name \ + --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ + --prefix PATH : $out/bin \ + --prefix LD_LIBRARY_PATH : "$out/lib:$out/lib/postgresql" \ + --prefix PKG_CONFIG_PATH : "$out/lib/pkgconfig" \ + --set PGDATA "/var/lib/postgresql/${postgresql.psqlSchema}" \ + --set PGHOST "/run/postgresql" + done + + # Ensure extension files are properly linked + for pkg in ${postgresql} ${toString selectedPackages}; do + # Link shared objects + if [ -d "$pkg/lib" ]; then + find "$pkg/lib" -type f -name '*.so' -exec ln -sf {} "$out/lib/postgresql/" \; + fi + + # Link extension control and SQL files + if [ -d "$pkg/share/postgresql/extension" ]; then + find "$pkg/share/postgresql/extension" -type f \( -name '*.control' -o -name '*.sql' \) \ + -exec ln -sf {} "$out/share/postgresql/extension/" \; + fi done - - rm -f $out/bin/postgres - makeWrapper ${postgresql}/bin/postgres $out/bin/postgres \ - --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ - --prefix PATH : $out/bin \ - --prefix LD_LIBRARY_PATH : $out/lib + + # Create pg_config if it doesn't exist + if [ ! -e "$out/bin/pg_config" ]; then + makeWrapper ${postgresql}/bin/pg_config $out/bin/pg_config \ + --set prefix "$out" \ + --set includedir "$out/include" \ + --set pkgincludedir "$out/include/postgresql" \ + --set includedir-server "$out/include/postgresql/server" \ + --set libdir "$out/lib" \ + --set pkglibdir "$out/lib/postgresql" \ + --set localedir "$out/share/locale" \ + --set mandir "$out/share/man" \ + --set sharedir "$out/share/postgresql" \ + --set sysconfdir "/etc/postgresql" \ + --set pgxs "$out/lib/postgresql/pgxs/src/makefiles/pgxs.mk" \ + --set configure "--enable-nls --with-openssl --with-libxml --with-libxslt" + fi ''; - }// { + } // { version = postgresql.version; }; in diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 1a634b8d6..a6b6ad7b1 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -64,11 +64,8 @@ session_preload_libraries = 'supautils'" \ remove_config_items() { if [ "$1" == "orioledb-16" ]; then - # Remove items from supautils.conf - sed -i '/supautils.privileged_extensions/d' "$DATDIR/supautils.conf" - # Remove items from postgresql.conf - sed -i 's/ timescaledb,//g' "$DATDIR/postgresql.conf" + sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" fi } From c2e2801e8a7c4e3a47a31e3345d13e197b885b05 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 23 Oct 2024 19:09:17 -0400 Subject: [PATCH 38/47] chore: consolidate nix code that handles building of postgres versions --- flake.nix | 196 +++++---------------------------- nix/postgresql/default.nix | 4 +- nix/postgresql/generic.nix | 21 +++- nix/postgresql/orioledb-16.nix | 20 ++++ 4 files changed, 62 insertions(+), 179 deletions(-) create mode 100644 nix/postgresql/orioledb-16.nix diff --git a/flake.nix b/flake.nix index 11d5425ea..5d25c3ef7 100644 --- a/flake.nix +++ b/flake.nix @@ -24,26 +24,6 @@ pgsqlSuperuser = "supabase_admin"; nix2img = nix2container.packages.${system}.nix2container; - # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which - # we can use to build our own images; it is the common name to refer to - # a copy of nixpkgs which contains all its packages. - # it also serves as a base for importing the orioldb/postgres overlay to - #build the orioledb postgres patched version of postgresql16 - oriole_pkgs = import nixpkgs { - config = { allowUnfree = true; }; - inherit system; - overlays = [ - # NOTE (aseipp): add any needed overlays here. in theory we could - # pull them from the overlays/ directory automatically, but we don't - # want to have an arbitrary order, since it might matter. being - # explicit is better. - (import ./nix/overlays/cargo-pgrx-0-11-3.nix) - (import ./nix/overlays/psql_16-oriole.nix) - - ]; - }; - #This variable works the same as 'oriole_pkgs' but builds using the upstream - #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { config = { allowUnfree = true; @@ -59,11 +39,7 @@ # explicit is better. (final: prev: { postgresql = final.callPackage ./nix/postgresql/default.nix { - inherit (final) lib; - inherit (final) stdenv; - inherit (final) fetchurl; - inherit (final) makeWrapper; - inherit (final) callPackage; + inherit (final) lib stdenv fetchurl makeWrapper callPackage buildEnv newScope; }; }) (import ./nix/overlays/cargo-pgrx-0-11-3.nix) @@ -141,17 +117,15 @@ x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && - x != ./nix/ext/pgrouting.nix && - x != ./nix/ext/pg_net.nix + x != ./nix/ext/pgrouting.nix ) ourExtensions; - orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; + orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - #this var is a convenience setting to import the orioledb patched version of postgresql - postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb; - #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: - pkgs.postgresql."postgresql_${version}"; + if version == "orioledb_16" + then pkgs.postgresql."postgresql_orioledb-16" + else pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -180,7 +154,7 @@ }; extensions = ourExts; - # NOTE (aseipp): this field can be used to do cache busting (e.g. + # NOTE this field can be used to do cache busting (e.g. # force a rebuild of the psql packages) but also to helpfully inform # tools what version of the schema is being used, for forwards and # backwards compatibility @@ -188,22 +162,18 @@ }; }; - makeOurOrioleDbPostgresPkgs = version: patchedPostgres: - let postgresql = patchedPostgres; - in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; + # makeOurOrioleDbPostgresPkgs = version: patchedPostgres: + # let postgresql = patchedPostgres; + # in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; makeOurPostgresPkgs = version: - let postgresql = getPostgresqlPackage version; - in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions; + let + postgresql = getPostgresqlPackage version; + extensionsToUse = if version == "orioledb-16" + then orioledbExtensions + else ourExtensions; + in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; - # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. - makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: - (builtins.listToAttrs (map - (drv: - { name = drv.pname; value = drv; } - ) - (makeOurOrioleDbPostgresPkgs version patchedPostgres))) - // { recurseForDerivations = true; }; # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: @@ -243,117 +213,6 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; - # makeOrioleDbPostgresBin = version: patchedPostgres: - # let - # postgresql = patchedPostgres; - # upstreamExts = map - # (ext: { - # name = postgresql.pkgs."${ext}".pname; - # version = postgresql.pkgs."${ext}".version; - # }) - # orioledbPsqlExtensions; - # ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); - - # pgbin = postgresql.withPackages (ps: - # (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) - # ); - # in - # pkgs.symlinkJoin { - # inherit (pgbin) name version; - # paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - # }; -makeOrioleDbPostgresBin = version: patchedPostgres: - let - postgresql = patchedPostgres; - upstreamExts = map - (ext: { - name = patchedPostgres.pkgs."${ext}".pname; - version = patchedPostgres.pkgs."${ext}".version; - }) - orioledbPsqlExtensions; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); - - scope = { - inherit (patchedPostgres) jitSupport; - inherit (oriole_pkgs.llvmPackages) llvm; - postgresql = patchedPostgres; - inherit (postgresql) stdenv'; - }; - newSelf = self // scope; - newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; - - selectedPackages = - (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ - (makeOurOrioleDbPostgresPkgs version postgresql); - - pgbin = oriole_pkgs.buildEnv { - name = "postgresql-and-plugins-${postgresql.version}"; - paths = [ postgresql ] ++ selectedPackages; - buildInputs = [ oriole_pkgs.makeWrapper ]; - passthru = postgresql.passthru // { - inherit postgresql; - args = selectedPackages; - }; - - postBuild = '' - # Ensure all necessary directories exist - mkdir -p $out/bin - mkdir -p $out/lib/postgresql - mkdir -p $out/share/postgresql/extension - - # Create proper links for all binaries with environment - for bin in ${postgresql}/bin/*; do - binary_name=$(basename $bin) - rm -f "$out/bin/$binary_name" - makeWrapper $bin $out/bin/$binary_name \ - --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ - --prefix PATH : $out/bin \ - --prefix LD_LIBRARY_PATH : "$out/lib:$out/lib/postgresql" \ - --prefix PKG_CONFIG_PATH : "$out/lib/pkgconfig" \ - --set PGDATA "/var/lib/postgresql/${postgresql.psqlSchema}" \ - --set PGHOST "/run/postgresql" - done - - # Ensure extension files are properly linked - for pkg in ${postgresql} ${toString selectedPackages}; do - # Link shared objects - if [ -d "$pkg/lib" ]; then - find "$pkg/lib" -type f -name '*.so' -exec ln -sf {} "$out/lib/postgresql/" \; - fi - - # Link extension control and SQL files - if [ -d "$pkg/share/postgresql/extension" ]; then - find "$pkg/share/postgresql/extension" -type f \( -name '*.control' -o -name '*.sql' \) \ - -exec ln -sf {} "$out/share/postgresql/extension/" \; - fi - done - - # Create pg_config if it doesn't exist - if [ ! -e "$out/bin/pg_config" ]; then - makeWrapper ${postgresql}/bin/pg_config $out/bin/pg_config \ - --set prefix "$out" \ - --set includedir "$out/include" \ - --set pkgincludedir "$out/include/postgresql" \ - --set includedir-server "$out/include/postgresql/server" \ - --set libdir "$out/lib" \ - --set pkglibdir "$out/lib/postgresql" \ - --set localedir "$out/share/locale" \ - --set mandir "$out/share/man" \ - --set sharedir "$out/share/postgresql" \ - --set sysconfdir "/etc/postgresql" \ - --set pgxs "$out/lib/postgresql/pgxs/src/makefiles/pgxs.mk" \ - --set configure "--enable-nls --with-openssl --with-libxml --with-libxslt" - fi - ''; - } // { - version = postgresql.version; - }; - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - }; - # Create an attribute set, containing all the relevant packages for a # PostgreSQL install, wrapped up with a bow on top. There are three # packages: @@ -368,11 +227,6 @@ makeOrioleDbPostgresBin = version: patchedPostgres: exts = makeOurPostgresPkgsSet version; recurseForDerivations = true; }; - makeOrioleDbPostgres = version: patchedPostgres: rec { - bin = makeOrioleDbPostgresBin version patchedPostgres; - exts = makeOurOrioleDbPostgresPkgsSet version patchedPostgres; - recurseForDerivations = true; - }; # The base set of packages that we export from this Nix Flake, that can # be used with 'nix build'. Don't use the names listed below; check the @@ -390,7 +244,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_oriole-16 = makeOrioleDbPostgres "16" postgresql_orioledb_16; + psql_orioledb-16 = makePostgres "orioledb-16" ; }; # Find the active PostgreSQL version @@ -406,20 +260,20 @@ makeOrioleDbPostgresBin = version: patchedPostgres: }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; - postgresql_oriole-16 = postgresql_orioledb_16; + postgresql_orioledb-16 = getPostgresqlPackage "orioledb-16"; in postgresVersions //{ supabase-groonga = supabase-groonga; # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - psql_oriole-16 = postgresVersions.psql_oriole-16; + psql_orioledb-16 = postgresVersions.psql_orioledb-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16 postgresql_oriole-16; + inherit postgresql_15 postgresql_16 postgresql_orioledb-16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; - postgresql_oriole-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb_16.debug else null; + postgresql_orioledb-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -464,9 +318,9 @@ makeOrioleDbPostgresBin = version: patchedPostgres: platforms = platforms.all; }; }; - postgresql_oriole-16_src = pkgs.stdenv.mkDerivation { + postgresql_orioledb-16_src = pkgs.stdenv.mkDerivation { pname = "postgresql-16-src"; - version = postgresql_oriole-16.version; + version = postgresql_orioledb-16.version; src = postgresql_16.src; @@ -545,7 +399,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -575,7 +429,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 5f92262b2..484cac777 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -1,11 +1,10 @@ self: let - #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; postgresql_16 = ./16.nix; + postgresql_orioledb-16 = ./orioledb-16.nix; }; - mkAttributes = jitSupport: self.lib.mapAttrs' (version: path: let @@ -15,7 +14,6 @@ let inherit jitSupport self; }) ) versions; - in # variations without and with JIT (mkAttributes false) // (mkAttributes true) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 323b65167..c03d1bcb3 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -7,6 +7,8 @@ let , glibc, zlib, readline, openssl, icu, lz4, zstd, systemd, libossp_uuid , pkg-config, libxml2, tzdata, libkrb5, substituteAll, darwin , linux-pam + #orioledb specific + , perl, bison, flex, docbook_xsl, docbook_xml_dtd_45, docbook_xsl_ns, libxslt # This is important to obtain a version of `libpq` that does not depend on systemd. , systemdSupport ? lib.meta.availableOn stdenv.hostPlatform systemd && !stdenv.hostPlatform.isStatic @@ -49,10 +51,16 @@ let inherit version; pname = pname + lib.optionalString jitSupport "-jit"; - src = fetchurl { - url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; - inherit hash; - }; + src = if (builtins.match "16_.*" version != null) then + fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; + inherit hash; + } + else + fetchurl { + url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; + inherit hash; + }; hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; @@ -74,7 +82,10 @@ let ++ lib.optionals pythonSupport [ python3 ] ++ lib.optionals gssSupport [ libkrb5 ] ++ lib.optionals stdenv'.isLinux [ linux-pam ] - ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ]; + ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] + ++ lib.optionals (builtins.match "16_.*" version != null) [ + perl bison flex docbook_xsl docbook_xml_dtd_45 docbook_xsl_ns libxslt + ]; nativeBuildInputs = [ makeWrapper diff --git a/nix/postgresql/orioledb-16.nix b/nix/postgresql/orioledb-16.nix new file mode 100644 index 000000000..50f8961f0 --- /dev/null +++ b/nix/postgresql/orioledb-16.nix @@ -0,0 +1,20 @@ +# { self, fetchurl, ... }: + +# let +# generic = import ./generic.nix rec { +# version = "16"; +# hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; +# src = fetchurl { +# url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; +# sha256 = hash; +# }; +# }; +# in +# generic.overrideAttrs (oldAttrs: { +# inherit generic; +# }) +# orioledb-16.nix +import ./generic.nix { + version = "16_31"; + hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; +} From 1cd20632576b47d5937561fa1f9c20a6e47ccf01 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 23 Oct 2024 19:16:12 -0400 Subject: [PATCH 39/47] fix: update name for bundle --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 78c843cd1..7213333f0 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,7 +10,7 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_oriole-16/bin -o psql_orioledb_16 +nix build .#psql_orioledb-16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 From 0e23e7d700ad673ae71458ca7dfe2620fc4d2258 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 23 Oct 2024 21:40:14 -0400 Subject: [PATCH 40/47] chore: small cleanup --- flake.nix | 6 +----- nix/postgresql/orioledb-16.nix | 16 ---------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/flake.nix b/flake.nix index 5d25c3ef7..eb590dc38 100644 --- a/flake.nix +++ b/flake.nix @@ -33,7 +33,7 @@ }; inherit system; overlays = [ - # NOTE (aseipp): add any needed overlays here. in theory we could + # NOTE: add any needed overlays here. in theory we could # pull them from the overlays/ directory automatically, but we don't # want to have an arbitrary order, since it might matter. being # explicit is better. @@ -162,10 +162,6 @@ }; }; - # makeOurOrioleDbPostgresPkgs = version: patchedPostgres: - # let postgresql = patchedPostgres; - # in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; - makeOurPostgresPkgs = version: let postgresql = getPostgresqlPackage version; diff --git a/nix/postgresql/orioledb-16.nix b/nix/postgresql/orioledb-16.nix index 50f8961f0..e9a66850e 100644 --- a/nix/postgresql/orioledb-16.nix +++ b/nix/postgresql/orioledb-16.nix @@ -1,19 +1,3 @@ -# { self, fetchurl, ... }: - -# let -# generic = import ./generic.nix rec { -# version = "16"; -# hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; -# src = fetchurl { -# url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; -# sha256 = hash; -# }; -# }; -# in -# generic.overrideAttrs (oldAttrs: { -# inherit generic; -# }) -# orioledb-16.nix import ./generic.nix { version = "16_31"; hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; From 9d48e503545709094b1eb9f336a898c5414fcd11 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 23 Oct 2024 23:27:26 -0400 Subject: [PATCH 41/47] chore: bump version release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 996049d78..b33eae2b0 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-7" + postgresoriole-16: "oriole-16.3.1.000-staging-8" # Non Postgres Extensions pgbouncer_release: "1.19.0" From fed90a0a1319057d56713c5c72b25a21d4e6058c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 00:30:52 -0400 Subject: [PATCH 42/47] chore: cleanup names --- ansible/tasks/stage2-setup-postgres.yml | 10 +++++----- ansible/tasks/test-image.yml | 4 ++-- ansible/vars.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index dbc61e5fd..685aa651d 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,16 +6,16 @@ - name: Check psql_version and modify supautils.conf and postgresql.conf if necessary block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Remove specified extensions from postgresql.conf if oriole-16 build ansible.builtin.command: cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_oriole_16 and stage2_nix + when: is_psql_orioledb_16 and stage2_nix become: yes - name: Remove specified extensions from supautils.conf if oriole-16 build @@ -222,9 +222,9 @@ - name: Check psql_version and run postgis linking if not oriole-16 block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir shell: > diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index f342d2e00..40a208136 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -65,9 +65,9 @@ - name: Check psql_version and modify migrations if orioe-16 block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Remove specified extensions from SQL file ansible.builtin.command: diff --git a/ansible/vars.yml b/ansible/vars.yml index b33eae2b0..41af00bde 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-8" + postgresoriole-16: "oriole-16.3.1.000-staging-9" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 9b078d03e1ecffeb53ead597d8be216e96ccf436 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 00:33:15 -0400 Subject: [PATCH 43/47] chore: correct names --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 41af00bde..e16af209a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -5,14 +5,14 @@ async_mode: true postgres_major: #- "15" #- "16" - - "oriole-16" + - "orioledb-16" # Full version strings for each major version # TOD PR uncomment these lines postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-9" + postgresoriole-16: "orioledb-16.3.1.000-staging-9" # Non Postgres Extensions pgbouncer_release: "1.19.0" From bc2b4cc592cd8517391c61c4e5a9b31fe1257b1c Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 06:10:56 -0400 Subject: [PATCH 44/47] fix: var name --- ansible/tasks/stage2-setup-postgres.yml | 4 ++-- ansible/vars.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 685aa651d..f1082347f 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -15,7 +15,7 @@ cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_orioledb_16 and stage2_nix + when: is_psql_oriole_16 and stage2_nix become: yes - name: Remove specified extensions from supautils.conf if oriole-16 build @@ -280,4 +280,4 @@ ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env line: 'GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins' - become: yes \ No newline at end of file + become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index e16af209a..74e26989e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "orioledb-16.3.1.000-staging-9" + postgresoriole-16: "orioledb-16.3.1.000-staging-10" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 12dc39708bd0eddb70398b9c92e22b38bd0115e7 Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 06:25:10 -0400 Subject: [PATCH 45/47] fix: one more var name --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 74e26989e..d427d64b1 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "orioledb-16.3.1.000-staging-10" + postgresorioledb-16: "orioledb-16.3.1.000-staging-10" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 73944e19e18608ca0d666d65a0c5eb76b48efa02 Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 07:31:31 -0400 Subject: [PATCH 46/47] chore: bump to release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d427d64b1..8a7a11b85 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresorioledb-16: "orioledb-16.3.1.000-staging-10" + postgresorioledb-16: "orioledb-16.3.1.000-staging-11" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 9ae9c9f9a08c8b77d85e7a3a213ddb344f20ee4c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 12:17:46 -0400 Subject: [PATCH 47/47] fix: reformatting metadata for clikchouse dep as git was not able to fetch --- ansible/vars.yml | 2 +- nix/ext/wrappers/default.nix | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 8a7a11b85..14b35ef69 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresorioledb-16: "orioledb-16.3.1.000-staging-11" + postgresorioledb-16: "orioledb-16.3.1.000-staging-12" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 74b94f9f9..d53604a5d 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -49,10 +49,46 @@ buildPgrxExtension_0_12_6 rec { CARGO_BUILD_JOBS = "2"; CARGO="${cargo}/bin/cargo"; + #CARGO_NET_GIT_FETCH_WITH_CLI = "true"; cargoLock = { lockFile = "${src}/Cargo.lock"; allowBuiltinFetchGit = true; }; + + preConfigure = '' + cd wrappers + + # update the clickhouse-rs dependency + # append the branch name to the git URL to help cargo locate the commit + # while maintaining the rev for reproducibility + awk -i inplace ' + /\[dependencies.clickhouse-rs\]/ { + print + getline + if ($0 ~ /git =/) { + print "git = \"https://github.com/suharev7/clickhouse-rs/async-await\"" + } else { + print + } + while ($0 !~ /^\[/ && NF > 0) { + getline + if ($0 ~ /rev =/) print + if ($0 ~ /^\[/) print + } + next + } + { print } + ' Cargo.toml + + # Verify the file is still valid TOML, break build with this error + # if it is not + if ! cargo verify-project 2>/dev/null; then + echo "Failed to maintain valid TOML syntax" + exit 1 + fi + + cd .. + ''; buildAndTestSubdir = "wrappers"; buildFeatures = [