diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 5d373ad8f..eb5f7a755 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -72,6 +72,9 @@ jobs: id: process_release_version run: | VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts diff --git a/Dockerfile b/Dockerfile index 9e98a73ee..8309c2585 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.5.0 ARG wal_g_release=2.0.1 #################### diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index e978b2a54..4d465036d 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -91,12 +91,24 @@ swap_postgres_and_supabase_admin() { alter database postgres connection limit 0; select pg_terminate_backend(pid) from pg_stat_activity where backend_type = 'client backend' and pid != pg_backend_pid(); EOSQL + + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres "" + fi + + retry 8 pg_isready -h localhost -U supabase_admin + run_sql <<'EOSQL' set statement_timeout = '600s'; begin; create role supabase_tmp superuser; set session authorization supabase_tmp; +-- to handle snowflakes that happened in the past +revoke supabase_admin from authenticator; + do $$ begin if exists (select from pg_extension where extname = 'timescaledb') then @@ -356,7 +368,8 @@ begin end , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' - else rec.grantee::regrole + when rec.grantee = 0 then 'public' + else rec.grantee::regrole::text end )); end if; @@ -382,7 +395,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , rec.grantee::regrole + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole::text end , case when rec.is_grantable then 'with grant option' else '' end )); end if; @@ -443,7 +456,10 @@ begin foreach obj in array functions loop if obj->>'owner' = 'postgres' then - execute(format('alter routine %s(%s) owner to postgres;', (obj->>'oid')::regproc, pg_get_function_identity_arguments((obj->>'oid')::regproc))); + execute(format('alter %s %s(%s) owner to postgres;' + , case when obj->>'kind' = 'p' then 'procedure' else 'function' end + , (obj->>'oid')::regproc + , pg_get_function_identity_arguments((obj->>'oid')::regproc))); end if; for rec in select grantor, grantee, privilege_type, is_grantable @@ -528,9 +544,6 @@ $$; alter database postgres connection limit -1; --- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -grant pg_read_all_data, pg_signal_backend to postgres; - set session authorization supabase_admin; drop role supabase_tmp; commit; diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index d19b72422..c8542c685 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -78,6 +78,9 @@ EOF run_sql -c "$RECREATE_PG_CRON_QUERY" fi + + # #incident-2024-09-12-project-upgrades-are-temporarily-disabled + run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres" } function complete_pg_upgrade { diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 9443d2acb..88518766b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -11,6 +11,7 @@ # them depending on regtypes referencing system OIDs or outdated library files. EXTENSIONS_TO_DISABLE=( "pg_graphql" + "pg_stat_monitor" ) PG14_EXTENSIONS_TO_DISABLE=( @@ -38,6 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -46,7 +48,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" PGBINOLD="/usr/lib/postgresql/bin" -PGLIBOLD="/usr/lib/postgresql/lib" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -119,20 +120,38 @@ cleanup() { CI_start_postgres fi + retry 8 pg_isready -h localhost -U supabase_admin + echo "Re-enabling extensions" if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then - run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + retry 5 run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT fi echo "Removing SUPERUSER grant from postgres" - run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" + retry 5 run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" echo "Resetting postgres database connection limit" - run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + + echo "Making sure postgres still has access to pg_shadow" + cat << EOF >> $POST_UPGRADE_POSTGRES_PERMS_SCRIPT +DO \$\$ +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end +\$\$; +grant pg_signal_backend to postgres; +EOF + + if [ -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT + fi if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" - umount $MOUNT_POINT + retry 3 umount $MOUNT_POINT fi echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status @@ -145,6 +164,14 @@ cleanup() { } function handle_extensions { + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres + fi + + retry 8 pg_isready -h localhost -U supabase_admin + rm -f $POST_UPGRADE_EXTENSION_SCRIPT touch $POST_UPGRADE_EXTENSION_SCRIPT @@ -178,58 +205,6 @@ EOF done } -function patch_wrappers { - local IS_NIX_UPGRADE=$1 - - WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") - if [ "$WRAPPERS_ENABLED" = "f" ]; then - echo "Wrappers extension not enabled. Skipping." - return - fi - - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [ -d "$PGLIBOLD" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - - find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do - if [ -f "$WRAPPERS_LIB_PATH" ]; then - WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - fi - fi - done - fi - else - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" - fi - fi - fi - fi - fi -} - function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -406,8 +381,6 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - patch_wrappers "$IS_NIX_UPGRADE" - echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" @@ -470,6 +443,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true + rm -rf "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index eada0a94b..ae5537715 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active commence-backup.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl start commence-backup.service %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf diff --git a/ansible/files/commence-backup.service.j2 b/ansible/files/commence-backup.service.j2 new file mode 100644 index 000000000..9d4ad0c68 --- /dev/null +++ b/ansible/files/commence-backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Async commence physical backup + +[Service] +Type=simple +User=adminapi +ExecStart=/usr/bin/admin-mgr commence-backup --run-as-service true +Restart=no +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/envoy_config/lds.yaml b/ansible/files/envoy_config/lds.yaml index 60a38f65d..2fc7cae13 100644 --- a/ansible/files/envoy_config/lds.yaml +++ b/ansible/files/envoy_config/lds.yaml @@ -254,8 +254,13 @@ resources: type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - match: safe_regex: + google_re2: + max_program_size: 150 regex: >- - /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)) + /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)|\.well-known/(openid-configuration|jwks\.json)) + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue regex_rewrite: @@ -269,6 +274,9 @@ resources: typed_per_filter_config: *ref_0 - match: prefix: /auth/v1/ + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue prefix_rewrite: / @@ -280,6 +288,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -293,6 +302,7 @@ resources: prefix: /rest/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -309,6 +319,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -321,6 +332,7 @@ resources: prefix: /rest-admin/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -330,18 +342,25 @@ resources: header: key: Content-Profile value: graphql_public + request_headers_to_remove: + - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: /rpc/graphql timeout: 125s - match: prefix: /admin/v1/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: / timeout: 600s - match: prefix: /customer/v1/privileged/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /privileged/ @@ -365,6 +384,8 @@ resources: treat_missing_header_as_empty: true - match: prefix: /metrics/aggregated + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /supabase-internal/metrics diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index ee16715a6..1604d94f9 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 2ab4257fd..5c0a1e214 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,12 +1,14 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 +# omitted because doesn't require superuser: pgmq supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' supautils.privileged_role_allowed_configs = 'auto_explain.log_min_duration, auto_explain.log_nested_statements, log_lock_waits, log_min_messages, pg_net.batch_size, pg_net.ttl, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing' -supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator' -supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' +supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, authenticator' +supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' diff --git a/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql b/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql new file mode 100644 index 000000000..8b126d403 --- /dev/null +++ b/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql @@ -0,0 +1,19 @@ +do $$ +declare + extoid oid := (select oid from pg_extension where extname = 'pgmq'); + r record; +begin + set local search_path = ''; + update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq'; + for r in (select * from pg_depend where refobjid = extoid) loop + if r.classid = 'pg_type'::regclass then + execute(format('alter type %s owner to postgres;', r.objid::regtype)); + elsif r.classid = 'pg_proc'::regclass then + execute(format('alter function %s(%s) owner to postgres;', r.objid::regproc, pg_get_function_identity_arguments(r.objid))); + elsif r.classid = 'pg_class'::regclass then + execute(format('alter table %s owner to postgres;', r.objid::regclass)); + else + raise exception 'error on pgmq after-create script: unexpected object type %', r.classid; + end if; + end loop; +end $$; diff --git a/ansible/manifest-playbook.yml b/ansible/manifest-playbook.yml index 93f0e15c5..5c1c65053 100644 --- a/ansible/manifest-playbook.yml +++ b/ansible/manifest-playbook.yml @@ -18,26 +18,10 @@ - name: gotrue - download commit archive get_url: url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-arm64.tar.gz" - dest: /tmp/gotrue.tar.gz + dest: /tmp/auth-v{{ gotrue_release }}-arm64.tar.gz checksum: "{{ gotrue_release_checksum }}" timeout: 60 - - name: gotrue - create /tmp/gotrue - file: - path: /tmp/gotrue - state: directory - mode: 0775 - - - name: gotrue - unpack archive in /tmp/gotrue - unarchive: - remote_src: yes - src: /tmp/gotrue.tar.gz - dest: /tmp/gotrue - - - name: gotrue - pack archive - shell: | - cd /tmp && tar -cJf gotrue-v{{ gotrue_release }}-arm64.tar.xz gotrue - - name: PostgREST - download ubuntu binary archive (arm) get_url: url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-ubuntu-aarch64.tar.xz" @@ -82,7 +66,7 @@ aws s3 cp /tmp/{{ item.file }} s3://{{ internal_artifacts_bucket }}/upgrades/{{ item.service }}/{{ item.file }} with_items: - service: gotrue - file: gotrue-v{{ gotrue_release }}-arm64.tar.xz + file: auth-v{{ gotrue_release }}-arm64.tar.gz - service: postgrest file: postgrest-{{ postgrest_release }}-arm64.tar.xz - service: supabase-admin-api diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index cea0109fd..da93fef62 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -79,6 +79,11 @@ src: files/adminapi.service.j2 dest: /etc/systemd/system/adminapi.service +- name: adminapi - create service file for commence backup process + template: + src: files/commence-backup.service.j2 + dest: /etc/systemd/system/commence-backup.service + - name: UFW - Allow connections to adminapi ports ufw: rule: allow diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 281e370bb..73cd6ee85 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,29 +1,47 @@ - name: Add apt repository for Saltstack (arm) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg arch=arm64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/{{ salt_minion_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=arm64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "arm64" - name: Add apt repository for Saltstack (amd) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/{{ salt_minon_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "amd64" - name: Salt minion install apt: name: salt-minion + state: present + update_cache: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index e76e99cb0..30f21c4c9 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.161.0 -gotrue_release_checksum: sha1:8e45f3511fee8f99a0b1567c73673991a0a5986c +gotrue_release: 2.165.1 +gotrue_release_checksum: sha1:bbd62327d8612ac756177dde81d5368b660ca4c8 aws_cli_release: "2.2.7" @@ -50,8 +50,8 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.68.0 -adminmgr_release: 0.22.1 +adminapi_release: 0.71.1 +adminmgr_release: 0.24.0 # Postgres Extensions postgis_release: "3.3.2" @@ -86,10 +86,10 @@ timescaledb_release_checksum: sha256:883638f2e79d25ec88ee58f603f3c81c999b6364cb4 wal2json_release: "2_5" wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e -supautils_release: "2.2.1" -supautils_release_arm64_deb_checksum: sha256:1a2d2b8fe604d38921ed9cf3a0d56dd142a274035d0dca17ad21cdc81ddd9569 -supautils_release_amd64_deb_checksum: sha256:4674fac146976dc179f5871271830718112f5eaa853bc461df9e303c8df2e1ab -supautils_release_tar_checksum: sha256:f1f33371390322ac830645b8b0b8e249cb8ca10b19fdeae917f383014ed01b5d +supautils_release: "2.5.0" +supautils_release_arm64_deb_checksum: sha256:406e4a816f719bd6c4b2143e9bb38078fbe60d7e85018ec0aed5d76924e28000 +supautils_release_amd64_deb_checksum: sha256:71f182b478d8aaf167609dd382875cdce3fbe992e888988b3d51cdad39e08202 +supautils_release_tar_checksum: sha256:07c41244e4374248da9c2df2822152f3ae8f1e74c8a92d361300480193219b63 pljava_release: master pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 @@ -141,8 +141,8 @@ hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c pg_repack_release: "1.5.0" pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 -pgvector_release: "0.7.4" -pgvector_release_checksum: sha256:0341edf89b1924ae0d552f617e14fb7f8867c0194ed775bcc44fa40288642583 +pgvector_release: "0.8.0" +pgvector_release_checksum: sha256:867a2c328d4928a5a9d6f052cd3bc78c7d60228a9b914ad32aa3db88e9de27b0 pg_tle_release: "1.3.2" pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index bacdd2c63..587e717cd 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.124" +postgres-version = "15.6.1.146" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 3c999dccd..871647b06 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.93" +postgres-version = "15.1.1.95" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index a8a8639e2..5035be062 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,6 +5,13 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi +SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') nix build .#psql_15/bin -o psql_15 nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +if [ "$SYSTEM" = "aarch64-linux" ]; then + nix build .#postgresql_15_debug -o ./postgresql_15_debug + nix build .#postgresql_15_src -o ./postgresql_15_src + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_debug-debug + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src +fi diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index 72bc9ba62..7adb7d74a 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.5.0 ARG wal_g_release=2.0.1 #################### diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 45cbc8ac6..3b0bb93c8 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -110,6 +110,11 @@ function format_and_mount_rootfs { fi mkfs.ext4 /dev/xvdh + + # Explicitly reserving 100MiB worth of blocks for the data volume + RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) + tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + mkdir -p /mnt/data mount -o defaults,discard /dev/xvdh /mnt/data } diff --git a/flake.nix b/flake.nix index 1f9d5f64d..82e76983b 100644 --- a/flake.nix +++ b/flake.nix @@ -107,9 +107,11 @@ ourExtensions = [ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix + ./nix/ext/timescaledb-2.9.1.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix + ./nix/ext/pgmq.nix ./nix/ext/pg_repack.nix ./nix/ext/pg-safeupdate.nix ./nix/ext/plpgsql-check.nix @@ -294,7 +296,7 @@ pg_regress = pg_regress; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; postgresql_15 = pkgs.postgresql_15; - + postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = pkgs.postgresql_15.version; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..3bf04ff94 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.10.0"; + version = "0.13.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-R9Mzw5gvV7b2R59LTOzuOc0AI99+3ncFNzijI4mySUg="; + hash = "sha256-FRaTZPCJQPYAFmsJg22hYJJ0+gH1tMdDQoCQgiqEnaA="; }; env.NIX_CFLAGS_COMPILE = "-Wno-error"; @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix new file mode 100644 index 000000000..97a3c27e3 --- /dev/null +++ b/nix/ext/pgmq.nix @@ -0,0 +1,33 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgmq"; + version = "1.4.4"; + buildInputs = [ postgresql ]; + src = fetchFromGitHub { + owner = "tembo-io"; + repo = pname; + rev = "v${version}"; + hash = "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="; + }; + + buildPhase = '' + cd pgmq-extension + ''; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + mv sql/pgmq.sql $out/share/postgresql/extension/pgmq--${version}.sql + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres."; + homepage = "https://github.com/tembo-io/pgmq"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 476f7606c..965be3656 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgvector"; - version = "0.7.4"; + version = "0.8.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "pgvector"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI="; + hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q="; }; installPhase = '' diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..338bba62e 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -103,16 +103,20 @@ stdenv.mkDerivation (finalAttrs: { # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - + mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so ''} ''; diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..82cf9bb20 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "supautils"; - version = "2.2.1"; + version = "2.5.0"; buildInputs = [ postgresql ]; @@ -10,13 +10,13 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE="; + hash = "sha256-NyAk+QpQEdibmFY4yceO/FzMOhRYhKXf4XUw9XJ5rOY="; }; installPhase = '' mkdir -p $out/lib - install -D supautils.so -t $out/lib + install -D supautils${postgresql.dlSuffix} -t $out/lib ''; meta = with lib; { diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix new file mode 100644 index 000000000..ad955e86f --- /dev/null +++ b/nix/ext/timescaledb-2.9.1.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: + +stdenv.mkDerivation rec { + pname = "timescaledb-apache"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + + # timescaledb-2.9.1.so already exists in the lib directory + # we have no need for the timescaledb.so or control file + postInstall = '' + rm $out/lib/timescaledb.so + rm $out/share/postgresql/extension/timescaledb.control + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 98d6a7cbc..1a4360044 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,24 +63,53 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi - rm git_tags.txt - ''; + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + ln -s $out/lib/wrappers.so $out/lib/wrappers-${version}.so + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi + + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index ab43f54e6..9b8e86fd5 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -88,6 +88,7 @@ order by pgaudit | t pgcrypto | t pgjwt | f + pgmq | f pgroonga | f pgroonga_database | f pgrouting | t @@ -121,7 +122,7 @@ order by vector | t wrappers | f xml2 | f -(81 rows) +(82 rows) /* @@ -1306,6 +1307,34 @@ order by pgjwt | public | url_decode | data text | bytea pgjwt | public | url_encode | data bytea | text pgjwt | public | verify | token text, secret text, algorithm text | TABLE(header json, payload json, valid boolean) + pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean + pgmq | pgmq | _ensure_pg_partman_installed | | void + pgmq | pgmq | _get_partition_col | partition_interval text | text + pgmq | pgmq | _get_pg_partman_major_version | | integer + pgmq | pgmq | _get_pg_partman_schema | | text + pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean + pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void + pgmq | pgmq | create | queue_name text | void + pgmq | pgmq | create_non_partitioned | queue_name text | void + pgmq | pgmq | create_partitioned | queue_name text, partition_interval text, retention_interval text | void + pgmq | pgmq | create_unlogged | queue_name text | void + pgmq | pgmq | delete | queue_name text, msg_id bigint | boolean + pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | detach_archive | queue_name text | void + pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean + pgmq | pgmq | format_table_name | queue_name text, prefix text | text + pgmq | pgmq | list_queues | | SETOF pgmq.queue_record + pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result + pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result + pgmq | pgmq | pop | queue_name text | SETOF pgmq.message_record + pgmq | pgmq | purge_queue | queue_name text | bigint + pgmq | pgmq | read | queue_name text, vt integer, qty integer | SETOF pgmq.message_record + pgmq | pgmq | read_with_poll | queue_name text, vt integer, qty integer, max_poll_seconds integer, poll_interval_ms integer | SETOF pgmq.message_record + pgmq | pgmq | send | queue_name text, msg jsonb, delay integer | SETOF bigint + pgmq | pgmq | send_batch | queue_name text, msgs jsonb[], delay integer | SETOF bigint + pgmq | pgmq | set_vt | queue_name text, msg_id bigint, vt integer | SETOF pgmq.message_record + pgmq | pgmq | validate_queue_name | queue_name text | void pgroonga | pgroonga | command | groongacommand text | text pgroonga | pgroonga | command | groongacommand text, arguments text[] | text pgroonga | pgroonga | command_escape_value | value text | text @@ -4994,6 +5023,10 @@ order by vector | public | array_to_halfvec | integer[], integer, boolean | halfvec vector | public | array_to_halfvec | numeric[], integer, boolean | halfvec vector | public | array_to_halfvec | real[], integer, boolean | halfvec + vector | public | array_to_sparsevec | double precision[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | integer[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | numeric[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | real[], integer, boolean | sparsevec vector | public | array_to_vector | double precision[], integer, boolean | vector vector | public | array_to_vector | integer[], integer, boolean | vector vector | public | array_to_vector | numeric[], integer, boolean | vector @@ -5156,7 +5189,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5005 rows) +(5037 rows) /* @@ -5378,6 +5411,10 @@ order by pg_tle | pgtle | feature_info | obj_identity pg_tle | pgtle | feature_info | proname pg_tle | pgtle | feature_info | schema_name + pgmq | pgmq | meta | created_at + pgmq | pgmq | meta | is_partitioned + pgmq | pgmq | meta | is_unlogged + pgmq | pgmq | meta | queue_name pgsodium | pgsodium | decrypted_key | associated_data pgsodium | pgsodium | decrypted_key | comment pgsodium | pgsodium | decrypted_key | created @@ -6078,8 +6115,8 @@ order by timescaledb | timescaledb_information | compression_settings | orderby_asc timescaledb | timescaledb_information | compression_settings | orderby_column_index timescaledb | _timescaledb_catalog | compression_settings | orderby_desc - timescaledb | _timescaledb_catalog | compression_settings | orderby_nullsfirst timescaledb | timescaledb_information | compression_settings | orderby_nullsfirst + timescaledb | _timescaledb_catalog | compression_settings | orderby_nullsfirst timescaledb | _timescaledb_catalog | compression_settings | relid timescaledb | _timescaledb_catalog | compression_settings | segmentby timescaledb | timescaledb_information | compression_settings | segmentby_column_index @@ -6271,5 +6308,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1082 rows) +(1086 rows) diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out new file mode 100644 index 000000000..9fb18191b --- /dev/null +++ b/nix/tests/expected/pgmq.out @@ -0,0 +1,141 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + create +-------- + +(1 row) + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + send +------ + 1 +(1 row) + +-- Test queue is not case sensitive +select + * +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + send +------ + 2 +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + msg_id | read_ct | message +--------+---------+----------------- + 1 | 1 | {"foo": "bar1"} +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + msg_id | read_ct | message +--------+---------+--------- +(0 rows) + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + archive +--------- + t +(1 row) + +select + pgmq.create('my_queue'); + create +-------- + +(1 row) + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + send_batch +------------ + 1 + 2 + 3 +(3 rows) + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + archive +--------- + 3 +(1 row) + +select + pgmq.delete('my_queue', 6); + delete +-------- + f +(1 row) + +select + pgmq.drop_queue('my_queue'); + drop_queue +------------ + t +(1 row) + +/* +-- Disabled until pg_partman goes back into the image +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); +*/ +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create('F$oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create($$F'oo$$); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..ef860afcb 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index f31558531..5406a9044 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -42,6 +42,7 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; +create extension pgmq; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql new file mode 100644 index 000000000..cd47cc34d --- /dev/null +++ b/nix/tests/sql/pgmq.sql @@ -0,0 +1,90 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + +-- Test queue is not case sensitive +select + * +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + + +select + pgmq.create('my_queue'); + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + +select + pgmq.delete('my_queue', 6); + + +select + pgmq.drop_queue('my_queue'); + +/* +-- Disabled until pg_partman goes back into the image +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); +*/ + + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +select pgmq.create('F$oo'); +select pgmq.create($$F'oo$$); + + + + diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh index ebbf49080..f2e19686d 100644 --- a/scripts/90-cleanup.sh +++ b/scripts/90-cleanup.sh @@ -36,7 +36,10 @@ elif [ -n "$(command -v apt-get)" ]; then libicu-dev \ libcgal-dev \ libgcc-9-dev \ - libgcc-8-dev + libgcc-8-dev \ + ansible + + add-apt-repository --yes --remove ppa:ansible/ansible source /etc/os-release apt-get -y remove --purge linux-headers-5.11.0-1021-aws diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..5ed678d7a 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -38,8 +38,12 @@ EOF $ARGS } +function cleanup_packages { + sudo apt-get -y remove --purge ansible + sudo add-apt-repository --yes --remove ppa:ansible/ansible +} install_packages install_nix execute_stage2_playbook - +cleanup_packages