From 4cd3e1d1ec046ca72d0b1e10be47d150c86446e5 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 14 Aug 2024 11:20:19 -0500 Subject: [PATCH 01/15] add pgmq SQL only extension --- flake.nix | 1 + nix/docs/migration-tests.md | 6 +++--- nix/ext/pgmq.nix | 35 +++++++++++++++++++++++++++++++ nix/tests/migrations/wrappers.sql | 6 ++++++ 4 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 nix/ext/pgmq.nix create mode 100644 nix/tests/migrations/wrappers.sql diff --git a/flake.nix b/flake.nix index 1f9d5f64d..4379b866c 100644 --- a/flake.nix +++ b/flake.nix @@ -110,6 +110,7 @@ ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix + ./nix/ext/pgmq.nix ./nix/ext/pg_repack.nix ./nix/ext/pg-safeupdate.nix ./nix/ext/plpgsql-check.nix diff --git a/nix/docs/migration-tests.md b/nix/docs/migration-tests.md index d04bfeb90..98a7f88c0 100644 --- a/nix/docs/migration-tests.md +++ b/nix/docs/migration-tests.md @@ -43,8 +43,8 @@ compare, and you could do something like the following: OLD_GIT_VERSION=... NEW_GIT_VERSION=... -nix run github:supabase/nix-postgres#migration-test \ - $(nix build "github:supabase/nix-postgres/$OLD_GIT_VERSION#psql_14/bin") \ - $(nix build "github:supabase/nix-postgres/$NEW_GIT_VERSION#psql_14/bin") \ +nix run github:supabase/nix-postgres#migration-test wrappers \ + $(nix build "github:supabase/postgres/$OLD_GIT_VERSION#psql_14/bin") \ + $(nix build "github:supabase/postgres/$NEW_GIT_VERSION#psql_14/bin") \ pg_upgrade ``` diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix new file mode 100644 index 000000000..b70483e4f --- /dev/null +++ b/nix/ext/pgmq.nix @@ -0,0 +1,35 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgmq"; + version = "1.4.0"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "tembo-io"; + repo = pname; + rev = "v${version}"; + hash = "sha256-k7iKp2CZY3M8POUqIOIbKxrofoOfn2FxfVW01KYojPA="; + }; + + buildPhase = '' + cd pgmq-extension + ''; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + mv sql/pgmq.sql $out/share/postgresql/extension/pgmq--${version}.sql + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres."; + homepage = "https://github.com/tembo-io/pgmq"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/tests/migrations/wrappers.sql b/nix/tests/migrations/wrappers.sql new file mode 100644 index 000000000..104168db1 --- /dev/null +++ b/nix/tests/migrations/wrappers.sql @@ -0,0 +1,6 @@ +if version > 0.3.1 + + create fdw .... stripe + +if version > 0.4.1 + create fdw ... wasm From 836c5f4fbe3169b51017b7bde8f340d637046fce Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 14 Aug 2024 11:26:19 -0500 Subject: [PATCH 02/15] revert accidental file inclusion --- nix/docs/migration-tests.md | 6 +++--- nix/tests/migrations/wrappers.sql | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 nix/tests/migrations/wrappers.sql diff --git a/nix/docs/migration-tests.md b/nix/docs/migration-tests.md index 98a7f88c0..d04bfeb90 100644 --- a/nix/docs/migration-tests.md +++ b/nix/docs/migration-tests.md @@ -43,8 +43,8 @@ compare, and you could do something like the following: OLD_GIT_VERSION=... NEW_GIT_VERSION=... -nix run github:supabase/nix-postgres#migration-test wrappers \ - $(nix build "github:supabase/postgres/$OLD_GIT_VERSION#psql_14/bin") \ - $(nix build "github:supabase/postgres/$NEW_GIT_VERSION#psql_14/bin") \ +nix run github:supabase/nix-postgres#migration-test \ + $(nix build "github:supabase/nix-postgres/$OLD_GIT_VERSION#psql_14/bin") \ + $(nix build "github:supabase/nix-postgres/$NEW_GIT_VERSION#psql_14/bin") \ pg_upgrade ``` diff --git a/nix/tests/migrations/wrappers.sql b/nix/tests/migrations/wrappers.sql deleted file mode 100644 index 104168db1..000000000 --- a/nix/tests/migrations/wrappers.sql +++ /dev/null @@ -1,6 +0,0 @@ -if version > 0.3.1 - - create fdw .... stripe - -if version > 0.4.1 - create fdw ... wasm From 408a9736601636146afd400a48cbd33de8d09a87 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 14 Aug 2024 11:29:48 -0500 Subject: [PATCH 03/15] add public interface test coverage --- common-nix.vars.pkr.hcl | 2 +- common.vars.pkr.hcl | 2 +- nix/ext/pgmq.nix | 4 +- .../expected/extensions_sql_interface.out | 48 +++++++++++++++++-- nix/tests/prime.sql | 12 ++--- 5 files changed, 52 insertions(+), 16 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 43c8c386a..e80a744d9 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.121" +postgres-version = "15.6.1.123" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 82123f85e..ce0d34bdb 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.90" +postgres-version = "15.1.1.91" diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix index b70483e4f..c18fa9bfe 100644 --- a/nix/ext/pgmq.nix +++ b/nix/ext/pgmq.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgmq"; - version = "1.4.0"; + version = "1.4.4"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "tembo-io"; repo = pname; rev = "v${version}"; - hash = "sha256-k7iKp2CZY3M8POUqIOIbKxrofoOfn2FxfVW01KYojPA="; + hash = ""; }; buildPhase = '' diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index bc544ab83..5146bd2ae 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -88,6 +88,7 @@ order by pgaudit | t pgcrypto | t pgjwt | f + pgmq | f pgroonga | f pgroonga_database | f pgrouting | t @@ -121,7 +122,11 @@ order by vector | t wrappers | f xml2 | f +<<<<<<< HEAD (81 rows) +======= +(79 rows) +>>>>>>> 95a20bc (add public interface test coverage) /* @@ -1302,6 +1307,31 @@ order by pgjwt | public | url_decode | data text | bytea pgjwt | public | url_encode | data bytea | text pgjwt | public | verify | token text, secret text, algorithm text | TABLE(header json, payload json, valid boolean) + pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean + pgmq | pgmq | _ensure_pg_partman_installed | | void + pgmq | pgmq | _get_partition_col | partition_interval text | text + pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean + pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void + pgmq | pgmq | create | queue_name text | void + pgmq | pgmq | create_non_partitioned | queue_name text | void + pgmq | pgmq | create_partitioned | queue_name text, partition_interval text, retention_interval text | void + pgmq | pgmq | create_unlogged | queue_name text | void + pgmq | pgmq | delete | queue_name text, msg_id bigint | boolean + pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | detach_archive | queue_name text | void + pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean + pgmq | pgmq | list_queues | | SETOF pgmq.queue_record + pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result + pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result + pgmq | pgmq | pop | queue_name text | SETOF pgmq.message_record + pgmq | pgmq | purge_queue | queue_name text | bigint + pgmq | pgmq | read | queue_name text, vt integer, qty integer | SETOF pgmq.message_record + pgmq | pgmq | read_with_poll | queue_name text, vt integer, qty integer, max_poll_seconds integer, poll_interval_ms integer | SETOF pgmq.message_record + pgmq | pgmq | send | queue_name text, msg jsonb, delay integer | SETOF bigint + pgmq | pgmq | send_batch | queue_name text, msgs jsonb[], delay integer | SETOF bigint + pgmq | pgmq | set_vt | queue_name text, msg_id bigint, vt integer | SETOF pgmq.message_record + pgmq | pgmq | validate_queue_name | queue_name text | void pgroonga | pgroonga | command | groongacommand text | text pgroonga | pgroonga | command | groongacommand text, arguments text[] | text pgroonga | pgroonga | command_escape_value | value text | text @@ -5108,7 +5138,15 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text +<<<<<<< HEAD (4957 rows) +======= +<<<<<<< HEAD +(4953 rows) +======= +(4966 rows) +>>>>>>> 95a20bc (add public interface test coverage) +>>>>>>> 21f844d (add public interface test coverage) /* @@ -5326,6 +5364,10 @@ order by pg_tle | pgtle | feature_info | obj_identity pg_tle | pgtle | feature_info | proname pg_tle | pgtle | feature_info | schema_name + pgmq | pgmq | meta | created_at + pgmq | pgmq | meta | is_partitioned + pgmq | pgmq | meta | is_unlogged + pgmq | pgmq | meta | queue_name pgsodium | pgsodium | decrypted_key | associated_data pgsodium | pgsodium | decrypted_key | comment pgsodium | pgsodium | decrypted_key | created @@ -6158,13 +6200,13 @@ order by timescaledb | _timescaledb_internal | job_errors | finish_time timescaledb | timescaledb_information | job_errors | job_id timescaledb | _timescaledb_internal | job_errors | job_id - timescaledb | _timescaledb_internal | job_errors | pid timescaledb | timescaledb_information | job_errors | pid + timescaledb | _timescaledb_internal | job_errors | pid timescaledb | timescaledb_information | job_errors | proc_name timescaledb | timescaledb_information | job_errors | proc_schema timescaledb | timescaledb_information | job_errors | sqlerrcode - timescaledb | _timescaledb_internal | job_errors | start_time timescaledb | timescaledb_information | job_errors | start_time + timescaledb | _timescaledb_internal | job_errors | start_time timescaledb | timescaledb_information | job_stats | hypertable_name timescaledb | timescaledb_information | job_stats | hypertable_schema timescaledb | timescaledb_information | job_stats | job_id @@ -6220,5 +6262,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1079 rows) +(1083 rows) diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index f31558531..8918ae8e0 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -28,7 +28,9 @@ create extension ltree; create extension moddatetime; create extension old_snapshot; create extension pageinspect; +create extension pg_backtrace; create extension pg_buffercache; +create extension pgmq; /* TODO: Does not enable locally mode @@ -42,6 +44,7 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; +create extension pg_partman; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; @@ -61,11 +64,9 @@ create extension pgsodium; create extension pgrowlocks; create extension pgstattuple; create extension plpgsql_check; - create extension plv8; create extension plcoffee; create extension plls; - create extension postgis; create extension postgis_raster; create extension postgis_sfcgal; @@ -88,10 +89,3 @@ create extension "uuid-ossp"; create extension vector; create extension wrappers; create extension xml2; - - - - - - -CREATE EXTENSION IF NOT EXISTS pg_backtrace; From e63d9eb6ce7b49606ce917e587b9634821cda43c Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 14 Aug 2024 11:37:09 -0500 Subject: [PATCH 04/15] order by namespace for tables in interface test to make timescale stable since they use 3 schemas --- .../expected/extensions_sql_interface.out | 267 +++++++++--------- nix/tests/sql/extensions_sql_interface.sql | 1 + 2 files changed, 135 insertions(+), 133 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 5146bd2ae..5315a2345 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -5175,6 +5175,7 @@ where and pc.relkind in ('r', 'v', 'm', 'i') order by e.extname, + n.nspname, pc.relname, pa.attname; extension_name | schema_name | entity_name | attname @@ -5946,42 +5947,6 @@ order by supabase_vault | vault | secrets | nonce supabase_vault | vault | secrets | secret supabase_vault | vault | secrets | updated_at - timescaledb | _timescaledb_config | bgw_job | application_name - timescaledb | _timescaledb_config | bgw_job | check_name - timescaledb | _timescaledb_config | bgw_job | check_schema - timescaledb | _timescaledb_config | bgw_job | config - timescaledb | _timescaledb_config | bgw_job | fixed_schedule - timescaledb | _timescaledb_config | bgw_job | hypertable_id - timescaledb | _timescaledb_config | bgw_job | id - timescaledb | _timescaledb_config | bgw_job | initial_start - timescaledb | _timescaledb_config | bgw_job | max_retries - timescaledb | _timescaledb_config | bgw_job | max_runtime - timescaledb | _timescaledb_config | bgw_job | owner - timescaledb | _timescaledb_config | bgw_job | proc_name - timescaledb | _timescaledb_config | bgw_job | proc_schema - timescaledb | _timescaledb_config | bgw_job | retry_period - timescaledb | _timescaledb_config | bgw_job | schedule_interval - timescaledb | _timescaledb_config | bgw_job | scheduled - timescaledb | _timescaledb_config | bgw_job | timezone - timescaledb | _timescaledb_internal | bgw_job_stat | consecutive_crashes - timescaledb | _timescaledb_internal | bgw_job_stat | consecutive_failures - timescaledb | _timescaledb_internal | bgw_job_stat | flags - timescaledb | _timescaledb_internal | bgw_job_stat | job_id - timescaledb | _timescaledb_internal | bgw_job_stat | last_finish - timescaledb | _timescaledb_internal | bgw_job_stat | last_run_success - timescaledb | _timescaledb_internal | bgw_job_stat | last_start - timescaledb | _timescaledb_internal | bgw_job_stat | last_successful_finish - timescaledb | _timescaledb_internal | bgw_job_stat | next_start - timescaledb | _timescaledb_internal | bgw_job_stat | total_crashes - timescaledb | _timescaledb_internal | bgw_job_stat | total_duration - timescaledb | _timescaledb_internal | bgw_job_stat | total_duration_failures - timescaledb | _timescaledb_internal | bgw_job_stat | total_failures - timescaledb | _timescaledb_internal | bgw_job_stat | total_runs - timescaledb | _timescaledb_internal | bgw_job_stat | total_successes - timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | chunk_id - timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | job_id - timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | last_time_job_run - timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | num_times_job_run timescaledb | _timescaledb_cache | cache_inval_bgw_job | timescaledb | _timescaledb_cache | cache_inval_extension | timescaledb | _timescaledb_cache | cache_inval_hypertable | @@ -6013,40 +5978,6 @@ order by timescaledb | _timescaledb_catalog | chunk_index | hypertable_id timescaledb | _timescaledb_catalog | chunk_index | hypertable_index_name timescaledb | _timescaledb_catalog | chunk_index | index_name - timescaledb | timescaledb_experimental | chunk_replication_status | chunk_name - timescaledb | timescaledb_experimental | chunk_replication_status | chunk_schema - timescaledb | timescaledb_experimental | chunk_replication_status | desired_num_replicas - timescaledb | timescaledb_experimental | chunk_replication_status | hypertable_name - timescaledb | timescaledb_experimental | chunk_replication_status | hypertable_schema - timescaledb | timescaledb_experimental | chunk_replication_status | non_replica_nodes - timescaledb | timescaledb_experimental | chunk_replication_status | num_replicas - timescaledb | timescaledb_experimental | chunk_replication_status | replica_nodes - timescaledb | timescaledb_information | chunks | chunk_name - timescaledb | timescaledb_information | chunks | chunk_schema - timescaledb | timescaledb_information | chunks | chunk_tablespace - timescaledb | timescaledb_information | chunks | data_nodes - timescaledb | timescaledb_information | chunks | hypertable_name - timescaledb | timescaledb_information | chunks | hypertable_schema - timescaledb | timescaledb_information | chunks | is_compressed - timescaledb | timescaledb_information | chunks | primary_dimension - timescaledb | timescaledb_information | chunks | primary_dimension_type - timescaledb | timescaledb_information | chunks | range_end - timescaledb | timescaledb_information | chunks | range_end_integer - timescaledb | timescaledb_information | chunks | range_start - timescaledb | timescaledb_information | chunks | range_start_integer - timescaledb | _timescaledb_internal | compressed_chunk_stats | chunk_name - timescaledb | _timescaledb_internal | compressed_chunk_stats | chunk_schema - timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_heap_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_index_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_toast_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_total_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | compression_status - timescaledb | _timescaledb_internal | compressed_chunk_stats | hypertable_name - timescaledb | _timescaledb_internal | compressed_chunk_stats | hypertable_schema - timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_heap_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_index_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_toast_size - timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_total_size timescaledb | _timescaledb_catalog | compression_algorithm | description timescaledb | _timescaledb_catalog | compression_algorithm | id timescaledb | _timescaledb_catalog | compression_algorithm | name @@ -6061,13 +5992,6 @@ order by timescaledb | _timescaledb_catalog | compression_chunk_size | uncompressed_heap_size timescaledb | _timescaledb_catalog | compression_chunk_size | uncompressed_index_size timescaledb | _timescaledb_catalog | compression_chunk_size | uncompressed_toast_size - timescaledb | timescaledb_information | compression_settings | attname - timescaledb | timescaledb_information | compression_settings | hypertable_name - timescaledb | timescaledb_information | compression_settings | hypertable_schema - timescaledb | timescaledb_information | compression_settings | orderby_asc - timescaledb | timescaledb_information | compression_settings | orderby_column_index - timescaledb | timescaledb_information | compression_settings | orderby_nullsfirst - timescaledb | timescaledb_information | compression_settings | segmentby_column_index timescaledb | _timescaledb_catalog | continuous_agg | bucket_width timescaledb | _timescaledb_catalog | continuous_agg | direct_view_name timescaledb | _timescaledb_catalog | continuous_agg | direct_view_schema @@ -6090,17 +6014,6 @@ order by timescaledb | _timescaledb_catalog | continuous_agg_migrate_plan_step | status timescaledb | _timescaledb_catalog | continuous_agg_migrate_plan_step | step_id timescaledb | _timescaledb_catalog | continuous_agg_migrate_plan_step | type - timescaledb | timescaledb_information | continuous_aggregates | compression_enabled - timescaledb | timescaledb_information | continuous_aggregates | finalized - timescaledb | timescaledb_information | continuous_aggregates | hypertable_name - timescaledb | timescaledb_information | continuous_aggregates | hypertable_schema - timescaledb | timescaledb_information | continuous_aggregates | materialization_hypertable_name - timescaledb | timescaledb_information | continuous_aggregates | materialization_hypertable_schema - timescaledb | timescaledb_information | continuous_aggregates | materialized_only - timescaledb | timescaledb_information | continuous_aggregates | view_definition - timescaledb | timescaledb_information | continuous_aggregates | view_name - timescaledb | timescaledb_information | continuous_aggregates | view_owner - timescaledb | timescaledb_information | continuous_aggregates | view_schema timescaledb | _timescaledb_catalog | continuous_aggs_bucket_function | bucket_width timescaledb | _timescaledb_catalog | continuous_aggs_bucket_function | experimental timescaledb | _timescaledb_catalog | continuous_aggs_bucket_function | mat_hypertable_id @@ -6115,9 +6028,6 @@ order by timescaledb | _timescaledb_catalog | continuous_aggs_materialization_invalidation_log | greatest_modified_value timescaledb | _timescaledb_catalog | continuous_aggs_materialization_invalidation_log | lowest_modified_value timescaledb | _timescaledb_catalog | continuous_aggs_materialization_invalidation_log | materialization_id - timescaledb | timescaledb_information | data_nodes | node_name - timescaledb | timescaledb_information | data_nodes | options - timescaledb | timescaledb_information | data_nodes | owner timescaledb | _timescaledb_catalog | dimension | aligned timescaledb | _timescaledb_catalog | dimension | column_name timescaledb | _timescaledb_catalog | dimension | column_type @@ -6137,16 +6047,6 @@ order by timescaledb | _timescaledb_catalog | dimension_slice | id timescaledb | _timescaledb_catalog | dimension_slice | range_end timescaledb | _timescaledb_catalog | dimension_slice | range_start - timescaledb | timescaledb_information | dimensions | column_name - timescaledb | timescaledb_information | dimensions | column_type - timescaledb | timescaledb_information | dimensions | dimension_number - timescaledb | timescaledb_information | dimensions | dimension_type - timescaledb | timescaledb_information | dimensions | hypertable_name - timescaledb | timescaledb_information | dimensions | hypertable_schema - timescaledb | timescaledb_information | dimensions | integer_interval - timescaledb | timescaledb_information | dimensions | integer_now_func - timescaledb | timescaledb_information | dimensions | num_partitions - timescaledb | timescaledb_information | dimensions | time_interval timescaledb | _timescaledb_catalog | hypertable | associated_schema_name timescaledb | _timescaledb_catalog | hypertable | associated_table_prefix timescaledb | _timescaledb_catalog | hypertable | chunk_sizing_func_name @@ -6159,6 +6059,74 @@ order by timescaledb | _timescaledb_catalog | hypertable | replication_factor timescaledb | _timescaledb_catalog | hypertable | schema_name timescaledb | _timescaledb_catalog | hypertable | table_name + timescaledb | _timescaledb_catalog | hypertable_compression | attname + timescaledb | _timescaledb_catalog | hypertable_compression | compression_algorithm_id + timescaledb | _timescaledb_catalog | hypertable_compression | hypertable_id + timescaledb | _timescaledb_catalog | hypertable_compression | orderby_asc + timescaledb | _timescaledb_catalog | hypertable_compression | orderby_column_index + timescaledb | _timescaledb_catalog | hypertable_compression | orderby_nullsfirst + timescaledb | _timescaledb_catalog | hypertable_compression | segmentby_column_index + timescaledb | _timescaledb_catalog | hypertable_data_node | block_chunks + timescaledb | _timescaledb_catalog | hypertable_data_node | hypertable_id + timescaledb | _timescaledb_catalog | hypertable_data_node | node_hypertable_id + timescaledb | _timescaledb_catalog | hypertable_data_node | node_name + timescaledb | _timescaledb_catalog | metadata | include_in_telemetry + timescaledb | _timescaledb_catalog | metadata | key + timescaledb | _timescaledb_catalog | metadata | value + timescaledb | _timescaledb_catalog | remote_txn | data_node_name + timescaledb | _timescaledb_catalog | remote_txn | remote_transaction_id + timescaledb | _timescaledb_catalog | tablespace | hypertable_id + timescaledb | _timescaledb_catalog | tablespace | id + timescaledb | _timescaledb_catalog | tablespace | tablespace_name + timescaledb | _timescaledb_config | bgw_job | application_name + timescaledb | _timescaledb_config | bgw_job | check_name + timescaledb | _timescaledb_config | bgw_job | check_schema + timescaledb | _timescaledb_config | bgw_job | config + timescaledb | _timescaledb_config | bgw_job | fixed_schedule + timescaledb | _timescaledb_config | bgw_job | hypertable_id + timescaledb | _timescaledb_config | bgw_job | id + timescaledb | _timescaledb_config | bgw_job | initial_start + timescaledb | _timescaledb_config | bgw_job | max_retries + timescaledb | _timescaledb_config | bgw_job | max_runtime + timescaledb | _timescaledb_config | bgw_job | owner + timescaledb | _timescaledb_config | bgw_job | proc_name + timescaledb | _timescaledb_config | bgw_job | proc_schema + timescaledb | _timescaledb_config | bgw_job | retry_period + timescaledb | _timescaledb_config | bgw_job | schedule_interval + timescaledb | _timescaledb_config | bgw_job | scheduled + timescaledb | _timescaledb_config | bgw_job | timezone + timescaledb | _timescaledb_internal | bgw_job_stat | consecutive_crashes + timescaledb | _timescaledb_internal | bgw_job_stat | consecutive_failures + timescaledb | _timescaledb_internal | bgw_job_stat | flags + timescaledb | _timescaledb_internal | bgw_job_stat | job_id + timescaledb | _timescaledb_internal | bgw_job_stat | last_finish + timescaledb | _timescaledb_internal | bgw_job_stat | last_run_success + timescaledb | _timescaledb_internal | bgw_job_stat | last_start + timescaledb | _timescaledb_internal | bgw_job_stat | last_successful_finish + timescaledb | _timescaledb_internal | bgw_job_stat | next_start + timescaledb | _timescaledb_internal | bgw_job_stat | total_crashes + timescaledb | _timescaledb_internal | bgw_job_stat | total_duration + timescaledb | _timescaledb_internal | bgw_job_stat | total_duration_failures + timescaledb | _timescaledb_internal | bgw_job_stat | total_failures + timescaledb | _timescaledb_internal | bgw_job_stat | total_runs + timescaledb | _timescaledb_internal | bgw_job_stat | total_successes + timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | chunk_id + timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | job_id + timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | last_time_job_run + timescaledb | _timescaledb_internal | bgw_policy_chunk_stats | num_times_job_run + timescaledb | _timescaledb_internal | compressed_chunk_stats | chunk_name + timescaledb | _timescaledb_internal | compressed_chunk_stats | chunk_schema + timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_heap_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_index_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_toast_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | compressed_total_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | compression_status + timescaledb | _timescaledb_internal | compressed_chunk_stats | hypertable_name + timescaledb | _timescaledb_internal | compressed_chunk_stats | hypertable_schema + timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_heap_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_index_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_toast_size + timescaledb | _timescaledb_internal | compressed_chunk_stats | uncompressed_total_size timescaledb | _timescaledb_internal | hypertable_chunk_local_size | chunk_id timescaledb | _timescaledb_internal | hypertable_chunk_local_size | chunk_name timescaledb | _timescaledb_internal | hypertable_chunk_local_size | chunk_schema @@ -6173,17 +6141,71 @@ order by timescaledb | _timescaledb_internal | hypertable_chunk_local_size | index_bytes timescaledb | _timescaledb_internal | hypertable_chunk_local_size | toast_bytes timescaledb | _timescaledb_internal | hypertable_chunk_local_size | total_bytes - timescaledb | _timescaledb_catalog | hypertable_compression | attname - timescaledb | _timescaledb_catalog | hypertable_compression | compression_algorithm_id - timescaledb | _timescaledb_catalog | hypertable_compression | hypertable_id - timescaledb | _timescaledb_catalog | hypertable_compression | orderby_asc - timescaledb | _timescaledb_catalog | hypertable_compression | orderby_column_index - timescaledb | _timescaledb_catalog | hypertable_compression | orderby_nullsfirst - timescaledb | _timescaledb_catalog | hypertable_compression | segmentby_column_index - timescaledb | _timescaledb_catalog | hypertable_data_node | block_chunks - timescaledb | _timescaledb_catalog | hypertable_data_node | hypertable_id - timescaledb | _timescaledb_catalog | hypertable_data_node | node_hypertable_id - timescaledb | _timescaledb_catalog | hypertable_data_node | node_name + timescaledb | _timescaledb_internal | job_errors | error_data + timescaledb | _timescaledb_internal | job_errors | finish_time + timescaledb | _timescaledb_internal | job_errors | job_id + timescaledb | _timescaledb_internal | job_errors | pid + timescaledb | _timescaledb_internal | job_errors | start_time + timescaledb | timescaledb_experimental | chunk_replication_status | chunk_name + timescaledb | timescaledb_experimental | chunk_replication_status | chunk_schema + timescaledb | timescaledb_experimental | chunk_replication_status | desired_num_replicas + timescaledb | timescaledb_experimental | chunk_replication_status | hypertable_name + timescaledb | timescaledb_experimental | chunk_replication_status | hypertable_schema + timescaledb | timescaledb_experimental | chunk_replication_status | non_replica_nodes + timescaledb | timescaledb_experimental | chunk_replication_status | num_replicas + timescaledb | timescaledb_experimental | chunk_replication_status | replica_nodes + timescaledb | timescaledb_experimental | policies | config + timescaledb | timescaledb_experimental | policies | hypertable_name + timescaledb | timescaledb_experimental | policies | hypertable_schema + timescaledb | timescaledb_experimental | policies | proc_name + timescaledb | timescaledb_experimental | policies | proc_schema + timescaledb | timescaledb_experimental | policies | relation_name + timescaledb | timescaledb_experimental | policies | relation_schema + timescaledb | timescaledb_experimental | policies | schedule_interval + timescaledb | timescaledb_information | chunks | chunk_name + timescaledb | timescaledb_information | chunks | chunk_schema + timescaledb | timescaledb_information | chunks | chunk_tablespace + timescaledb | timescaledb_information | chunks | data_nodes + timescaledb | timescaledb_information | chunks | hypertable_name + timescaledb | timescaledb_information | chunks | hypertable_schema + timescaledb | timescaledb_information | chunks | is_compressed + timescaledb | timescaledb_information | chunks | primary_dimension + timescaledb | timescaledb_information | chunks | primary_dimension_type + timescaledb | timescaledb_information | chunks | range_end + timescaledb | timescaledb_information | chunks | range_end_integer + timescaledb | timescaledb_information | chunks | range_start + timescaledb | timescaledb_information | chunks | range_start_integer + timescaledb | timescaledb_information | compression_settings | attname + timescaledb | timescaledb_information | compression_settings | hypertable_name + timescaledb | timescaledb_information | compression_settings | hypertable_schema + timescaledb | timescaledb_information | compression_settings | orderby_asc + timescaledb | timescaledb_information | compression_settings | orderby_column_index + timescaledb | timescaledb_information | compression_settings | orderby_nullsfirst + timescaledb | timescaledb_information | compression_settings | segmentby_column_index + timescaledb | timescaledb_information | continuous_aggregates | compression_enabled + timescaledb | timescaledb_information | continuous_aggregates | finalized + timescaledb | timescaledb_information | continuous_aggregates | hypertable_name + timescaledb | timescaledb_information | continuous_aggregates | hypertable_schema + timescaledb | timescaledb_information | continuous_aggregates | materialization_hypertable_name + timescaledb | timescaledb_information | continuous_aggregates | materialization_hypertable_schema + timescaledb | timescaledb_information | continuous_aggregates | materialized_only + timescaledb | timescaledb_information | continuous_aggregates | view_definition + timescaledb | timescaledb_information | continuous_aggregates | view_name + timescaledb | timescaledb_information | continuous_aggregates | view_owner + timescaledb | timescaledb_information | continuous_aggregates | view_schema + timescaledb | timescaledb_information | data_nodes | node_name + timescaledb | timescaledb_information | data_nodes | options + timescaledb | timescaledb_information | data_nodes | owner + timescaledb | timescaledb_information | dimensions | column_name + timescaledb | timescaledb_information | dimensions | column_type + timescaledb | timescaledb_information | dimensions | dimension_number + timescaledb | timescaledb_information | dimensions | dimension_type + timescaledb | timescaledb_information | dimensions | hypertable_name + timescaledb | timescaledb_information | dimensions | hypertable_schema + timescaledb | timescaledb_information | dimensions | integer_interval + timescaledb | timescaledb_information | dimensions | integer_now_func + timescaledb | timescaledb_information | dimensions | num_partitions + timescaledb | timescaledb_information | dimensions | time_interval timescaledb | timescaledb_information | hypertables | compression_enabled timescaledb | timescaledb_information | hypertables | data_nodes timescaledb | timescaledb_information | hypertables | hypertable_name @@ -6195,18 +6217,13 @@ order by timescaledb | timescaledb_information | hypertables | replication_factor timescaledb | timescaledb_information | hypertables | tablespaces timescaledb | timescaledb_information | job_errors | err_message - timescaledb | _timescaledb_internal | job_errors | error_data timescaledb | timescaledb_information | job_errors | finish_time - timescaledb | _timescaledb_internal | job_errors | finish_time timescaledb | timescaledb_information | job_errors | job_id - timescaledb | _timescaledb_internal | job_errors | job_id timescaledb | timescaledb_information | job_errors | pid - timescaledb | _timescaledb_internal | job_errors | pid timescaledb | timescaledb_information | job_errors | proc_name timescaledb | timescaledb_information | job_errors | proc_schema timescaledb | timescaledb_information | job_errors | sqlerrcode timescaledb | timescaledb_information | job_errors | start_time - timescaledb | _timescaledb_internal | job_errors | start_time timescaledb | timescaledb_information | job_stats | hypertable_name timescaledb | timescaledb_information | job_stats | hypertable_schema timescaledb | timescaledb_information | job_stats | job_id @@ -6237,22 +6254,6 @@ order by timescaledb | timescaledb_information | jobs | retry_period timescaledb | timescaledb_information | jobs | schedule_interval timescaledb | timescaledb_information | jobs | scheduled - timescaledb | _timescaledb_catalog | metadata | include_in_telemetry - timescaledb | _timescaledb_catalog | metadata | key - timescaledb | _timescaledb_catalog | metadata | value - timescaledb | timescaledb_experimental | policies | config - timescaledb | timescaledb_experimental | policies | hypertable_name - timescaledb | timescaledb_experimental | policies | hypertable_schema - timescaledb | timescaledb_experimental | policies | proc_name - timescaledb | timescaledb_experimental | policies | proc_schema - timescaledb | timescaledb_experimental | policies | relation_name - timescaledb | timescaledb_experimental | policies | relation_schema - timescaledb | timescaledb_experimental | policies | schedule_interval - timescaledb | _timescaledb_catalog | remote_txn | data_node_name - timescaledb | _timescaledb_catalog | remote_txn | remote_transaction_id - timescaledb | _timescaledb_catalog | tablespace | hypertable_id - timescaledb | _timescaledb_catalog | tablespace | id - timescaledb | _timescaledb_catalog | tablespace | tablespace_name wrappers | public | wrappers_fdw_stats | bytes_in wrappers | public | wrappers_fdw_stats | bytes_out wrappers | public | wrappers_fdw_stats | create_times diff --git a/nix/tests/sql/extensions_sql_interface.sql b/nix/tests/sql/extensions_sql_interface.sql index ce7580248..947a4a612 100644 --- a/nix/tests/sql/extensions_sql_interface.sql +++ b/nix/tests/sql/extensions_sql_interface.sql @@ -97,5 +97,6 @@ where and pc.relkind in ('r', 'v', 'm', 'i') order by e.extname, + n.nspname, pc.relname, pa.attname; From efbb71d1acdff3210bbcfcf5f86e1230e9f375b4 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 19 Aug 2024 08:42:16 -0500 Subject: [PATCH 05/15] pgmq in prime --- nix/ext/pgmq.nix | 2 -- nix/tests/prime.sql | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix index c18fa9bfe..bcd512ab5 100644 --- a/nix/ext/pgmq.nix +++ b/nix/ext/pgmq.nix @@ -3,9 +3,7 @@ stdenv.mkDerivation rec { pname = "pgmq"; version = "1.4.4"; - buildInputs = [ postgresql ]; - src = fetchFromGitHub { owner = "tembo-io"; repo = pname; diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 8918ae8e0..7ad34bc00 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -30,7 +30,7 @@ create extension old_snapshot; create extension pageinspect; create extension pg_backtrace; create extension pg_buffercache; -create extension pgmq; +create extension pg_backtrace; /* TODO: Does not enable locally mode @@ -56,6 +56,7 @@ create extension pg_visibility; create extension pg_walinspect; create extension pgaudit; create extension pgcrypto; +create extension pgmq; create extension pgtap; create extension pgjwt; create extension pgroonga; From 0fa765b642873ffd8c752fae3ac60927a8097872 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 19 Aug 2024 08:46:53 -0500 Subject: [PATCH 06/15] update to pgmq 1.4.2 --- nix/tests/expected/extensions_sql_interface.out | 11 ++++++----- nix/tests/prime.sql | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 5315a2345..af4198981 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -122,11 +122,7 @@ order by vector | t wrappers | f xml2 | f -<<<<<<< HEAD -(81 rows) -======= -(79 rows) ->>>>>>> 95a20bc (add public interface test coverage) +(82 rows) /* @@ -1321,6 +1317,7 @@ order by pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint pgmq | pgmq | detach_archive | queue_name text | void pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean + pgmq | pgmq | format_table_name | queue_name text, prefix text | text pgmq | pgmq | list_queues | | SETOF pgmq.queue_record pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result @@ -5139,6 +5136,7 @@ order by xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text <<<<<<< HEAD +<<<<<<< HEAD (4957 rows) ======= <<<<<<< HEAD @@ -5147,6 +5145,9 @@ order by (4966 rows) >>>>>>> 95a20bc (add public interface test coverage) >>>>>>> 21f844d (add public interface test coverage) +======= +(4979 rows) +>>>>>>> fe3449e (update to pgmq 1.4.2) /* diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 7ad34bc00..3891817e1 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -30,7 +30,6 @@ create extension old_snapshot; create extension pageinspect; create extension pg_backtrace; create extension pg_buffercache; -create extension pg_backtrace; /* TODO: Does not enable locally mode From 8b2bc6c716bd7a65d7e1e7ec1362a4d6b945dbb2 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 19 Aug 2024 10:14:42 -0500 Subject: [PATCH 07/15] regression tests for pgmq --- nix/tests/expected/pgmq.out | 132 ++++++++++++++++++++++++++++++++++++ nix/tests/sql/pgmq.sql | 79 +++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 nix/tests/expected/pgmq.out create mode 100644 nix/tests/sql/pgmq.sql diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out new file mode 100644 index 000000000..ae3326338 --- /dev/null +++ b/nix/tests/expected/pgmq.out @@ -0,0 +1,132 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + create +-------- + +(1 row) + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + send +------ + 1 +(1 row) + +-- Test queue is not case sensitive +select + msg_id, + read_ct, + message +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); +ERROR: column "msg_id" does not exist +LINE 2: msg_id, + ^ +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + msg_id | read_ct | message +--------+---------+----------------- + 1 | 1 | {"foo": "bar1"} +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + msg_id | read_ct | message +--------+---------+--------- +(0 rows) + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + archive +--------- + f +(1 row) + +select + pgmq.create('my_queue'); + create +-------- + +(1 row) + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + send_batch +------------ + 1 + 2 + 3 +(3 rows) + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + archive +--------- + 3 +(1 row) + +select + pgmq.delete('my_queue', 6); + delete +-------- + f +(1 row) + +select + pgmq.drop_queue('my_queue'); + drop_queue +------------ + t +(1 row) + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create('F$oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create($$F'oo$$); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql new file mode 100644 index 000000000..623585fdc --- /dev/null +++ b/nix/tests/sql/pgmq.sql @@ -0,0 +1,79 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + +-- Test queue is not case sensitive +select + msg_id, + read_ct, + message +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + + +select + pgmq.create('my_queue'); + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + +select + pgmq.delete('my_queue', 6); + + +select + pgmq.drop_queue('my_queue'); + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +select pgmq.create('F$oo'); +select pgmq.create($$F'oo$$); + + From 6415640301a54d06385e6467331421832ea41fcf Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 19 Aug 2024 10:15:40 -0500 Subject: [PATCH 08/15] add pgmq migration test script --- nix/tests/migrations/pgmq.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 nix/tests/migrations/pgmq.sql diff --git a/nix/tests/migrations/pgmq.sql b/nix/tests/migrations/pgmq.sql new file mode 100644 index 000000000..d0121a269 --- /dev/null +++ b/nix/tests/migrations/pgmq.sql @@ -0,0 +1,12 @@ +select + pgmq.create('Foo'); + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + + From 107d7b69f9db9066b1a61a8962dd56b03ec19c75 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Tue, 10 Sep 2024 10:28:50 -0500 Subject: [PATCH 09/15] add pg_partman --- flake.nix | 1 + nix/ext/pg_partman.nix | 34 ++++++ .../expected/extensions_sql_interface.out | 107 +++++++++++++++++- nix/tests/expected/pg_partman.out | 101 +++++++++++++++++ nix/tests/postgresql.conf.in | 2 +- nix/tests/prime.sql | 1 + nix/tests/sql/pg_partman.sql | 82 ++++++++++++++ 7 files changed, 323 insertions(+), 5 deletions(-) create mode 100644 nix/ext/pg_partman.nix create mode 100644 nix/tests/expected/pg_partman.out create mode 100644 nix/tests/sql/pg_partman.sql diff --git a/flake.nix b/flake.nix index 4379b866c..9c1aa6b1f 100644 --- a/flake.nix +++ b/flake.nix @@ -127,6 +127,7 @@ ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix + ./nix/ext/pg_partman.nix ./nix/ext/pg_stat_monitor.nix ./nix/ext/pg_jsonschema.nix ./nix/ext/pgvector.nix diff --git a/nix/ext/pg_partman.nix b/nix/ext/pg_partman.nix new file mode 100644 index 000000000..1ece63386 --- /dev/null +++ b/nix/ext/pg_partman.nix @@ -0,0 +1,34 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pg_partman"; + version = "5.1.0"; + + buildInputs = [ postgresql ]; + + src = fetchFromGitHub { + owner = "pgpartman"; + repo = pname; + rev = "refs/tags/v${version}"; + sha256 = "sha256-GrVOJ5ywZMyqyDroYDLdKkXDdIJSDGhDfveO/ZvrmYs="; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + cp src/*${postgresql.dlSuffix} $out/lib + cp updates/* $out/share/postgresql/extension + cp -r sql/* $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "Partition management extension for PostgreSQL"; + homepage = "https://github.com/pgpartman/pg_partman"; + changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + broken = versionOlder postgresql.version "14"; + }; +} diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index af4198981..b98346564 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -76,6 +76,7 @@ order by pg_hashids | t pg_jsonschema | f pg_net | f + pg_partman | f pg_prewarm | t pg_repack | f pg_stat_monitor | t @@ -122,7 +123,7 @@ order by vector | t wrappers | f xml2 | f -(82 rows) +(83 rows) /* @@ -150,8 +151,8 @@ order by n.nspname, p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid); - extension_name | schema_name | function_name | argument_types | return_type -------------------------+--------------------------+--------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + extension_name | schema_name | function_name | argument_types | return_type +------------------------+--------------------------+--------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ address_standardizer | public | parse_address | text, OUT num text, OUT street text, OUT street2 text, OUT address1 text, OUT city text, OUT state text, OUT zip text, OUT zipplus text, OUT country text | record address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, address text | stdaddr address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, micro text, macro text | stdaddr @@ -1148,6 +1149,45 @@ order by pg_net | net | http_get | url text, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | http_post | url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | worker_restart | | boolean + pg_partman | public | apply_cluster | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text | void + pg_partman | public | apply_constraints | p_parent_table text, p_child_table text, p_analyze boolean, p_job_id bigint | void + pg_partman | public | apply_privileges | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text, p_job_id bigint | void + pg_partman | public | autovacuum_off | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean + pg_partman | public | autovacuum_reset | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean + pg_partman | public | calculate_time_partition_info | p_time_interval interval, p_start_time timestamp with time zone, p_date_trunc_interval text, OUT base_timestamp timestamp with time zone, OUT datetime_string text | record + pg_partman | public | check_automatic_maintenance_value | p_automatic_maintenance text | boolean + pg_partman | public | check_control_type | p_parent_schema text, p_parent_tablename text, p_control text | TABLE(general_type text, exact_type text) + pg_partman | public | check_default | p_exact_count boolean | SETOF check_default_table + pg_partman | public | check_epoch_type | p_type text | boolean + pg_partman | public | check_name_length | p_object_name text, p_suffix text, p_table_partition boolean | text + pg_partman | public | check_partition_type | p_type text | boolean + pg_partman | public | check_subpart_sameconfig | p_parent_table text | TABLE(sub_control text, sub_partition_interval text, sub_partition_type text, sub_premake integer, sub_automatic_maintenance text, sub_template_table text, sub_retention text, sub_retention_schema text, sub_retention_keep_index boolean, sub_retention_keep_table boolean, sub_epoch text, sub_constraint_cols text[], sub_optimize_constraint integer, sub_infinite_time_partitions boolean, sub_jobmon boolean, sub_inherit_privileges boolean, sub_constraint_valid boolean, sub_date_trunc_interval text, sub_ignore_default_data boolean, sub_default_table boolean, sub_maintenance_order integer, sub_retention_keep_publication boolean) + pg_partman | public | check_subpartition_limits | p_parent_table text, p_type text, OUT sub_min text, OUT sub_max text | record + pg_partman | public | create_parent | p_parent_table text, p_control text, p_interval text, p_type text, p_epoch text, p_premake integer, p_start_partition text, p_default_table boolean, p_automatic_maintenance text, p_constraint_cols text[], p_template_table text, p_jobmon boolean, p_date_trunc_interval text | boolean + pg_partman | public | create_partition_id | p_parent_table text, p_partition_ids bigint[], p_start_partition text | boolean + pg_partman | public | create_partition_time | p_parent_table text, p_partition_times timestamp with time zone[], p_start_partition text | boolean + pg_partman | public | create_sub_parent | p_top_parent text, p_control text, p_interval text, p_type text, p_default_table boolean, p_declarative_check text, p_constraint_cols text[], p_premake integer, p_start_partition text, p_epoch text, p_jobmon boolean, p_date_trunc_interval text | boolean + pg_partman | public | drop_constraints | p_parent_table text, p_child_table text, p_debug boolean | void + pg_partman | public | drop_partition_id | p_parent_table text, p_retention bigint, p_keep_table boolean, p_keep_index boolean, p_retention_schema text | integer + pg_partman | public | drop_partition_time | p_parent_table text, p_retention interval, p_keep_table boolean, p_keep_index boolean, p_retention_schema text, p_reference_timestamp timestamp with time zone | integer + pg_partman | public | dump_partitioned_table_definition | p_parent_table text, p_ignore_template_table boolean | text + pg_partman | public | inherit_replica_identity | p_parent_schemaname text, p_parent_tablename text, p_child_tablename text | void + pg_partman | public | inherit_template_properties | p_parent_table text, p_child_schema text, p_child_tablename text | boolean + pg_partman | public | partition_data_id | p_parent_table text, p_batch_count integer, p_batch_interval bigint, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint + pg_partman | public | partition_data_proc | IN p_parent_table text, IN p_loop_count integer, IN p_interval text, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_order text, IN p_source_table text, IN p_ignored_columns text[], IN p_quiet boolean | + pg_partman | public | partition_data_time | p_parent_table text, p_batch_count integer, p_batch_interval interval, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint + pg_partman | public | partition_gap_fill | p_parent_table text | integer + pg_partman | public | reapply_constraints_proc | IN p_parent_table text, IN p_drop_constraints boolean, IN p_apply_constraints boolean, IN p_wait integer, IN p_dryrun boolean | + pg_partman | public | reapply_privileges | p_parent_table text | void + pg_partman | public | run_analyze | IN p_skip_locked boolean, IN p_quiet boolean, IN p_parent_table text | + pg_partman | public | run_maintenance | p_parent_table text, p_analyze boolean, p_jobmon boolean | void + pg_partman | public | run_maintenance_proc | IN p_wait integer, IN p_analyze boolean, IN p_jobmon boolean | + pg_partman | public | show_partition_info | p_child_table text, p_partition_interval text, p_parent_table text, OUT child_start_time timestamp with time zone, OUT child_end_time timestamp with time zone, OUT child_start_id bigint, OUT child_end_id bigint, OUT suffix text | record + pg_partman | public | show_partition_name | p_parent_table text, p_value text, OUT partition_schema text, OUT partition_table text, OUT suffix_timestamp timestamp with time zone, OUT suffix_id bigint, OUT table_exists boolean | record + pg_partman | public | show_partitions | p_parent_table text, p_order text, p_include_default boolean | TABLE(partition_schemaname text, partition_tablename text) + pg_partman | public | stop_sub_partition | p_parent_table text, p_jobmon boolean | boolean + pg_partman | public | undo_partition | p_parent_table text, p_target_table text, p_loop_count integer, p_batch_interval text, p_keep_table boolean, p_lock_wait numeric, p_ignored_columns text[], p_drop_cascade boolean, OUT partitions_undone integer, OUT rows_undone bigint | record + pg_partman | public | undo_partition_proc | IN p_parent_table text, IN p_target_table text, IN p_loop_count integer, IN p_interval text, IN p_keep_table boolean, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_ignored_columns text[], IN p_drop_cascade boolean, IN p_quiet boolean | pg_prewarm | public | autoprewarm_dump_now | | bigint pg_prewarm | public | autoprewarm_start_worker | | void pg_prewarm | public | pg_prewarm | regclass, mode text, fork text, first_block bigint, last_block bigint | bigint @@ -5137,6 +5177,7 @@ order by xml2 | public | xslt_process | text, text, text | text <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD (4957 rows) ======= <<<<<<< HEAD @@ -5148,6 +5189,9 @@ order by ======= (4979 rows) >>>>>>> fe3449e (update to pgmq 1.4.2) +======= +(5018 rows) +>>>>>>> 29ef673 (add pg_partman) /* @@ -5230,6 +5274,61 @@ order by pg_net | net | http_request_queue | method pg_net | net | http_request_queue | timeout_milliseconds pg_net | net | http_request_queue | url + pg_partman | public | part_config | automatic_maintenance + pg_partman | public | part_config | constraint_cols + pg_partman | public | part_config | constraint_valid + pg_partman | public | part_config | control + pg_partman | public | part_config | date_trunc_interval + pg_partman | public | part_config | datetime_string + pg_partman | public | part_config | default_table + pg_partman | public | part_config | epoch + pg_partman | public | part_config | ignore_default_data + pg_partman | public | part_config | infinite_time_partitions + pg_partman | public | part_config | inherit_privileges + pg_partman | public | part_config | jobmon + pg_partman | public | part_config | maintenance_last_run + pg_partman | public | part_config | maintenance_order + pg_partman | public | part_config | optimize_constraint + pg_partman | public | part_config | parent_table + pg_partman | public | part_config | partition_interval + pg_partman | public | part_config | partition_type + pg_partman | public | part_config | premake + pg_partman | public | part_config | retention + pg_partman | public | part_config | retention_keep_index + pg_partman | public | part_config | retention_keep_publication + pg_partman | public | part_config | retention_keep_table + pg_partman | public | part_config | retention_schema + pg_partman | public | part_config | sub_partition_set_full + pg_partman | public | part_config | template_table + pg_partman | public | part_config | undo_in_progress + pg_partman | public | part_config_sub | sub_automatic_maintenance + pg_partman | public | part_config_sub | sub_constraint_cols + pg_partman | public | part_config_sub | sub_constraint_valid + pg_partman | public | part_config_sub | sub_control + pg_partman | public | part_config_sub | sub_date_trunc_interval + pg_partman | public | part_config_sub | sub_default_table + pg_partman | public | part_config_sub | sub_epoch + pg_partman | public | part_config_sub | sub_ignore_default_data + pg_partman | public | part_config_sub | sub_infinite_time_partitions + pg_partman | public | part_config_sub | sub_inherit_privileges + pg_partman | public | part_config_sub | sub_jobmon + pg_partman | public | part_config_sub | sub_maintenance_order + pg_partman | public | part_config_sub | sub_optimize_constraint + pg_partman | public | part_config_sub | sub_parent + pg_partman | public | part_config_sub | sub_partition_interval + pg_partman | public | part_config_sub | sub_partition_type + pg_partman | public | part_config_sub | sub_premake + pg_partman | public | part_config_sub | sub_retention + pg_partman | public | part_config_sub | sub_retention_keep_index + pg_partman | public | part_config_sub | sub_retention_keep_publication + pg_partman | public | part_config_sub | sub_retention_keep_table + pg_partman | public | part_config_sub | sub_retention_schema + pg_partman | public | part_config_sub | sub_template_table + pg_partman | public | table_privs | grantee + pg_partman | public | table_privs | grantor + pg_partman | public | table_privs | privilege_type + pg_partman | public | table_privs | table_name + pg_partman | public | table_privs | table_schema pg_repack | repack | primary_keys | indexrelid pg_repack | repack | primary_keys | indrelid pg_repack | repack | tables | alter_col_storage @@ -6264,5 +6363,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1083 rows) +(1138 rows) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out new file mode 100644 index 000000000..527d58b12 --- /dev/null +++ b/nix/tests/expected/pg_partman.out @@ -0,0 +1,101 @@ +create schema if not exists partman_test; +/* +Simple Time Based: 1 Partition Per Day + +For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. +*/ +create table partman_test.time_taptest_table( + col1 int, + col2 text default 'stuff', + col3 timestamptz not null default now() +) + partition by range (col3); +create index on partman_test.time_tap (col3); +ERROR: relation "partman_test.time_tap" does not exist +/* +Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. + +Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. +*/ +create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); +alter table partman_test.time_taptest_table_template add primary key (col1); +/* +Review tables in the partman_test schema +*/ +select + table_name, + table_type +from + information_schema.tables +where + table_schema = 'partman_test'; + table_name | table_type +-----------------------------+------------ + time_taptest_table | BASE TABLE + time_taptest_table_template | BASE TABLE +(2 rows) + +select public.create_parent( + p_parent_table := 'partman_test.time_taptest_table', + p_control := 'col3', + p_interval := '1 day', + p_template_table := 'partman_test.time_taptest_table_template' +); + create_parent +--------------- + t +(1 row) + +/* +Review tables in the partman_test schema, which should now include daily partitions +*/ +select + -- dates in partition names are variable, so reduced to the prefix + substring(table_name, 1, 21) as table_prefix, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name; + table_prefix | table_type +-----------------------+------------ + time_taptest_table | BASE TABLE + time_taptest_table_de | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_te | BASE TABLE +(12 rows) + +/* +Confirm maintenance proc runs without issue +*/ +call public.run_maintenance_proc(); +/* +Make sure the background worker is NOT enabled. +This is intentional. We document using pg_cron to schedule calls to +public.run_maintenance_proc(). That is consistent with other providers. +*/ +select + application_name +from + pg_stat_activity +where + application_name = 'pg_partman_bgw'; + application_name +------------------ +(0 rows) + +-- Cleanup +drop schema partman_test cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table partman_test.time_taptest_table +drop cascades to table partman_test.time_taptest_table_template diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..25e412239 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace, pg_partman_bgw' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 3891817e1..633870d5c 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -42,6 +42,7 @@ create extension pg_net; create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; +create extension pg_partman; create extension pg_prewarm; create extension pg_partman; create extension pg_jsonschema; diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql new file mode 100644 index 000000000..621e1a7c9 --- /dev/null +++ b/nix/tests/sql/pg_partman.sql @@ -0,0 +1,82 @@ +create schema if not exists partman_test; + +/* +Simple Time Based: 1 Partition Per Day + +For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. +*/ + +create table partman_test.time_taptest_table( + col1 int, + col2 text default 'stuff', + col3 timestamptz not null default now() +) + partition by range (col3); + +create index on partman_test.time_tap (col3); + +/* +Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. + +Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. +*/ + +create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); + +alter table partman_test.time_taptest_table_template add primary key (col1); + +/* +Review tables in the partman_test schema +*/ + +select + table_name, + table_type +from + information_schema.tables +where + table_schema = 'partman_test'; + + +select public.create_parent( + p_parent_table := 'partman_test.time_taptest_table', + p_control := 'col3', + p_interval := '1 day', + p_template_table := 'partman_test.time_taptest_table_template' +); + +/* +Review tables in the partman_test schema, which should now include daily partitions +*/ + +select + -- dates in partition names are variable, so reduced to the prefix + substring(table_name, 1, 21) as table_prefix, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name; + + +/* +Confirm maintenance proc runs without issue +*/ +call public.run_maintenance_proc(); + +/* +Make sure the background worker is NOT enabled. +This is intentional. We document using pg_cron to schedule calls to +public.run_maintenance_proc(). That is consistent with other providers. +*/ +select + application_name +from + pg_stat_activity +where + application_name = 'pg_partman_bgw'; + +-- Cleanup +drop schema partman_test cascade; From 1a416c9ff7db5afc0f3c5d3eb1a841e3c6ab72b0 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 11 Sep 2024 12:15:20 -0500 Subject: [PATCH 10/15] remove pg_partman_bgq from test suite postgresql.conf --- nix/tests/postgresql.conf.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 25e412239..d2d667b9a 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace, pg_partman_bgw' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use From 0222a126e4212a1b25c02f13cd0962dc71eec446 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 11 Sep 2024 15:00:11 -0500 Subject: [PATCH 11/15] checkin broken test --- nix/tests/expected/pgmq.out | 17 +++++++++++++++++ nix/tests/sql/pgmq.sql | 6 ++++++ 2 files changed, 23 insertions(+) diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out index ae3326338..b2dd8e0e9 100644 --- a/nix/tests/expected/pgmq.out +++ b/nix/tests/expected/pgmq.out @@ -111,6 +111,23 @@ select t (1 row) +select + pgmq.create_partitioned('my_partitioned_queue'); +ERROR: 10000 is not a valid partitioning type for pg_partman +CONTEXT: PL/pgSQL function create_parent(text,text,text,text,text,integer,text,boolean,text,text[],text,boolean,text) line 119 at RAISE +SQL statement "SELECT public.create_parent( + fq_qtable, + partition_col, 'native', partition_interval + )" +PL/pgSQL function pgmq.create_partitioned(text,text,text) line 33 at PERFORM +DETAIL: +HINT: +CONTEXT: PL/pgSQL function create_parent(text,text,text,text,text,integer,text,boolean,text,text[],text,boolean,text) line 647 at RAISE +SQL statement "SELECT public.create_parent( + fq_qtable, + partition_col, 'native', partition_interval + )" +PL/pgSQL function pgmq.create_partitioned(text,text,text) line 33 at PERFORM -- Make sure SQLI enabling characters are blocked select pgmq.create('F--oo'); ERROR: queue name contains invalid characters: $, ;, --, or \' diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql index 623585fdc..ae3df7641 100644 --- a/nix/tests/sql/pgmq.sql +++ b/nix/tests/sql/pgmq.sql @@ -71,9 +71,15 @@ select select pgmq.drop_queue('my_queue'); +select + pgmq.create_partitioned('my_partitioned_queue'); + + -- Make sure SQLI enabling characters are blocked select pgmq.create('F--oo'); select pgmq.create('F$oo'); select pgmq.create($$F'oo$$); + + From 67b8f1ec7e2641c350c3d7640afde15013b1f322 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Thu, 12 Sep 2024 09:26:43 -0500 Subject: [PATCH 12/15] checkin broken test 2 --- ansible/files/postgresql_config/supautils.conf.j2 | 2 +- nix/tests/expected/pgmq.out | 8 ++++++-- nix/tests/sql/pgmq.sql | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 2ab4257fd..87105c7be 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -3,7 +3,7 @@ supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identitie # full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out index b2dd8e0e9..223577536 100644 --- a/nix/tests/expected/pgmq.out +++ b/nix/tests/expected/pgmq.out @@ -112,8 +112,12 @@ select (1 row) select - pgmq.create_partitioned('my_partitioned_queue'); -ERROR: 10000 is not a valid partitioning type for pg_partman + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); +ERROR: 5 seconds is not a valid partitioning type for pg_partman CONTEXT: PL/pgSQL function create_parent(text,text,text,text,text,integer,text,boolean,text,text[],text,boolean,text) line 119 at RAISE SQL statement "SELECT public.create_parent( fq_qtable, diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql index ae3df7641..4d4045484 100644 --- a/nix/tests/sql/pgmq.sql +++ b/nix/tests/sql/pgmq.sql @@ -72,7 +72,11 @@ select pgmq.drop_queue('my_queue'); select - pgmq.create_partitioned('my_partitioned_queue'); + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); -- Make sure SQLI enabling characters are blocked From 53d79c36e4b02762e0c26ff944500f1a94526569 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 18 Sep 2024 15:51:37 -0500 Subject: [PATCH 13/15] bump pgmq to 1.4.4 --- nix/ext/pgmq.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix index bcd512ab5..97a3c27e3 100644 --- a/nix/ext/pgmq.nix +++ b/nix/ext/pgmq.nix @@ -8,7 +8,7 @@ stdenv.mkDerivation rec { owner = "tembo-io"; repo = pname; rev = "v${version}"; - hash = ""; + hash = "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="; }; buildPhase = '' From 880db799f8cf784f481f14a3642c49aa2db79bae Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 18 Sep 2024 16:03:46 -0500 Subject: [PATCH 14/15] sync test outputs and pg_partman with 1.4.4 --- .../expected/extensions_sql_interface.out | 20 +++---------------- nix/tests/expected/pgmq.out | 20 +++++-------------- nix/tests/prime.sql | 1 - 3 files changed, 8 insertions(+), 33 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index b98346564..b7fd273af 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -1346,6 +1346,8 @@ order by pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean pgmq | pgmq | _ensure_pg_partman_installed | | void pgmq | pgmq | _get_partition_col | partition_interval text | text + pgmq | pgmq | _get_pg_partman_major_version | | integer + pgmq | pgmq | _get_pg_partman_schema | | text pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void @@ -5175,23 +5177,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -(4957 rows) -======= -<<<<<<< HEAD -(4953 rows) -======= -(4966 rows) ->>>>>>> 95a20bc (add public interface test coverage) ->>>>>>> 21f844d (add public interface test coverage) -======= -(4979 rows) ->>>>>>> fe3449e (update to pgmq 1.4.2) -======= -(5018 rows) ->>>>>>> 29ef673 (add pg_partman) +(5024 rows) /* diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out index 223577536..b2886d9fa 100644 --- a/nix/tests/expected/pgmq.out +++ b/nix/tests/expected/pgmq.out @@ -117,21 +117,11 @@ select '5 seconds', '10 seconds' ); -ERROR: 5 seconds is not a valid partitioning type for pg_partman -CONTEXT: PL/pgSQL function create_parent(text,text,text,text,text,integer,text,boolean,text,text[],text,boolean,text) line 119 at RAISE -SQL statement "SELECT public.create_parent( - fq_qtable, - partition_col, 'native', partition_interval - )" -PL/pgSQL function pgmq.create_partitioned(text,text,text) line 33 at PERFORM -DETAIL: -HINT: -CONTEXT: PL/pgSQL function create_parent(text,text,text,text,text,integer,text,boolean,text,text[],text,boolean,text) line 647 at RAISE -SQL statement "SELECT public.create_parent( - fq_qtable, - partition_col, 'native', partition_interval - )" -PL/pgSQL function pgmq.create_partitioned(text,text,text) line 33 at PERFORM + create_partitioned +-------------------- + +(1 row) + -- Make sure SQLI enabling characters are blocked select pgmq.create('F--oo'); ERROR: queue name contains invalid characters: $, ;, --, or \' diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 633870d5c..3891817e1 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -42,7 +42,6 @@ create extension pg_net; create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; -create extension pg_partman; create extension pg_prewarm; create extension pg_partman; create extension pg_jsonschema; From b73d7dfafad65ceecc84480b9350a802e30a5f4c Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Wed, 18 Sep 2024 16:05:29 -0500 Subject: [PATCH 15/15] add pgmq to supautils privileged extensions --- ansible/files/postgresql_config/supautils.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 87105c7be..aa01cd8aa 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -3,7 +3,7 @@ supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identitie # full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres'