From c822ba46b6c1d76babf09baa430759b49078cb31 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Fri, 25 Oct 2024 16:19:58 -0500 Subject: [PATCH 01/10] remove pgmq and pg_partman (unreleased) until permissions issues resolved --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 3014f9032..69a80b8ab 100644 --- a/flake.nix +++ b/flake.nix @@ -135,7 +135,9 @@ ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix - ./nix/ext/pgmq.nix + # pending resolution of a permissions issue. Use 15.5.1.004 on staging to test + # ./nix/ext/pgmq.nix + # ./nix/ext/pg_partman.nix ./nix/ext/pg_repack.nix ./nix/ext/pg-safeupdate.nix ./nix/ext/plpgsql-check.nix @@ -151,7 +153,6 @@ ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix - ./nix/ext/pg_partman.nix ./nix/ext/pg_stat_monitor.nix ./nix/ext/pg_jsonschema.nix ./nix/ext/pgvector.nix From 5bf9fc16d0ecde6f181c406c2de5774a0a41b541 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Sat, 26 Oct 2024 07:20:30 -0500 Subject: [PATCH 02/10] align ext tests --- .../expected/extensions_sql_interface.out | 138 +---------------- nix/tests/expected/pg_partman.out | 104 ------------- nix/tests/expected/pgmq.out | 143 ------------------ nix/tests/prime.sql | 4 +- nix/tests/sql/pg_partman.sql | 85 ----------- nix/tests/sql/pgmq.sql | 89 ----------- 6 files changed, 7 insertions(+), 556 deletions(-) delete mode 100644 nix/tests/expected/pg_partman.out delete mode 100644 nix/tests/expected/pgmq.out delete mode 100644 nix/tests/sql/pg_partman.sql delete mode 100644 nix/tests/sql/pgmq.sql diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 07c0d6650..01f736115 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -76,7 +76,6 @@ order by pg_hashids | t pg_jsonschema | f pg_net | f - pg_partman | f pg_prewarm | t pg_repack | f pg_stat_monitor | t @@ -89,7 +88,6 @@ order by pgaudit | t pgcrypto | t pgjwt | f - pgmq | f pgroonga | f pgroonga_database | f pgrouting | t @@ -123,7 +121,7 @@ order by vector | t wrappers | f xml2 | f -(83 rows) +(81 rows) /* @@ -163,8 +161,8 @@ order by n.nspname, p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid); - extension_name | schema_name | function_name | argument_types | return_type -------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + extension_name | schema_name | function_name | argument_types | return_type +------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- address_standardizer | public | parse_address | text, OUT num text, OUT street text, OUT street2 text, OUT address1 text, OUT city text, OUT state text, OUT zip text, OUT zipplus text, OUT country text | record address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, address text | stdaddr address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, micro text, macro text | stdaddr @@ -1160,45 +1158,6 @@ order by pg_net | net | http_get | url text, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | http_post | url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | worker_restart | | boolean - pg_partman | public | apply_cluster | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text | void - pg_partman | public | apply_constraints | p_parent_table text, p_child_table text, p_analyze boolean, p_job_id bigint | void - pg_partman | public | apply_privileges | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text, p_job_id bigint | void - pg_partman | public | autovacuum_off | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean - pg_partman | public | autovacuum_reset | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean - pg_partman | public | calculate_time_partition_info | p_time_interval interval, p_start_time timestamp with time zone, p_date_trunc_interval text, OUT base_timestamp timestamp with time zone, OUT datetime_string text | record - pg_partman | public | check_automatic_maintenance_value | p_automatic_maintenance text | boolean - pg_partman | public | check_control_type | p_parent_schema text, p_parent_tablename text, p_control text | TABLE(general_type text, exact_type text) - pg_partman | public | check_default | p_exact_count boolean | SETOF check_default_table - pg_partman | public | check_epoch_type | p_type text | boolean - pg_partman | public | check_name_length | p_object_name text, p_suffix text, p_table_partition boolean | text - pg_partman | public | check_partition_type | p_type text | boolean - pg_partman | public | check_subpart_sameconfig | p_parent_table text | TABLE(sub_control text, sub_partition_interval text, sub_partition_type text, sub_premake integer, sub_automatic_maintenance text, sub_template_table text, sub_retention text, sub_retention_schema text, sub_retention_keep_index boolean, sub_retention_keep_table boolean, sub_epoch text, sub_constraint_cols text[], sub_optimize_constraint integer, sub_infinite_time_partitions boolean, sub_jobmon boolean, sub_inherit_privileges boolean, sub_constraint_valid boolean, sub_date_trunc_interval text, sub_ignore_default_data boolean, sub_default_table boolean, sub_maintenance_order integer, sub_retention_keep_publication boolean) - pg_partman | public | check_subpartition_limits | p_parent_table text, p_type text, OUT sub_min text, OUT sub_max text | record - pg_partman | public | create_parent | p_parent_table text, p_control text, p_interval text, p_type text, p_epoch text, p_premake integer, p_start_partition text, p_default_table boolean, p_automatic_maintenance text, p_constraint_cols text[], p_template_table text, p_jobmon boolean, p_date_trunc_interval text | boolean - pg_partman | public | create_partition_id | p_parent_table text, p_partition_ids bigint[], p_start_partition text | boolean - pg_partman | public | create_partition_time | p_parent_table text, p_partition_times timestamp with time zone[], p_start_partition text | boolean - pg_partman | public | create_sub_parent | p_top_parent text, p_control text, p_interval text, p_type text, p_default_table boolean, p_declarative_check text, p_constraint_cols text[], p_premake integer, p_start_partition text, p_epoch text, p_jobmon boolean, p_date_trunc_interval text | boolean - pg_partman | public | drop_constraints | p_parent_table text, p_child_table text, p_debug boolean | void - pg_partman | public | drop_partition_id | p_parent_table text, p_retention bigint, p_keep_table boolean, p_keep_index boolean, p_retention_schema text | integer - pg_partman | public | drop_partition_time | p_parent_table text, p_retention interval, p_keep_table boolean, p_keep_index boolean, p_retention_schema text, p_reference_timestamp timestamp with time zone | integer - pg_partman | public | dump_partitioned_table_definition | p_parent_table text, p_ignore_template_table boolean | text - pg_partman | public | inherit_replica_identity | p_parent_schemaname text, p_parent_tablename text, p_child_tablename text | void - pg_partman | public | inherit_template_properties | p_parent_table text, p_child_schema text, p_child_tablename text | boolean - pg_partman | public | partition_data_id | p_parent_table text, p_batch_count integer, p_batch_interval bigint, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint - pg_partman | public | partition_data_proc | IN p_parent_table text, IN p_loop_count integer, IN p_interval text, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_order text, IN p_source_table text, IN p_ignored_columns text[], IN p_quiet boolean | - pg_partman | public | partition_data_time | p_parent_table text, p_batch_count integer, p_batch_interval interval, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint - pg_partman | public | partition_gap_fill | p_parent_table text | integer - pg_partman | public | reapply_constraints_proc | IN p_parent_table text, IN p_drop_constraints boolean, IN p_apply_constraints boolean, IN p_wait integer, IN p_dryrun boolean | - pg_partman | public | reapply_privileges | p_parent_table text | void - pg_partman | public | run_analyze | IN p_skip_locked boolean, IN p_quiet boolean, IN p_parent_table text | - pg_partman | public | run_maintenance | p_parent_table text, p_analyze boolean, p_jobmon boolean | void - pg_partman | public | run_maintenance_proc | IN p_wait integer, IN p_analyze boolean, IN p_jobmon boolean | - pg_partman | public | show_partition_info | p_child_table text, p_partition_interval text, p_parent_table text, OUT child_start_time timestamp with time zone, OUT child_end_time timestamp with time zone, OUT child_start_id bigint, OUT child_end_id bigint, OUT suffix text | record - pg_partman | public | show_partition_name | p_parent_table text, p_value text, OUT partition_schema text, OUT partition_table text, OUT suffix_timestamp timestamp with time zone, OUT suffix_id bigint, OUT table_exists boolean | record - pg_partman | public | show_partitions | p_parent_table text, p_order text, p_include_default boolean | TABLE(partition_schemaname text, partition_tablename text) - pg_partman | public | stop_sub_partition | p_parent_table text, p_jobmon boolean | boolean - pg_partman | public | undo_partition | p_parent_table text, p_target_table text, p_loop_count integer, p_batch_interval text, p_keep_table boolean, p_lock_wait numeric, p_ignored_columns text[], p_drop_cascade boolean, OUT partitions_undone integer, OUT rows_undone bigint | record - pg_partman | public | undo_partition_proc | IN p_parent_table text, IN p_target_table text, IN p_loop_count integer, IN p_interval text, IN p_keep_table boolean, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_ignored_columns text[], IN p_drop_cascade boolean, IN p_quiet boolean | pg_prewarm | public | autoprewarm_dump_now | | bigint pg_prewarm | public | autoprewarm_start_worker | | void pg_prewarm | public | pg_prewarm | regclass, mode text, fork text, first_block bigint, last_block bigint | bigint @@ -1356,34 +1315,6 @@ order by pgjwt | public | url_decode | data text | bytea pgjwt | public | url_encode | data bytea | text pgjwt | public | verify | token text, secret text, algorithm text | TABLE(header json, payload json, valid boolean) - pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean - pgmq | pgmq | _ensure_pg_partman_installed | | void - pgmq | pgmq | _get_partition_col | partition_interval text | text - pgmq | pgmq | _get_pg_partman_major_version | | integer - pgmq | pgmq | _get_pg_partman_schema | | text - pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean - pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint - pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void - pgmq | pgmq | create | queue_name text | void - pgmq | pgmq | create_non_partitioned | queue_name text | void - pgmq | pgmq | create_partitioned | queue_name text, partition_interval text, retention_interval text | void - pgmq | pgmq | create_unlogged | queue_name text | void - pgmq | pgmq | delete | queue_name text, msg_id bigint | boolean - pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint - pgmq | pgmq | detach_archive | queue_name text | void - pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean - pgmq | pgmq | format_table_name | queue_name text, prefix text | text - pgmq | pgmq | list_queues | | SETOF pgmq.queue_record - pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result - pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result - pgmq | pgmq | pop | queue_name text | SETOF pgmq.message_record - pgmq | pgmq | purge_queue | queue_name text | bigint - pgmq | pgmq | read | queue_name text, vt integer, qty integer | SETOF pgmq.message_record - pgmq | pgmq | read_with_poll | queue_name text, vt integer, qty integer, max_poll_seconds integer, poll_interval_ms integer | SETOF pgmq.message_record - pgmq | pgmq | send | queue_name text, msg jsonb, delay integer | SETOF bigint - pgmq | pgmq | send_batch | queue_name text, msgs jsonb[], delay integer | SETOF bigint - pgmq | pgmq | set_vt | queue_name text, msg_id bigint, vt integer | SETOF pgmq.message_record - pgmq | pgmq | validate_queue_name | queue_name text | void pgroonga | pgroonga | command | groongacommand text | text pgroonga | pgroonga | command | groongacommand text, arguments text[] | text pgroonga | pgroonga | command_escape_value | value text | text @@ -5234,7 +5165,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5069 rows) +(5002 rows) /* @@ -5317,61 +5248,6 @@ order by pg_net | net | http_request_queue | method pg_net | net | http_request_queue | timeout_milliseconds pg_net | net | http_request_queue | url - pg_partman | public | part_config | automatic_maintenance - pg_partman | public | part_config | constraint_cols - pg_partman | public | part_config | constraint_valid - pg_partman | public | part_config | control - pg_partman | public | part_config | date_trunc_interval - pg_partman | public | part_config | datetime_string - pg_partman | public | part_config | default_table - pg_partman | public | part_config | epoch - pg_partman | public | part_config | ignore_default_data - pg_partman | public | part_config | infinite_time_partitions - pg_partman | public | part_config | inherit_privileges - pg_partman | public | part_config | jobmon - pg_partman | public | part_config | maintenance_last_run - pg_partman | public | part_config | maintenance_order - pg_partman | public | part_config | optimize_constraint - pg_partman | public | part_config | parent_table - pg_partman | public | part_config | partition_interval - pg_partman | public | part_config | partition_type - pg_partman | public | part_config | premake - pg_partman | public | part_config | retention - pg_partman | public | part_config | retention_keep_index - pg_partman | public | part_config | retention_keep_publication - pg_partman | public | part_config | retention_keep_table - pg_partman | public | part_config | retention_schema - pg_partman | public | part_config | sub_partition_set_full - pg_partman | public | part_config | template_table - pg_partman | public | part_config | undo_in_progress - pg_partman | public | part_config_sub | sub_automatic_maintenance - pg_partman | public | part_config_sub | sub_constraint_cols - pg_partman | public | part_config_sub | sub_constraint_valid - pg_partman | public | part_config_sub | sub_control - pg_partman | public | part_config_sub | sub_date_trunc_interval - pg_partman | public | part_config_sub | sub_default_table - pg_partman | public | part_config_sub | sub_epoch - pg_partman | public | part_config_sub | sub_ignore_default_data - pg_partman | public | part_config_sub | sub_infinite_time_partitions - pg_partman | public | part_config_sub | sub_inherit_privileges - pg_partman | public | part_config_sub | sub_jobmon - pg_partman | public | part_config_sub | sub_maintenance_order - pg_partman | public | part_config_sub | sub_optimize_constraint - pg_partman | public | part_config_sub | sub_parent - pg_partman | public | part_config_sub | sub_partition_interval - pg_partman | public | part_config_sub | sub_partition_type - pg_partman | public | part_config_sub | sub_premake - pg_partman | public | part_config_sub | sub_retention - pg_partman | public | part_config_sub | sub_retention_keep_index - pg_partman | public | part_config_sub | sub_retention_keep_publication - pg_partman | public | part_config_sub | sub_retention_keep_table - pg_partman | public | part_config_sub | sub_retention_schema - pg_partman | public | part_config_sub | sub_template_table - pg_partman | public | table_privs | grantee - pg_partman | public | table_privs | grantor - pg_partman | public | table_privs | privilege_type - pg_partman | public | table_privs | table_name - pg_partman | public | table_privs | table_schema pg_repack | repack | primary_keys | indexrelid pg_repack | repack | primary_keys | indrelid pg_repack | repack | tables | alter_col_storage @@ -5512,10 +5388,6 @@ order by pg_tle | pgtle | feature_info | obj_identity pg_tle | pgtle | feature_info | proname pg_tle | pgtle | feature_info | schema_name - pgmq | pgmq | meta | created_at - pgmq | pgmq | meta | is_partitioned - pgmq | pgmq | meta | is_unlogged - pgmq | pgmq | meta | queue_name pgsodium | pgsodium | decrypted_key | associated_data pgsodium | pgsodium | decrypted_key | comment pgsodium | pgsodium | decrypted_key | created @@ -6409,5 +6281,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1141 rows) +(1082 rows) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out deleted file mode 100644 index 25aa80d2e..000000000 --- a/nix/tests/expected/pg_partman.out +++ /dev/null @@ -1,104 +0,0 @@ -create schema if not exists partman_test; -/* -Simple Time Based: 1 Partition Per Day - -For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. -*/ -create table partman_test.time_taptest_table( - col1 int, - col2 text default 'stuff', - col3 timestamptz not null default now() -) - partition by range (col3); -create index on partman_test.time_tap (col3); -ERROR: relation "partman_test.time_tap" does not exist -/* -Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. - -Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. -*/ -create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); -alter table partman_test.time_taptest_table_template add primary key (col1); -/* -Review tables in the partman_test schema -*/ -select - table_name, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name, - table_type; - table_name | table_type ------------------------------+------------ - time_taptest_table | BASE TABLE - time_taptest_table_template | BASE TABLE -(2 rows) - -select public.create_parent( - p_parent_table := 'partman_test.time_taptest_table', - p_control := 'col3', - p_interval := '1 day', - p_template_table := 'partman_test.time_taptest_table_template' -); - create_parent ---------------- - t -(1 row) - -/* -Review tables in the partman_test schema, which should now include daily partitions -*/ -select - -- dates in partition names are variable, so reduced to the prefix - substring(table_name, 1, 21) as table_prefix, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name; - table_prefix | table_type ------------------------+------------ - time_taptest_table | BASE TABLE - time_taptest_table_de | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_p2 | BASE TABLE - time_taptest_table_te | BASE TABLE -(12 rows) - -/* -Confirm maintenance proc runs without issue -*/ -call public.run_maintenance_proc(); -/* -Make sure the background worker is NOT enabled. -This is intentional. We document using pg_cron to schedule calls to -public.run_maintenance_proc(). That is consistent with other providers. -*/ -select - application_name -from - pg_stat_activity -where - application_name = 'pg_partman_bgw'; - application_name ------------------- -(0 rows) - --- Cleanup -drop schema partman_test cascade; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to table partman_test.time_taptest_table -drop cascades to table partman_test.time_taptest_table_template diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out deleted file mode 100644 index b2886d9fa..000000000 --- a/nix/tests/expected/pgmq.out +++ /dev/null @@ -1,143 +0,0 @@ --- Test the standard flow -select - pgmq.create('Foo'); - create --------- - -(1 row) - -select - * -from - pgmq.send( - queue_name:='Foo', - msg:='{"foo": "bar1"}' - ); - send ------- - 1 -(1 row) - --- Test queue is not case sensitive -select - msg_id, - read_ct, - message -from - pgmq.send( - queue_name:='foo', -- note: lowercase useage - msg:='{"foo": "bar2"}', - delay:=5 - ); -ERROR: column "msg_id" does not exist -LINE 2: msg_id, - ^ -select - msg_id, - read_ct, - message -from - pgmq.read( - queue_name:='Foo', - vt:=30, - qty:=2 - ); - msg_id | read_ct | message ---------+---------+----------------- - 1 | 1 | {"foo": "bar1"} -(1 row) - -select - msg_id, - read_ct, - message -from - pgmq.pop('Foo'); - msg_id | read_ct | message ---------+---------+--------- -(0 rows) - --- Archive message with msg_id=2. -select - pgmq.archive( - queue_name:='Foo', - msg_id:=2 - ); - archive ---------- - f -(1 row) - -select - pgmq.create('my_queue'); - create --------- - -(1 row) - -select - pgmq.send_batch( - queue_name:='my_queue', - msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] -); - send_batch ------------- - 1 - 2 - 3 -(3 rows) - -select - pgmq.archive( - queue_name:='my_queue', - msg_ids:=array[3, 4, 5] - ); - archive ---------- - 3 -(1 row) - -select - pgmq.delete('my_queue', 6); - delete --------- - f -(1 row) - -select - pgmq.drop_queue('my_queue'); - drop_queue ------------- - t -(1 row) - -select - pgmq.create_partitioned( - 'my_partitioned_queue', - '5 seconds', - '10 seconds' -); - create_partitioned --------------------- - -(1 row) - --- Make sure SQLI enabling characters are blocked -select pgmq.create('F--oo'); -ERROR: queue name contains invalid characters: $, ;, --, or \' -CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE -PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization -SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" -PL/pgSQL function pgmq."create"(text) line 3 at PERFORM -select pgmq.create('F$oo'); -ERROR: queue name contains invalid characters: $, ;, --, or \' -CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE -PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization -SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" -PL/pgSQL function pgmq."create"(text) line 3 at PERFORM -select pgmq.create($$F'oo$$); -ERROR: queue name contains invalid characters: $, ;, --, or \' -CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE -PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization -SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" -PL/pgSQL function pgmq."create"(text) line 3 at PERFORM diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 3891817e1..09fa8b273 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -43,7 +43,7 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; -create extension pg_partman; +-- create extension pg_partman; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; @@ -55,7 +55,7 @@ create extension pg_visibility; create extension pg_walinspect; create extension pgaudit; create extension pgcrypto; -create extension pgmq; +-- create extension pgmq; create extension pgtap; create extension pgjwt; create extension pgroonga; diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql deleted file mode 100644 index d1fb9f4ba..000000000 --- a/nix/tests/sql/pg_partman.sql +++ /dev/null @@ -1,85 +0,0 @@ -create schema if not exists partman_test; - -/* -Simple Time Based: 1 Partition Per Day - -For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. -*/ - -create table partman_test.time_taptest_table( - col1 int, - col2 text default 'stuff', - col3 timestamptz not null default now() -) - partition by range (col3); - -create index on partman_test.time_tap (col3); - -/* -Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. - -Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. -*/ - -create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); - -alter table partman_test.time_taptest_table_template add primary key (col1); - -/* -Review tables in the partman_test schema -*/ - -select - table_name, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name, - table_type; - - -select public.create_parent( - p_parent_table := 'partman_test.time_taptest_table', - p_control := 'col3', - p_interval := '1 day', - p_template_table := 'partman_test.time_taptest_table_template' -); - -/* -Review tables in the partman_test schema, which should now include daily partitions -*/ - -select - -- dates in partition names are variable, so reduced to the prefix - substring(table_name, 1, 21) as table_prefix, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name; - - -/* -Confirm maintenance proc runs without issue -*/ -call public.run_maintenance_proc(); - -/* -Make sure the background worker is NOT enabled. -This is intentional. We document using pg_cron to schedule calls to -public.run_maintenance_proc(). That is consistent with other providers. -*/ -select - application_name -from - pg_stat_activity -where - application_name = 'pg_partman_bgw'; - --- Cleanup -drop schema partman_test cascade; diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql deleted file mode 100644 index 4d4045484..000000000 --- a/nix/tests/sql/pgmq.sql +++ /dev/null @@ -1,89 +0,0 @@ --- Test the standard flow -select - pgmq.create('Foo'); - -select - * -from - pgmq.send( - queue_name:='Foo', - msg:='{"foo": "bar1"}' - ); - --- Test queue is not case sensitive -select - msg_id, - read_ct, - message -from - pgmq.send( - queue_name:='foo', -- note: lowercase useage - msg:='{"foo": "bar2"}', - delay:=5 - ); - -select - msg_id, - read_ct, - message -from - pgmq.read( - queue_name:='Foo', - vt:=30, - qty:=2 - ); - -select - msg_id, - read_ct, - message -from - pgmq.pop('Foo'); - - --- Archive message with msg_id=2. -select - pgmq.archive( - queue_name:='Foo', - msg_id:=2 - ); - - -select - pgmq.create('my_queue'); - -select - pgmq.send_batch( - queue_name:='my_queue', - msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] -); - -select - pgmq.archive( - queue_name:='my_queue', - msg_ids:=array[3, 4, 5] - ); - -select - pgmq.delete('my_queue', 6); - - -select - pgmq.drop_queue('my_queue'); - -select - pgmq.create_partitioned( - 'my_partitioned_queue', - '5 seconds', - '10 seconds' -); - - --- Make sure SQLI enabling characters are blocked -select pgmq.create('F--oo'); -select pgmq.create('F$oo'); -select pgmq.create($$F'oo$$); - - - - From 85bc04e19b6633e6c783d90ef8a50bcc9965add2 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Sat, 26 Oct 2024 07:23:45 -0500 Subject: [PATCH 03/10] bump 15.8 and 16.3 images w/o pgmq and pg_partman --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 129e1a238..fbc9bf6ba 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.004" - postgres16: "16.3.1.010" + postgres15: "15.8.1.005" + postgres16: "16.3.1.011" # Non Postgres Extensions pgbouncer_release: "1.19.0" From d24fb7f7582a70b63c43aa228634066c48b03a63 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 10:30:30 -0500 Subject: [PATCH 04/10] convert ownership to postgres --- .../files/postgresql_config/supautils.conf.j2 | 4 +- ansible/vars.yml | 4 +- flake.nix | 5 +- .../expected/extensions_sql_interface.out | 138 ++++++++++++++++- nix/tests/expected/pg_partman.out | 104 +++++++++++++ nix/tests/expected/pgmq.out | 143 ++++++++++++++++++ nix/tests/prime.sql | 4 +- nix/tests/sql/pg_partman.sql | 85 +++++++++++ nix/tests/sql/pgmq.sql | 89 +++++++++++ 9 files changed, 563 insertions(+), 13 deletions(-) create mode 100644 nix/tests/expected/pg_partman.out create mode 100644 nix/tests/expected/pgmq.out create mode 100644 nix/tests/sql/pg_partman.sql create mode 100644 nix/tests/sql/pgmq.sql diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index aa01cd8aa..3637aef86 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -3,7 +3,9 @@ supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identitie # full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' + +# Adding extensions to this list causes them to be owned by supabase_admin vs the default of postgres +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' diff --git a/ansible/vars.yml b/ansible/vars.yml index fbc9bf6ba..8d556a407 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.005" - postgres16: "16.3.1.011" + postgres15: "15.8.1.005-staging-1" + postgres16: "16.3.1.011-staging-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/flake.nix b/flake.nix index 69a80b8ab..3014f9032 100644 --- a/flake.nix +++ b/flake.nix @@ -135,9 +135,7 @@ ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix - # pending resolution of a permissions issue. Use 15.5.1.004 on staging to test - # ./nix/ext/pgmq.nix - # ./nix/ext/pg_partman.nix + ./nix/ext/pgmq.nix ./nix/ext/pg_repack.nix ./nix/ext/pg-safeupdate.nix ./nix/ext/plpgsql-check.nix @@ -153,6 +151,7 @@ ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix + ./nix/ext/pg_partman.nix ./nix/ext/pg_stat_monitor.nix ./nix/ext/pg_jsonschema.nix ./nix/ext/pgvector.nix diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 01f736115..07c0d6650 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -76,6 +76,7 @@ order by pg_hashids | t pg_jsonschema | f pg_net | f + pg_partman | f pg_prewarm | t pg_repack | f pg_stat_monitor | t @@ -88,6 +89,7 @@ order by pgaudit | t pgcrypto | t pgjwt | f + pgmq | f pgroonga | f pgroonga_database | f pgrouting | t @@ -121,7 +123,7 @@ order by vector | t wrappers | f xml2 | f -(81 rows) +(83 rows) /* @@ -161,8 +163,8 @@ order by n.nspname, p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid); - extension_name | schema_name | function_name | argument_types | return_type -------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + extension_name | schema_name | function_name | argument_types | return_type +------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ address_standardizer | public | parse_address | text, OUT num text, OUT street text, OUT street2 text, OUT address1 text, OUT city text, OUT state text, OUT zip text, OUT zipplus text, OUT country text | record address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, address text | stdaddr address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, micro text, macro text | stdaddr @@ -1158,6 +1160,45 @@ order by pg_net | net | http_get | url text, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | http_post | url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | worker_restart | | boolean + pg_partman | public | apply_cluster | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text | void + pg_partman | public | apply_constraints | p_parent_table text, p_child_table text, p_analyze boolean, p_job_id bigint | void + pg_partman | public | apply_privileges | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text, p_job_id bigint | void + pg_partman | public | autovacuum_off | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean + pg_partman | public | autovacuum_reset | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean + pg_partman | public | calculate_time_partition_info | p_time_interval interval, p_start_time timestamp with time zone, p_date_trunc_interval text, OUT base_timestamp timestamp with time zone, OUT datetime_string text | record + pg_partman | public | check_automatic_maintenance_value | p_automatic_maintenance text | boolean + pg_partman | public | check_control_type | p_parent_schema text, p_parent_tablename text, p_control text | TABLE(general_type text, exact_type text) + pg_partman | public | check_default | p_exact_count boolean | SETOF check_default_table + pg_partman | public | check_epoch_type | p_type text | boolean + pg_partman | public | check_name_length | p_object_name text, p_suffix text, p_table_partition boolean | text + pg_partman | public | check_partition_type | p_type text | boolean + pg_partman | public | check_subpart_sameconfig | p_parent_table text | TABLE(sub_control text, sub_partition_interval text, sub_partition_type text, sub_premake integer, sub_automatic_maintenance text, sub_template_table text, sub_retention text, sub_retention_schema text, sub_retention_keep_index boolean, sub_retention_keep_table boolean, sub_epoch text, sub_constraint_cols text[], sub_optimize_constraint integer, sub_infinite_time_partitions boolean, sub_jobmon boolean, sub_inherit_privileges boolean, sub_constraint_valid boolean, sub_date_trunc_interval text, sub_ignore_default_data boolean, sub_default_table boolean, sub_maintenance_order integer, sub_retention_keep_publication boolean) + pg_partman | public | check_subpartition_limits | p_parent_table text, p_type text, OUT sub_min text, OUT sub_max text | record + pg_partman | public | create_parent | p_parent_table text, p_control text, p_interval text, p_type text, p_epoch text, p_premake integer, p_start_partition text, p_default_table boolean, p_automatic_maintenance text, p_constraint_cols text[], p_template_table text, p_jobmon boolean, p_date_trunc_interval text | boolean + pg_partman | public | create_partition_id | p_parent_table text, p_partition_ids bigint[], p_start_partition text | boolean + pg_partman | public | create_partition_time | p_parent_table text, p_partition_times timestamp with time zone[], p_start_partition text | boolean + pg_partman | public | create_sub_parent | p_top_parent text, p_control text, p_interval text, p_type text, p_default_table boolean, p_declarative_check text, p_constraint_cols text[], p_premake integer, p_start_partition text, p_epoch text, p_jobmon boolean, p_date_trunc_interval text | boolean + pg_partman | public | drop_constraints | p_parent_table text, p_child_table text, p_debug boolean | void + pg_partman | public | drop_partition_id | p_parent_table text, p_retention bigint, p_keep_table boolean, p_keep_index boolean, p_retention_schema text | integer + pg_partman | public | drop_partition_time | p_parent_table text, p_retention interval, p_keep_table boolean, p_keep_index boolean, p_retention_schema text, p_reference_timestamp timestamp with time zone | integer + pg_partman | public | dump_partitioned_table_definition | p_parent_table text, p_ignore_template_table boolean | text + pg_partman | public | inherit_replica_identity | p_parent_schemaname text, p_parent_tablename text, p_child_tablename text | void + pg_partman | public | inherit_template_properties | p_parent_table text, p_child_schema text, p_child_tablename text | boolean + pg_partman | public | partition_data_id | p_parent_table text, p_batch_count integer, p_batch_interval bigint, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint + pg_partman | public | partition_data_proc | IN p_parent_table text, IN p_loop_count integer, IN p_interval text, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_order text, IN p_source_table text, IN p_ignored_columns text[], IN p_quiet boolean | + pg_partman | public | partition_data_time | p_parent_table text, p_batch_count integer, p_batch_interval interval, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint + pg_partman | public | partition_gap_fill | p_parent_table text | integer + pg_partman | public | reapply_constraints_proc | IN p_parent_table text, IN p_drop_constraints boolean, IN p_apply_constraints boolean, IN p_wait integer, IN p_dryrun boolean | + pg_partman | public | reapply_privileges | p_parent_table text | void + pg_partman | public | run_analyze | IN p_skip_locked boolean, IN p_quiet boolean, IN p_parent_table text | + pg_partman | public | run_maintenance | p_parent_table text, p_analyze boolean, p_jobmon boolean | void + pg_partman | public | run_maintenance_proc | IN p_wait integer, IN p_analyze boolean, IN p_jobmon boolean | + pg_partman | public | show_partition_info | p_child_table text, p_partition_interval text, p_parent_table text, OUT child_start_time timestamp with time zone, OUT child_end_time timestamp with time zone, OUT child_start_id bigint, OUT child_end_id bigint, OUT suffix text | record + pg_partman | public | show_partition_name | p_parent_table text, p_value text, OUT partition_schema text, OUT partition_table text, OUT suffix_timestamp timestamp with time zone, OUT suffix_id bigint, OUT table_exists boolean | record + pg_partman | public | show_partitions | p_parent_table text, p_order text, p_include_default boolean | TABLE(partition_schemaname text, partition_tablename text) + pg_partman | public | stop_sub_partition | p_parent_table text, p_jobmon boolean | boolean + pg_partman | public | undo_partition | p_parent_table text, p_target_table text, p_loop_count integer, p_batch_interval text, p_keep_table boolean, p_lock_wait numeric, p_ignored_columns text[], p_drop_cascade boolean, OUT partitions_undone integer, OUT rows_undone bigint | record + pg_partman | public | undo_partition_proc | IN p_parent_table text, IN p_target_table text, IN p_loop_count integer, IN p_interval text, IN p_keep_table boolean, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_ignored_columns text[], IN p_drop_cascade boolean, IN p_quiet boolean | pg_prewarm | public | autoprewarm_dump_now | | bigint pg_prewarm | public | autoprewarm_start_worker | | void pg_prewarm | public | pg_prewarm | regclass, mode text, fork text, first_block bigint, last_block bigint | bigint @@ -1315,6 +1356,34 @@ order by pgjwt | public | url_decode | data text | bytea pgjwt | public | url_encode | data bytea | text pgjwt | public | verify | token text, secret text, algorithm text | TABLE(header json, payload json, valid boolean) + pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean + pgmq | pgmq | _ensure_pg_partman_installed | | void + pgmq | pgmq | _get_partition_col | partition_interval text | text + pgmq | pgmq | _get_pg_partman_major_version | | integer + pgmq | pgmq | _get_pg_partman_schema | | text + pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean + pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void + pgmq | pgmq | create | queue_name text | void + pgmq | pgmq | create_non_partitioned | queue_name text | void + pgmq | pgmq | create_partitioned | queue_name text, partition_interval text, retention_interval text | void + pgmq | pgmq | create_unlogged | queue_name text | void + pgmq | pgmq | delete | queue_name text, msg_id bigint | boolean + pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | detach_archive | queue_name text | void + pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean + pgmq | pgmq | format_table_name | queue_name text, prefix text | text + pgmq | pgmq | list_queues | | SETOF pgmq.queue_record + pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result + pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result + pgmq | pgmq | pop | queue_name text | SETOF pgmq.message_record + pgmq | pgmq | purge_queue | queue_name text | bigint + pgmq | pgmq | read | queue_name text, vt integer, qty integer | SETOF pgmq.message_record + pgmq | pgmq | read_with_poll | queue_name text, vt integer, qty integer, max_poll_seconds integer, poll_interval_ms integer | SETOF pgmq.message_record + pgmq | pgmq | send | queue_name text, msg jsonb, delay integer | SETOF bigint + pgmq | pgmq | send_batch | queue_name text, msgs jsonb[], delay integer | SETOF bigint + pgmq | pgmq | set_vt | queue_name text, msg_id bigint, vt integer | SETOF pgmq.message_record + pgmq | pgmq | validate_queue_name | queue_name text | void pgroonga | pgroonga | command | groongacommand text | text pgroonga | pgroonga | command | groongacommand text, arguments text[] | text pgroonga | pgroonga | command_escape_value | value text | text @@ -5165,7 +5234,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5002 rows) +(5069 rows) /* @@ -5248,6 +5317,61 @@ order by pg_net | net | http_request_queue | method pg_net | net | http_request_queue | timeout_milliseconds pg_net | net | http_request_queue | url + pg_partman | public | part_config | automatic_maintenance + pg_partman | public | part_config | constraint_cols + pg_partman | public | part_config | constraint_valid + pg_partman | public | part_config | control + pg_partman | public | part_config | date_trunc_interval + pg_partman | public | part_config | datetime_string + pg_partman | public | part_config | default_table + pg_partman | public | part_config | epoch + pg_partman | public | part_config | ignore_default_data + pg_partman | public | part_config | infinite_time_partitions + pg_partman | public | part_config | inherit_privileges + pg_partman | public | part_config | jobmon + pg_partman | public | part_config | maintenance_last_run + pg_partman | public | part_config | maintenance_order + pg_partman | public | part_config | optimize_constraint + pg_partman | public | part_config | parent_table + pg_partman | public | part_config | partition_interval + pg_partman | public | part_config | partition_type + pg_partman | public | part_config | premake + pg_partman | public | part_config | retention + pg_partman | public | part_config | retention_keep_index + pg_partman | public | part_config | retention_keep_publication + pg_partman | public | part_config | retention_keep_table + pg_partman | public | part_config | retention_schema + pg_partman | public | part_config | sub_partition_set_full + pg_partman | public | part_config | template_table + pg_partman | public | part_config | undo_in_progress + pg_partman | public | part_config_sub | sub_automatic_maintenance + pg_partman | public | part_config_sub | sub_constraint_cols + pg_partman | public | part_config_sub | sub_constraint_valid + pg_partman | public | part_config_sub | sub_control + pg_partman | public | part_config_sub | sub_date_trunc_interval + pg_partman | public | part_config_sub | sub_default_table + pg_partman | public | part_config_sub | sub_epoch + pg_partman | public | part_config_sub | sub_ignore_default_data + pg_partman | public | part_config_sub | sub_infinite_time_partitions + pg_partman | public | part_config_sub | sub_inherit_privileges + pg_partman | public | part_config_sub | sub_jobmon + pg_partman | public | part_config_sub | sub_maintenance_order + pg_partman | public | part_config_sub | sub_optimize_constraint + pg_partman | public | part_config_sub | sub_parent + pg_partman | public | part_config_sub | sub_partition_interval + pg_partman | public | part_config_sub | sub_partition_type + pg_partman | public | part_config_sub | sub_premake + pg_partman | public | part_config_sub | sub_retention + pg_partman | public | part_config_sub | sub_retention_keep_index + pg_partman | public | part_config_sub | sub_retention_keep_publication + pg_partman | public | part_config_sub | sub_retention_keep_table + pg_partman | public | part_config_sub | sub_retention_schema + pg_partman | public | part_config_sub | sub_template_table + pg_partman | public | table_privs | grantee + pg_partman | public | table_privs | grantor + pg_partman | public | table_privs | privilege_type + pg_partman | public | table_privs | table_name + pg_partman | public | table_privs | table_schema pg_repack | repack | primary_keys | indexrelid pg_repack | repack | primary_keys | indrelid pg_repack | repack | tables | alter_col_storage @@ -5388,6 +5512,10 @@ order by pg_tle | pgtle | feature_info | obj_identity pg_tle | pgtle | feature_info | proname pg_tle | pgtle | feature_info | schema_name + pgmq | pgmq | meta | created_at + pgmq | pgmq | meta | is_partitioned + pgmq | pgmq | meta | is_unlogged + pgmq | pgmq | meta | queue_name pgsodium | pgsodium | decrypted_key | associated_data pgsodium | pgsodium | decrypted_key | comment pgsodium | pgsodium | decrypted_key | created @@ -6281,5 +6409,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1082 rows) +(1141 rows) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out new file mode 100644 index 000000000..25aa80d2e --- /dev/null +++ b/nix/tests/expected/pg_partman.out @@ -0,0 +1,104 @@ +create schema if not exists partman_test; +/* +Simple Time Based: 1 Partition Per Day + +For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. +*/ +create table partman_test.time_taptest_table( + col1 int, + col2 text default 'stuff', + col3 timestamptz not null default now() +) + partition by range (col3); +create index on partman_test.time_tap (col3); +ERROR: relation "partman_test.time_tap" does not exist +/* +Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. + +Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. +*/ +create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); +alter table partman_test.time_taptest_table_template add primary key (col1); +/* +Review tables in the partman_test schema +*/ +select + table_name, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name, + table_type; + table_name | table_type +-----------------------------+------------ + time_taptest_table | BASE TABLE + time_taptest_table_template | BASE TABLE +(2 rows) + +select public.create_parent( + p_parent_table := 'partman_test.time_taptest_table', + p_control := 'col3', + p_interval := '1 day', + p_template_table := 'partman_test.time_taptest_table_template' +); + create_parent +--------------- + t +(1 row) + +/* +Review tables in the partman_test schema, which should now include daily partitions +*/ +select + -- dates in partition names are variable, so reduced to the prefix + substring(table_name, 1, 21) as table_prefix, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name; + table_prefix | table_type +-----------------------+------------ + time_taptest_table | BASE TABLE + time_taptest_table_de | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_p2 | BASE TABLE + time_taptest_table_te | BASE TABLE +(12 rows) + +/* +Confirm maintenance proc runs without issue +*/ +call public.run_maintenance_proc(); +/* +Make sure the background worker is NOT enabled. +This is intentional. We document using pg_cron to schedule calls to +public.run_maintenance_proc(). That is consistent with other providers. +*/ +select + application_name +from + pg_stat_activity +where + application_name = 'pg_partman_bgw'; + application_name +------------------ +(0 rows) + +-- Cleanup +drop schema partman_test cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table partman_test.time_taptest_table +drop cascades to table partman_test.time_taptest_table_template diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out new file mode 100644 index 000000000..b2886d9fa --- /dev/null +++ b/nix/tests/expected/pgmq.out @@ -0,0 +1,143 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + create +-------- + +(1 row) + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + send +------ + 1 +(1 row) + +-- Test queue is not case sensitive +select + msg_id, + read_ct, + message +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); +ERROR: column "msg_id" does not exist +LINE 2: msg_id, + ^ +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + msg_id | read_ct | message +--------+---------+----------------- + 1 | 1 | {"foo": "bar1"} +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + msg_id | read_ct | message +--------+---------+--------- +(0 rows) + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + archive +--------- + f +(1 row) + +select + pgmq.create('my_queue'); + create +-------- + +(1 row) + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + send_batch +------------ + 1 + 2 + 3 +(3 rows) + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + archive +--------- + 3 +(1 row) + +select + pgmq.delete('my_queue', 6); + delete +-------- + f +(1 row) + +select + pgmq.drop_queue('my_queue'); + drop_queue +------------ + t +(1 row) + +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); + create_partitioned +-------------------- + +(1 row) + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create('F$oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create($$F'oo$$); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 09fa8b273..3891817e1 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -43,7 +43,7 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; --- create extension pg_partman; +create extension pg_partman; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; @@ -55,7 +55,7 @@ create extension pg_visibility; create extension pg_walinspect; create extension pgaudit; create extension pgcrypto; --- create extension pgmq; +create extension pgmq; create extension pgtap; create extension pgjwt; create extension pgroonga; diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql new file mode 100644 index 000000000..d1fb9f4ba --- /dev/null +++ b/nix/tests/sql/pg_partman.sql @@ -0,0 +1,85 @@ +create schema if not exists partman_test; + +/* +Simple Time Based: 1 Partition Per Day + +For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. +*/ + +create table partman_test.time_taptest_table( + col1 int, + col2 text default 'stuff', + col3 timestamptz not null default now() +) + partition by range (col3); + +create index on partman_test.time_tap (col3); + +/* +Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. + +Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. +*/ + +create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); + +alter table partman_test.time_taptest_table_template add primary key (col1); + +/* +Review tables in the partman_test schema +*/ + +select + table_name, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name, + table_type; + + +select public.create_parent( + p_parent_table := 'partman_test.time_taptest_table', + p_control := 'col3', + p_interval := '1 day', + p_template_table := 'partman_test.time_taptest_table_template' +); + +/* +Review tables in the partman_test schema, which should now include daily partitions +*/ + +select + -- dates in partition names are variable, so reduced to the prefix + substring(table_name, 1, 21) as table_prefix, + table_type +from + information_schema.tables +where + table_schema = 'partman_test' +order by + table_name; + + +/* +Confirm maintenance proc runs without issue +*/ +call public.run_maintenance_proc(); + +/* +Make sure the background worker is NOT enabled. +This is intentional. We document using pg_cron to schedule calls to +public.run_maintenance_proc(). That is consistent with other providers. +*/ +select + application_name +from + pg_stat_activity +where + application_name = 'pg_partman_bgw'; + +-- Cleanup +drop schema partman_test cascade; diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql new file mode 100644 index 000000000..4d4045484 --- /dev/null +++ b/nix/tests/sql/pgmq.sql @@ -0,0 +1,89 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + +-- Test queue is not case sensitive +select + msg_id, + read_ct, + message +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + + +select + pgmq.create('my_queue'); + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + +select + pgmq.delete('my_queue', 6); + + +select + pgmq.drop_queue('my_queue'); + +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); + + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +select pgmq.create('F$oo'); +select pgmq.create($$F'oo$$); + + + + From fd60061fef1d3b7652fed979289629cb8b503e86 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 10:35:39 -0500 Subject: [PATCH 05/10] minor test bugfix --- nix/tests/expected/pg_partman.out | 3 +-- nix/tests/sql/pg_partman.sql | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out index 25aa80d2e..5dee2b748 100644 --- a/nix/tests/expected/pg_partman.out +++ b/nix/tests/expected/pg_partman.out @@ -10,8 +10,7 @@ create table partman_test.time_taptest_table( col3 timestamptz not null default now() ) partition by range (col3); -create index on partman_test.time_tap (col3); -ERROR: relation "partman_test.time_tap" does not exist +create index on partman_test.time_taptest_table (col3); /* Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql index d1fb9f4ba..6c4998ccb 100644 --- a/nix/tests/sql/pg_partman.sql +++ b/nix/tests/sql/pg_partman.sql @@ -13,7 +13,7 @@ create table partman_test.time_taptest_table( ) partition by range (col3); -create index on partman_test.time_tap (col3); +create index on partman_test.time_taptest_table (col3); /* Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. From e262bd35f7b60d6b448d688903240f45eac890f1 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 11:15:35 -0500 Subject: [PATCH 06/10] 15 release only --- .github/workflows/ami-release-nix.yml | 2 +- ansible/files/postgresql_config/supautils.conf.j2 | 7 +++---- ansible/vars.yml | 8 ++++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 643f26fc7..206f4c31d 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -55,7 +55,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 3637aef86..af54510f3 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,11 +1,10 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 - -# Adding extensions to this list causes them to be owned by supabase_admin vs the default of postgres -supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +# omitted because doesn't require superuser: pgmq, pg_partman +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' diff --git a/ansible/vars.yml b/ansible/vars.yml index 8d556a407..2e451561a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -7,12 +7,12 @@ async_mode: true # postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c postgres_major: - "15" - - "16" + # - "16" # Full version strings for each major version postgres_release: - postgres15: "15.8.1.005-staging-1" - postgres16: "16.3.1.011-staging-1" + postgres15: "15.8.1.004-pgmq-perms-5" + # postgres16: "16.3.1.010-pgmq-perms" # Non Postgres Extensions pgbouncer_release: "1.19.0" @@ -141,7 +141,7 @@ groonga_release_checksum: sha256:1c2d1a6981c1ad3f02a11aff202b15ba30cb1c6147f1fa9 pgroonga_release: "3.0.7" pgroonga_release_checksum: sha256:885ff3878cc30e9030e5fc56d561bc8b66df3ede1562c9d802bc0ea04fe5c203 -wrappers_release: "0.4.3" +wrappers_release: "0.4.2" hypopg_release: "1.4.1" hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c6c89282bcdedd6bc From a5e8462da7b5f12f2c12db52c2b001acfc769c15 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 13:26:50 -0500 Subject: [PATCH 07/10] remove partman --- .../files/postgresql_config/supautils.conf.j2 | 4 +- flake.nix | 4 +- nix/tests/prime.sql | 1 - nix/tests/sql/pg_partman.sql | 85 ------------------- 4 files changed, 5 insertions(+), 89 deletions(-) delete mode 100644 nix/tests/sql/pg_partman.sql diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index af54510f3..b2112cf98 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,9 +1,9 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 -# omitted because doesn't require superuser: pgmq, pg_partman +# omitted because doesn't require superuser: pgmq supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' diff --git a/flake.nix b/flake.nix index 3014f9032..c58fe5109 100644 --- a/flake.nix +++ b/flake.nix @@ -151,7 +151,9 @@ ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix - ./nix/ext/pg_partman.nix + # Requires superuser but also uses dynamic sql so need to be owned by posgtres. + # investigating options + #./nix/ext/pg_partman.nix ./nix/ext/pg_stat_monitor.nix ./nix/ext/pg_jsonschema.nix ./nix/ext/pgvector.nix diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 3891817e1..a72f3bf48 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -43,7 +43,6 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; -create extension pg_partman; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql deleted file mode 100644 index 6c4998ccb..000000000 --- a/nix/tests/sql/pg_partman.sql +++ /dev/null @@ -1,85 +0,0 @@ -create schema if not exists partman_test; - -/* -Simple Time Based: 1 Partition Per Day - -For native partitioning, you must start with a parent table that has already been set up to be partitioned in the desired type. Currently pg_partman only supports the RANGE type of partitioning (both for time & id). You cannot turn a non-partitioned table into the parent table of a partitioned set, which can make migration a challenge. This document will show you some techniques for how to manage this later. For now, we will start with a brand new table in this example. Any non-unique indexes can also be added to the parent table in PG11+ and they will automatically be created on all child tables. -*/ - -create table partman_test.time_taptest_table( - col1 int, - col2 text default 'stuff', - col3 timestamptz not null default now() -) - partition by range (col3); - -create index on partman_test.time_taptest_table (col3); - -/* -Unique indexes (including primary keys) cannot be created on a natively partitioned parent unless they include the partition key. For time-based partitioning that generally doesn't work out since that would limit only a single timestamp value in each child table. pg_partman helps to manage this by using a template table to manage properties that currently are not supported by native partitioning. Note that this does not solve the issue of the constraint not being enforced across the entire partition set. See the main documentation to see which properties are managed by the template. - -Manually create the template table first so that when we run create_parent() the initial child tables that are created will have a primary key. If you do not supply a template table to pg_partman, it will create one for you in the schema that you installed the extension to. However properties you add to that template are only then applied to newly created child tables after that point. You will have to retroactively apply those properties manually to any child tables that already existed. -*/ - -create table partman_test.time_taptest_table_template (like partman_test.time_taptest_table); - -alter table partman_test.time_taptest_table_template add primary key (col1); - -/* -Review tables in the partman_test schema -*/ - -select - table_name, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name, - table_type; - - -select public.create_parent( - p_parent_table := 'partman_test.time_taptest_table', - p_control := 'col3', - p_interval := '1 day', - p_template_table := 'partman_test.time_taptest_table_template' -); - -/* -Review tables in the partman_test schema, which should now include daily partitions -*/ - -select - -- dates in partition names are variable, so reduced to the prefix - substring(table_name, 1, 21) as table_prefix, - table_type -from - information_schema.tables -where - table_schema = 'partman_test' -order by - table_name; - - -/* -Confirm maintenance proc runs without issue -*/ -call public.run_maintenance_proc(); - -/* -Make sure the background worker is NOT enabled. -This is intentional. We document using pg_cron to schedule calls to -public.run_maintenance_proc(). That is consistent with other providers. -*/ -select - application_name -from - pg_stat_activity -where - application_name = 'pg_partman_bgw'; - --- Cleanup -drop schema partman_test cascade; From 10784848ee085c75c86d00011ac4b84cf508b6e9 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 13:27:15 -0500 Subject: [PATCH 08/10] bump version num --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 2e451561a..d7b754eac 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.004-pgmq-perms-5" + postgres15: "15.8.1.004-pgmq-perms-6" # postgres16: "16.3.1.010-pgmq-perms" # Non Postgres Extensions From e3f6e6316723483ad80d5566354e381c8d95d499 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 28 Oct 2024 13:30:33 -0500 Subject: [PATCH 09/10] align output for removing pg_partman --- .../expected/extensions_sql_interface.out | 105 +----------------- nix/tests/expected/pgmq.out | 8 +- nix/tests/sql/pgmq.sql | 3 + 3 files changed, 11 insertions(+), 105 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 07c0d6650..fff273a88 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -76,7 +76,6 @@ order by pg_hashids | t pg_jsonschema | f pg_net | f - pg_partman | f pg_prewarm | t pg_repack | f pg_stat_monitor | t @@ -123,7 +122,7 @@ order by vector | t wrappers | f xml2 | f -(83 rows) +(82 rows) /* @@ -163,8 +162,8 @@ order by n.nspname, p.proname, pg_catalog.pg_get_function_identity_arguments(p.oid); - extension_name | schema_name | function_name | argument_types | return_type -------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + extension_name | schema_name | function_name | argument_types | return_type +------------------------+--------------------------+--------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- address_standardizer | public | parse_address | text, OUT num text, OUT street text, OUT street2 text, OUT address1 text, OUT city text, OUT state text, OUT zip text, OUT zipplus text, OUT country text | record address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, address text | stdaddr address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, micro text, macro text | stdaddr @@ -1160,45 +1159,6 @@ order by pg_net | net | http_get | url text, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | http_post | url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer | bigint pg_net | net | worker_restart | | boolean - pg_partman | public | apply_cluster | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text | void - pg_partman | public | apply_constraints | p_parent_table text, p_child_table text, p_analyze boolean, p_job_id bigint | void - pg_partman | public | apply_privileges | p_parent_schema text, p_parent_tablename text, p_child_schema text, p_child_tablename text, p_job_id bigint | void - pg_partman | public | autovacuum_off | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean - pg_partman | public | autovacuum_reset | p_parent_schema text, p_parent_tablename text, p_source_schema text, p_source_tablename text | boolean - pg_partman | public | calculate_time_partition_info | p_time_interval interval, p_start_time timestamp with time zone, p_date_trunc_interval text, OUT base_timestamp timestamp with time zone, OUT datetime_string text | record - pg_partman | public | check_automatic_maintenance_value | p_automatic_maintenance text | boolean - pg_partman | public | check_control_type | p_parent_schema text, p_parent_tablename text, p_control text | TABLE(general_type text, exact_type text) - pg_partman | public | check_default | p_exact_count boolean | SETOF check_default_table - pg_partman | public | check_epoch_type | p_type text | boolean - pg_partman | public | check_name_length | p_object_name text, p_suffix text, p_table_partition boolean | text - pg_partman | public | check_partition_type | p_type text | boolean - pg_partman | public | check_subpart_sameconfig | p_parent_table text | TABLE(sub_control text, sub_partition_interval text, sub_partition_type text, sub_premake integer, sub_automatic_maintenance text, sub_template_table text, sub_retention text, sub_retention_schema text, sub_retention_keep_index boolean, sub_retention_keep_table boolean, sub_epoch text, sub_constraint_cols text[], sub_optimize_constraint integer, sub_infinite_time_partitions boolean, sub_jobmon boolean, sub_inherit_privileges boolean, sub_constraint_valid boolean, sub_date_trunc_interval text, sub_ignore_default_data boolean, sub_default_table boolean, sub_maintenance_order integer, sub_retention_keep_publication boolean) - pg_partman | public | check_subpartition_limits | p_parent_table text, p_type text, OUT sub_min text, OUT sub_max text | record - pg_partman | public | create_parent | p_parent_table text, p_control text, p_interval text, p_type text, p_epoch text, p_premake integer, p_start_partition text, p_default_table boolean, p_automatic_maintenance text, p_constraint_cols text[], p_template_table text, p_jobmon boolean, p_date_trunc_interval text | boolean - pg_partman | public | create_partition_id | p_parent_table text, p_partition_ids bigint[], p_start_partition text | boolean - pg_partman | public | create_partition_time | p_parent_table text, p_partition_times timestamp with time zone[], p_start_partition text | boolean - pg_partman | public | create_sub_parent | p_top_parent text, p_control text, p_interval text, p_type text, p_default_table boolean, p_declarative_check text, p_constraint_cols text[], p_premake integer, p_start_partition text, p_epoch text, p_jobmon boolean, p_date_trunc_interval text | boolean - pg_partman | public | drop_constraints | p_parent_table text, p_child_table text, p_debug boolean | void - pg_partman | public | drop_partition_id | p_parent_table text, p_retention bigint, p_keep_table boolean, p_keep_index boolean, p_retention_schema text | integer - pg_partman | public | drop_partition_time | p_parent_table text, p_retention interval, p_keep_table boolean, p_keep_index boolean, p_retention_schema text, p_reference_timestamp timestamp with time zone | integer - pg_partman | public | dump_partitioned_table_definition | p_parent_table text, p_ignore_template_table boolean | text - pg_partman | public | inherit_replica_identity | p_parent_schemaname text, p_parent_tablename text, p_child_tablename text | void - pg_partman | public | inherit_template_properties | p_parent_table text, p_child_schema text, p_child_tablename text | boolean - pg_partman | public | partition_data_id | p_parent_table text, p_batch_count integer, p_batch_interval bigint, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint - pg_partman | public | partition_data_proc | IN p_parent_table text, IN p_loop_count integer, IN p_interval text, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_order text, IN p_source_table text, IN p_ignored_columns text[], IN p_quiet boolean | - pg_partman | public | partition_data_time | p_parent_table text, p_batch_count integer, p_batch_interval interval, p_lock_wait numeric, p_order text, p_analyze boolean, p_source_table text, p_ignored_columns text[] | bigint - pg_partman | public | partition_gap_fill | p_parent_table text | integer - pg_partman | public | reapply_constraints_proc | IN p_parent_table text, IN p_drop_constraints boolean, IN p_apply_constraints boolean, IN p_wait integer, IN p_dryrun boolean | - pg_partman | public | reapply_privileges | p_parent_table text | void - pg_partman | public | run_analyze | IN p_skip_locked boolean, IN p_quiet boolean, IN p_parent_table text | - pg_partman | public | run_maintenance | p_parent_table text, p_analyze boolean, p_jobmon boolean | void - pg_partman | public | run_maintenance_proc | IN p_wait integer, IN p_analyze boolean, IN p_jobmon boolean | - pg_partman | public | show_partition_info | p_child_table text, p_partition_interval text, p_parent_table text, OUT child_start_time timestamp with time zone, OUT child_end_time timestamp with time zone, OUT child_start_id bigint, OUT child_end_id bigint, OUT suffix text | record - pg_partman | public | show_partition_name | p_parent_table text, p_value text, OUT partition_schema text, OUT partition_table text, OUT suffix_timestamp timestamp with time zone, OUT suffix_id bigint, OUT table_exists boolean | record - pg_partman | public | show_partitions | p_parent_table text, p_order text, p_include_default boolean | TABLE(partition_schemaname text, partition_tablename text) - pg_partman | public | stop_sub_partition | p_parent_table text, p_jobmon boolean | boolean - pg_partman | public | undo_partition | p_parent_table text, p_target_table text, p_loop_count integer, p_batch_interval text, p_keep_table boolean, p_lock_wait numeric, p_ignored_columns text[], p_drop_cascade boolean, OUT partitions_undone integer, OUT rows_undone bigint | record - pg_partman | public | undo_partition_proc | IN p_parent_table text, IN p_target_table text, IN p_loop_count integer, IN p_interval text, IN p_keep_table boolean, IN p_lock_wait integer, IN p_lock_wait_tries integer, IN p_wait integer, IN p_ignored_columns text[], IN p_drop_cascade boolean, IN p_quiet boolean | pg_prewarm | public | autoprewarm_dump_now | | bigint pg_prewarm | public | autoprewarm_start_worker | | void pg_prewarm | public | pg_prewarm | regclass, mode text, fork text, first_block bigint, last_block bigint | bigint @@ -5234,7 +5194,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5069 rows) +(5030 rows) /* @@ -5317,61 +5277,6 @@ order by pg_net | net | http_request_queue | method pg_net | net | http_request_queue | timeout_milliseconds pg_net | net | http_request_queue | url - pg_partman | public | part_config | automatic_maintenance - pg_partman | public | part_config | constraint_cols - pg_partman | public | part_config | constraint_valid - pg_partman | public | part_config | control - pg_partman | public | part_config | date_trunc_interval - pg_partman | public | part_config | datetime_string - pg_partman | public | part_config | default_table - pg_partman | public | part_config | epoch - pg_partman | public | part_config | ignore_default_data - pg_partman | public | part_config | infinite_time_partitions - pg_partman | public | part_config | inherit_privileges - pg_partman | public | part_config | jobmon - pg_partman | public | part_config | maintenance_last_run - pg_partman | public | part_config | maintenance_order - pg_partman | public | part_config | optimize_constraint - pg_partman | public | part_config | parent_table - pg_partman | public | part_config | partition_interval - pg_partman | public | part_config | partition_type - pg_partman | public | part_config | premake - pg_partman | public | part_config | retention - pg_partman | public | part_config | retention_keep_index - pg_partman | public | part_config | retention_keep_publication - pg_partman | public | part_config | retention_keep_table - pg_partman | public | part_config | retention_schema - pg_partman | public | part_config | sub_partition_set_full - pg_partman | public | part_config | template_table - pg_partman | public | part_config | undo_in_progress - pg_partman | public | part_config_sub | sub_automatic_maintenance - pg_partman | public | part_config_sub | sub_constraint_cols - pg_partman | public | part_config_sub | sub_constraint_valid - pg_partman | public | part_config_sub | sub_control - pg_partman | public | part_config_sub | sub_date_trunc_interval - pg_partman | public | part_config_sub | sub_default_table - pg_partman | public | part_config_sub | sub_epoch - pg_partman | public | part_config_sub | sub_ignore_default_data - pg_partman | public | part_config_sub | sub_infinite_time_partitions - pg_partman | public | part_config_sub | sub_inherit_privileges - pg_partman | public | part_config_sub | sub_jobmon - pg_partman | public | part_config_sub | sub_maintenance_order - pg_partman | public | part_config_sub | sub_optimize_constraint - pg_partman | public | part_config_sub | sub_parent - pg_partman | public | part_config_sub | sub_partition_interval - pg_partman | public | part_config_sub | sub_partition_type - pg_partman | public | part_config_sub | sub_premake - pg_partman | public | part_config_sub | sub_retention - pg_partman | public | part_config_sub | sub_retention_keep_index - pg_partman | public | part_config_sub | sub_retention_keep_publication - pg_partman | public | part_config_sub | sub_retention_keep_table - pg_partman | public | part_config_sub | sub_retention_schema - pg_partman | public | part_config_sub | sub_template_table - pg_partman | public | table_privs | grantee - pg_partman | public | table_privs | grantor - pg_partman | public | table_privs | privilege_type - pg_partman | public | table_privs | table_name - pg_partman | public | table_privs | table_schema pg_repack | repack | primary_keys | indexrelid pg_repack | repack | primary_keys | indrelid pg_repack | repack | tables | alter_col_storage @@ -6409,5 +6314,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1141 rows) +(1086 rows) diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out index b2886d9fa..aa58dfcb3 100644 --- a/nix/tests/expected/pgmq.out +++ b/nix/tests/expected/pgmq.out @@ -111,17 +111,15 @@ select t (1 row) +/* +-- Disabled until pg_partman goes back into the image select pgmq.create_partitioned( 'my_partitioned_queue', '5 seconds', '10 seconds' ); - create_partitioned --------------------- - -(1 row) - +*/ -- Make sure SQLI enabling characters are blocked select pgmq.create('F--oo'); ERROR: queue name contains invalid characters: $, ;, --, or \' diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql index 4d4045484..c8313f0a9 100644 --- a/nix/tests/sql/pgmq.sql +++ b/nix/tests/sql/pgmq.sql @@ -71,12 +71,15 @@ select select pgmq.drop_queue('my_queue'); +/* +-- Disabled until pg_partman goes back into the image select pgmq.create_partitioned( 'my_partitioned_queue', '5 seconds', '10 seconds' ); +*/ -- Make sure SQLI enabling characters are blocked From be4e27a699f1881cc9dfa16db1ec14315ee9cfe2 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Tue, 29 Oct 2024 08:07:57 -0500 Subject: [PATCH 10/10] re-enable partman (not added back into tests yet) --- ansible/files/postgresql_config/supautils.conf.j2 | 4 ++-- ansible/vars.yml | 2 +- flake.nix | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index b2112cf98..af54510f3 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,9 +1,9 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_partman, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 -# omitted because doesn't require superuser: pgmq +# omitted because doesn't require superuser: pgmq, pg_partman supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' diff --git a/ansible/vars.yml b/ansible/vars.yml index d7b754eac..34a0f46ea 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.004-pgmq-perms-6" + postgres15: "15.8.1.004-pgmq-perms-7" # postgres16: "16.3.1.010-pgmq-perms" # Non Postgres Extensions diff --git a/flake.nix b/flake.nix index c58fe5109..3014f9032 100644 --- a/flake.nix +++ b/flake.nix @@ -151,9 +151,7 @@ ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix - # Requires superuser but also uses dynamic sql so need to be owned by posgtres. - # investigating options - #./nix/ext/pg_partman.nix + ./nix/ext/pg_partman.nix ./nix/ext/pg_stat_monitor.nix ./nix/ext/pg_jsonschema.nix ./nix/ext/pgvector.nix