Updated roles: init_dbserver, and setup_pgd

roles/init_dbserver/tasks/verify_tde_vars.yml

@@ -1,5 +1,5 @@
 ---
-- name: Verify edb_key_wrap_cmd and edb_key_unwrap_cmd
+- name: Verify edb_key_wrap_cmd and edb_key_unwrap_cmd are defined
   ansible.builtin.fail:
     msg: "edb_key_wrap_cmd and edb_key_unwrap_cmd are must for initializing a cluster with TDE"
   when: edb_key_wrap_cmd|length < 1 or edb_key_unwrap_cmd|length < 1

roles/setup_pgd/README.md

@@ -63,6 +63,43 @@ The rest of the variables can be configured and are available in the:
   * [roles/setup_pgd/vars/PG_RedHat.yml](./vars/PG_RedHat.yml)
   * [roles/setup_pgd/vars/EPAS_RedHat.yml](./vars/EPAS_RedHat.yml)
 
+### `PGD Commit Scopes Configuration`
+
+See examples for PGD Commit Scopes available at: [EDB PGD v5](https://www.enterprisedb.com/docs/pgd/5/durability/commit-scopes/).
+
+The code below is part of the [roles/setup_pgd/defaults/main.yml](./defaults/main.yml), and 
+example for configuring two PGD commit scopes is listed below.
+
+The configuration requirements for PGD through the configuration setting variables are:
+  1. The length of the `member_nodes` for a `camo` commit scope is exactly `two`
+  2. No node in `member_nodes` for either commit scope can belong to the other commit scope
+  3. All nodes in `member_nodes` must belong to a `parent_group`
+  4. The `cs_rule` parameter must be: valid, correctly formatted, and adhere to the correct syntax
+
+COMMIT AT MOST ONCE SCOPE - CAMO
+```yaml
+pgd_commit_scopes:
+  - cs_name: 'camo_scope_1'
+    cs_type: 'CAMO'
+    parent_group: 'pgd_cluster'
+    cs_origin_node_group: 'pgd_two_nodes'
+    member_nodes: ['edb-primary1', 'edb-primary2']
+    default_group_cs: true
+    cs_rule: "ALL ( pgd_two_nodes ) ON visible CAMO DEGRADE ON (timeout=500s) TO ASYNC"
+```
+
+GROUP COMMIT SCOPE
+```yaml
+pgd_commit_scopes:
+  - cs_name: 'groupcommit_scope_1'
+    cs_type: 'GROUP_COMMIT'
+    parent_group: 'pgd_cluster'
+    cs_origin_node_group: 'pgd_remaining_nodes'
+    member_nodes: ['edb-primary3']
+    default_group_cs: true
+    cs_rule: "ALL ( pgd_remaining_nodes ) GROUP COMMIT"
+```
+
 Host Variables
 --------------
 
@@ -93,6 +130,8 @@ This role does not have any dependencies, but package repositories should have b
 configured beforehand with the `setup_repo` role. At least one lead primary must exist
 and a database cluster must be initialized on that node. 
 
+
+
 Example Playbook
 ----------------
 
@@ -196,5 +235,6 @@ Author:
 
   * Vibhor Kumar
   * Hannah Stoik
+  * Doug Ortiz
   * EDB Postgres
-  * [email protected] www.enterprisedb.com
+  * [email protected] www.enterprisedb.com

roles/setup_pgd/defaults/main.yml

@@ -34,7 +34,6 @@ pass_dir: "~/.edb"
 pg_local_wal_archive_dir: ""
 edb_audit_directory: ""
 
-
 # unix socket domain directories
 pg_unix_socket_directories:
   - "/var/run/postgresql"
@@ -92,6 +91,29 @@ pgd_cluster_nodes: []
 local_node_dsn: ""
 lead_primary_dsn: ""
 
+# For CAMO Commit Scopes
+max_prepared_transactions: 100
+
+# The following parameters must be configured in the 'pgd_commit_scopes' list
+# the values below are examples and should be adjusted to your requirements
+#  for CAMO
+#  - cs_type: 'CAMO'
+#  - cs_name: 'camo_scope_1'
+#  - parent_group: 'pgd_cluster'
+#  - cs_origin_node_group: 'pgd_two_nodes'
+#  - member_nodes: ['edb-primary1', 'edb-primary2']
+#  - default_group_cs: true
+#  - cs_rule: "ALL ( pgd_two_nodes ) ON visible CAMO DEGRADE ON (timeout=500s) TO ASYNC"
+#  for GROUP_COMMIT
+#  - cs_type: 'GROUP_COMMIT'
+#  - cs_name: 'group_commit_scope_1'
+#  - parent_group: 'pgd_cluster'
+#  - cs_origin_node_group: 'pgd_gc_nodes'
+#  - member_nodes: ['edb-primary3']
+#  - default_group_cs: true
+#  - cs_rule: "ALL ( pgd_remaining_nodes ) GROUP COMMIT"
+pgd_commit_scopes: ""
+
 etc_hosts_lists: []
 
 supported_os:

roles/setup_pgd/tasks/pg_hba_config.yml

@@ -39,7 +39,6 @@
           "databases": pgd_cluster_database
           },
           {
-          "contype": "host",
           "users": pgd_replication_user,
           "source": node.private_ip + "/32",
           "databases": "replication"

roles/setup_pgd/tasks/pgd_camo_commitscope_configuration.yml

@@ -0,0 +1,147 @@
+---
+# CAMO Commit Scope
+- name: Run query to check if PGD parent group exists or not
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT COUNT(1) FROM bdr.node_group WHERE node_group_name  = '{{ item.parent_group }}'"
+        db: "{{ pgd_cluster_database }}"
+
+- name: Store status of existence of PGD parent group
+  ansible.builtin.set_fact:
+    _pgd_camo_parentgroup_exists: "{{ sql_query_output.results[0].query_result[0].count | int }}"
+
+- name: Run query to check if PGD CAMO commit scope origin node group
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT COUNT(1) FROM bdr.node_group WHERE node_group_name  = '{{ item.cs_origin_node_group }}'"
+        db: "{{ pgd_cluster_database }}"
+
+- name: Store status of existence of PGD commit scope origin node group
+  ansible.builtin.set_fact:
+    _pgd_camo_originnodegroup_exists: "{{ sql_query_output.results[0].query_result[0].count | int }}"
+
+- name: Create PGD Cluster commit scope origin node group
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT bdr.create_node_group(node_group_name := '{{ item.cs_origin_node_group }}',
+                                      parent_group_name := '{{ item.parent_group }}',
+                                      join_node_group := false
+                                      )"
+        db: "{{ pgd_cluster_database }}"
+    ignore_query_execution_error: false
+  when:
+    - inventory_hostname == item.member_nodes[0]
+    - _pgd_camo_originnodegroup_exists|int == 0 and _pgd_camo_parentgroup_exists|int == 1
+
+- name: Run query to re-check if PGD CAMO commit scope origin node group
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT COUNT(1) FROM bdr.node_group WHERE node_group_name  = '{{ item.cs_origin_node_group }}'"
+        db: "{{ pgd_cluster_database }}"
+
+- name: Store status of existence of PGD commit scope origin node group
+  ansible.builtin.set_fact:
+    _pgd_camo_originnodegroup_exists: "{{ sql_query_output.results[0].query_result[0].count | int }}"
+
+# FIX ME: Get 'join' to work
+- name: Switch primary node towards PGD node sub group with two nodes when CAMO
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT bdr.switch_node_group(node_group_name := '{{ item.cs_origin_node_group }}',
+                                      wait_for_completion := true
+                                      )"
+        db: "{{ pgd_cluster_database }}"
+    ignore_query_execution_error: false
+  when:
+    - inventory_hostname == node
+    - _pgd_camo_originnodegroup_exists|int == 1 and _pgd_camo_parentgroup_exists|int == 1
+  loop: "{{ item.member_nodes }}"
+  loop_control:
+    loop_var: node
+
+- name: Configure PGD Cluster with a CAMO Commit Scope
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+    apply:
+      throttle: 1
+  vars:
+    pg_query:
+      - query: "SELECT bdr.add_commit_scope(commit_scope_name := '{{ item.cs_name }}',
+                                          origin_node_group := '{{ item.cs_origin_node_group }}',
+                                          rule := '{{ item.cs_rule }}'
+                                         )"
+        db: "{{ pgd_cluster_database }}"
+    ignore_query_execution_error: false
+  when:
+    - inventory_hostname == node
+    - _pgd_camo_originnodegroup_exists|int == 1 and _pgd_camo_parentgroup_exists|int == 1
+  loop: "{{ item.member_nodes }}"
+  loop_control:
+    loop_var: node
+
+- name: Run query to check if PGD CAMO commit scope exists or not
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+  vars:
+    pg_query:
+      - query: "SELECT COUNT(1) FROM bdr.commit_scopes WHERE commit_scope_name  = '{{ item.cs_name }}'"
+        db: "{{ pgd_cluster_database }}"
+
+- name: Store status of existence of PGD CAMO commit scope
+  ansible.builtin.set_fact:
+    _pgd_camoscope_exists: "{{ sql_query_output.results[0].query_result[0].count | int }}"
+
+- name: Configure PGD Cluster default CAMO Commit Scope
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+    apply:
+      throttle: 1
+  vars:
+    pg_query:
+      - query: "SELECT bdr.alter_node_group_option(node_group_name := '{{ item.cs_origin_node_group }}',
+                                      config_key := 'default_commit_scope',
+                                      config_value := '{{ item.cs_name }}'
+                                      )"
+        db: "{{ pgd_cluster_database }}"
+    ignore_query_execution_error: false
+  when:
+    - _pgd_camoscope_exists|int == 1 and _pgd_camo_originnodegroup_exists|int == 1
+
+- name: ALTER max_prepared_transactions
+  ansible.builtin.include_role:
+    name: manage_dbserver
+    tasks_from: execute_sql_scripts
+    apply:
+      throttle: 1
+  vars:
+    pg_query:  
+      - query: "ALTER SYSTEM SET max_prepared_transactions = '{{ max_prepared_transactions }}'"
+        db: "{{ pg_database }}"
+        autocommit: true
+    ignore_query_execution_error: false
+  when:
+    - _pgd_camoscope_exists|int == 1 and _pgd_camo_originnodegroup_exists|int == 1
+
+- name: Restart the pg service
+  ansible.builtin.systemd:
+    name: "{{ pg_service }}"
+    state: restarted
+  become: true

roles/setup_pgd/tasks/pgd_cluster_database.yml

@@ -7,15 +7,15 @@
     pg_databases:
       - name: "{{ pgd_cluster_database }}"
         owner: "{{ pgd_cluster_database_owner }}"
-  when: 
+  when:
     - pgd_cluster_database != pg_database
 
 - name: Ensure BDR extension exists in {{ pgd_cluster_database }}
   ansible.builtin.include_role:
-     name: manage_dbserver
-     tasks_from: manage_extensions
+    name: manage_dbserver
+    tasks_from: manage_extensions
   vars:
     pg_extensions:
-       - name: bdr
-         state: present
-         database: "{{ pgd_cluster_database }}"
+      - name: bdr
+        state: present
+        database: "{{ pgd_cluster_database }}"

roles/setup_pgd/tasks/pgd_commit_scopes.yml

@@ -0,0 +1,23 @@
+---
+# since we know only one is populated based on verify_pgd_settings in init_dbserver
+# set both and only enter the one that is populated
+- name: Filter items based on cs_type
+  set_fact:
+    filtered_items_camo: "{{ pgd_commit_scopes | selectattr('cs_type', 'equalto', 'CAMO') | list }}"
+    filtered_items_group_commit: "{{ pgd_commit_scopes | selectattr('cs_type', 'equalto', 'GROUP_COMMIT') | list }}"
+
+- name: Include tasks to setup CAMO
+  ansible.builtin.include_tasks: pgd_camo_commitscope_configuration.yml
+  loop: "{{ filtered_items_camo }}"
+  loop_control:
+    label: "{{ item.cs_name }}"
+  when:
+    - filtered_items_camo|length > 0
+
+- name: Include tasks to setup GROUP_COMMIT
+  ansible.builtin.include_tasks: pgd_group_commitscope_configuration.yml
+  loop: "{{ filtered_items_group_commit }}"
+  loop_control:
+    label: "{{ item.cs_name }}"
+  when:
+    - filtered_items_group_commit|length > 0