[DPE-5833] Create new PostgreSQL version 16 charm

.github/workflows/ci.yaml

@@ -45,9 +45,10 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v23.0.5
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@lucas/fix-collect-bases
     with:
-      cache: true
+      cache: false
+      charmcraft-snap-channel: 3.x/stable
 
   integration-test:
     strategy:

.github/workflows/release.yaml

@@ -5,7 +5,7 @@ name: Release to Charmhub
 on:
   push:
     branches:
-      - main
+      - 16/edge
     paths-ignore:
       - 'tests/**'
       - 'docs/**'
@@ -14,6 +14,8 @@ on:
       - '.github/workflows/ci.yaml'
       - '.github/workflows/lib-check.yaml'
       - '.github/workflows/sync_docs.yaml'
+  # for testing purposes:
+  workflow_dispatch:
 
 jobs:
   ci-tests:
@@ -42,16 +44,19 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@lucas/fix-collect-bases
+    with:
+      cache: false
+      charmcraft-snap-channel: 3.x/stable
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v23.0.5
+    uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@lucas/fix-collect-bases
     with:
-      channel: 14/edge
+      channel: 16/edge
       artifact-prefix: ${{ needs.build.outputs.artifact-prefix }}
     secrets:
       charmhub-token: ${{ secrets.CHARMHUB_TOKEN }}

CONTRIBUTING.md

@@ -65,7 +65,7 @@ juju model-config logging-config="<root>=INFO;unit=DEBUG"
 microk8s enable rbac
 
 # Deploy the charm
-juju deploy ./postgresql-k8s_ubuntu-22.04-amd64.charm --trust \
+juju deploy ./postgresql-k8s_ubuntu-24.04-amd64.charm --trust \
     --resource postgresql-image=$(yq '(.resources.postgresql-image.upstream-source)' metadata.yaml)
 ```
 

README.md

@@ -16,7 +16,7 @@ Bootstrap a Kubernetes (e.g. [Multipass-based MicroK8s](https://discourse.charmh
 
 ```shell
 juju add-model postgresql-k8s
-juju deploy postgresql-k8s --channel 14 --trust
+juju deploy postgresql-k8s --channel 16/edge --trust
 ```
 
 **Note:** the `--trust` flag is required because the charm and Patroni need to create some K8s resources.
@@ -62,7 +62,7 @@ Adding a relation is accomplished with `juju relate` (or `juju integrate` for Ju
 
 ```shell
 # Deploy Charmed PostgreSQL cluster with 3 nodes
-juju deploy postgresql-k8s -n 3 --trust --channel 14
+juju deploy postgresql-k8s -n 3 --trust --channel 16/edge
 
 # Deploy the relevant application charms
 juju deploy mycharm
@@ -87,7 +87,7 @@ juju status --relations
 This charm supports legacy interface `pgsql` from the previous [PostgreSQL charm](https://launchpad.net/postgresql-charm):
 
 ```shell
-juju deploy postgresql-k8s --trust --channel 14
+juju deploy postgresql-k8s --trust --channel 16/edge
 juju deploy finos-waltz-k8s --channel edge
 juju relate postgresql-k8s:db finos-waltz-k8s
 ```

charmcraft.yaml

@@ -1,11 +1,9 @@
 type: charm
-bases:
-  - name: ubuntu
-    channel: "22.04"
-    architectures: [amd64]
-  - name: ubuntu
-    channel: "22.04"
-    architectures: [arm64]
+base: [email protected]
+
+platforms:
+  amd64:
+  arm64:
 parts:
   charm:
     build-snaps:

lib/charms/postgresql_k8s/v0/postgresql.py

@@ -36,7 +36,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 39
+LIBPATCH = 40
 
 INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
 
@@ -311,7 +311,7 @@ def delete_user(self, user: str) -> None:
             logger.error(f"Failed to delete user: {e}")
             raise PostgreSQLDeleteUserError()
 
-    def enable_disable_extensions(self, extensions: Dict[str, bool], database: str = None) -> None:
+    def enable_disable_extensions(self, extensions: Dict[str, bool], database: str = None) -> None:  # noqa: C901
         """Enables or disables a PostgreSQL extension.
 
         Args:
@@ -339,10 +339,12 @@ def enable_disable_extensions(self, extensions: Dict[str, bool], database: str =
 
             # Enable/disabled the extension in each database.
             for database in databases:
-                with self._connect_to_database(
-                    database=database
-                ) as connection, connection.cursor() as cursor:
+                connection = self._connect_to_database(database=database)
+                connection.autocommit = True
+                with connection.cursor() as cursor:
                     for extension, enable in ordered_extensions.items():
+                        if extension == "postgis":
+                            cursor.execute("SET pgaudit.log = 'none';")
                         cursor.execute(
                             f"CREATE EXTENSION IF NOT EXISTS {extension};"
                             if enable
@@ -364,6 +366,7 @@ def _generate_database_privileges_statements(
     ) -> List[Composed]:
         """Generates a list of databases privileges statements."""
         statements = []
+        statements.append(sql.SQL("GRANT USAGE, CREATE ON SCHEMA public TO admin;"))
         if relations_accessing_this_database == 1:
             statements.append(
                 sql.SQL(
@@ -418,12 +421,10 @@ def _generate_database_privileges_statements(
                     sql.SQL("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA {} TO {};").format(
                         schema, sql.Identifier(user)
                     ),
-                    sql.SQL("GRANT USAGE ON SCHEMA {} TO {};").format(
-                        schema, sql.Identifier(user)
-                    ),
-                    sql.SQL("GRANT CREATE ON SCHEMA {} TO {};").format(
+                    sql.SQL("GRANT USAGE, CREATE ON SCHEMA {} TO {};").format(
                         schema, sql.Identifier(user)
                     ),
+                    sql.SQL("GRANT USAGE, CREATE ON SCHEMA {} TO admin;").format(schema),
                 ])
         return statements
 

metadata.yaml

@@ -28,7 +28,7 @@ resources:
   postgresql-image:
     type: oci-image
     description: OCI image for PostgreSQL
-    upstream-source: ghcr.io/canonical/charmed-postgresql@sha256:3abdbc00413b065fbfea8c6a3aaad8e137790ebb3e7bf5e1f42e19cbc1861926 # renovate: oci-image tag: 14.13-22.04_edge
+    upstream-source: ghcr.io/canonical/charmed-postgresql:16.4-24.04_edge
 
 peers:
   database-peers:

pyproject.toml

@@ -66,7 +66,7 @@ lightkube = "^0.15.5"
 pytest = "^8.3.3"
 pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.0.5", subdirectory = "python/pytest_plugins/github_secrets"}
 pytest-operator = "^0.38.0"
-pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.0.5", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
+pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", branch = "lucas/fix-collect-bases", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
 pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.0.5", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v23.0.5", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
 # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940

src/charm.py

@@ -1875,6 +1875,12 @@ def update_config(self, is_creating_backup: bool = False) -> bool:
             return True
 
         if not self._patroni.member_started:
+            if self.is_tls_enabled:
+                logger.debug(
+                    "Early exit update_config: patroni not responding but TLS is enabled."
+                )
+                self._handle_postgresql_restart_need()
+                return True
             logger.debug("Early exit update_config: Patroni not started yet")
             return False
 
@@ -1936,8 +1942,14 @@ def _validate_config_options(self) -> None:
 
     def _handle_postgresql_restart_need(self):
         """Handle PostgreSQL restart need based on the TLS configuration and configuration changes."""
-        restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled()
-        self._patroni.reload_patroni_configuration()
+        if self._can_connect_to_postgresql:
+            restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled()
+        else:
+            restart_postgresql = False
+        try:
+            self._patroni.reload_patroni_configuration()
+        except Exception as e:
+            logger.error(f"Reload patroni call failed! error: {e!s}")
         # Wait for some more time than the Patroni's loop_wait default value (10 seconds),
         # which tells how much time Patroni will wait before checking the configuration
         # file again to reload it.

src/dependency.json

@@ -8,7 +8,7 @@
   "rock": {
     "dependencies": {},
     "name": "charmed-postgresql",
-    "upgrade_supported": "^14",
-    "version": "14.11"
+    "upgrade_supported": "^16",
+    "version": "16.4"
   }
 }

src/patroni.py

@@ -498,7 +498,7 @@ def render_patroni_yml_file(
         )
         self._render_file(f"{self._storage_path}/patroni.yml", rendered, 0o644)
 
-    @retry(stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30))
+    @retry(stop=stop_after_attempt(20), wait=wait_exponential(multiplier=1, min=2, max=30))
     def reload_patroni_configuration(self) -> None:
         """Reloads the configuration after it was updated in the file."""
         requests.post(f"{self._patroni_url}/reload", verify=self._verify, auth=self._patroni_auth)

templates/patroni.yml.j2

@@ -155,11 +155,26 @@ postgresql:
   authentication:
     replication:
       password: {{ replication_password }}
+      {%- if enable_tls %}
+      sslrootcert: {{ conf_path }}/ca.pem
+      sslcert: {{ conf_path }}/cert.pem
+      sslkey: {{ conf_path }}/key.pem
+      {%- endif %}
     rewind:
       username: {{ rewind_user }}
       password: {{ rewind_password }}
+      {%- if enable_tls %}
+      sslrootcert: {{ conf_path }}/ca.pem
+      sslcert: {{ conf_path }}/cert.pem
+      sslkey: {{ conf_path }}/key.pem
+      {%- endif %}
     superuser:
       password: {{ superuser_password }}
+      {%- if enable_tls %}
+      sslrootcert: {{ conf_path }}/ca.pem
+      sslcert: {{ conf_path }}/cert.pem
+      sslkey: {{ conf_path }}/key.pem
+      {%- endif %}
 use_endpoints: true
 use_unix_socket: true
 {%- if is_no_sync_member or is_creating_backup %}

terraform/variables.tf

@@ -12,7 +12,7 @@ variable "app_name" {
 variable "channel" {
   description = "Charm channel to use when deploying"
   type        = string
-  default     = "14/stable"
+  default     = "16/stable"
 }
 
 variable "revision" {
@@ -24,7 +24,7 @@ variable "revision" {
 variable "base" {
   description = "Application base"
   type        = string
-  default     = "ubuntu@22.04"
+  default     = "ubuntu@24.04"
 }
 
 variable "units" {

tests/integration/ha_tests/test_async_replication.py

@@ -18,7 +18,6 @@
 from .. import architecture, markers
 from ..helpers import (
     APPLICATION_NAME,
-    CHARM_BASE,
     DATABASE_APP_NAME,
     build_and_deploy,
     get_leader_unit,
@@ -113,12 +112,8 @@ async def test_deploy_async_replication_setup(
     """Build and deploy two PostgreSQL cluster in two separate models to test async replication."""
     await build_and_deploy(ops_test, CLUSTER_SIZE, wait_for_idle=False)
     await build_and_deploy(ops_test, CLUSTER_SIZE, wait_for_idle=False, model=second_model)
-    await ops_test.model.deploy(
-        APPLICATION_NAME, channel="latest/edge", num_units=1, base=CHARM_BASE
-    )
-    await second_model.deploy(
-        APPLICATION_NAME, channel="latest/edge", num_units=1, base=CHARM_BASE
-    )
+    await ops_test.model.deploy(APPLICATION_NAME, channel="latest/edge", num_units=1)
+    await second_model.deploy(APPLICATION_NAME, channel="latest/edge", num_units=1)
 
     async with ops_test.fast_forward(), fast_forward(second_model):
         await gather(

tests/integration/ha_tests/test_replication.py

@@ -9,7 +9,6 @@
 
 from ..helpers import (
     APPLICATION_NAME,
-    CHARM_BASE,
     app_name,
     build_and_deploy,
     db_connect,
@@ -43,7 +42,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             await ops_test.model.deploy(
                 APPLICATION_NAME,
                 application_name=APPLICATION_NAME,
-                base=CHARM_BASE,
                 channel="edge",
             )
 

tests/integration/ha_tests/test_rollback_to_master_label.py

@@ -37,6 +37,7 @@
 
 
 @pytest.mark.group(1)
+@pytest.mark.unstable
 @markers.juju3
 @markers.amd64_only  # TODO: remove after arm64 stable release
 @pytest.mark.abort_on_fail
@@ -46,7 +47,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None:
         ops_test.model.deploy(
             DATABASE_APP_NAME,
             num_units=3,
-            channel="14/stable",
+            channel="16/stable",
             revision=LABEL_REVISION,
             base=CHARM_BASE,
             trust=True,
@@ -55,7 +56,6 @@ async def test_deploy_stable(ops_test: OpsTest) -> None:
             APPLICATION_NAME,
             num_units=1,
             channel="latest/edge",
-            base=CHARM_BASE,
         ),
     )
     logger.info("Wait for applications to become active")
@@ -71,6 +71,7 @@ async def test_deploy_stable(ops_test: OpsTest) -> None:
 
 
 @pytest.mark.group(1)
+@pytest.mark.unstable
 @markers.juju3
 @markers.amd64_only  # TODO: remove after arm64 stable release
 async def test_fail_and_rollback(ops_test, continuous_writes) -> None:

tests/integration/ha_tests/test_self_healing.py

@@ -12,7 +12,6 @@
 from .. import markers
 from ..helpers import (
     APPLICATION_NAME,
-    CHARM_BASE,
     METADATA,
     app_name,
     build_and_deploy,
@@ -69,7 +68,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             await ops_test.model.deploy(
                 APPLICATION_NAME,
                 application_name=APPLICATION_NAME,
-                base=CHARM_BASE,
                 channel="edge",
             )