Merge branch 'zalando:master' into master

public-repos-backup · Dec 11, 2024 · 3788cac · 3788cac
2 parents 89f8e8a + 517daf7
commit 3788cac
Show file tree

Hide file tree

Showing 17 changed files with 437 additions and 170 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -5,21 +5,22 @@ on:
   push:
     branches:
     - master
+    - trigger
 
 jobs:
   tests:
     runs-on: ubuntu-22.04
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up packages
-      run: sudo apt-get install -y shellcheck
+      run: sudo apt-get install -y shellcheck docker-compose
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.7
-    - name: Install flake8 and docker-compose
-      run: python -m pip install flake8 docker-compose==1.17.1
+        python-version: '3.10'
+    - name: Install flake8
+      run: python -m pip install flake8
     - name: Run shellcheck
       run: find postgres-appliance -name '*.sh' -print0 | xargs -0 shellcheck
     - name: Run flake8

diff --git a/ENVIRONMENT.rst b/ENVIRONMENT.rst
@@ -90,7 +90,9 @@ Environment Configuration Settings
 - **AZURE_TENANT_ID**: (optional) Tenant ID of the Service Principal
 - **CALLBACK_SCRIPT**: the callback script to run on various cluster actions (on start, on stop, on restart, on role change). The script will receive the cluster name, connection string and the current action. See `Patroni <http://patroni.readthedocs.io/en/latest/SETTINGS.html?highlight=callback#postgresql>`__ documentation for details.
 - **LOG_S3_BUCKET**: path to the S3 bucket used for PostgreSQL daily log files (i.e. foobar, without `s3://` prefix). Spilo will add `/spilo/{LOG_BUCKET_SCOPE_PREFIX}{SCOPE}{LOG_BUCKET_SCOPE_SUFFIX}/log/` to that path. Logs are shipped if this variable is set.
-- **LOG_SHIP_SCHEDULE**: cron schedule for shipping compressed logs from ``pg_log`` (if this feature is enabled, '00 02 * * *' by default)
+- **LOG_S3_TAGS**: map of key value pairs to be used for tagging files uploaded to S3. Values should be referencing existing environment variables e.g. ``{"ClusterName": "SCOPE", "Namespace": "POD_NAMESPACE"}``
+- **LOG_SHIP_HOURLY**: if true, log rotation in Postgres is set to 1h incl. foreign tables for every hour (schedule `1 */1 * * *`)
+- **LOG_SHIP_SCHEDULE**: cron schedule for shipping compressed logs from ``pg_log`` (``1 0 * * *`` by default)
 - **LOG_ENV_DIR**: directory to store environment variables necessary for log shipping
 - **LOG_TMPDIR**: directory to store temporary compressed daily log files. PGROOT/../tmp by default.
 - **LOG_S3_ENDPOINT**: (optional) S3 Endpoint to use with Boto3
@@ -99,7 +101,9 @@ Environment Configuration Settings
 - **LOG_GROUP_BY_DATE**: (optional) enable grouping log by date. Default is False - group the log files based on the instance ID.
 - **DCS_ENABLE_KUBERNETES_API**: a non-empty value forces Patroni to use Kubernetes as a DCS. Default is empty.
 - **KUBERNETES_USE_CONFIGMAPS**: a non-empty value makes Patroni store its metadata in ConfigMaps instead of Endpoints when running on Kubernetes. Default is empty.
-- **KUBERNETES_ROLE_LABEL**: name of the label containing Postgres role when running on Kubernetens. Default is 'spilo-role'.
+- **KUBERNETES_ROLE_LABEL**: name of the label containing Postgres role when running on Kubernetes. Default is 'spilo-role'.
+- **KUBERNETES_LEADER_LABEL_VALUE**: value of the pod label if Postgres role is primary when running on Kubernetes. Default is 'master'.
+- **KUBERNETES_STANDBY_LEADER_LABEL_VALUE**: value of the pod label if Postgres role is standby_leader when running on Kubernetes. Default is 'master'.
 - **KUBERNETES_SCOPE_LABEL**: name of the label containing cluster name. Default is 'version'.
 - **KUBERNETES_LABELS**: a JSON describing names and values of other labels used by Patroni on Kubernetes to locate its metadata. Default is '{"application": "spilo"}'.
 - **INITDB_LOCALE**: database cluster's default UTF-8 locale (en_US by default)

diff --git a/delivery.yaml b/delivery.yaml
@@ -1,12 +1,77 @@
 version: "2017-09-20"
 allow_concurrent_steps: true
 
+build_env: &BUILD_ENV
+  BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04
+  PGVERSION: 17
+  MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid
+
 pipeline:
-- id: push-spilo-ecr
+- id: push-spilo-cdp-pr
+  env:
+    <<: *BUILD_ENV
+  type: script
+  requires_human_approval: true
+  when:
+    - event: pull_request
+  timeout: 10h
+  vm_config:
+    type: linux
+    size: large
+  commands:
+  - desc: Tag and push spilo-cdp image without promotion
+    cmd: |
+      cd postgres-appliance
+
+      PATRONIVERSION=$(sed -n 's/^ENV PATRONIVERSION=\([1-9][0-9]*\.[0-9]*\).*$/\1/p' Dockerfile)
+      ECR_TEST_IMAGE="$MULTI_ARCH_REGISTRY/spilo-cdp-pr$CDP_PULL_REQUEST_NUMBER-$PGVERSION:$PATRONIVERSION-p$CDP_PULL_REQUEST_COUNTER"
+
+      # create a Buildkit builder with CDP specific configuration
+      docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use
+
+      # single platform build for PR images!
+      docker buildx build --platform "linux/amd64" \
+                          --build-arg PGVERSION="$PGVERSION" \
+                          --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
+                          -t "$ECR_TEST_IMAGE" \
+                          --push .
+
+- id: push-spilo-cdp
+  env:
+    <<: *BUILD_ENV
+  type: script
+  when:
+    - event: push
+      branch: trigger
+  timeout: 10h
+  vm_config:
+    type: linux
+    size: extra_large
+  commands:
+  - desc: Tag and push spilo-cdp image
+    cmd: |
+      cd postgres-appliance
+
+      PATRONIVERSION=$(sed -n 's/^ENV PATRONIVERSION=\([1-9][0-9]*\.[0-9]*\).*$/\1/p' Dockerfile)
+      ECR_TEST_IMAGE="$MULTI_ARCH_REGISTRY/spilo-cdp-$PGVERSION:trigger-$PATRONIVERSION-p$CDP_TARGET_BRANCH_COUNTER"
+
+      # create a Buildkit builder with CDP specific configuration
+      docker buildx create --config /etc/cdp-buildkitd.toml --driver-opt network=host --bootstrap --use
+
+      docker buildx build --platform "linux/amd64,linux/arm64" \
+                          --build-arg PGVERSION="$PGVERSION" \
+                          --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
+                          -t "$ECR_TEST_IMAGE" \
+                          --push .
+      cdp-promote-image "$ECR_TEST_IMAGE"
+
+- id: push-spilo-cdp-master
   env:
-    BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04
-    PGVERSION: 16
-    MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid
+    <<: *BUILD_ENV
   type: script
   requires_human_approval: true
   when:
@@ -30,12 +95,13 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
       cdp-promote-image "$ECR_TEST_IMAGE"
 
 - id: tag-spilo
-  depends_on: [push-spilo-ecr]
   type: script
   requires_human_approval: true
   when:

diff --git a/kubernetes/spilo_kubernetes.yaml b/kubernetes/spilo_kubernetes.yaml
@@ -79,6 +79,10 @@ spec:
           value: spilo-cluster
         - name: KUBERNETES_ROLE_LABEL
           value: role
+        - name: KUBERNETES_LEADER_LABEL_VALUE
+          value: master
+        - name: KUBERNETES_STANDBY_LEADER_LABEL_VALUE
+          value: master
         - name: SPILO_CONFIGURATION
           value: | ## https://github.com/zalando/patroni#yaml-configuration
             bootstrap:

diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile
@@ -1,6 +1,6 @@
 ARG BASE_IMAGE=ubuntu:22.04
-ARG PGVERSION=16
-ARG TIMESCALEDB="2.11.2 2.14.2"
+ARG PGVERSION=17
+ARG TIMESCALEDB="2.15.3 2.17.2"
 ARG DEMO=false
 ARG COMPRESS=false
 ARG ADDITIONAL_LOCALES=
@@ -48,21 +48,21 @@ ARG TIMESCALEDB
 ARG TIMESCALEDB_APACHE_ONLY=true
 ARG TIMESCALEDB_TOOLKIT=true
 ARG COMPRESS
-ARG PGOLDVERSIONS="12 13 14 15"
+ARG PGOLDVERSIONS="13 14 15 16"
 ARG WITH_PERL=false
 
 ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION"
 
 # Install PostgreSQL, extensions and contribs
-ENV POSTGIS_VERSION=3.4 \
-    BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \
-    PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \
-    PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \
-    SET_USER=REL4_0_1 \
-    PLPROFILER=REL4_2_4 \
-    PG_PROFILE=4.6 \
+ENV POSTGIS_VERSION=3.5 \
+    BG_MON_COMMIT=7f5887218790b263fe3f42f85f4ddc9c8400b154 \
+    PG_AUTH_MON_COMMIT=fe099eef7662cbc85b0b79191f47f52f1e96b779 \
+    PG_MON_COMMIT=ead1de70794ed62ca1e34d4022f6165ff36e9a91 \
+    SET_USER=REL4_1_0 \
+    PLPROFILER=REL4_2_5 \
+    PG_PROFILE=4.7 \
     PAM_OAUTH2=v1.0.1 \
-    PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925
+    PG_PERMISSIONS_COMMIT=f4b7c18676fa64236a1c8e28d34a35764e4a70e2
 
 WORKDIR /builddeps
 RUN bash base.sh
@@ -73,7 +73,7 @@ COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/
 COPY build_scripts/patroni_wale.sh build_scripts/compress_build.sh /builddeps/
 
 # Install patroni and wal-e
-ENV PATRONIVERSION=3.3.2
+ENV PATRONIVERSION=4.0.4
 ENV WALE_VERSION=1.1.1
 
 WORKDIR /
@@ -92,7 +92,6 @@ FROM builder-${COMPRESS}
 LABEL maintainer="Team ACID @ Zalando <[email protected]>"
 
 ARG PGVERSION
-ARG TIMESCALEDB
 ARG DEMO
 ARG COMPRESS
 
@@ -102,7 +101,6 @@ ENV LC_ALL=en_US.utf-8 \
     PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \
     PGHOME=/home/postgres \
     RW_DIR=/run \
-    TIMESCALEDB=$TIMESCALEDB \
     DEMO=$DEMO
 
 ENV WALE_ENV_DIR=$RW_DIR/etc/wal-e.d/env \

diff --git a/postgres-appliance/bootstrap/maybe_pg_upgrade.py b/postgres-appliance/bootstrap/maybe_pg_upgrade.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import datetime
+import glob
 import logging
 import os
 import subprocess
@@ -8,29 +8,14 @@
 logger = logging.getLogger(__name__)
 
 
-def tail_postgres_log(weekday):
+def tail_postgres_logs():
     logdir = os.environ.get('PGLOG', '/home/postgres/pgdata/pgroot/pg_log')
-    logfile = os.path.join(logdir, 'postgresql-{0}.csv'.format(weekday))
+    csv_files = glob.glob(os.path.join(logdir, '*.csv'))
+    # Find the last modified CSV file
+    logfile = max(csv_files, key=os.path.getmtime)
     return subprocess.check_output(['tail', '-n5', logfile]).decode('utf-8')
 
 
-def tail_postgres_logs():
-    weekday = datetime.datetime.today().isoweekday()
-    try:
-        ret = tail_postgres_log(weekday)
-    except Exception:
-        ret = ''
-    if not ret:
-        weekday += 6
-        if weekday > 7:
-            weekday %= 7
-        try:
-            ret = tail_postgres_log(weekday)  # maybe log just switched? try yesterday
-        except Exception:
-            ret = ''
-    return ret
-
-
 def wait_end_of_recovery(postgresql):
     from patroni.utils import polling_loop
 

diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh
@@ -56,7 +56,6 @@ curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.
 curl -sL "https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz" | tar xz
 curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz
 git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git
-git clone https://github.com/timescale/timescaledb.git
 
 apt-get install -y \
     postgresql-common \
@@ -81,8 +80,8 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
                 "postgresql-${version}-first-last-agg"
                 "postgresql-${version}-hll"
                 "postgresql-${version}-hypopg"
-                "postgresql-${version}-plproxy"
                 "postgresql-${version}-partman"
+                "postgresql-${version}-plproxy"
                 "postgresql-${version}-pgaudit"
                 "postgresql-${version}-pldebugger"
                 "postgresql-${version}-pglogical"
@@ -105,6 +104,12 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
 
     fi
 
+    if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then
+        EXTRAS+=("timescaledb-2-oss-postgresql-${version}")
+    else
+        EXTRAS+=("timescaledb-2-postgresql-${version}")
+    fi
+
     # Install PostgreSQL binaries, contrib, plproxy and multiple pl's
     apt-get install --allow-downgrades -y \
         "postgresql-${version}-cron" \
@@ -116,39 +121,28 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
         "postgresql-${version}-pg-stat-kcache" \
         "${EXTRAS[@]}"
 
-    # Install 3rd party stuff
+    # Clean up timescaledb versions except the highest compatible version
+    exclude_patterns=()
+    exclude_patterns_tsl=()
+    for ts_version in ${TIMESCALEDB}; do
+        exclude_patterns+=(! -name timescaledb-"${ts_version}".so)
+        exclude_patterns_tsl+=(! -name timescaledb-tsl-"${ts_version}".so)
+    done
+    find /usr/lib/postgresql/"${version}"/lib/ -name 'timescaledb-2.*.so' "${exclude_patterns[@]}" -delete;
 
-    # use subshell to avoid having to cd back (SC2103)
-    (
-        cd timescaledb
-        for v in $TIMESCALEDB; do
-            git checkout "$v"
-            sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt
-            if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \
-                    -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \
-                    -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then
-                make -C build install
-                strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so
-            fi
-            git reset --hard
-            git clean -f -d
-        done
-    )
+    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ]; then
+        find /usr/lib/postgresql/"${version}"/lib/ -name 'timescaledb-tsl-2.*.so' "${exclude_patterns_tsl[@]}" -delete;
+    fi
 
-    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then
-        __versionCodename=$(sed </etc/os-release -ne 's/^VERSION_CODENAME=//p')
-        echo "deb [signed-by=/usr/share/keyrings/timescale_E7391C94080429FF.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${__versionCodename} main" | tee /etc/apt/sources.list.d/timescaledb.list
-        curl -L https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor > /usr/share/keyrings/timescale_E7391C94080429FF.gpg
+    # Install 3rd party stuff
 
+    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then
         apt-get update
         if [ "$(apt-cache search --names-only "^timescaledb-toolkit-postgresql-${version}$" | wc -l)" -eq 1 ]; then
             apt-get install "timescaledb-toolkit-postgresql-$version"
         else
             echo "Skipping timescaledb-toolkit-postgresql-$version as it's not found in the repository"
         fi
-
-        rm /etc/apt/sources.list.d/timescaledb.list
-        rm /usr/share/keyrings/timescale_E7391C94080429FF.gpg
     fi
 
     EXTRA_EXTENSIONS=()

diff --git a/postgres-appliance/build_scripts/prepare.sh b/postgres-appliance/build_scripts/prepare.sh
@@ -40,6 +40,10 @@ for t in deb deb-src; do
 done
 curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg
 
+# add TimescaleDB repository
+echo "deb [signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${DISTRIB_CODENAME} main" | tee /etc/apt/sources.list.d/timescaledb.list
+curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor | tee /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg > /dev/null
+
 # Clean up
 apt-get purge -y libcap2-bin
 apt-get autoremove -y

diff --git a/postgres-appliance/launch.sh b/postgres-appliance/launch.sh
@@ -34,8 +34,16 @@ fi
 ## Ensure all logfiles exist, most appliances will have
 ## a foreign data wrapper pointing to these files
 for i in $(seq 0 7); do
-    if [ ! -f "${PGLOG}/postgresql-$i.csv" ]; then
-        touch "${PGLOG}/postgresql-$i.csv"
+    if [ "$LOG_SHIP_HOURLY" != "true" ]; then
+        if [ ! -f "${PGLOG}/postgresql-${i}.csv" ]; then
+            touch "${PGLOG}/postgresql-${i}.csv"
+        fi
+    else
+        for h in $(seq -w 0 23); do
+            if [ ! -f "${PGLOG}/postgresql-${i}-${h}.csv" ]; then
+                touch "${PGLOG}/postgresql-${i}-${h}.csv"
+            fi
+        done
     fi
 done
 chown -R postgres: "$PGROOT" "$RW_DIR/certs"

diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py
@@ -204,7 +204,7 @@ def pg_upgrade(self, check=False):
     def prepare_new_pgdata(self, version):
         from spilo_commons import append_extensions
 
-        locale = self.query('SHOW lc_collate')[0][0]
+        locale = self.query("SELECT datcollate FROM pg_database WHERE datname='template1';")[0][0]
         encoding = self.query('SHOW server_encoding')[0][0]
         initdb_config = [{'locale': locale}, {'encoding': encoding}]
         if self.query("SELECT current_setting('data_checksums')::bool")[0][0]: