K8SPXC-1475 use aws cli

percona-xtradb-cluster-5.7-backup/Dockerfile

@@ -27,13 +27,17 @@ LABEL org.opencontainers.image.version=${PXC_VERSION}
 # check repository package signature in secure way
 RUN set -ex; \
 	export GNUPGHOME="$(mktemp -d)"; \
-	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F; \
+	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F FB5DB77FD5C118B80511ADA8A6310ACC4672475C; \
 	gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \
 	gpg --batch --export --armor 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F > ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
-	rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY  ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
+	gpg --batch --export --armor FB5DB77FD5C118B80511ADA8A6310ACC4672475C > ${GNUPGHOME}/RPM-GPG-KEY-aws; \
+	rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle ${GNUPGHOME}/RPM-GPG-KEY-aws; \
 	microdnf install -y findutils; \
 	curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \
 	rpmkeys --checksig /tmp/percona-release.rpm; \
+	curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip;  \
+	curl -o /tmp/awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig; \
+	gpg --verify /tmp/awscliv2.sig /tmp/awscliv2.zip; \
 	rpm -i /tmp/percona-release.rpm; \
 	rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \
 	rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY; \
@@ -61,14 +65,18 @@ RUN set -ex; \
 	iputils \
 	procps-ng \
 	util-linux \
+	unzip \
 	procps-ng \
 	qpress \
 	tar \
 	cracklib-dicts \
 	libatomic \
 	libaio; \
 	microdnf clean all; \
-	rm -rf /var/cache/dnf /var/cache/yum
+	unzip -q /tmp/awscliv2.zip -d /tmp/; \
+	/tmp/aws/install; \
+	aws --version; \
+	rm -rf /var/cache/dnf /var/cache/yum /tmp/awscliv2.zip /tmp/awscliv2.sig /tmp/aws
 
 # create mysql user/group before mysql installation
 RUN groupadd -g 1001 mysql; \
@@ -102,15 +110,6 @@ RUN set -ex; \
 RUN install -d -o 1001 -g 0 -m 0775 /backup; \
 	mkdir /usr/lib/pxc
 
-ENV MC_VERSION=RELEASE.2024-07-08T20-59-24Z
-ENV MC_SHA256SUM=e111d2b4bea05aadbffaa3fc8d2436a3fefedf030cd1318568bccb72810024f0
-RUN set -ex; \
-    curl -o /usr/bin/mc -O https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.${MC_VERSION} \
-	&& chmod +x /usr/bin/mc \
-	&& echo "${MC_SHA256SUM} /usr/bin/mc" | sha256sum -c - \
-	&& curl -o /licenses/LICENSE.mc \
-	https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
-
 COPY lib/pxc /usr/lib/pxc
 COPY recovery-*.sh backup.sh get-pxc-state /usr/bin/
 

percona-xtradb-cluster-5.7-backup/backup.sh

@@ -5,6 +5,7 @@ set -o xtrace
 
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
 GARBD_OPTS=""
 SOCAT_OPTS="TCP-LISTEN:4444,reuseaddr,retry=30"
@@ -141,22 +142,12 @@ backup_volume() {
 	echo '[INFO] Backup was finished successfully'
 }
 
-is_object_exist() {
-	local bucket="$1"
-	local object="$2"
-
-	if [[ -n "$(mc -C /tmp/mc ${INSECURE_ARG} --json ls "dest/$bucket/$object" | jq '.status')" ]]; then
-		return 1
-	fi
-}
-
 backup_s3() {
 	S3_BUCKET_PATH=${S3_BUCKET_PATH:-$PXC_SERVICE-$(date +%F-%H-%M)-xtrabackup.stream}
 
 	echo "[INFO] Backup to s3://$S3_BUCKET/$S3_BUCKET_PATH started"
 	{ set +x; } 2>/dev/null
-	echo "+ mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" ACCESS_KEY_ID SECRET_ACCESS_KEY"
-	mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+	s3_add_bucket_dest
 	set -x
 	is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" || xbcloud delete ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME"
 	is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH" || xbcloud delete ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH"
@@ -176,8 +167,8 @@ backup_s3() {
 		| xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
-	mc -C /tmp/mc ${INSECURE_ARG} stat "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
-	md5_size=$(mc -C /tmp/mc ${INSECURE_ARG} stat --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+	aws $AWS_S3_NO_VERIFY_SSL s3 ls s3://$S3_BUCKET/$S3_BUCKET_PATH.md5
+	md5_size=$(aws $AWS_S3_NO_VERIFY_SSL --output json s3api list-objects --bucket "$S3_BUCKET" --prefix "$S3_BUCKET_PATH.md5" --query 'Contents[0].Size' | sed -e 's/.*"size":\([0-9]*\).*/\1/')
 	if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
 		echo '[ERROR] Backup is empty'
 		echo '[ERROR] Backup was finished unsuccessfully'

percona-xtradb-cluster-5.7-backup/recovery-cloud.sh

@@ -6,21 +6,19 @@ set -o xtrace
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/check-version.sh
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
-MC_ARGS='-C /tmp/mc'
 XBCLOUD_ARGS="$XBCLOUD_EXTRA_ARGS"
 
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
 	XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
-	MC_ARGS="${MC_ARGS} --insecure"
 fi
 
 if [ -n "$S3_BUCKET_URL" ]; then
 	{ set +x; } 2>/dev/null
-	echo "+ mc ${MC_ARGS} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY"
-	mc ${MC_ARGS} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+	s3_add_bucket_dest
 	set -x
-	mc ${MC_ARGS} ls "dest/${S3_BUCKET_URL}"
+	aws $AWS_S3_NO_VERIFY_SSL s3 ls "${S3_BUCKET_URL}"
 elif [ -n "${BACKUP_PATH}" ]; then
 	XBCLOUD_ARGS="${XBCLOUD_ARGS} --storage=azure"
 fi

percona-xtradb-cluster-8.0-backup/Dockerfile

@@ -32,13 +32,17 @@ LABEL org.opencontainers.image.version=${PXC_VERSION}
 # check repository package signature in secure way
 RUN set -ex; \
 	export GNUPGHOME="$(mktemp -d)"; \
-	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F; \
+	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F FB5DB77FD5C118B80511ADA8A6310ACC4672475C; \
 	gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \
 	gpg --batch --export --armor 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F > ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
-	rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY  ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
+	gpg --batch --export --armor FB5DB77FD5C118B80511ADA8A6310ACC4672475C > ${GNUPGHOME}/RPM-GPG-KEY-aws; \
+	rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle ${GNUPGHOME}/RPM-GPG-KEY-aws; \
 	microdnf install -y findutils; \
 	curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \
 	rpmkeys --checksig /tmp/percona-release.rpm; \
+	curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip;  \
+	curl -o /tmp/awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig; \
+	gpg --verify /tmp/awscliv2.sig /tmp/awscliv2.zip; \
 	rpm -i /tmp/percona-release.rpm; \
 	rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \
 	rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
@@ -66,9 +70,13 @@ RUN set -ex; \
 	iputils \
 	procps-ng \
 	util-linux \
+	unzip \
 	findutils; \
 	microdnf clean all; \
-	rm -rf /var/cache/dnf /var/cache/yum
+	unzip -q /tmp/awscliv2.zip -d /tmp/; \
+	/tmp/aws/install; \
+	aws --version; \
+	rm -rf /var/cache/dnf /var/cache/yum /tmp/awscliv2.zip /tmp/awscliv2.sig /tmp/aws
 
 # create mysql user/group before mysql installation
 RUN groupadd -g 1001 mysql; \
@@ -90,7 +98,8 @@ RUN set -ex; \
 
 COPY LICENSE /licenses/LICENSE.Dockerfile
 RUN cp /usr/share/doc/percona-xtrabackup-80/LICENSE /licenses/LICENSE.xtrabackup; \
-	cp /usr/share/doc/percona-xtradb-cluster-garbd-3/COPYING /licenses/LICENSE.garbd
+	cp /usr/share/doc/percona-xtradb-cluster-garbd-3/COPYING /licenses/LICENSE.garbd; \
+	curl -o /licenses/LICENSE.aws-cli https://raw.githubusercontent.com/aws/aws-cli/refs/heads/master/LICENSE.txt
 
 RUN set -ex; \
     curl -o /usr/bin/kubectl -LO  \
@@ -103,15 +112,6 @@ RUN set -ex; \
 RUN install -d -o 1001 -g 0 -m 0775 /backup; \
 	mkdir /usr/lib/pxc
 
-ENV MC_VERSION=RELEASE.2024-07-08T20-59-24Z
-ENV MC_SHA256SUM=e111d2b4bea05aadbffaa3fc8d2436a3fefedf030cd1318568bccb72810024f0
-RUN set -ex; \
-    curl -o /usr/bin/mc -O https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.${MC_VERSION} \
-	&& chmod +x /usr/bin/mc \
-	&& echo "${MC_SHA256SUM} /usr/bin/mc" | sha256sum -c - \
-	&& curl -o /licenses/LICENSE.mc \
-	https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
-
 COPY lib/pxc /usr/lib/pxc
 COPY recovery-*.sh run_backup.sh backup.sh post_backup.sh get-pxc-state /usr/bin/
 

percona-xtradb-cluster-8.0-backup/lib/pxc/aws.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o errexit
+
+export AWS_SHARED_CREDENTIALS_FILE='/tmp/aws-credfile'
+export AWS_ENDPOINT_URL="${ENDPOINT:-https://s3.amazonaws.com}"
+export AWS_REGION="${DEFAULT_REGION:-us-west-2}"
+
+if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
+	AWS_S3_NO_VERIFY_SSL='--no-verify-ssl'
+fi
+
+is_object_exist() {
+	local bucket="$1"
+	local object="$2"
+
+	aws $AWS_S3_NO_VERIFY_SSL s3api head-object  --bucket $bucket --key "$object" || NOT_EXIST=true
+	if [[ -z "$NOT_EXIST" ]]; then
+		return 1
+	fi
+}
+
+s3_add_bucket_dest() {
+	{ set +x; } 2>/dev/null
+	aws configure set aws_access_key_id "$ACCESS_KEY_ID"
+	aws configure set aws_secret_access_key "$SECRET_ACCESS_KEY"
+	set -x
+}
+

percona-xtradb-cluster-8.0-backup/lib/pxc/backup.sh

@@ -2,13 +2,15 @@
 
 set -o errexit
 
+LIB_PATH='/usr/lib/pxc'
+. ${LIB_PATH}/aws.sh
+
 SST_INFO_NAME=sst_info
 XBCLOUD_ARGS="--curl-retriable-errors=7 $XBCLOUD_EXTRA_ARGS"
-
 INSECURE_ARG=""
+
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
-	INSECURE_ARG="--insecure"
-	XBCLOUD_ARGS="${INSECURE_ARG} ${XBCLOUD_ARGS}"
+	XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
 fi
 
 S3_BUCKET_PATH=${S3_BUCKET_PATH:-$PXC_SERVICE-$(date +%F-%H-%M)-xtrabackup.stream}
@@ -24,24 +26,8 @@ log() {
 	set -x
 }
 
-is_object_exist() {
-	local bucket="$1"
-	local object="$2"
-
-	if [[ -n "$(mc -C /tmp/mc ${INSECURE_ARG} --json ls "dest/$bucket/$object" | jq '.status')" ]]; then
-		return 1
-	fi
-}
-
-mc_add_bucket_dest() {
-	echo "+ mc -C /tmp/mc ${INSECURE_ARG} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY "
-	{ set +x; } 2>/dev/null
-	mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
-	set -x
-}
-
 clean_backup_s3() {
-	mc_add_bucket_dest
+	s3_add_bucket_dest
 
 	is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" || xbcloud delete ${XBCLOUD_ARGS} --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME"
 	is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH/" || xbcloud delete ${XBCLOUD_ARGS} --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH"

percona-xtradb-cluster-8.0-backup/post_backup.sh

@@ -7,6 +7,7 @@ set -m
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
 . ${LIB_PATH}/backup.sh
+. ${LIB_PATH}/aws.sh
 
 handle_sigterm() {
     log 'INFO' 'Post recv script was finished'
@@ -28,12 +29,12 @@ backup_volume() {
 
 backup_s3() {
         log 'INFO' 'Checking backup in S3'
-        mc -C /tmp/mc stat ${INSECURE_ARG} "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
-        md5_size=$(mc -C /tmp/mc stat ${INSECURE_ARG} --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
-        if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
-                log 'ERROR' 'Backup is empty'
-                log 'ERROR' 'Backup was finished unsuccessfull'
-                exit 1
+        aws $AWS_S3_NO_VERIFY_SSL s3 ls s3://$S3_BUCKET/$S3_BUCKET_PATH.md5
+        md5_size=$(aws $AWS_S3_NO_VERIFY_SSL --output json s3api list-objects --bucket "$S3_BUCKET" --prefix "$S3_BUCKET_PATH.md5" --query 'Contents[0].Size' | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+        if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)) ; then
+            log 'ERROR' 'Backup is empty'
+            log 'ERROR' 'Backup was finished unsuccessfull'
+            exit 1
         fi
 }
 

percona-xtradb-cluster-8.0-backup/recovery-cloud.sh

@@ -6,23 +6,20 @@ set -o xtrace
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/check-version.sh
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
 # temporary fix for PXB-2784
 XBCLOUD_ARGS="--curl-retriable-errors=7 $XBCLOUD_EXTRA_ARGS"
 
-MC_ARGS='-C /tmp/mc'
-
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
 	XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
-	MC_ARGS="${MC_ARGS} --insecure"
 fi
 
 if [ -n "$S3_BUCKET_URL" ]; then
 	{ set +x; } 2>/dev/null
-	echo "+ mc ${MC_ARGS} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY"
-	mc ${MC_ARGS} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+	s3_add_bucket_dest
 	set -x
-	mc ${MC_ARGS} ls "dest/${S3_BUCKET_URL}"
+	aws $AWS_S3_NO_VERIFY_SSL s3 ls "${S3_BUCKET_URL}"
 elif [ -n "${BACKUP_PATH}" ]; then
 	XBCLOUD_ARGS="${XBCLOUD_ARGS} --storage=azure"
 fi

percona-xtradb-cluster-8.0-backup/run_backup.sh

@@ -7,6 +7,7 @@ set -m
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
 . ${LIB_PATH}/backup.sh
+. ${LIB_PATH}/aws.sh
 
 SOCAT_OPTS="TCP-LISTEN:4444,reuseaddr,retry=30"
 
@@ -91,7 +92,7 @@ backup_volume() {
 }
 
 backup_s3() {
-	mc_add_bucket_dest
+	s3_add_bucket_dest
 
 	socat -u "$SOCAT_OPTS" stdio | xbstream -x -C /tmp $XBSTREAM_EXTRA_ARGS &
 	wait $!