From 411068b1b68e4d624a32ac25709e041f178b4fcc Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 11:30:42 +0200
Subject: [PATCH 01/10] use latest backup image to run bats test

---
 Makefile                                  | 10 ++--------
 test/bats/1-deploy.bats                   |  2 ++
 test/bats/2-deploy-with-more-options.bats |  2 ++
 test/bats/3-deploy-with-webhook.bats      |  2 ++
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index bdd8d4e37..e4ef73b29 100644
--- a/Makefile
+++ b/Makefile
@@ -377,22 +377,16 @@ kind-clean: ## Delete kind cluster
 	@echo "+ $@"
 	kind delete cluster --name $(KIND_CLUSTER_NAME)
 
-.PHONY: kind-load-backup
-kind-load-backup:
-	@echo "+ $@"
-	make -C
-	kind delete cluster --name $(KIND_CLUSTER_NAME)
-
 .PHONY: bats-tests
 IMAGE_NAME := quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY):$(GITCOMMIT)-amd64
 BUILD_PRESENT := $(shell docker images |grep -q ${IMAGE_NAME})
 ifndef BUILD_PRESENT
-bats-tests: container-runtime-build-amd64 ## Run bats tests
+bats-tests: backup-kind-load container-runtime-build-amd64 ## Run bats tests
 	@echo "+ $@"
 	kind load docker-image ${IMAGE_NAME} --name $(KIND_CLUSTER_NAME)
 	OPERATOR_IMAGE="${IMAGE_NAME}" TERM=xterm bats -T -p test/bats
 else
-bats-tests: ## Run bats tests
+bats-tests: backup-kind-load
 	@echo "+ $@"
 	OPERATOR_IMAGE="${IMAGE_NAME}" TERM=xterm bats -T -p test/bats
 endif
diff --git a/test/bats/1-deploy.bats b/test/bats/1-deploy.bats
index 2155f1ac7..3625957b2 100644
--- a/test/bats/1-deploy.bats
+++ b/test/bats/1-deploy.bats
@@ -35,6 +35,7 @@ diag() {
     --set jenkins.latestPlugins=true \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=false \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     --set jenkins.seedJobs[0].id=seed-job \
     --set jenkins.seedJobs[0].targets="cicd/jobs/*.jenkins" \
     --set jenkins.seedJobs[0].description="jobs-from-operator-repo" \
@@ -152,6 +153,7 @@ diag() {
     --set jenkins.latestPlugins=true \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=false \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     chart/jenkins-operator
   assert_success
   assert ${HELM} status default
diff --git a/test/bats/2-deploy-with-more-options.bats b/test/bats/2-deploy-with-more-options.bats
index d4ce27d35..1fbb65dd7 100644
--- a/test/bats/2-deploy-with-more-options.bats
+++ b/test/bats/2-deploy-with-more-options.bats
@@ -31,6 +31,7 @@ setup() {
     --set jenkins.nodeSelector.batstest=yep \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=false \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     jenkins-operator/jenkins-operator --version=$(cat VERSION.txt | sed 's/v//')
   assert_success
   assert ${HELM} status options
@@ -103,6 +104,7 @@ setup() {
     --set jenkins.nodeSelector.batstest=yep \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=false \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     chart/jenkins-operator
   assert_success
   assert ${HELM} status options
diff --git a/test/bats/3-deploy-with-webhook.bats b/test/bats/3-deploy-with-webhook.bats
index c832f82fe..858229ed8 100644
--- a/test/bats/3-deploy-with-webhook.bats
+++ b/test/bats/3-deploy-with-webhook.bats
@@ -31,6 +31,7 @@ setup() {
     --set jenkins.latestPlugins=true \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=true \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     --set webhook.enabled=true \
     jenkins-operator/jenkins-operator --version=$(cat VERSION.txt | sed 's/v//')
   assert_success
@@ -88,6 +89,7 @@ setup() {
     --set jenkins.latestPlugins=true \
     --set jenkins.image="jenkins/jenkins:2.452.1-lts" \
     --set jenkins.backup.makeBackupBeforePodDeletion=true \
+    --set jenkins.backup.image=quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test \
     --set webhook.enabled=true \
     chart/jenkins-operator
   assert_success

From 90b00bd6e8fe36bf61d374c9d8c5804206a7b9aa Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 17:17:50 +0200
Subject: [PATCH 02/10] feat(backup): log everything stdout/err, implement
 lockfile for both backup/restore

---
 backup/pvc/VERSION.txt       |  2 +-
 backup/pvc/bin/backup.sh     | 55 ++++++++++++++++++++++++++----------
 backup/pvc/bin/get-latest.sh |  8 ++----
 backup/pvc/bin/restore.sh    | 42 ++++++++++++++++++---------
 backup/pvc/bin/run.sh        | 11 ++++----
 backup/pvc/bin/utils.sh      | 13 +++++++++
 6 files changed, 91 insertions(+), 40 deletions(-)
 create mode 100644 backup/pvc/bin/utils.sh

diff --git a/backup/pvc/VERSION.txt b/backup/pvc/VERSION.txt
index 268b0334e..fb7a04cff 100644
--- a/backup/pvc/VERSION.txt
+++ b/backup/pvc/VERSION.txt
@@ -1 +1 @@
-v0.3.0
+v0.4.0
diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index eaa648f98..6aa9dd5a7 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -1,39 +1,64 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source "$(dirname "$0")/utils.sh"
+set -x
 
-[[ ! $# -eq 1 ]] && echo "Usage: $0 backup_number" && exit 1;
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
-[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
+[[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 BACKUP_NUMBER" && exit 1
+[[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1
+[[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1
+RETRY_COUNT=${RETRY_COUNT:-3}
+RETRY_INTERVAL=${RETRY_INTERVAL:-60}
+BACKUP_NUMBER=$1
+TRAP_FILE="${BACKUP_DIR}/_backup_${BACKUP_NUMBER}_is_running"
 
+# --> Check if another backup process is running (operator restart/crash)
+for ((i=0; i<RETRY_COUNT; i++)); do
+    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "Backup: no other backup process are running" && break
+    _log "INFO" "Backup is already running. Waiting for ${RETRY_INTERVAL} seconds..."
+    sleep "${RETRY_INTERVAL}"
+done
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Backup is stil running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+# --< Done
+
+_log "INFO" "Running backup ${BACKUP_NUMBER}"
+touch "${TRAP_FILE}"
 # create temp dir on the same filesystem with a BACKUP_DIR to be able use atomic mv enstead of copy
-BACKUP_TMP_DIR=$(mktemp -d --tmpdir=${BACKUP_DIR})
-trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT SIGINT SIGTERM
+BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
+
+_clean(){
+    test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"
+    test -f "${TRAP_FILE}" && rm -f "${TRAP_FILE}"
+}
+
+_trap(){
+    _clean
+    _log "ERROR" "Backup: something wrong happened, check the logs"
+}
 
-backup_number=$1
-echo "Running backup"
+trap '_trap' SIGQUIT SIGINT SIGTERM
 
 # config.xml in a job directory is a config file that shouldn't be backed up
 # config.xml in child directories is state that should. For example-
 # branches/myorg/branches/myrepo/branches/master/config.xml should be retained while
 # branches/myorg/config.xml should not
-tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" \
+tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd" \
     --exclude jobs/*/workspace* \
     --no-wildcards-match-slash --anchored \
     --ignore-failed-read \
     --exclude jobs/*/config.xml -c jobs || ret=$?
 
 if [[ "$ret" -eq 0 ]]; then
-  echo "Backup was completed without warnings"
+  _log "INFO" "Backup ${BACKUP_NUMBER} was completed without warnings"
 elif [[ "$ret" -eq 1 ]]; then
-  echo "Backup was completed with some warnings"
+  _log "INFO" "Backup ${BACKUP_NUMBER} was completed with some warnings"
 fi
 
-# atomically create a backup file
-mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd"
+mv "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd" "${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd"
 
-rm -rf "${BACKUP_TMP_DIR}"
-[[ ! -s ${BACKUP_DIR}/${backup_number}.tar.zstd ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.zstd' is empty" && exit 1;
+_log "INFO" "Cleaning ${BACKUP_TMP_DIR} and trap file ${TRAP_FILE}"
+_clean
+[[ ! -s ${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd ]] && _log "ERROR" "Backup file '${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd' is empty" && exit 1
 
-echo Done
+_log "INFO" "Backup ${BACKUP_NUMBER} done"
 exit 0
diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh
index da8055f4e..f3e016957 100644
--- a/backup/pvc/bin/get-latest.sh
+++ b/backup/pvc/bin/get-latest.sh
@@ -1,30 +1,26 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source "$(dirname "$0")/utils.sh"
 
 is_backup_not_exist() {
     local backup_dir="$1"
     # Save the current value of 'set -e'
     local previous_e
     previous_e=$(set +e; :; echo $?)
-
     # Temporarily turn off 'set -e'
     set +e
-
     # Run ls command to check if any files matching the pattern exist
     ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
-
     # Store the exit status of the ls command
     local ls_exit_status=$?
-
     # Restore the previous value of 'set -e'
     [ "$previous_e" = "0" ] && set -e
-
     # Return true if ls command succeeded (no files found), otherwise return false
     [ $ls_exit_status -ne 0 ]
 }
 
-[[ -z "${BACKUP_DIR}" ]] && { echo "Required 'BACKUP_DIR' env not set"; exit 1; }
+[[ -z "${BACKUP_DIR}" ]] && { _log "ERROR" "Required 'BACKUP_DIR' env not set"; exit 1; }
 
 # Check if we have any backup
 if is_backup_not_exist "${BACKUP_DIR}"; then
diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh
index 994208d33..10984cd1b 100644
--- a/backup/pvc/bin/restore.sh
+++ b/backup/pvc/bin/restore.sh
@@ -1,29 +1,45 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source "$(dirname "$0")/utils.sh"
 
-[[ ! $# -eq 1 ]] && echo "Usage: $0 backup_number" && exit 1
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
-[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
+[[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 <backup number>" && exit 1
+[[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1
+[[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1
+BACKUP_NUMBER=$1
+RETRY_COUNT=${RETRY_COUNT:-3}
+RETRY_INTERVAL=${RETRY_INTERVAL:-60}
 
-backup_number=$1
-backup_file="${BACKUP_DIR}/${backup_number}"
-echo "Running restore backup with backup number #${backup_number}"
+# --> Check if another restore process is running (operator restart/crash)
+TRAP_FILE="${BACKUP_DIR}/_restore_${BACKUP_NUMBER}_is_running"
+trap "rm -f ${TRAP_FILE}" SIGINT SIGTERM
 
-if [[ -f "$backup_file.tar.gz" ]]; then
-    echo "Old format tar.gz found, restoring it"
+for ((i=0; i<RETRY_COUNT; i++)); do
+    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "Restore: No other process are running, restoring" && break
+    _log "INFO" "Restore is already running. Waiting for ${RETRY_INTERVAL} seconds..."
+    sleep "${RETRY_INTERVAL}"
+done
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Restore is stil running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+# --< Done
+
+_log "INFO" "Running restore backup with backup number #${BACKUP_NUMBER}"
+touch "${TRAP_FILE}"
+BACKUP_FILE="${BACKUP_DIR}/${BACKUP_NUMBER}"
+
+if [[ -f "$BACKUP_FILE.tar.gz" ]]; then
+    _log "INFO" "Restore: ld format tar.gz found, restoring it"
     OPTS=""
     EXT="tar.gz"
-elif [[ -f "$backup_file.tar.zstd" ]]; then
-    echo "Backup file found, proceeding"
+elif [[ -f "$BACKUP_FILE.tar.zstd" ]]; then
+    _log "INFO" "Restore: Backup file found, proceeding"
     OPTS="--zstd"
     EXT="tar.zstd"
 else
-  echo "ERR: Backup file not found: $backup_file"
+  _log "ERROR" "Restore: Backup file not found: $BACKUP_FILE"
   exit 1
 fi
 
-tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${backup_number}.${EXT}"
+tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${BACKUP_NUMBER}.${EXT}"
 
-echo Done
+_log "INFO" "Restore: ${BACKUP_NUMBER} Done"
 exit 0
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 4b6052019..5486dbaaa 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+source "$(dirname "$0")/utils.sh"
 
 # Use 60 as default in case BACKUP_CLEANUP_INTERVAL did not set
 BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60}
@@ -8,7 +9,7 @@ BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60}
 # Ensure required environment variables are set
 check_env_var() {
     if [[ -z "${!1}" ]]; then
-        echo "Required '$1' environment variable is not set"
+        _log "ERROR" "Required '$1' environment variable is not set"
         exit 1
     fi
 }
@@ -41,7 +42,7 @@ find_exceeding_backups() {
     local backup_count="$2"
     # Check if we have any backup
     if is_backup_not_exist "${backup_dir}"; then
-        echo "backups not found in ${backup_dir}" >&2
+        _log "ERROR" "backups not found in ${backup_dir}"
         return
     fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
@@ -51,9 +52,9 @@ check_env_var "BACKUP_DIR"
 check_env_var "JENKINS_HOME"
 
 if [[ -z "${BACKUP_COUNT}" ]]; then
-    echo "ATTENTION! No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
+    _log "WARNING" "No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
 else
-    echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
+    _log "INFO" "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
 fi
 
 while true;
@@ -62,7 +63,7 @@ do
     if [[ -n "${BACKUP_COUNT}" ]]; then
         exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
         if [[ -n "$exceeding_backups" ]]; then
-            echo "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
+            _log "INFO" "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
             echo "$exceeding_backups" | while read -r file; do
                 rm "${BACKUP_DIR}/${file}"
             done
diff --git a/backup/pvc/bin/utils.sh b/backup/pvc/bin/utils.sh
new file mode 100644
index 000000000..5d9342385
--- /dev/null
+++ b/backup/pvc/bin/utils.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Common utils
+
+_log() {
+    local level="$1"
+    local message="$2"
+    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
+    if [[ "$level" =~ ^(ERROR|ERR|error|err)$ ]]; then
+        echo "${timestamp} [${level}] ${message}" > /proc/1/fd/2
+    else
+        echo "${timestamp} [${level}] ${message}" > /proc/1/fd/1
+    fi
+}

From 9cd9d84c0a85a3c1f6637e763e442825a77d83d6 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 17:22:50 +0200
Subject: [PATCH 03/10] fix typo

---
 backup/pvc/bin/backup.sh  | 2 +-
 backup/pvc/bin/restore.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index 6aa9dd5a7..c8d4d3821 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -18,7 +18,7 @@ for ((i=0; i<RETRY_COUNT; i++)); do
     _log "INFO" "Backup is already running. Waiting for ${RETRY_INTERVAL} seconds..."
     sleep "${RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Backup is stil running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Backup is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
 _log "INFO" "Running backup ${BACKUP_NUMBER}"
diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh
index 10984cd1b..48d2b4862 100644
--- a/backup/pvc/bin/restore.sh
+++ b/backup/pvc/bin/restore.sh
@@ -19,7 +19,7 @@ for ((i=0; i<RETRY_COUNT; i++)); do
     _log "INFO" "Restore is already running. Waiting for ${RETRY_INTERVAL} seconds..."
     sleep "${RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Restore is stil running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Restore is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
 _log "INFO" "Running restore backup with backup number #${BACKUP_NUMBER}"

From f0a7db391104011bdb887dbc4dca21c47519f27b Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 17:24:56 +0200
Subject: [PATCH 04/10] update default backup image

---
 chart/jenkins-operator/README.md   | 2 +-
 chart/jenkins-operator/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/chart/jenkins-operator/README.md b/chart/jenkins-operator/README.md
index 2c2b2fc11..729cbe644 100644
--- a/chart/jenkins-operator/README.md
+++ b/chart/jenkins-operator/README.md
@@ -30,7 +30,7 @@ Kubernetes native operator which fully manages Jenkins on Kubernetes
 | jenkins.backup.env[2].name | string | `"BACKUP_COUNT"` |  |
 | jenkins.backup.env[2].value | string | `"3"` |  |
 | jenkins.backup.getLatestAction[0] | string | `"/home/user/bin/get-latest.sh"` |  |
-| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6"` |  |
+| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.3.1"` |  |
 | jenkins.backup.interval | int | `30` |  |
 | jenkins.backup.makeBackupBeforePodDeletion | bool | `true` |  |
 | jenkins.backup.pvc.className | string | `""` |  |
diff --git a/chart/jenkins-operator/values.yaml b/chart/jenkins-operator/values.yaml
index cc878194b..ab7d6612c 100644
--- a/chart/jenkins-operator/values.yaml
+++ b/chart/jenkins-operator/values.yaml
@@ -214,7 +214,7 @@ jenkins:
 
     # image used by backup feature
     # By default using prebuilt backup PVC image
-    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6
+    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.3.1
 
     # containerName is backup container name
     containerName: backup

From 7a4cbf98d7fae5b81ebd972312252ccc37ec4442 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 18:00:25 +0200
Subject: [PATCH 05/10] remove -x

---
 backup/pvc/bin/backup.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index c8d4d3821..d9a2196ea 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -2,7 +2,6 @@
 
 set -eo pipefail
 source "$(dirname "$0")/utils.sh"
-set -x
 
 [[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 BACKUP_NUMBER" && exit 1
 [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1

From fd5d09d2b695a107c63758a216008e6554e7b285 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 19:17:11 +0200
Subject: [PATCH 06/10] cosmetic log change, in case of error also send it to
 current process stdout

---
 backup/pvc/bin/utils.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backup/pvc/bin/utils.sh b/backup/pvc/bin/utils.sh
index 5d9342385..718dfcf1a 100644
--- a/backup/pvc/bin/utils.sh
+++ b/backup/pvc/bin/utils.sh
@@ -6,8 +6,9 @@ _log() {
     local message="$2"
     local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
     if [[ "$level" =~ ^(ERROR|ERR|error|err)$ ]]; then
-        echo "${timestamp} [${level}] ${message}" > /proc/1/fd/2
+        echo "${timestamp} - ${level} - ${message}" > /proc/1/fd/2
     else
-        echo "${timestamp} [${level}] ${message}" > /proc/1/fd/1
+        echo "${timestamp} - ${level} - ${message}" > /proc/1/fd/1
+        echo "${timestamp} - ${level} - ${message}" >&2
     fi
 }

From a947d127b8435d63e7137c2d3e18f9c85f2ca179 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sat, 22 Jun 2024 19:24:18 +0200
Subject: [PATCH 07/10] cosmetic changes

---
 backup/pvc/bin/backup.sh  | 20 ++++++++++----------
 backup/pvc/bin/restore.sh | 16 ++++++++--------
 backup/pvc/bin/run.sh     |  8 ++++----
 3 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index d9a2196ea..94dcdb1ad 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -13,14 +13,14 @@ TRAP_FILE="${BACKUP_DIR}/_backup_${BACKUP_NUMBER}_is_running"
 
 # --> Check if another backup process is running (operator restart/crash)
 for ((i=0; i<RETRY_COUNT; i++)); do
-    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "Backup: no other backup process are running" && break
-    _log "INFO" "Backup is already running. Waiting for ${RETRY_INTERVAL} seconds..."
+    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[backup] no other backup process are running" && break
+    _log "INFO" "[backup] backup is already running. Waiting for ${RETRY_INTERVAL} seconds..."
     sleep "${RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Backup is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[backup] backup is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
-_log "INFO" "Running backup ${BACKUP_NUMBER}"
+_log "INFO" "[backup] running backup ${BACKUP_NUMBER}"
 touch "${TRAP_FILE}"
 # create temp dir on the same filesystem with a BACKUP_DIR to be able use atomic mv enstead of copy
 BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
@@ -32,7 +32,7 @@ _clean(){
 
 _trap(){
     _clean
-    _log "ERROR" "Backup: something wrong happened, check the logs"
+    _log "ERROR" "[backup] something wrong happened, check the logs"
 }
 
 trap '_trap' SIGQUIT SIGINT SIGTERM
@@ -48,16 +48,16 @@ tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd
     --exclude jobs/*/config.xml -c jobs || ret=$?
 
 if [[ "$ret" -eq 0 ]]; then
-  _log "INFO" "Backup ${BACKUP_NUMBER} was completed without warnings"
+  _log "INFO" "[backup] backup ${BACKUP_NUMBER} was completed without warnings"
 elif [[ "$ret" -eq 1 ]]; then
-  _log "INFO" "Backup ${BACKUP_NUMBER} was completed with some warnings"
+  _log "INFO" "[backup] backup ${BACKUP_NUMBER} was completed with some warnings"
 fi
 
 mv "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd" "${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd"
 
-_log "INFO" "Cleaning ${BACKUP_TMP_DIR} and trap file ${TRAP_FILE}"
+_log "INFO" "[backup] cleaning ${BACKUP_TMP_DIR} and trap file ${TRAP_FILE}"
 _clean
-[[ ! -s ${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd ]] && _log "ERROR" "Backup file '${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd' is empty" && exit 1
+[[ ! -s ${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd ]] && _log "ERROR" "[backup] file '${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd' is empty" && exit 1
 
-_log "INFO" "Backup ${BACKUP_NUMBER} done"
+_log "INFO" "[backup] ${BACKUP_NUMBER} done"
 exit 0
diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh
index 48d2b4862..f75779bbc 100644
--- a/backup/pvc/bin/restore.sh
+++ b/backup/pvc/bin/restore.sh
@@ -15,31 +15,31 @@ TRAP_FILE="${BACKUP_DIR}/_restore_${BACKUP_NUMBER}_is_running"
 trap "rm -f ${TRAP_FILE}" SIGINT SIGTERM
 
 for ((i=0; i<RETRY_COUNT; i++)); do
-    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "Restore: No other process are running, restoring" && break
-    _log "INFO" "Restore is already running. Waiting for ${RETRY_INTERVAL} seconds..."
+    [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[restore] no other process are running, restoring" && break
+    _log "INFO" "[restore] is already running. Waiting for ${RETRY_INTERVAL} seconds..."
     sleep "${RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "Restore is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[restore] is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
-_log "INFO" "Running restore backup with backup number #${BACKUP_NUMBER}"
+_log "INFO" "[restore] restore backup with backup number #${BACKUP_NUMBER}"
 touch "${TRAP_FILE}"
 BACKUP_FILE="${BACKUP_DIR}/${BACKUP_NUMBER}"
 
 if [[ -f "$BACKUP_FILE.tar.gz" ]]; then
-    _log "INFO" "Restore: ld format tar.gz found, restoring it"
+    _log "INFO" "[restore] old format tar.gz found, restoring it"
     OPTS=""
     EXT="tar.gz"
 elif [[ -f "$BACKUP_FILE.tar.zstd" ]]; then
-    _log "INFO" "Restore: Backup file found, proceeding"
+    _log "INFO" "[restore] Backup file found, proceeding"
     OPTS="--zstd"
     EXT="tar.zstd"
 else
-  _log "ERROR" "Restore: Backup file not found: $BACKUP_FILE"
+  _log "ERROR" "[restore] backup file not found: $BACKUP_FILE"
   exit 1
 fi
 
 tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${BACKUP_NUMBER}.${EXT}"
 
-_log "INFO" "Restore: ${BACKUP_NUMBER} Done"
+_log "INFO" "[restore] restoring ${BACKUP_NUMBER} Done"
 exit 0
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 5486dbaaa..56a535d3d 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -42,7 +42,7 @@ find_exceeding_backups() {
     local backup_count="$2"
     # Check if we have any backup
     if is_backup_not_exist "${backup_dir}"; then
-        _log "ERROR" "backups not found in ${backup_dir}"
+        _log "ERROR" "[run] backups not found in ${backup_dir}"
         return
     fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
@@ -52,9 +52,9 @@ check_env_var "BACKUP_DIR"
 check_env_var "JENKINS_HOME"
 
 if [[ -z "${BACKUP_COUNT}" ]]; then
-    _log "WARNING" "No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
+    _log "WARNING" "[run] no BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
 else
-    _log "INFO" "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
+    _log "INFO" "[run] retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
 fi
 
 while true;
@@ -63,7 +63,7 @@ do
     if [[ -n "${BACKUP_COUNT}" ]]; then
         exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
         if [[ -n "$exceeding_backups" ]]; then
-            _log "INFO" "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
+            _log "INFO" "[run] removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
             echo "$exceeding_backups" | while read -r file; do
                 rm "${BACKUP_DIR}/${file}"
             done

From f535053858e30142819173dda6a056f7104d5e3f Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sun, 23 Jun 2024 11:20:27 +0200
Subject: [PATCH 08/10] fix restore, use tmp in case jenkins pod is restarted,
 not the operator

---
 backup/pvc/bin/backup.sh  | 2 +-
 backup/pvc/bin/restore.sh | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index 94dcdb1ad..ccd7b50bb 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -9,7 +9,7 @@ source "$(dirname "$0")/utils.sh"
 RETRY_COUNT=${RETRY_COUNT:-3}
 RETRY_INTERVAL=${RETRY_INTERVAL:-60}
 BACKUP_NUMBER=$1
-TRAP_FILE="${BACKUP_DIR}/_backup_${BACKUP_NUMBER}_is_running"
+TRAP_FILE="/tmp/_backup_${BACKUP_NUMBER}_is_running"
 
 # --> Check if another backup process is running (operator restart/crash)
 for ((i=0; i<RETRY_COUNT; i++)); do
diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh
index f75779bbc..c6a1789ed 100644
--- a/backup/pvc/bin/restore.sh
+++ b/backup/pvc/bin/restore.sh
@@ -7,11 +7,11 @@ source "$(dirname "$0")/utils.sh"
 [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1
 [[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1
 BACKUP_NUMBER=$1
-RETRY_COUNT=${RETRY_COUNT:-3}
-RETRY_INTERVAL=${RETRY_INTERVAL:-60}
+RETRY_COUNT=${RETRY_COUNT:-10}
+RETRY_INTERVAL=${RETRY_INTERVAL:-10}
 
 # --> Check if another restore process is running (operator restart/crash)
-TRAP_FILE="${BACKUP_DIR}/_restore_${BACKUP_NUMBER}_is_running"
+TRAP_FILE="/tmp/_restore_${BACKUP_NUMBER}_is_running"
 trap "rm -f ${TRAP_FILE}" SIGINT SIGTERM
 
 for ((i=0; i<RETRY_COUNT; i++)); do
@@ -41,5 +41,7 @@ fi
 
 tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${BACKUP_NUMBER}.${EXT}"
 
+_log "INFO" "[restore] deleting lock file ${TRAP_FILE}"
+test -f "${TRAP_FILE}" && rm -f "${TRAP_FILE}"
 _log "INFO" "[restore] restoring ${BACKUP_NUMBER} Done"
 exit 0

From 5f5c8e17142d1cf9912350adacc02fbd9828bcac Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sun, 23 Jun 2024 12:05:58 +0200
Subject: [PATCH 09/10] backup and restore retry to different envs

---
 backup/pvc/bin/backup.sh  | 12 ++++++------
 backup/pvc/bin/restore.sh | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/backup/pvc/bin/backup.sh b/backup/pvc/bin/backup.sh
index ccd7b50bb..13d2f0963 100644
--- a/backup/pvc/bin/backup.sh
+++ b/backup/pvc/bin/backup.sh
@@ -6,18 +6,18 @@ source "$(dirname "$0")/utils.sh"
 [[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 BACKUP_NUMBER" && exit 1
 [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1
 [[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1
-RETRY_COUNT=${RETRY_COUNT:-3}
-RETRY_INTERVAL=${RETRY_INTERVAL:-60}
+BACKUP_RETRY_COUNT=${BACKUP_RETRY_COUNT:-3}
+BACKUP_RETRY_INTERVAL=${BACKUP_RETRY_INTERVAL:-60}
 BACKUP_NUMBER=$1
 TRAP_FILE="/tmp/_backup_${BACKUP_NUMBER}_is_running"
 
 # --> Check if another backup process is running (operator restart/crash)
-for ((i=0; i<RETRY_COUNT; i++)); do
+for ((i=0; i<BACKUP_RETRY_COUNT; i++)); do
     [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[backup] no other backup process are running" && break
-    _log "INFO" "[backup] backup is already running. Waiting for ${RETRY_INTERVAL} seconds..."
-    sleep "${RETRY_INTERVAL}"
+    _log "INFO" "[backup] backup is already running. Waiting for ${BACKUP_RETRY_INTERVAL} seconds..."
+    sleep "${BACKUP_RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[backup] backup is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[backup] backup is still running after waiting ${BACKUP_RETRY_COUNT} time ${BACKUP_RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
 _log "INFO" "[backup] running backup ${BACKUP_NUMBER}"
diff --git a/backup/pvc/bin/restore.sh b/backup/pvc/bin/restore.sh
index c6a1789ed..3a692625d 100644
--- a/backup/pvc/bin/restore.sh
+++ b/backup/pvc/bin/restore.sh
@@ -7,19 +7,19 @@ source "$(dirname "$0")/utils.sh"
 [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1
 [[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1
 BACKUP_NUMBER=$1
-RETRY_COUNT=${RETRY_COUNT:-10}
-RETRY_INTERVAL=${RETRY_INTERVAL:-10}
+RESTORE_RETRY_COUNT=${RESTORE_RETRY_COUNT:-10}
+RESTORE_RETRY_INTERVAL=${RESTORE_RETRY_INTERVAL:-10}
 
 # --> Check if another restore process is running (operator restart/crash)
 TRAP_FILE="/tmp/_restore_${BACKUP_NUMBER}_is_running"
 trap "rm -f ${TRAP_FILE}" SIGINT SIGTERM
 
-for ((i=0; i<RETRY_COUNT; i++)); do
+for ((i=0; i<RESTORE_RETRY_COUNT; i++)); do
     [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[restore] no other process are running, restoring" && break
-    _log "INFO" "[restore] is already running. Waiting for ${RETRY_INTERVAL} seconds..."
-    sleep "${RETRY_INTERVAL}"
+    _log "INFO" "[restore] is already running. Waiting for ${RESTORE_RETRY_INTERVAL} seconds..."
+    sleep "${RESTORE_RETRY_INTERVAL}"
 done
-[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[restore] is still running after waiting ${RETRY_COUNT} time ${RETRY_INTERVAL}s. Exiting."; exit 1; }
+[[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[restore] is still running after waiting ${RESTORE_RETRY_COUNT} time ${RESTORE_RETRY_INTERVAL}s. Exiting."; exit 1; }
 # --< Done
 
 _log "INFO" "[restore] restore backup with backup number #${BACKUP_NUMBER}"

From 4ab4a5a968a7332cf6e2cd5df468daa78f8283f3 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Tue, 25 Jun 2024 19:20:56 +0200
Subject: [PATCH 10/10] add docs

---
 chart/jenkins-operator/README.md               |  2 +-
 chart/jenkins-operator/values.yaml             | 16 +++++++++++++++-
 nix/website-shell.nix                          |  1 +
 .../latest/configuring-backup-and-restore.md   | 18 +++++++++++++++++-
 4 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/chart/jenkins-operator/README.md b/chart/jenkins-operator/README.md
index 729cbe644..e61ec2f6f 100644
--- a/chart/jenkins-operator/README.md
+++ b/chart/jenkins-operator/README.md
@@ -30,7 +30,7 @@ Kubernetes native operator which fully manages Jenkins on Kubernetes
 | jenkins.backup.env[2].name | string | `"BACKUP_COUNT"` |  |
 | jenkins.backup.env[2].value | string | `"3"` |  |
 | jenkins.backup.getLatestAction[0] | string | `"/home/user/bin/get-latest.sh"` |  |
-| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.3.1"` |  |
+| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.4.1"` |  |
 | jenkins.backup.interval | int | `30` |  |
 | jenkins.backup.makeBackupBeforePodDeletion | bool | `true` |  |
 | jenkins.backup.pvc.className | string | `""` |  |
diff --git a/chart/jenkins-operator/values.yaml b/chart/jenkins-operator/values.yaml
index ab7d6612c..2ecdb5796 100644
--- a/chart/jenkins-operator/values.yaml
+++ b/chart/jenkins-operator/values.yaml
@@ -214,7 +214,7 @@ jenkins:
 
     # image used by backup feature
     # By default using prebuilt backup PVC image
-    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.3.1
+    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.4.1
 
     # containerName is backup container name
     containerName: backup
@@ -262,6 +262,11 @@ jenkins:
     # BACKUP_DIR - path for storing backup files (default: "/backup")
     # JENKINS_HOME - path to jenkins home (default: "/jenkins-home")
     # BACKUP_COUNT - define how much recent backups will be kept
+    # Optional in case you want to modify the backup and restore retry logic
+    # BACKUP_RETRY_COUNT
+    # BACKUP_RETRY_INTERVAL
+    # RESTORE_RETRY_COUNT
+    # RESTORE_RETRY_INTERVAL
     env:
       - name: BACKUP_DIR
         value: /backup
@@ -269,6 +274,15 @@ jenkins:
         value: /jenkins-home
       - name: BACKUP_COUNT
         value: "3" # keep only the 3 most recent backups
+      #- name: BACKUP_RETRY_COUNT
+      #  value: "3"
+      #- name: BACKUP_RETRY_INTERVAL
+      #  value: "60"
+      #- name: RESTORE_RETRY_COUNT
+      #  value: "10"
+      #- name: RESTORE_RETRY_INTERVAL
+      #  value: "10"
+
 
     # volumeMounts holds the mount points for volumes
     volumeMounts:
diff --git a/nix/website-shell.nix b/nix/website-shell.nix
index cda6748cb..725425855 100644
--- a/nix/website-shell.nix
+++ b/nix/website-shell.nix
@@ -4,6 +4,7 @@ let
   devShellPackages = [
     hugo_099_pkgs.hugo #hugo pre-v100
     pkgs.nodejs_21 #Node 1.21
+    pkgs.helm-docs
   ];
   baseUrl = ((builtins.fromTOML (builtins.readFile ../website/config.toml)).baseURL);
 in
diff --git a/website/content/en/docs/Getting Started/latest/configuring-backup-and-restore.md b/website/content/en/docs/Getting Started/latest/configuring-backup-and-restore.md
index 35acf8660..86cccde8d 100644
--- a/website/content/en/docs/Getting Started/latest/configuring-backup-and-restore.md	
+++ b/website/content/en/docs/Getting Started/latest/configuring-backup-and-restore.md	
@@ -2,7 +2,7 @@
 title: "Configuring backup and restore"
 linkTitle: "Configuring backup and restore"
 weight: 5
-date: 2023-01-08
+date: 2024-06-25
 description: >
   Prevent loss of job history
 ---
@@ -115,3 +115,19 @@ spec:
         command:
           - /home/user/bin/get-latest.sh # this command is invoked on "backup" container to get last backup number before pod deletion; not having it in the CR may cause loss of data
 ```
+
+#### Customizing pvc backup behaviour
+
+To prevent situations where the operator crashes or gets killed during a backup and restore process, a retry logic has been implemented.
+
+This logic can be customized by adjusting the following environment variables:
+
+* **Backup**: total time wait until giving up by default: 180s
+  * `BACKUP_RETRY_COUNT`: by default is `3`
+  * `BACKUP_RETRY_INTERVAL`: by default is `60`
+
+* **Restore**: total time wait until giving up by default: 100s
+  * `RESTORE_RETRY_COUNT`: by default is `10`
+  * `RESTORE_RETRY_INTERVAL`: by default is `10`
+
+You can adjust the retry logic based on the size of your backup and the duration of the restore process.