From fbfcfe929c16a074f83e3ef680a02a498a4d4645 Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 09:51:44 +0100
Subject: [PATCH 01/10] fix(backup): find race and disable search in subdir

---
 backup/pvc/bin/get-latest.sh |  2 +-
 backup/pvc/bin/run.sh        | 41 ++++++++++++++++++++++++++++++------
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh
index 519a4c6e9..1893683e5 100644
--- a/backup/pvc/bin/get-latest.sh
+++ b/backup/pvc/bin/get-latest.sh
@@ -4,7 +4,7 @@ set -eo pipefail
 
 [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
 # Search for all the tar.* inside the backup dir to support the migration between gzip vs zstd
-latest=$(find ${BACKUP_DIR} -name '*.tar.*' -exec basename {} \; | sort -g | tail -n 1)
+latest=$(find "${BACKUP_DIR}"/*.tar.* -maxdepth 0 -exec basename {} \; | sort -g | tail -n 1)
 
 if [[ "${latest}" == "" ]]; then
   echo "-1"
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 50123d5e2..dad3621eb 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -2,15 +2,42 @@
 
 set -eo pipefail
 
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
-[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
+INTERVAL=60
+
+# Ensure required environment variables are set
+check_env_var() {
+    if [[ -z "${!1}" ]]; then
+        echo "Required '$1' environment variable is not set"
+        exit 1
+    fi
+}
+
+# Function to find exceeding backups
+find_exceeding_backups() {
+    local backup_dir="$1"
+    local backup_count="$2"
+    find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
+}
+
+check_env_var "BACKUP_DIR"
+check_env_var "JENKINS_HOME"
+
+if [[ -z "${BACKUP_COUNT}" ]]; then
+    echo "ATTENTION! No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
+else
+    echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${INTERVAL} seconds"
+fi
 
 while true;
 do
-    sleep 10
-    if [[ ! -z "${BACKUP_COUNT}" ]]; then
-        echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup"
-        #TODO: add the list of exceeding backup before delete
-        find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.zstd' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}'
+    sleep $INTERVAL
+    if [[ -n "${BACKUP_COUNT}" ]]; then
+        exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
+        if [[ -n "$exceeding_backups" ]]; then
+            echo "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
+            echo "$exceeding_backups" | while read -r file; do
+                rm "${BACKUP_DIR}/${file}"
+            done
+        fi
     fi
 done

From 1e039557f23e9b974551cde9517132723f464393 Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 11:26:02 +0100
Subject: [PATCH 02/10] fix and improve backup related tests

---
 backup/pvc/bin/get-latest.sh                  | 28 ++++++++++++++++
 backup/pvc/bin/run.sh                         | 33 +++++++++++++++++--
 backup/pvc/e2e/get-latest/test.sh             | 15 +++++++++
 backup/pvc/e2e/limit_backup_count/test.sh     | 11 ++++---
 .../e2e/limit_backup_count_no_backups/test.sh |  4 +--
 .../test.sh                                   | 12 ++++++-
 6 files changed, 93 insertions(+), 10 deletions(-)

diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh
index 1893683e5..9e3ee6db2 100644
--- a/backup/pvc/bin/get-latest.sh
+++ b/backup/pvc/bin/get-latest.sh
@@ -2,7 +2,35 @@
 
 set -eo pipefail
 
+check_backup_exist() {
+    local backup_dir="$1"
+    # Save the current value of 'set -e'
+    local previous_e=$(set +e; :; echo $?)
+
+    # Temporarily turn off 'set -e'
+    set +e
+
+    # Run ls command to check if any files matching the pattern exist
+    ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
+
+    # Store the exit status of the ls command
+    local ls_exit_status=$?
+
+    # Restore the previous value of 'set -e'
+    [ "$previous_e" = "0" ] && set -e
+
+    # Return true if ls command succeeded (files found), otherwise return false
+    return $ls_exit_status
+}
+
 [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
+
+# Check if we have any backup
+if check_backup_exist "${BACKUP_DIR}"; then
+  echo "-1"
+  exit 0
+fi
+
 # Search for all the tar.* inside the backup dir to support the migration between gzip vs zstd
 latest=$(find "${BACKUP_DIR}"/*.tar.* -maxdepth 0 -exec basename {} \; | sort -g | tail -n 1)
 
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index dad3621eb..8ae497943 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -2,7 +2,8 @@
 
 set -eo pipefail
 
-INTERVAL=60
+# Use 60 as default in case BACKUP_CLEANUP_INTERVAL did not set
+BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60}
 
 # Ensure required environment variables are set
 check_env_var() {
@@ -12,10 +13,36 @@ check_env_var() {
     fi
 }
 
+check_backup_exist() {
+    local backup_dir="$1"
+    # Save the current value of 'set -e'
+    local previous_e=$(set +e; :; echo $?)
+
+    # Temporarily turn off 'set -e'
+    set +e
+
+    # Run ls command to check if any files matching the pattern exist
+    ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
+
+    # Store the exit status of the ls command
+    local ls_exit_status=$?
+
+    # Restore the previous value of 'set -e'
+    [ "$previous_e" = "0" ] && set -e
+
+    # Return true if ls command succeeded (files found), otherwise return false
+    return $ls_exit_status
+}
+
 # Function to find exceeding backups
 find_exceeding_backups() {
     local backup_dir="$1"
     local backup_count="$2"
+    # Check if we have any backup
+    if check_backup_exist "${BACKUP_DIR}"; then
+        # return ""
+        echo "backups not found"
+    fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
 }
 
@@ -25,12 +52,12 @@ check_env_var "JENKINS_HOME"
 if [[ -z "${BACKUP_COUNT}" ]]; then
     echo "ATTENTION! No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
 else
-    echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${INTERVAL} seconds"
+    echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
 fi
 
 while true;
 do
-    sleep $INTERVAL
+    sleep $BACKUP_CLEANUP_INTERVAL
     if [[ -n "${BACKUP_COUNT}" ]]; then
         exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
         if [[ -n "$exceeding_backups" ]]; then
diff --git a/backup/pvc/e2e/get-latest/test.sh b/backup/pvc/e2e/get-latest/test.sh
index a0aabe9a2..85da7acef 100755
--- a/backup/pvc/e2e/get-latest/test.sh
+++ b/backup/pvc/e2e/get-latest/test.sh
@@ -30,6 +30,9 @@ touch ${BACKUP_DIR}/8.tar.zstd
 touch ${BACKUP_DIR}/9.tar.zstd
 touch ${BACKUP_DIR}/10.tar.zstd
 touch ${BACKUP_DIR}/11.tar.zstd
+# Emulate backup creation
+BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
+touch ${BACKUP_TMP_DIR}/12.tar.zstd
 
 # Create an instance of the container under testing
 cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
@@ -38,9 +41,17 @@ echo "Docker container ID '${cid}'"
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
 
+echo "Try to get latest against 11 backups and one in progress"
 latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
 rm ${BACKUP_DIR}/*.tar.zstd
+echo "Try to get latest against one in progress"
 empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+rmdir ${BACKUP_DIR}/lost+found
+rm ${BACKUP_TMP_DIR}/*.tar.zstd
+rmdir ${BACKUP_TMP_DIR}
+echo "Try to get latest against empty dir"
+empty_dir_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+
 
 if [[ "${DEBUG}" ]]; then
     docker logs ${cid}
@@ -55,5 +66,9 @@ if [[ ! "${empty_latest}" == "-1" ]]; then
     echo "Latest backup number should be '-1' but is '${empty_latest}'"
     exit 1
 fi
+if [[ ! "${empty_dir_latest}" == "-1" ]]; then
+    echo "Latest backup number should be '-1' but is '${empty_dir_latest}'"
+    exit 1
+fi
 
 echo PASS
diff --git a/backup/pvc/e2e/limit_backup_count/test.sh b/backup/pvc/e2e/limit_backup_count/test.sh
index 469b3e44c..1fe346e9d 100755
--- a/backup/pvc/e2e/limit_backup_count/test.sh
+++ b/backup/pvc/e2e/limit_backup_count/test.sh
@@ -30,17 +30,20 @@ touch ${BACKUP_DIR}/8.tar.zstd
 touch ${BACKUP_DIR}/9.tar.zstd
 touch ${BACKUP_DIR}/10.tar.zstd
 touch ${BACKUP_DIR}/11.tar.zstd
+# Emulate backup creation
+BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
+touch ${BACKUP_TMP_DIR}/12.tar.zstd
 
 # Create an instance of the container under testing
-cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
 
-sleep 11
-touch ${BACKUP_DIR}/12.tar.zstd
-sleep 11
+sleep 2
+mv ${BACKUP_TMP_DIR}/12.tar.zstd ${BACKUP_DIR}/
+sleep 2
 
 if [[ "${DEBUG}" ]]; then
     docker logs ${cid}
diff --git a/backup/pvc/e2e/limit_backup_count_no_backups/test.sh b/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
index 531ac3da2..1754f8982 100755
--- a/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
+++ b/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
@@ -19,7 +19,7 @@ mkdir -p ${BACKUP_DIR}
 mkdir -p ${JENKINS_HOME}
 
 # Create an instance of the container under testing
-cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
@@ -27,7 +27,7 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}
 
 # container should be running
 echo 'Checking if container is running'
-sleep 11
+sleep 3
 docker exec ${cid} echo
 echo 'Container is running'
 
diff --git a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
index 662809c89..483730666 100755
--- a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
+++ b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
@@ -18,16 +18,26 @@ BACKUP_DIR="$(pwd)/backup"
 mkdir -p ${BACKUP_DIR}
 
 # Create an instance of the container under testing
-cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
+cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
 echo "Docker container ID '${cid}'"
 
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR}" EXIT
 
+# check cleanup against empty backup dir
+sleep 2
+restart_count=$(docker inspect --format='{{.RestartCount}}' "$cid")
+if [ "$restart_count" -eq 0 ]; then
+    echo "The container has been restarted $restart_count times."
+    exit 1
+fi
+
 backup_number=1
 docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
 
 [ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1;
+# We should also check backup directory, since after #1000 we create temp directory at backup filesystem
+[ "$(docker exec ${cid} ls ${BACKUP_DIR} | grep 'tmp')" ] && echo "backup dir consists temp directory" && exit 1;
 
 backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd"
 [[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1;

From 6e1099bbcf22421aea965a547993faff8fa18a31 Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 11:40:24 +0100
Subject: [PATCH 03/10] fix is_backup_not_exist

---
 backup/pvc/bin/get-latest.sh      | 8 ++++----
 backup/pvc/bin/run.sh             | 8 ++++----
 backup/pvc/e2e/get-latest/test.sh | 2 ++
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh
index 9e3ee6db2..09a804dc6 100644
--- a/backup/pvc/bin/get-latest.sh
+++ b/backup/pvc/bin/get-latest.sh
@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-check_backup_exist() {
+is_backup_not_exist() {
     local backup_dir="$1"
     # Save the current value of 'set -e'
     local previous_e=$(set +e; :; echo $?)
@@ -19,14 +19,14 @@ check_backup_exist() {
     # Restore the previous value of 'set -e'
     [ "$previous_e" = "0" ] && set -e
 
-    # Return true if ls command succeeded (files found), otherwise return false
-    return $ls_exit_status
+    # Return true if ls command succeeded (no files found), otherwise return false
+    [ $ls_exit_status -ne 0 ]
 }
 
 [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
 
 # Check if we have any backup
-if check_backup_exist "${BACKUP_DIR}"; then
+if is_backup_not_exist "${BACKUP_DIR}"; then
   echo "-1"
   exit 0
 fi
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 8ae497943..a772de827 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -13,7 +13,7 @@ check_env_var() {
     fi
 }
 
-check_backup_exist() {
+is_backup_not_exist() {
     local backup_dir="$1"
     # Save the current value of 'set -e'
     local previous_e=$(set +e; :; echo $?)
@@ -30,8 +30,8 @@ check_backup_exist() {
     # Restore the previous value of 'set -e'
     [ "$previous_e" = "0" ] && set -e
 
-    # Return true if ls command succeeded (files found), otherwise return false
-    return $ls_exit_status
+    # Return true if ls command succeeded (no files found), otherwise return false
+    [ $ls_exit_status -ne 0 ]
 }
 
 # Function to find exceeding backups
@@ -39,7 +39,7 @@ find_exceeding_backups() {
     local backup_dir="$1"
     local backup_count="$2"
     # Check if we have any backup
-    if check_backup_exist "${BACKUP_DIR}"; then
+    if is_backup_not_exist "${BACKUP_DIR}"; then
         # return ""
         echo "backups not found"
     fi
diff --git a/backup/pvc/e2e/get-latest/test.sh b/backup/pvc/e2e/get-latest/test.sh
index 85da7acef..e8b4eb480 100755
--- a/backup/pvc/e2e/get-latest/test.sh
+++ b/backup/pvc/e2e/get-latest/test.sh
@@ -43,9 +43,11 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}
 
 echo "Try to get latest against 11 backups and one in progress"
 latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+
 rm ${BACKUP_DIR}/*.tar.zstd
 echo "Try to get latest against one in progress"
 empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
+
 rmdir ${BACKUP_DIR}/lost+found
 rm ${BACKUP_TMP_DIR}/*.tar.zstd
 rmdir ${BACKUP_TMP_DIR}

From 66136e4cca05af3bbcc8bfc6d8be3912646690c6 Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 11:48:42 +0100
Subject: [PATCH 04/10] handle empty backup dir cleanup

---
 backup/pvc/bin/run.sh                                     | 4 ++--
 .../pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh   | 8 --------
 2 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index a772de827..f1cbc77b4 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -40,8 +40,8 @@ find_exceeding_backups() {
     local backup_count="$2"
     # Check if we have any backup
     if is_backup_not_exist "${BACKUP_DIR}"; then
-        # return ""
-        echo "backups not found"
+        echo "backups not found in ${BACKUP_DIR}"
+        return ""
     fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
 }
diff --git a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
index 483730666..0d08f000f 100755
--- a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
+++ b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
@@ -24,14 +24,6 @@ echo "Docker container ID '${cid}'"
 # Remove test directory and container afterwards
 trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR}" EXIT
 
-# check cleanup against empty backup dir
-sleep 2
-restart_count=$(docker inspect --format='{{.RestartCount}}' "$cid")
-if [ "$restart_count" -eq 0 ]; then
-    echo "The container has been restarted $restart_count times."
-    exit 1
-fi
-
 backup_number=1
 docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
 

From 702a911f81f0f7b86e16dafd2610ec0f960035eb Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 16:28:27 +0100
Subject: [PATCH 05/10] show container logs in limit_backup_count_no_backups
 test

---
 backup/pvc/e2e/limit_backup_count/test.sh              |  2 ++
 backup/pvc/e2e/limit_backup_count_no_backups/test.sh   | 10 ++++++++++
 .../e2e/tmp_dir_clean_after_backup_creation/test.sh    |  2 ++
 3 files changed, 14 insertions(+)

diff --git a/backup/pvc/e2e/limit_backup_count/test.sh b/backup/pvc/e2e/limit_backup_count/test.sh
index 1fe346e9d..cbe1ecd41 100755
--- a/backup/pvc/e2e/limit_backup_count/test.sh
+++ b/backup/pvc/e2e/limit_backup_count/test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running limit_backup_count e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script
diff --git a/backup/pvc/e2e/limit_backup_count_no_backups/test.sh b/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
index 1754f8982..1b90acd82 100755
--- a/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
+++ b/backup/pvc/e2e/limit_backup_count_no_backups/test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running limit_backup_count_no_backups e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script
@@ -28,7 +30,15 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}
 # container should be running
 echo 'Checking if container is running'
 sleep 3
+set +e
 docker exec ${cid} echo
+exit_code=$?
+set -e
+if [ $exit_code -ne 0 ]; then
+    echo "container terminated with following logs:"
+    docker logs "${cid}"
+    exit 1
+fi
 echo 'Container is running'
 
 echo PASS
diff --git a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
index 0d08f000f..f373543c8 100755
--- a/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
+++ b/backup/pvc/e2e/tmp_dir_clean_after_backup_creation/test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eo pipefail
 
+echo "Running tmp_dir_clean_after_backup_creation e2e test..."
+
 [[ "${DEBUG}" ]] && set -x
 
 # set current working directory to the directory of the script

From 6349e1d1a202c6c4861504233ad69f6b665f484a Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 16:32:19 +0100
Subject: [PATCH 06/10] fix find_exceeding_backups

---
 backup/pvc/bin/run.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index f1cbc77b4..5fb371739 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -39,9 +39,9 @@ find_exceeding_backups() {
     local backup_dir="$1"
     local backup_count="$2"
     # Check if we have any backup
-    if is_backup_not_exist "${BACKUP_DIR}"; then
-        echo "backups not found in ${BACKUP_DIR}"
-        return ""
+    if is_backup_not_exist "${backup_dir}"; then
+        echo "backups not found in ${backup_dir}"
+        return
     fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
 }

From 60e297dafc77fb4a1e5cb74be1347fa78edafb5a Mon Sep 17 00:00:00 2001
From: skillcoder <mail@skillcoder.com>
Date: Wed, 8 May 2024 16:36:17 +0100
Subject: [PATCH 07/10] fix warning in find_exceeding_backups

---
 backup/pvc/bin/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 5fb371739..441902dd0 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -40,7 +40,7 @@ find_exceeding_backups() {
     local backup_count="$2"
     # Check if we have any backup
     if is_backup_not_exist "${backup_dir}"; then
-        echo "backups not found in ${backup_dir}"
+        echo "backups not found in ${backup_dir}" >&2
         return
     fi
     find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))

From 1ecd21e9ad2572212888be1037a034e96191c0c3 Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Sun, 12 May 2024 11:15:57 +0200
Subject: [PATCH 08/10] bump version, shellcheck, cosmetic changes

---
 backup/pvc/VERSION.txt             | 2 +-
 backup/pvc/bin/get-latest.sh       | 5 +++--
 backup/pvc/bin/run.sh              | 5 +++--
 chart/jenkins-operator/README.md   | 2 +-
 chart/jenkins-operator/values.yaml | 2 +-
 5 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/backup/pvc/VERSION.txt b/backup/pvc/VERSION.txt
index b88fb90fb..400feeb94 100644
--- a/backup/pvc/VERSION.txt
+++ b/backup/pvc/VERSION.txt
@@ -1 +1 @@
-v0.2.5
+v0.2.6
diff --git a/backup/pvc/bin/get-latest.sh b/backup/pvc/bin/get-latest.sh
index 09a804dc6..da8055f4e 100644
--- a/backup/pvc/bin/get-latest.sh
+++ b/backup/pvc/bin/get-latest.sh
@@ -5,7 +5,8 @@ set -eo pipefail
 is_backup_not_exist() {
     local backup_dir="$1"
     # Save the current value of 'set -e'
-    local previous_e=$(set +e; :; echo $?)
+    local previous_e
+    previous_e=$(set +e; :; echo $?)
 
     # Temporarily turn off 'set -e'
     set +e
@@ -23,7 +24,7 @@ is_backup_not_exist() {
     [ $ls_exit_status -ne 0 ]
 }
 
-[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
+[[ -z "${BACKUP_DIR}" ]] && { echo "Required 'BACKUP_DIR' env not set"; exit 1; }
 
 # Check if we have any backup
 if is_backup_not_exist "${BACKUP_DIR}"; then
diff --git a/backup/pvc/bin/run.sh b/backup/pvc/bin/run.sh
index 441902dd0..4b6052019 100644
--- a/backup/pvc/bin/run.sh
+++ b/backup/pvc/bin/run.sh
@@ -16,7 +16,8 @@ check_env_var() {
 is_backup_not_exist() {
     local backup_dir="$1"
     # Save the current value of 'set -e'
-    local previous_e=$(set +e; :; echo $?)
+    local previous_e
+    previous_e=$(set +e; :; echo $?)
 
     # Temporarily turn off 'set -e'
     set +e
@@ -57,7 +58,7 @@ fi
 
 while true;
 do
-    sleep $BACKUP_CLEANUP_INTERVAL
+    sleep "$BACKUP_CLEANUP_INTERVAL"
     if [[ -n "${BACKUP_COUNT}" ]]; then
         exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
         if [[ -n "$exceeding_backups" ]]; then
diff --git a/chart/jenkins-operator/README.md b/chart/jenkins-operator/README.md
index 96e510919..c7e5ad847 100644
--- a/chart/jenkins-operator/README.md
+++ b/chart/jenkins-operator/README.md
@@ -30,7 +30,7 @@ Kubernetes native operator which fully manages Jenkins on Kubernetes
 | jenkins.backup.env[2].name | string | `"BACKUP_COUNT"` |  |
 | jenkins.backup.env[2].value | string | `"3"` |  |
 | jenkins.backup.getLatestAction[0] | string | `"/home/user/bin/get-latest.sh"` |  |
-| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6"` |  |
+| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.7"` |  |
 | jenkins.backup.interval | int | `30` |  |
 | jenkins.backup.makeBackupBeforePodDeletion | bool | `true` |  |
 | jenkins.backup.pvc.className | string | `""` |  |
diff --git a/chart/jenkins-operator/values.yaml b/chart/jenkins-operator/values.yaml
index 388de22ec..236f6e800 100644
--- a/chart/jenkins-operator/values.yaml
+++ b/chart/jenkins-operator/values.yaml
@@ -210,7 +210,7 @@ jenkins:
 
     # image used by backup feature
     # By default using prebuilt backup PVC image
-    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6
+    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.7
 
     # containerName is backup container name
     containerName: backup

From 965d0fd80756b4ced322c57c1432f0ee183eea5c Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Mon, 3 Jun 2024 11:06:08 +0200
Subject: [PATCH 09/10] bump minor version of backup img after this change

---
 backup/pvc/VERSION.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backup/pvc/VERSION.txt b/backup/pvc/VERSION.txt
index 400feeb94..268b0334e 100644
--- a/backup/pvc/VERSION.txt
+++ b/backup/pvc/VERSION.txt
@@ -1 +1 @@
-v0.2.6
+v0.3.0

From 86d41557cb94b704e2436c88d06cf1b214b3aa1a Mon Sep 17 00:00:00 2001
From: brokenpip3 <brokenpip3@gmail.com>
Date: Mon, 3 Jun 2024 11:50:45 +0200
Subject: [PATCH 10/10] always test latest backup in e2e

---
 Makefile                           | 15 ++++++++++++++-
 backup/pvc/Makefile                |  8 +++++++-
 backup/pvc/config.env              |  1 +
 chart/jenkins-operator/README.md   |  2 +-
 chart/jenkins-operator/values.yaml |  2 +-
 test/e2e/restorebackup_test.go     |  2 +-
 6 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 10c3e0475..bdd8d4e37 100644
--- a/Makefile
+++ b/Makefile
@@ -89,11 +89,18 @@ test: ## Runs the go tests
 	@RUNNING_TESTS=1 go test -tags "$(BUILDTAGS) cgo" $(PACKAGES_FOR_UNIT_TESTS)
 
 .PHONY: e2e
-e2e: deepcopy-gen manifests ## Runs e2e tests, you can use EXTRA_ARGS
+e2e: deepcopy-gen manifests backup-kind-load ## Runs e2e tests, you can use EXTRA_ARGS
 	@echo "+ $@"
 	RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -ginkgo.v -tags "$(BUILDTAGS) cgo" -v -timeout 60m -run "$(E2E_TEST_SELECTOR)" \
 		-jenkins-api-hostname=$(JENKINS_API_HOSTNAME) -jenkins-api-port=$(JENKINS_API_PORT) -jenkins-api-use-nodeport=$(JENKINS_API_USE_NODEPORT) $(E2E_TEST_ARGS)
 
+## Backup Section
+
+.PHONY: backup-kind-load
+backup-kind-load: ## Load latest backup image in the cluster
+	@echo "+ $@"
+	make -C backup/pvc backup-kind-load
+
 ## HELM Section
 
 .PHONY: helm
@@ -370,6 +377,12 @@ kind-clean: ## Delete kind cluster
 	@echo "+ $@"
 	kind delete cluster --name $(KIND_CLUSTER_NAME)
 
+.PHONY: kind-load-backup
+kind-load-backup:
+	@echo "+ $@"
+	make -C
+	kind delete cluster --name $(KIND_CLUSTER_NAME)
+
 .PHONY: bats-tests
 IMAGE_NAME := quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY):$(GITCOMMIT)-amd64
 BUILD_PRESENT := $(shell docker images |grep -q ${IMAGE_NAME})
diff --git a/backup/pvc/Makefile b/backup/pvc/Makefile
index d8ec544fb..4999ae7a4 100644
--- a/backup/pvc/Makefile
+++ b/backup/pvc/Makefile
@@ -118,6 +118,12 @@ docker-release-latest: docker-build ## Release image with latest tags (in additi
 docker-release: docker-release-version docker-release-latest ## Release image with version and latest tags (in addition to build tag)
 	@echo "+ $@"
 
+.PHONY: backup-kind-load
+backup-kind-load: docker-build ## Build and load backup img in kind with e2e-test tag
+	@echo "+ $@"
+	docker tag quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test
+	kind load docker-image quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test --name $(KIND_CLUSTER_NAME)
+
 # if this session isn't interactive, then we don't want to allocate a
 # TTY, which would fail, but if it is interactive, we do want to attach
 # so that the user can send e.g. ^C through.
@@ -127,7 +133,7 @@ ifeq ($(INTERACTIVE), 1)
 endif
 
 .PHONY: docker-run
-docker-run: docker-build ## Run the container in docker, you can use EXTRA_ARGS
+docker-run: docker-build
 	@echo "+ $@"
 	docker run --rm -i $(DOCKER_FLAGS) \
 		quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) $(ARGS)
diff --git a/backup/pvc/config.env b/backup/pvc/config.env
index 64a407549..e2b892251 100644
--- a/backup/pvc/config.env
+++ b/backup/pvc/config.env
@@ -4,3 +4,4 @@ QUAY_ORGANIZATION=jenkins-kubernetes-operator
 QUAY_REGISTRY=backup
 UID=1000
 GID=1000
+KIND_CLUSTER_NAME=jenkins
diff --git a/chart/jenkins-operator/README.md b/chart/jenkins-operator/README.md
index 198940970..ccaaae9c4 100644
--- a/chart/jenkins-operator/README.md
+++ b/chart/jenkins-operator/README.md
@@ -30,7 +30,7 @@ Kubernetes native operator which fully manages Jenkins on Kubernetes
 | jenkins.backup.env[2].name | string | `"BACKUP_COUNT"` |  |
 | jenkins.backup.env[2].value | string | `"3"` |  |
 | jenkins.backup.getLatestAction[0] | string | `"/home/user/bin/get-latest.sh"` |  |
-| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.7"` |  |
+| jenkins.backup.image | string | `"quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6"` |  |
 | jenkins.backup.interval | int | `30` |  |
 | jenkins.backup.makeBackupBeforePodDeletion | bool | `true` |  |
 | jenkins.backup.pvc.className | string | `""` |  |
diff --git a/chart/jenkins-operator/values.yaml b/chart/jenkins-operator/values.yaml
index 9f244889d..ed1e401d8 100644
--- a/chart/jenkins-operator/values.yaml
+++ b/chart/jenkins-operator/values.yaml
@@ -210,7 +210,7 @@ jenkins:
 
     # image used by backup feature
     # By default using prebuilt backup PVC image
-    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.7
+    image: quay.io/jenkins-kubernetes-operator/backup-pvc:v0.2.6
 
     # containerName is backup container name
     containerName: backup
diff --git a/test/e2e/restorebackup_test.go b/test/e2e/restorebackup_test.go
index aa90f6ae5..43f2f5b46 100644
--- a/test/e2e/restorebackup_test.go
+++ b/test/e2e/restorebackup_test.go
@@ -157,7 +157,7 @@ func createJenkinsWithBackupAndRestoreConfigured(name, namespace string) *v1alph
 					},
 					{
 						Name:            containerName,
-						Image:           "virtuslab/jenkins-operator-backup-pvc:v0.1.1",
+						Image:           "quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test",
 						ImagePullPolicy: corev1.PullIfNotPresent,
 						Env: []corev1.EnvVar{
 							{