forked from VirtusLab/jenkins-operator
-
Notifications
You must be signed in to change notification settings - Fork 236
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
214 changed files
with
442 additions
and
165,289 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,11 +23,11 @@ jobs: | |
| name: PreCommit | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/[email protected].0 | ||
| - uses: DeterminateSystems/nix-installer-action@v10 | ||
| - uses: actions/[email protected].1 | ||
| - uses: DeterminateSystems/nix-installer-action@v13 | ||
| with: | ||
| diagnostic-endpoint: "" | ||
| - uses: DeterminateSystems/magic-nix-cache-action@v4 | ||
| - uses: DeterminateSystems/magic-nix-cache-action@v7 | ||
| with: | ||
| diagnostic-endpoint: "" | ||
| - name: Checkout | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| v0.8.0 | ||
| v0.8.1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| v0.2.5 | ||
| v0.4.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,39 +1,63 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eo pipefail | ||
| source "$(dirname "$0")/utils.sh" | ||
|
|
||
| [[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 BACKUP_NUMBER" && exit 1 | ||
| [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1 | ||
| [[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1 | ||
| BACKUP_RETRY_COUNT=${BACKUP_RETRY_COUNT:-3} | ||
| BACKUP_RETRY_INTERVAL=${BACKUP_RETRY_INTERVAL:-60} | ||
| BACKUP_NUMBER=$1 | ||
| TRAP_FILE="/tmp/_backup_${BACKUP_NUMBER}_is_running" | ||
|
|
||
| # --> Check if another backup process is running (operator restart/crash) | ||
| for ((i=0; i<BACKUP_RETRY_COUNT; i++)); do | ||
| [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[backup] no other backup process are running" && break | ||
| _log "INFO" "[backup] backup is already running. Waiting for ${BACKUP_RETRY_INTERVAL} seconds..." | ||
| sleep "${BACKUP_RETRY_INTERVAL}" | ||
| done | ||
| [[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[backup] backup is still running after waiting ${BACKUP_RETRY_COUNT} time ${BACKUP_RETRY_INTERVAL}s. Exiting."; exit 1; } | ||
| # --< Done | ||
|
|
||
| _log "INFO" "[backup] running backup ${BACKUP_NUMBER}" | ||
| touch "${TRAP_FILE}" | ||
| # create temp dir on the same filesystem with a BACKUP_DIR to be able use atomic mv enstead of copy | ||
| BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}") | ||
|
|
||
| [[ ! $# -eq 1 ]] && echo "Usage: $0 backup_number" && exit 1; | ||
| [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1; | ||
| [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; | ||
| _clean(){ | ||
| test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}" | ||
| test -f "${TRAP_FILE}" && rm -f "${TRAP_FILE}" | ||
| } | ||
|
|
||
| # create temp dir on the same filesystem with a BACKUP_DIR to be able use atomic mv enstead of copy | ||
| BACKUP_TMP_DIR=$(mktemp -d --tmpdir=${BACKUP_DIR}) | ||
| trap "test -d "${BACKUP_TMP_DIR}" && rm -fr "${BACKUP_TMP_DIR}"" EXIT SIGINT SIGTERM | ||
| _trap(){ | ||
| _clean | ||
| _log "ERROR" "[backup] something wrong happened, check the logs" | ||
| } | ||
|
|
||
| backup_number=$1 | ||
| echo "Running backup" | ||
| trap '_trap' SIGQUIT SIGINT SIGTERM | ||
|
|
||
| # config.xml in a job directory is a config file that shouldn't be backed up | ||
| # config.xml in child directories is state that should. For example- | ||
| # branches/myorg/branches/myrepo/branches/master/config.xml should be retained while | ||
| # branches/myorg/config.xml should not | ||
| tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" \ | ||
| tar --zstd -C "${JENKINS_HOME}" -cf "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd" \ | ||
| --exclude jobs/*/workspace* \ | ||
| --no-wildcards-match-slash --anchored \ | ||
| --ignore-failed-read \ | ||
| --exclude jobs/*/config.xml -c jobs || ret=$? | ||
|
|
||
| if [[ "$ret" -eq 0 ]]; then | ||
| echo "Backup was completed without warnings" | ||
| _log "INFO" "[backup] backup ${BACKUP_NUMBER} was completed without warnings" | ||
| elif [[ "$ret" -eq 1 ]]; then | ||
| echo "Backup was completed with some warnings" | ||
| _log "INFO" "[backup] backup ${BACKUP_NUMBER} was completed with some warnings" | ||
| fi | ||
|
|
||
| # atomically create a backup file | ||
| mv "${BACKUP_TMP_DIR}/${backup_number}.tar.zstd" "${BACKUP_DIR}/${backup_number}.tar.zstd" | ||
| mv "${BACKUP_TMP_DIR}/${BACKUP_NUMBER}.tar.zstd" "${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd" | ||
|
|
||
| rm -rf "${BACKUP_TMP_DIR}" | ||
| [[ ! -s ${BACKUP_DIR}/${backup_number}.tar.zstd ]] && echo "backup file '${BACKUP_DIR}/${backup_number}.tar.zstd' is empty" && exit 1; | ||
| _log "INFO" "[backup] cleaning ${BACKUP_TMP_DIR} and trap file ${TRAP_FILE}" | ||
| _clean | ||
| [[ ! -s ${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd ]] && _log "ERROR" "[backup] file '${BACKUP_DIR}/${BACKUP_NUMBER}.tar.zstd' is empty" && exit 1 | ||
|
|
||
| echo Done | ||
| _log "INFO" "[backup] ${BACKUP_NUMBER} done" | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,29 +1,47 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eo pipefail | ||
| source "$(dirname "$0")/utils.sh" | ||
|
|
||
| [[ ! $# -eq 1 ]] && echo "Usage: $0 backup_number" && exit 1 | ||
| [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1; | ||
| [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; | ||
| [[ ! $# -eq 1 ]] && _log "ERROR" "Usage: $0 <backup number>" && exit 1 | ||
| [[ -z "${BACKUP_DIR}" ]] && _log "ERROR" "Required 'BACKUP_DIR' env not set" && exit 1 | ||
| [[ -z "${JENKINS_HOME}" ]] && _log "ERROR" "Required 'JENKINS_HOME' env not set" && exit 1 | ||
| BACKUP_NUMBER=$1 | ||
| RESTORE_RETRY_COUNT=${RESTORE_RETRY_COUNT:-10} | ||
| RESTORE_RETRY_INTERVAL=${RESTORE_RETRY_INTERVAL:-10} | ||
|
|
||
| backup_number=$1 | ||
| backup_file="${BACKUP_DIR}/${backup_number}" | ||
| echo "Running restore backup with backup number #${backup_number}" | ||
| # --> Check if another restore process is running (operator restart/crash) | ||
| TRAP_FILE="/tmp/_restore_${BACKUP_NUMBER}_is_running" | ||
| trap "rm -f ${TRAP_FILE}" SIGINT SIGTERM | ||
|
|
||
| if [[ -f "$backup_file.tar.gz" ]]; then | ||
| echo "Old format tar.gz found, restoring it" | ||
| for ((i=0; i<RESTORE_RETRY_COUNT; i++)); do | ||
| [[ ! -f "${TRAP_FILE}" ]] && _log "INFO" "[restore] no other process are running, restoring" && break | ||
| _log "INFO" "[restore] is already running. Waiting for ${RESTORE_RETRY_INTERVAL} seconds..." | ||
| sleep "${RESTORE_RETRY_INTERVAL}" | ||
| done | ||
| [[ -f "${TRAP_FILE}" ]] && { _log "ERROR" "[restore] is still running after waiting ${RESTORE_RETRY_COUNT} time ${RESTORE_RETRY_INTERVAL}s. Exiting."; exit 1; } | ||
| # --< Done | ||
|
|
||
| _log "INFO" "[restore] restore backup with backup number #${BACKUP_NUMBER}" | ||
| touch "${TRAP_FILE}" | ||
| BACKUP_FILE="${BACKUP_DIR}/${BACKUP_NUMBER}" | ||
|
|
||
| if [[ -f "$BACKUP_FILE.tar.gz" ]]; then | ||
| _log "INFO" "[restore] old format tar.gz found, restoring it" | ||
| OPTS="" | ||
| EXT="tar.gz" | ||
| elif [[ -f "$backup_file.tar.zstd" ]]; then | ||
| echo "Backup file found, proceeding" | ||
| elif [[ -f "$BACKUP_FILE.tar.zstd" ]]; then | ||
| _log "INFO" "[restore] Backup file found, proceeding" | ||
| OPTS="--zstd" | ||
| EXT="tar.zstd" | ||
| else | ||
| echo "ERR: Backup file not found: $backup_file" | ||
| _log "ERROR" "[restore] backup file not found: $BACKUP_FILE" | ||
| exit 1 | ||
| fi | ||
|
|
||
| tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${backup_number}.${EXT}" | ||
| tar $OPTS -C "${JENKINS_HOME}" -xf "${BACKUP_DIR}/${BACKUP_NUMBER}.${EXT}" | ||
|
|
||
| echo Done | ||
| _log "INFO" "[restore] deleting lock file ${TRAP_FILE}" | ||
| test -f "${TRAP_FILE}" && rm -f "${TRAP_FILE}" | ||
| _log "INFO" "[restore] restoring ${BACKUP_NUMBER} Done" | ||
| exit 0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,72 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| set -eo pipefail | ||
| source "$(dirname "$0")/utils.sh" | ||
|
|
||
| [[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1; | ||
| [[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1; | ||
| # Use 60 as default in case BACKUP_CLEANUP_INTERVAL did not set | ||
| BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60} | ||
|
|
||
| # Ensure required environment variables are set | ||
| check_env_var() { | ||
| if [[ -z "${!1}" ]]; then | ||
| _log "ERROR" "Required '$1' environment variable is not set" | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| is_backup_not_exist() { | ||
| local backup_dir="$1" | ||
| # Save the current value of 'set -e' | ||
| local previous_e | ||
| previous_e=$(set +e; :; echo $?) | ||
|
|
||
| # Temporarily turn off 'set -e' | ||
| set +e | ||
|
|
||
| # Run ls command to check if any files matching the pattern exist | ||
| ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1 | ||
|
|
||
| # Store the exit status of the ls command | ||
| local ls_exit_status=$? | ||
|
|
||
| # Restore the previous value of 'set -e' | ||
| [ "$previous_e" = "0" ] && set -e | ||
|
|
||
| # Return true if ls command succeeded (no files found), otherwise return false | ||
| [ $ls_exit_status -ne 0 ] | ||
| } | ||
|
|
||
| # Function to find exceeding backups | ||
| find_exceeding_backups() { | ||
| local backup_dir="$1" | ||
| local backup_count="$2" | ||
| # Check if we have any backup | ||
| if is_backup_not_exist "${backup_dir}"; then | ||
| _log "ERROR" "[run] backups not found in ${backup_dir}" | ||
| return | ||
| fi | ||
| find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1)) | ||
| } | ||
|
|
||
| check_env_var "BACKUP_DIR" | ||
| check_env_var "JENKINS_HOME" | ||
|
|
||
| if [[ -z "${BACKUP_COUNT}" ]]; then | ||
| _log "WARNING" "[run] no BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script" | ||
| else | ||
| _log "INFO" "[run] retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds" | ||
| fi | ||
|
|
||
| while true; | ||
| do | ||
| sleep 10 | ||
| if [[ ! -z "${BACKUP_COUNT}" ]]; then | ||
| echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup" | ||
| #TODO: add the list of exceeding backup before delete | ||
| find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.zstd' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}' | ||
| sleep "$BACKUP_CLEANUP_INTERVAL" | ||
| if [[ -n "${BACKUP_COUNT}" ]]; then | ||
| exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}") | ||
| if [[ -n "$exceeding_backups" ]]; then | ||
| _log "INFO" "[run] removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')" | ||
| echo "$exceeding_backups" | while read -r file; do | ||
| rm "${BACKUP_DIR}/${file}" | ||
| done | ||
| fi | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| #!/usr/bin/env bash | ||
| # Common utils | ||
|
|
||
| _log() { | ||
| local level="$1" | ||
| local message="$2" | ||
| local timestamp=$(date '+%Y-%m-%d %H:%M:%S') | ||
| if [[ "$level" =~ ^(ERROR|ERR|error|err)$ ]]; then | ||
| echo "${timestamp} - ${level} - ${message}" > /proc/1/fd/2 | ||
| else | ||
| echo "${timestamp} - ${level} - ${message}" > /proc/1/fd/1 | ||
| echo "${timestamp} - ${level} - ${message}" >&2 | ||
| fi | ||
| } |
Oops, something went wrong.