pbr.host

#!/usr/bin/env bash

set -e
set -o pipefail

umask o=rwx,go=

show_help() {
    echo "# pbr.sh — provisioning & backup & restore cycle for small hosts"
    echo
    echo "> pbr.host — host machine pbr.sh"
    echo
    echo "Usage: $0 {backup|restore|help}"
    echo
    echo "Examples:"
    echo
    echo "  pbr.host backup                        - do backup"
    echo "  pbr.host restore                       - do restore"
    echo "  pbr.host help                          - display help message"
    echo
}

# Regexp for removing unnecessary lines from logs (for notifications)
backup_log_start_pattern="using parent|start scan"

# Variables
restore_folder="./restore"
log_file_path="./pbr_log.txt"
log_separator="--------------------"
mysql_credentials_config_name="backup.cnf"
mysql_backup_file_name="docker_mysql_dump.sql"

# All working files used in sciprt should be listed in this array,
# so we can clean them up after script error / exit
working_files=()
is_mysql_credentials_uploaded_to_container=0

# Echo error string to stderr
# Exit if not error message provided
echo_err() {
  [ -z "$1" ] && {
    echo 'Error message not provided via $1' >&2
    exit 1
  }

  printf "$1\n" >&2
}

# TODO: ability to see help here (before any checks)

# Check for required host toolset
check_host() {
  required_host_programms=(restic curl docker)
  for prog in $required_host_programms; do
    if ! [ -n "$(command -v $prog)" ]; then
      echo_err "$prog not installed or missing on host machine"
      exit 1
    fi
  done
}

load_config() {
  if [ -n "$1" ]; then
    if [ -f "$1" ]; then
      secrets_file="$1"
    elif [ -d "$1" ]; then
      if [ -f "$1/settings.conf" ]; then
        secrets_file="$1/settings.conf"
      else
        echo_err "Can't find settings.conf file in secrets directory"
        exit 1
      fi
    else
      echo_err "Can't access $1"
      exit 1
    fi
  else
    secrets_file=${SECRETS_FILE:-~/.config/.secrets/settings.conf}
  fi

  source $secrets_file || {
    echo_err "Can't load secrets file"
    exit 1
  }
}

setup_variables() {
  if [ -z "$restoration_mode" ]; then
    restoration_mode=${RESTORATION_MODE:-false}
  fi
}

# Remove all created working files on script exit
finalize() {
  # Remove files from host
  for file in "${working_files[@]}"; do
    # echo "removing working file $file"
    rm -rf "${file}"
  done

  working_files=()

  # Remove files from containers
  if [[ $is_mysql_credentials_uploaded_to_container == 1 ]]; then
    mysql_id=$(get_mysql_container_id 2> /dev/null)
    if [ -n "$mysql_id" ]; then
      remove_mysql_credentials_from_container $mysql_id
    fi
  fi
}

trap finalize EXIT ERR

# $1 -> repository name
init_backup_repository() {
  # if repo not exists
  if ! restic --password-file=<(echo $backup_storage_password) --repo="$1" cat config > /dev/null; then
    # then init new one
    restic --password-file=<(echo $backup_storage_password) --repo="$1" init || {
      echo_err "Failed initializing restic repository $1"
      exit 1
    }
  fi
}

# $1 -> bold title message
# $2 -> log message
# No return 1 or abort here, silently fail
notify_to_telegram() {
  if [ -z "$1" ] || [ -z "$2" ] || [ -z "$telegram_bot_token" ] || [ -z "$telegram_chat_id" ]; then
    echo_err "Skipping telegram notifications (reason: not configured)"
    return 0
  fi

  text="<b>$1</b>\n$2"
  message_json="{
    \"chat_id\": $telegram_chat_id,
    \"text\": \"$text\",
    \"parse_mode\": \"html\",
    \"disable_notification\": true
  }"

  echo "$text"

  curl --silent --show-error --fail -X POST \
    -H 'Content-Type: application/json' \
    -d "$message_json" https://api.telegram.org/bot${telegram_bot_token}/sendMessage > /dev/null || \
      echo_err "sending to telegram failed"

  return 0
}

# Convert "a,b" string to list
# "a, b,c" => a b c
# "a ,b" => a b
#
# "a a,b" => return 1, we don't support names with spaces like "my volume"
split_by_comma() {
  [ -z "$1" ] && {
    echo_err 'String for splitting not provided via $1'
    return 1
  }

  local volumes=()

  local OLD_IFS="$IFS";
  local IFS=','

  for v in $1; do
    # remove space before and after volume name
    volume_name=$(echo "$v" | sed 's/\(^ \| $\)//g') || {
      echo_err "volume name filtering failed"
      return 1
    }

    # if still found any space then abort script
    if [[ "$volume_name" =~ " " ]]; then
      echo_err "Names with spaces are forbidden"
      return 1
    fi

    volumes+=("$volume_name")
  done

  local IFS="$OLD_IFS"

  echo ${volumes[@]}
  return 0
}

# Make directories argument list
# convert "a,b" to
#   /a /b
#
# if --with-path-glob-ending provided in $2 then convert to:
#   /a/* /b/*
volumes_string_to_mounted_dir_list() {
  local volumes_list=()
  local dir_list=()

  volumes_list=$(split_by_comma "$1") || return 1

  for v in $volumes_list; do
    if [ "$2" = "--with-path-glob-ending" ]; then
      dir_list+=("/$v/*")
    else
      dir_list+=("/$v")
    fi
  done

  echo ${dir_list[@]}
  return 0
}

folders_string_to_mounted_dir_list() {
  local volumes_list=()
  local dir_list=()

  volumes_list=$(split_by_comma "$1") || return 1
  dir_list=()

  for v in $volumes_list; do
    dir_list+=("/host$v")
  done

  echo ${dir_list[@]}
  return 0
}

# Make docker volumes mount argument list
# convert from "a,b" to
#   -v a:/a/ -v b:/b/
# if --readonly provided in $2 then convert to:
#   -v a:/a/:ro -v b:/b/:ro
volumes_string_to_docker_volumes_arg() {
  local volumes_list=()

  volumes_list=$(split_by_comma "$1") || return 1
  args_list=()

  for v in $volumes_list; do
    args_list+=("-v")

    if [ "$2" = "--readonly" ]; then
      args_list+=("$v:/$v/:ro")
    else
      args_list+=("$v:/$v/")
    fi
  done

  echo ${args_list[@]}
  return 0
}

# Make host folders mount argument list
# convert from "/home/user/folder" to
#   -v /home/user/folder:/host/home/user/folder/
folders_string_to_docker_volumes_arg() {
  local volumes_list=()

  volumes_list=$(split_by_comma "$1") || return 1
  args_list=()

  for v in $volumes_list; do
    args_list+=("-v")
    args_list+=("$v:/host$v:ro")
  done

  echo ${args_list[@]}
  return 0
}

# Remove old restore folder is exists
# Create new one and return absolute path via stdin
prepare_restore_folder() {
  [ -d $restore_folder ] && rm -r $restore_folder      # ensure no restore folder present
  mkdir $restore_folder                                  # create new one

  restore_folder_abs_path=$(realpath $restore_folder)  # construct absolute path
  echo $restore_folder_abs_path                        # echo absolute path to stdout

  return 0
}

get_mysql_container_id() {
  mysql_id=$(docker ps --filter "ancestor=mysql" -q) || {
    echo_err "Can't fetch mysql docker container id"
    return 1
  }

  [ -z "$mysql_id" ] && {
    echo_err "Mysql container not running"
    return 1
  }

  echo $mysql_id
}

# Create config file suitable for mysql
# Copy this file to container
#
# $1 -> mysql container id
# $2 -> if --restore provided, then restore config will be used (for mysql binary)
#
# NOTICE: config file should be deleted after with remove_mysql_credentials_from_container
copy_mysql_credentials_to_container() {
  [ -z "$1" ] && { echo_err 'mysql container id not provided via $1' && exit 1; }

  # Determintate type of process, is it restore or backup?
  if [ "$2" = "--restore" ]; then binary_name="mysql"; else binary_name="mysqldump"; fi

  # Consturct mysql configuration file with credentials
  echo -e "[${binary_name}]\nuser=$docker_mysql_user\npassword=$docker_mysql_password" > \
    ./${mysql_credentials_config_name}

  working_files+=("./${mysql_credentials_config_name}")
  is_mysql_credentials_uploaded_to_container=1

  # Copy configuration file to container
  docker cp ./${mysql_credentials_config_name} ${1}:/${mysql_credentials_config_name} || {
    echo_err "Docker copy failed"
    return 1
  }
}

# $1 -> mysql container id
remove_mysql_credentials_from_container() {
  docker exec "$1" sh -e -c "rm -f /${mysql_credentials_config_name}" || {
    echo_err "Can't remove config file from container"
    return 1
  }
}

# $1 -> volumes list to check
check_docker_volumes_presence() {
  [ -z "$1" ] && { echo_err "Volumes list not provided in $1" && return 1; }

  local volumes_list

  volumes_list="$(docker volume list -q)" || {
    echo_err "Can't fetch docker volumes list"
    return 1
  }

  for v in "pechorindev"; do
    if ! [[ "$volumes_list" =~ "$v" ]]; then
      echo_err "Docker volume not created on host machine"
      return 1
    fi
  done

  return 0
}

do_data_backup() {
  if [ -z "$data_rclone_repo" ]; then
    echo "Skipping data backup, data_rclone_repo not configured"
    return 0
  fi

  if [ -z "$docker_backup_volumes" ] && [ -z "$host_backup_folders" ]; then
    echo "Skipping data backup, docker_backup_volumes nor host_backup_folders not set"
    return 0
  fi

  init_backup_repository "$data_rclone_repo" || return 1

  local volumes_list=()
  local volumes_arg_list=()
  local mounted_dir_list=()

  if [ -n "$docker_backup_volumes" ]; then
    local tmp_list=()

    tmp_list=$(split_by_comma "$docker_backup_volumes") || return 1
    volumes_list+=(${tmp_list[@]})

    check_docker_volumes_presence "$tmp_list" || return 1

    tmp_list=$(volumes_string_to_docker_volumes_arg "$docker_backup_volumes" --readonly) || return 1
    volumes_arg_list+=(${tmp_list[@]})

    tmp_list=$(volumes_string_to_mounted_dir_list "$docker_backup_volumes") || return 1
    mounted_dir_list+=(${tmp_list[@]})
  fi

  if [ -n "$host_backup_folders" ]; then
    local tmp_list=()

    tmp_list=$(split_by_comma "$host_backup_folders") || return 1
    volumes_list+=(${tmp_list[@]})

    tmp_list=$(folders_string_to_docker_volumes_arg "$host_backup_folders") || return 1
    volumes_arg_list+=(${tmp_list[@]})

    tmp_list=$(folders_string_to_mounted_dir_list "$host_backup_folders") || return 1
    mounted_dir_list+=(${tmp_list[@]})
  fi

  # if only host folders backup needed then do not create container for backups
  # this is mainly for perfomance on osx hosts backups
  if [ -n "$host_backup_folders" ] && [ -z "$docker_backup_volumes" ]; then
    echo "running host-only folders backup"
    local host_folders
    host_folders=$(split_by_comma "$host_backup_folders") || return 1

    echo "backup args -> $restic_backup_args"

    backup_log=$(restic -v --password-file=<(echo $backup_storage_password) $restic_backup_args \
                           --host=$backup_name \
                           --repo=$data_rclone_repo backup ${host_folders[@]})
  else
    echo "running docker backup"
    # otherwise:
    # Bakcup command will run in container:
    backup_command="restic -v --password-file=<(echo \$backup_storage_password) $restic_backup_args \
                              --host=\$backup_name \
                              --repo=\$data_rclone_repo backup ${mounted_dir_list[@]}"

    # Run backup inside container
    # - mount backup volumes
    # - mount rclone config
    # - mount backup settings
    # - run backup inside alpine container, but use bash instead of ash internally
    #   because we need bash process substituion <() for safety providing password to processes
    backup_log=$(docker run --rm --name "volumes_backup" \
      ${volumes_arg_list[@]} \
      -v "$(realpath ~/.config/rclone/rclone.conf):/root/.config/rclone/rclone.conf:ro" \
      -v "$(realpath "$secrets_file"):/settings.conf:ro" \
      frolvlad/alpine-bash bash -e -c "
        apk update && apk add rclone restic
        source /settings.conf
        $backup_command
    ") || {
      echo_err "docker volumes backup failed: $backup_log"
      return 1
    }
  fi

  # Log to stdout
  printf "$backup_log\n"

  # remove unnecessary data from telegram message and notify
  filtered_log=$(printf "$backup_log" | sed -n -E -e "/$backup_log_start_pattern/,\$p")
  notify_to_telegram "$backup_name volumes backup" "$filtered_log"

  return 0
}

do_data_restore() {
  if [ -z "$data_rclone_repo" ]; then
    echo "Skipping data restore, data_rclone_repo not configured"
    return 0
  fi

  local volumes_list

  if [ -n "$docker_backup_volumes" ]; then
    volumes_list=$(split_by_comma "$docker_backup_volumes") || return 1
    check_docker_volumes_presence "$volumes_list" || return 1

    volumes_arg_list=$(volumes_string_to_docker_volumes_arg $docker_backup_volumes) || return 1
    mounted_dir_list=$(volumes_string_to_mounted_dir_list $docker_backup_volumes) || return 1
    mounted_dir_list_with_glob=$(volumes_string_to_mounted_dir_list $docker_backup_volumes --with-path-glob-ending) || return 1
  fi

  restore_folder_path=$(prepare_restore_folder)
  working_files+=($restore_folder_path)

  # Make this configurable?
  restic --password-file=<(echo $backup_storage_password) -v --repo="$data_rclone_repo" check || return 1

  # Do restore on host folder
  restic --password-file=<(echo $backup_storage_password) -v --repo="$data_rclone_repo" restore \
           latest --target="$restore_folder_path" || return 1

  restored_size=($(du -sh $restore_folder_path))

  # Get snapshot id
  restic_snapshot=$(restic --password-file=<(echo $backup_storage_password) -v --repo="$data_rclone_repo" snapshots -c --latest 1) || return 1
  snapshot_id=$(echo "$restic_snapshot" | grep -E -e '^[a-z0-9]{8}' | cut -f 1 -d ' ') || {
    echo_err "can't fetch snapshot id"
    return 1
  }

  if [ "$restoration_mode" = 'true' ]; then
    if [ -n "$docker_backup_volumes" ]; then
      # Mount restore folder and docker volumes in container, and then do restoration
      restore_log=$(docker run --rm --name "restore" \
        ${volumes_arg_list[@]} \
        -v "$restore_folder_path:/restore/" \
        frolvlad:alpine-bash bash -e -c "
          apk add bash
          rm -rf ${mounted_dir_list_with_glob[@]}
          for v in ${volumes_list[@]}; do
            cp -r /restore/\${v}/* /\${v}/
          done
      ") || {
        echo_err "docker volumes restore failed: $restore_log"
        return 1
      }

      # Log to stdout
      printf "$restore_log\n"
    fi

    if [ -n "$host_backup_folders" ]; then
      local host_restore_to="$HOME/pbr.restore/host-folders-restore-$snapshot_id"

      echo "Restoration host folders is not implemented, please do it by yourself"
      echo "Processing regular restoring to $host_restore_to"

      folders_list=$(split_by_comma "$host_backup_folders") || return 1

      for folder in ${folder_list[@]}; do
        mkdir -p "$host_restore_to"
        mv -v "$restore_folder_path/$folder" "$host_restore_to/$folder"
      done
    fi
  else
    mkdir -p $HOME/pbr.restore/
    mv "$restore_folder_path" "$HOME/pbr.restore/data-restore-$snapshot_id"
  fi

  notify_to_telegram "$backup_name $snapshot_id data restore finished" "total size: ${restored_size[0]}"

  return 0
}

do_db_backup() {
  if [ -z "$db_rclone_repo" ]; then
    echo "Skipping db backup"
    return 0
  fi

  init_backup_repository "$db_rclone_repo" || return 1

  dumps=()

  # Mysql docker backup
  if [ -n "$docker_mysql_db_names" ]; then
    mysql_id=$(get_mysql_container_id) || return 1
    copy_mysql_credentials_to_container $mysql_id || return 1

    # Run backup
    # also ensure what config file removed fron container
    mysql_databases=$(split_by_comma $docker_mysql_db_names) || return 1
    docker exec $mysql_id sh -e -c "
      mysqldump --defaults-extra-file=/${mysql_credentials_config_name} \
                --databases $mysql_databases " > ./${mysql_backup_file_name} || {
      echo_err "mysql dump failed"
      return 1
    }

    dumps+=($mysql_backup_file_name)
    working_files+=("./$mysql_backup_file_name")
  fi

  # Postgresql docker backup
  if [ -n "$host_postgresql_db_names" ]; then
    pg_databases=$(split_by_comma $host_postgresql_db_names) || return 1

    # TODO: move to prerequirements part?
    if [ -z "$(command -v pg_dump)" ]; then
      echo_err "pg_dump not installed or missing on host machine"
      return 1
    fi

    for db in $pg_databases; do
      dump_path="./host_postgresql_${db}_dump.sql"

      pg_dump --clean $db > $dump_path

      dumps+=(host_postgresql_${db}_dump.sql)
      working_files+=($dump_path)
    done
  fi

  # if no backups was processed then return
  [ 0 -eq ${#dumps[@]} ] && return 0;

  # otherwise do backup
  backup_log=$(restic -v --password-file=<(echo $backup_storage_password) \
                         --host=$backup_name \
                         --repo=$db_rclone_repo backup ${dumps[@]}) || { echo_err "db backup failed"; return 1; }

  # Log to stdout
  printf "$backup_log\n"

  # remove unnecessary data from telegram message and notify
  filtered_log=$(printf "$backup_log" | sed -n -E -e "/$backup_log_start_pattern/,\$p")
  notify_to_telegram "$backup_name DB backup" "$filtered_log"

  return 0
}

do_db_restore() {
  if [ -z "$db_rclone_repo" ]; then
    echo "Skipping db restore"
    return 0
  fi

  restore_folder_path=$(prepare_restore_folder)
  working_files+=($restore_folder_path)

  # Make this configurable?
  restic --password-file=<(echo $backup_storage_password) -v --repo="$db_rclone_repo" check || return 1

  # Do restore on host folder
  restic --password-file=<(echo $backup_storage_password) -v --repo="$db_rclone_repo" restore latest \
         --target="$restore_folder_path" || return 1

  restored_size=($(du -sh $restore_folder_path))

  if [ "$restoration_mode" = "true" ]; then
    if [ -n "$docker_mysql_db_names" ]; then
      mysql_id=$(get_mysql_container_id) || return 1
      copy_mysql_credentials_to_container $mysql_id --restore || return 1

      # Run restore
      # also ensure what config file removed fron container
      docker exec -i $mysql_id sh -c "mysql --defaults-extra-file=/${mysql_credentials_config_name}" \
        < ${restore_folder_path}/${mysql_backup_file_name} || {
          echo_err "mysql restore failed"
          return 1
      }
    fi

    if [ -n "$host_postgresql_db_names" ]; then
      pg_databases=$(split_by_comma $host_postgresql_db_names) || return 1

      for db in $pg_databases; do
        psql $db < ${restore_folder_path}/host_postgresql_${db}_dump.sql
      done
    fi
  else
    local host_restore_to="$HOME/pbr.restore/db-restore-$snapshot_id"

    mkdir -p $host_restore_to
    mv $restore_folder_path/* $host_restore_to/
  fi

  notify_to_telegram "Restoring $backup_name db backup" "total size: ${restored_size[0]}"
}

# Execution part
if [ "$1" = "backup" ]; then
  current_action="backup"
elif [ "$1" = "restore" ]; then
  current_action="restore"
elif [ "$1" = "help" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
  show_help
  exit 1
else
  # TODO: add usage
  echo_err "action not provided: pbr.host backup\n"
  show_help
  exit 1
fi

run() {
  echo -e "\n$log_separator"
  echo "Starting $current_action at $(date)"

  if [ "$current_action" = "restore" ]; then
    echo "Restoration mode enabled: $restoration_mode"

    do_data_restore || return 1
    do_db_restore || return 1
  elif  [ "$current_action" = "backup" ]; then
    do_data_backup || return 1
    do_db_backup || return 1
  fi

  echo -e "$current_action completed at $(date)"
}


echo "Running pbr.sh (on host machine)"

check_host
load_config "$2"
setup_variables

echo "at $ssh_connection_string"

run $1 &>> $log_file_path || {
  notify_to_telegram "$current_action script failed" \
                     "inspect errors on host ~/$log_file_path"
}

echo "pbr.sh finished, log can be found in $log_file_path"