functions

#!/bin/bash

set -euE
set -o pipefail

# Library of functions for the kayobe development environment.

###############################################################################
# Context variables
# These get set according to execution context. They may not have been set
# if you haven't reach a certain point in the execution. Instead of using
# globals we could use assosociate arrays to namespace these, but it is
# unclear if it is worth the effort.

# Set after calling activate_kayobe_env and points to the env that is
# currently activated.
export KAYOBE_AUTOMATION_CONTEXT_ENV_PATH=

###############################################################################
# Globals

FUNCTIONS_PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export KAYOBE_AUTOMATION_UTILS_PATH="$FUNCTIONS_PARENT/utils"
export KAYOBE_AUTOMATION_REPO_ROOT="$FUNCTIONS_PARENT/.."
export KAYOBE_AUTOMATION_CONFIG_PATH="${KAYOBE_AUTOMATION_CONFIG_PATH:-$KAYOBE_AUTOMATION_REPO_ROOT/.automation.conf}"

export _LOG_LEVEL_DEBUG=4000
export _LOG_LEVEL_INFO=3000
export _LOG_LEVEL_WARN=2000
export _LOG_LEVEL_ERROR=1000

###############################################################################
# Logging

function log_level {
    case "${KAYOBE_AUTOMATION_LOG_LEVEL:-warn}" in
    debug)
        echo -n "$_LOG_LEVEL_DEBUG"
        ;;
    info)
        echo -n "$_LOG_LEVEL_INFO"
        ;;

    error)
        echo -n "$_LOG_LEVEL_ERROR"
        ;;

    *)
        echo -n "$_LOG_LEVEL_WARN"
        ;;
    esac
}

function log_info {
    if [ "$(log_level)" -ge $_LOG_LEVEL_INFO ]; then
        msg="[INFO]: $@"
        echo "$msg" 1>&2
        if [ ! -z ${LOGDIR:+x} ]; then
            echo "$msg" >>"${LOGDIR}/kayobe-automation.log"
        fi
    fi
}

function log_warn {
    if [ "$(log_level)" -ge $_LOG_LEVEL_WARN ]; then
        msg="[WARNING]: $@"
        echo "$msg" 1>&2
        if [ ! -z ${LOGDIR:+x} ]; then
            echo "$msg" >>"${LOGDIR}/kayobe-automation.log"
        fi
    fi
}

function log_debug {
    if [ "$(log_level)" -ge $_LOG_LEVEL_DEBUG ]; then
        msg="[DEBUG]: $@"
        echo "$msg" 1>&2
        if [ ! -z ${LOGDIR:+x} ]; then
            echo "$msg" >>"${LOGDIR}/kayobe-automation.log"
        fi
    fi
}

function log_error {
    if [ "$(log_level)" -ge $_LOG_LEVEL_ERROR ]; then
        msg="[ERROR]: $@"
        echo "$msg" 1>&2
        if [ ! -z ${LOGDIR:+x} ]; then
            echo "$msg" >>"${LOGDIR}/kayobe-automation.log"
        fi
    fi
}

###############################################################################
# Configuration

function config_defaults {
    export USER=${USER:-$(whoami)}
    # Set default values for kayobe automation.
    # Deprecated: Add a requirements.txt to kayobe-config instead
    export KAYOBE_URI="${KAYOBE_URI:-https://github.com/openstack/kayobe}"

    KAYOBE_PIP_INSTALL_ARGS_DEFAULT="${KAYOBE_PIP_INSTALL_ARGS_EXTRA:-}"
    export KAYOBE_PIP_INSTALL_ARGS="${KAYOBE_PIP_INSTALL_ARGS-$KAYOBE_PIP_INSTALL_ARGS_DEFAULT}"

    export KAYOBE_CONFIG_SOURCE_PATH="${KAYOBE_CONFIG_SOURCE_PATH:-${FUNCTIONS_PARENT}/..}"

    # Additional arguments to pass to kayobe commands.
    export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-}

    # Additional arguments to pass to kayobe that are safe from shell injection.
    export KAYOBE_TAGS=${KAYOBE_TAGS:-}
    export KAYOBE_LIMIT=${KAYOBE_LIMIT:-}
    export KOLLA_TAGS=${KOLLA_TAGS:-}
    export KOLLA_LIMIT=${KOLLA_LIMIT:-}

    export KAYOBE_AUTOMATION_LOG_LEVEL=${KAYOBE_AUTOMATION_LOG_LEVEL:-warn}
    # When to pause execution. This is useful for debugging. Can be one of:
    #   - on-error
    # You must run the script with errtrace (-E) if using errexit (-e) for the
    # on-error trap to function.
    export KAYOBE_AUTOMATION_BREAK=${KAYOBE_AUTOMATION_BREAK:-}

    # Log directory in case of errors
    export LOGDIR=${LOGDIR:-/tmp/logs}

    # Ansible defaults
    export ANSIBLE_FORCE_COLOR=${ANSIBLE_FORCE_COLOR:-True}

    # SSH key to use. This exists for documentation purposes. It is not recommended
    # to set it in config.sh
    export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY=${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY:-}

    # Name of the SSH key to be used when connecting to kayobe hosts.
    export KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME=${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME:-id_rsa}

    # TODO: validate/auto detect?
    # Determines which helper to use: gitlab, github, etc, defaults to none.
    export KAYOBE_AUTOMATION_PR_TYPE=${KAYOBE_AUTOMATION_PR_TYPE:-}

    # Kayobe environment to use. Note this is overridden by .environment.
    export KAYOBE_ENVIRONMENT=${KAYOBE_ENVIRONMENT:-}

    # Flag to control whether images are pushed. Can be one of: 0, 1. Where
    # zero === false and one === true.
    export KAYOBE_AUTOMATION_PUSH_IMAGE=${KAYOBE_AUTOMATION_PUSH_IMAGE:-0}
    # Flag to pass --skip-prechecks as CLI option
    export KAYOBE_AUTOMATION_SKIP_PRECHECKS=${KAYOBE_AUTOMATION_SKIP_PRECHECKS:-0}

    if [[ ! -z "$KAYOBE_AUTOMATION_PR_TYPE" ]]; then
        config_extension_pull_request
    fi
}

function config_extension_pull_request {
    # These are only set if KAYOBE_AUTOMATION_PR_TYPE is set.

    # Which branch should we create a PR into?
    export KAYOBE_AUTOMATION_PR_TARGET_BRANCH=${KAYOBE_AUTOMATION_PR_TARGET_BRANCH:-}
    # For convienience strip refs/heads prefix
    export KAYOBE_AUTOMATION_PR_TARGET_BRANCH=${KAYOBE_AUTOMATION_PR_TARGET_BRANCH#refs/heads/}
    # Auth token to used to authenticate against the API
    export KAYOBE_AUTOMATION_PR_AUTH_TOKEN=${KAYOBE_AUTOMATION_PR_AUTH_TOKEN:-}
    # Do we only care about a subset of the files? e.g "**/*.yml"
    export KAYOBE_AUTOMATION_PR_PATHSPEC=${KAYOBE_AUTOMATION_PR_PATHSPEC:-}

    export KAYOBE_AUTOMATION_PR_URL_DEFAULT=
    if [[ "${KAYOBE_AUTOMATION_PR_TYPE,,}" == "gitlab" ]]; then
        # default to gitlab.com
        export KAYOBE_AUTOMATION_PR_URL_DEFAULT="https://gitlab.com/api/v4/projects/${KAYOBE_AUTOMATION_PR_GITLAB_PROJECT_ID:-}/merge_requests"
    fi

    # API endpoint used to created pull request
    export KAYOBE_AUTOMATION_PR_URL=${KAYOBE_AUTOMATION_PR_URL:-$KAYOBE_AUTOMATION_PR_URL_DEFAULT}
    # commit author
    export KAYOBE_AUTOMATION_PR_USERNAME=${KAYOBE_AUTOMATION_PR_USERNAME:-kayobe-automation}
    # commit email
    export KAYOBE_AUTOMATION_PR_EMAIL=${KAYOBE_AUTOMATION_PR_EMAIL:-kayobe-automation@example.org}
    # Where to push the commit
    export KAYOBE_AUTOMATION_PR_REMOTE=${KAYOBE_AUTOMATION_PR_REMOTE:-}
}

function config_set {
    # Source the configuration file, config.sh
    source "${KAYOBE_AUTOMATION_CONFIG_PATH}/config.sh"
}

function config_check {
    # Check the configuration environment variables.
    set +u
    if [[ ! -z "$KAYOBE_BRANCH" ]]; then
        log_info "KAYOBE_BRANCH and KAYOBE_URI are deprecated. Add a requirements.txt file to your kayobe config instead."
    fi
    if [[ -z "$KAYOBE_CONFIG_SOURCE_PATH" ]]; then
        if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then
            die $LINENO "KAYOBE_CONFIG_SOURCE_PATH must be set"
        fi
    fi
    if [[ ! -e "$KAYOBE_CONFIG_SOURCE_PATH" ]]; then
        if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then
            die $LINENO "Kayobe configuration path $KAYOBE_CONFIG_SOURCE_PATH does not exist"
        fi
    fi

    if [[ "$KAYOBE_AUTOMATION_PUSH_IMAGE" != 0 ]] && [[ "$KAYOBE_AUTOMATION_PUSH_IMAGE" != 1 ]]; then
        die $LINENO "KAYOBE_AUTOMATION_PUSH_IMAGE must be set to either: 0, or 1"
    fi

    if [[ "$KAYOBE_AUTOMATION_SKIP_PRECHECKS" != 0 ]] && [[ "$KAYOBE_AUTOMATION_SKIP_PRECHECKS" != 1 ]]; then
        die $LINENO "KAYOBE_AUTOMATION_SKIP_PRECHECKS must be set to either: 0, or 1"
    fi

    # PR validation
    if [[ ! -z "$KAYOBE_AUTOMATION_PR_TYPE" ]] && [[ "$KAYOBE_AUTOMATION_PR_TYPE" != "disabled" ]]; then
        # Shared validation
        if [[ -z "${KAYOBE_AUTOMATION_PR_TARGET_BRANCH}" ]]; then
            die $LINENO "KAYOBE_AUTOMATION_PR_TARGET_BRANCH must be set"
        fi
        if [[ -z "${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}" ]]; then
            die $LINENO "KAYOBE_AUTOMATION_PR_AUTH_TOKEN must be set"
        fi
        if [[ -z "${KAYOBE_AUTOMATION_PR_TITLE}" ]]; then
            die $LINENO "KAYOBE_AUTOMATION_PR_TITLE must be set"
        fi
        if [[ -z "${KAYOBE_AUTOMATION_PR_URL}" ]]; then
            die $LINENO "KAYOBE_AUTOMATION_PR_URL must be set"
        fi
        if [[ -z "${KAYOBE_AUTOMATION_PR_REMOTE}" ]]; then
            die $LINENO "KAYOBE_AUTOMATION_PR_REMOTE must be set"
        fi
        # Gitlab specific valiation
        if [[ "${KAYOBE_AUTOMATION_PR_TYPE,,}" == "gitlab" ]]; then
            if [[ -z "${KAYOBE_AUTOMATION_PR_GITLAB_PROJECT_ID}" ]]; then
                die $LINENO "KAYOBE_AUTOMATION_PR_GITLAB_PROJECT_ID must be set"
            fi
        fi
        # Github specific valiation
        if [[ "${KAYOBE_AUTOMATION_PR_TYPE,,}" == "github" ]]; then
            if [[ -z "${KAYOBE_AUTOMATION_PR_GITHUB_USER}" ]]; then
                die $LINENO "KAYOBE_AUTOMATION_PR_GITHUB_USER must be set"
            fi
        fi
    fi

    set -u
}

function config_init {
    call_with_hooks config_set
    call_with_hooks config_defaults
    call_with_hooks config_check
    # Ensure log file exists as we try and append to it in the logging functions.
    mkdir -p "$LOGDIR"
    touch "$LOGDIR"/kayobe-automation.log
}

function validate {
    true
}

###############################################################################
# Git utilities, see:
#    - https://stackoverflow.com/questions/2657935/checking-for-a-dirty-index-or-untracked-files-with-git

function git_modified_count {
    echo $(git status --porcelain 2>/dev/null | egrep "^(M| M)" | wc -l)
}

function git_untracked_count {
    echo $(git status --porcelain 2>/dev/null | grep "^??" | wc -l)
}

function is_git_dirty {
    local path_spec=()
    if [ "$#" -gt 0 ]; then
        local path_spec=("--" "$@")
    fi
    log_debug "git_is_dirty::path_spec: " ${path_spec[@]}
    let result="$(git status --porcelain "${path_spec[@]}" 2>/dev/null | wc -l)"
    log_debug "git_is_dirty::result: $result"
    if [ $result -eq 0 ]; then
        return 1
    fi
    return 0
}

###############################################################################
# Pull request utilities

function _git_commit {
    local branch_name="$1"

    if [ ! -z ${KAYOBE_AUTOMATION_PR_PATHSPEC:+x} ]; then
        for filespec in ${KAYOBE_AUTOMATION_PR_PATHSPEC}; do
            # It is accept for one or more filespec to not match, so we
            # swallow the failure
            git add "${filespec}" ||
                log_info "_git_commit::filespec: $filespec matches zero files"
        done
    else
        git add -A
    fi

    git config --local user.name ${KAYOBE_AUTOMATION_PR_USERNAME}
    git config --local user.email ${KAYOBE_AUTOMATION_PR_EMAIL}

    git commit -m "${KAYOBE_AUTOMATION_PR_TITLE}"

    git remote add kayobe-automation ${KAYOBE_AUTOMATION_PR_REMOTE}

    git push kayobe-automation HEAD:refs/heads/$branch_name
}

function gitlab_merge_request {
    # TODO: optional params
    #         \"assignee_id\":\"${GITLAB_USER_ID}\"

    local branch_name="$(pull_request_branch_name)"

    _git_commit "${branch_name}"

    local body="{
        \"id\": ${KAYOBE_AUTOMATION_PR_GITLAB_PROJECT_ID},
        \"source_branch\": \"${branch_name}\",
        \"target_branch\": \"${KAYOBE_AUTOMATION_PR_TARGET_BRANCH}\",
        \"remove_source_branch\": true,
        \"title\": \"${KAYOBE_AUTOMATION_PR_TITLE}\"
    }"

    log_info "Submitting merge request to: $KAYOBE_AUTOMATION_PR_URL"

    curl -X POST "$KAYOBE_AUTOMATION_PR_URL" \
        --header "PRIVATE-TOKEN:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}" \
        --header "Content-Type: application/json" \
        --fail \
        --data "${body}" | jq "."

    log_info "Merge request created sucessfully"
}

function github_pull_request {
    local branch_name="$(pull_request_branch_name)"

    _git_commit "${branch_name}"

    local body="{
        \"head\": \"${branch_name}\",
        \"base\": \"${KAYOBE_AUTOMATION_PR_TARGET_BRANCH}\",
        \"title\": \"${KAYOBE_AUTOMATION_PR_TITLE}\"
    }"

    log_debug "Pull request body: $body"

    # Support 0Auth token? Current method requires a personal access token, but
    # we might have access to 0Auth token in the gitlab runner? Recommended to
    # to setup a service account otherwise.
    # --header "Authorization: token ${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}" \

    log_info "Submitting merge request to: $KAYOBE_AUTOMATION_PR_URL"

    # https://docs.github.com/en/rest/reference/pulls
    curl -X POST "$KAYOBE_AUTOMATION_PR_URL" \
        --user "${KAYOBE_AUTOMATION_PR_GITHUB_USER}:${KAYOBE_AUTOMATION_PR_AUTH_TOKEN}" \
        --header "Accept: application/vnd.github.v3+json" \
        --header "Content-Type: application/json" \
        --fail \
        --data "${body}" | jq "."

    log_info "Pull request created sucessfully"
}

function pull_request_branch_name {
    local uuid=$(uuidgen)
    local branch_name="$(basename $0)/${uuid}"
    echo "$branch_name"
}

function _pull_request {
    if [[ -z "$KAYOBE_AUTOMATION_PR_TYPE" ]] || [[ "$KAYOBE_AUTOMATION_PR_TYPE" == "disabled" ]]; then
        log_info "Not creating PR as KAYOBE_AUTOMATION_PR_TYPE is not set"
        return
    elif [[ "${KAYOBE_AUTOMATION_PR_TYPE,,}" == "gitlab" ]]; then
        gitlab_merge_request
        return
    elif [[ "${KAYOBE_AUTOMATION_PR_TYPE,,}" == "github" ]]; then
        github_pull_request
        return
    fi
    die $LINENO "KAYOBE_AUTOMATION_PR_TYPE: $KAYOBE_AUTOMATION_PR_TYPE is an invalid value"
}

###############################################################################
# General purpose utility functions

# Prints backtrace info
# filename:lineno:function
# backtrace level
function backtrace {
    local level=$1
    local deep
    deep=$((${#BASH_SOURCE[@]} - 1))
    echo "[Call Trace]"
    while [ $level -le $deep ]; do
        echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep - 1]}:${FUNCNAME[$deep - 1]}"
        deep=$((deep - 1))
    done
}

# Prints line number and "message" then exits
# die $LINENO "message"
function die {
    local exitcode=$?
    set +o xtrace
    local line=$1
    shift
    if [ $exitcode == 0 ]; then
        exitcode=1
    fi
    backtrace 2
    _err $line "$*"
    # Give buffers a second to flush
    sleep 1
    exit $exitcode
}

# Prints line number and "message" in error format
# _err $LINENO "message"
function _err {
    local exitcode=$?
    local xtrace
    xtrace=$(set +o | grep xtrace)
    set +o xtrace
    local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
    echo "$msg" 1>&2
    if [ ! -z ${LOGDIR:+x} ]; then
        echo "$msg" >>"${LOGDIR}/kayobe-automation.log"
    fi
    $xtrace
    return $exitcode
}

function environment_diagnostics {
    log_info "KAYOBE_ENVIRONMENT: $KAYOBE_ENVIRONMENT"
    set -x
    whoami
    set +x
}

function pause {
    echo "Press [Enter] to continue..."
    read -p "$*"
}

function function_exists {
    declare -f -F $1 >/dev/null
    return $?
}

function call {
    if function_exists $1; then
        log_debug "Entering $1"
        "${@}"
        log_debug "Exiting $1"
    fi
}

function call_with_hooks {
    call "pre_$1"
    call "$@"
    call "post_$1"
}

function is_absolute_path {
    path="$1"
    case "$path" in
    /*) true ;;
    *) false ;;
    esac
}

function clean_copy {
    local src=$1
    local dest=$2
    if [ -d "$dest" ]; then
        rm -rf "$dest"
    fi
    sudo_if_available cp -rf "$src" "$dest"
    sudo_if_available chown -Rf $USER:$USER "$dest"
}

##############################################################################
# Traps

function on_error {
    if [ ${KAYOBE_AUTOMATION_BREAK:-invalid,,} == "on-error" ]; then
        log_error "An error occured, pausing to allow debug access"
        pause
    fi
}

###############################################################################
# Installation

function is_dnf {
    test -e /usr/bin/dnf
}

function workaround_start_sshd {
    # Kayobe will try and keyscan localhost which will fail if sshd not running
    # See: https://github.com/openstack/kayobe/blob/869185ea7be5d6b5b21c964a620839d5475196fd/ansible/roles/bootstrap/tasks/main.yml#L31
    if ! pgrep -x "sshd" >/dev/null; then
        sudo_if_available /usr/bin/ssh-keygen -A
        sudo_if_available /usr/sbin/sshd &
    fi
}

function workaround_git_config {
    # Workaround: fatal: unable to auto-detect email address (got 'stack@runner-bkqvjp4m-project-21118777-concurrent-0.(none)')
    # Might not be the best place to put this, but it is causing control_host_bootstrap to fail.
    git config --global user.name "Kayobe Automation"
    git config --global user.email "kayobe-automation@example.org"
}

function workarounds {
    call_with_hooks workaround_start_sshd
    call_with_hooks workaround_git_config
}

function sudo_if_available {
    if [[ -e "/usr/bin/sudo" ]]; then
        sudo --preserve-env=http_proxy,https_proxy,no_proxy,ftp_proxy,HTTP_PROXY,HTTPS_PROXY,NO_PROXY "$@"
        return
    fi
    "$@"
}

function install_dependencies {
    echo "Installing package dependencies for kayobe"
    if is_dnf; then
        sudo_if_available dnf -y install gcc git vim python3-pyyaml libffi-devel openssh-clients jq
    else
        sudo_if_available apt install -y python-dev python3-venv gcc git libffi-dev openssh-client jq
    fi
}

function setup_ssh_agent {
    if [ ! -z ${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY:+x} ]; then
        log_info "Setting up SSH agent to use private key"
        eval $(ssh-agent -s)
        echo "$KAYOBE_AUTOMATION_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    fi
}

function inject_ssh_keys {
    # These are read when generating kolla passwords
    if [ ! -z ${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY:+x} ]; then
        echo "${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY}" >~/.ssh/"${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME}"
        chmod 600 ~/.ssh/"${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME}"
        ssh-keygen -y -f ~/.ssh/"${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME}" >~/.ssh/"${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME}.pub"
        chmod 600 ~/.ssh/"${KAYOBE_AUTOMATION_SSH_PRIVATE_KEY_NAME}.pub"
    fi
}

function install_venv {
    # Install a virtualenv at $1. The rest of the arguments are passed
    # directly to pip.
    venv_path="$1"
    shift
    pip_args="$@"
    local venv_parent="$(dirname ${venv_path})"
    if [[ ! -d "$venv_parent" ]]; then
        mkdir -p "$venv_parent"
    fi
    if [[ ! -f "${venv_path}/bin/activate" ]]; then
        echo "Creating virtual environment in ${venv_path}"
        python3 -m venv "${venv_path}"
        # NOTE: Virtualenv's activate and deactivate scripts reference an
        # unbound variable.
        set +u
        source "${venv_path}/bin/activate"
        pip install -U pip
        pip install $pip_args
        deactivate
        set -u
    else
        echo "Using existing virtual environment in ${venv_path}"
    fi
}

function install_kayobe_venv {
    local KAYOBE_ANSIBLE_PATH="$1/share/kayobe/ansible"
    # Install the Kayobe venv.
    install_venv "$1" $2
    # We need access to group_vars and plugins and if this venv can move it means that we can't
    # create a symlink to a known location. Plugins could be set with an environment variable.
    cp -rfp "$FUNCTIONS_PARENT"/ansible/ "$KAYOBE_ANSIBLE_PATH/.."
}

function create_kayobe_environment {
    local env=$1
    local kayobe_config_source_path=${2:-$KAYOBE_CONFIG_SOURCE_PATH}
    local allow_unclean_kayobe_config=${KAYOBE_AUTOMATION_ALLOW_UNCLEAN_KAYOBE_CONFIG:-0}
    mkdir -p "$env"/{src/kayobe,src/kayobe-config,venvs/kayobe}

    if [ "$allow_unclean_kayobe_config" == 0 ]; then
        # Ensure we have an unmodified copy of kayobe-config
        if [[ $(realpath "$kayobe_config_source_path") != $(realpath "$env/src/kayobe-config") ]]; then
            clean_copy "$kayobe_config_source_path" "$env/src/kayobe-config"
        fi
    fi

    if [ -f "$kayobe_config_source_path/requirements.txt" ]; then
        # Requirements file gets precedence
        install_kayobe_venv "$env/venvs/kayobe" "$KAYOBE_PIP_INSTALL_ARGS -r $kayobe_config_source_path/requirements.txt"
        # Custom playbooks require symlinks to be in a certain location.
        # A common custom is to reference a kayobe source checkout using
        # a canonical layout. This adds a level of indirection to support
        # the following layout:
        #  - env/src/kayobe
        #  - env/src/kayobe-config
        pushd "$env/src/kayobe"
        if [ ! -e ansible ]; then
            ln -s ../../venvs/kayobe/share/kayobe/ansible ansible
        fi
        popd
    elif [ ! "$(ls -A $env/src/kayobe)" ]; then
        # Fallback to Legacy KAYOBE_URI
        git clone -b $KAYOBE_BRANCH --single-branch $KAYOBE_URI "$env/src/kayobe"
        install_kayobe_venv "$env/venvs/kayobe" "$KAYOBE_PIP_INSTALL_ARGS $env/src/kayobe"
    else
        # Kayobe pre-checked out
        install_kayobe_venv "$env/venvs/kayobe" "$KAYOBE_PIP_INSTALL_ARGS $env/src/kayobe"
    fi
}

###############################################################################
# Runtime

function environment_setup {
    env="$HOME/kayobe-automation-env"
    create_kayobe_environment "$env"
    activate_kayobe_env "$env"
}

function activate_kayobe_env {
    local env=$1
    shift
    # NOTE: Virtualenv's activate script references an unbound variable.
    set +u
    . "$env/venvs/kayobe/bin/activate"
    set -u
    . "$env/src/kayobe-config/kayobe-env"
    export KAYOBE_AUTOMATION_CONTEXT_ENV_PATH="$env"
    log_info "Context variable: KAYOBE_AUTOMATION_CONTEXT_ENV_PATH set to $KAYOBE_AUTOMATION_CONTEXT_ENV_PATH"
}

function run_kayobe {
    # Run a kayobe command, including extra arguments provided via
    # $KAYOBE_EXTRA_ARGS, $KAYOBE_TAGS, $KOLLA_TAGS, $KOLLA_LIMIT,
    # $KAYOBE_LIMIT.

    declare -a kolla_limit
    if [ ! -z ${KOLLA_LIMIT:+x} ]; then
        kolla_limit=(--kolla-limit "$KOLLA_LIMIT")
    fi

    declare -a kolla_tags
    if [ ! -z ${KOLLA_TAGS:+x} ]; then
        kolla_tags=(--kolla-tags "$KOLLA_TAGS")
    fi

    declare -a kayobe_tags
    if [ ! -z ${KAYOBE_TAGS:+x} ]; then
        kayobe_tags=(--tags "$KAYOBE_TAGS")
    fi

    declare -a kayobe_limit
    if [ ! -z ${KAYOBE_LIMIT:+x} ]; then
        kayobe_limit=(--limit "$KAYOBE_LIMIT")
    fi

    kayobe "${@}" "${kayobe_limit[@]}" "${kayobe_tags[@]}" "${kolla_limit[@]}" "${kolla_tags[@]}" ${KAYOBE_EXTRA_ARGS}
}

function run_kayobe_automation_playbook {
    # Run a kayobe command, including extra arguments provided via
    # $KAYOBE_EXTRA_ARGS.
    local KAYOBE_ANSIBLE_PATH="$KAYOBE_AUTOMATION_CONTEXT_ENV_PATH/venvs/kayobe/share/kayobe/ansible"
    run_kayobe playbook run "$KAYOBE_ANSIBLE_PATH/""$@"
}

function control_host_bootstrap {
    echo "Bootstrapping the Ansible control host"
    local KAYOBE_EXTRA_ARGS=${KAYOBE_BOOTSTRAP_EXTRA_ARGS:-}
    for i in $(seq 1 3); do
        if run_kayobe control host bootstrap; then
            chb_success=1
            break
        fi
        echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
    done
    if [[ -z ${chb_success+x} ]]; then
        die $LINENO "Failed to bootstrap control host"
    fi
    echo "Bootstrapped control host after $i attempts"
}

function pull_request {
    # Little bit dirty to use this global var, but do we really want to pass
    # the env around
    pushd $1 >/dev/null
    if is_git_dirty ${KAYOBE_AUTOMATION_PR_PATHSPEC:-}; then
        # Delegate to the correct helper
        log_info "Git repository is dirty, going to make a PR"
        _pull_request
    else
        log_info "Repository is clean, no need to submit a PR..."
    fi
    popd >/dev/null
}

function kayobe_install {
    call_with_hooks config_init
    call_with_hooks validate
    call_with_hooks install_dependencies
    call_with_hooks environment_setup
    call_with_hooks workarounds
    call_with_hooks control_host_bootstrap
}

function kayobe_init {
    trap on_error ERR
    call_with_hooks config_init
    call_with_hooks validate
    call_with_hooks environment_diagnostics
    call_with_hooks environment_setup
    call_with_hooks workarounds
    call_with_hooks inject_ssh_keys
}