build.yaml

# vim: ai ts=4 sts=4 et sw=4 ft=yaml et
#
# build.yaml
#
# This file contains the Jenkins job definitions for the Flocker project.
#
# We add new or reconfigure existing jobs in Jenkins using the Job DSL plugin.
# https://github.com/jenkinsci/job-dsl-plugin
#
# That plugin consumes 'groovy Job DSL' code through FreeStyle Jenkins job
# types which contain a 'Process Job DSLs' step.
#
# As part of the provisioning process for the Jenkins Master, we configure a
# job called 'setup_ClusterHQ_Flocker' which is responsible for querying the
# github clusterhq/flocker respository and retrieving this build.yaml file, as
# well as the jobs.groovy.j2 jinja file.
#
# As part of the provisioning process for the Jenkins Master, we also deploy
# a small python script '/usr/local/bin/render.py', this script simply contains
# code to read a YAML file into a dictionary and expand a Jinja2 template using
# the k,v in that dict.
#
# Our job 'setup_ClusterHQ_Flocker' when running, will run the 'render.py' with
# this build.yaml file and produce a jobs.groovy file.
# We do this, because we'd rather configure our jobs with YAML than with Groovy
#
# The next step in the job is a 'Process JOB DSL's' step which consumes the
# jobs.groovy file, and generates all the jenkins folders and jobs.
#
#
# We pass the branch name as a parameter to the setup_clusterhq_flocker job,
# the parameter is shown as 'RECONFIGURE_BRANCH'.
#
# The setup job only produces jobs for a single branch . We don't produce jobs
# for every branch due to the large number of branches in the repository, which
# would generate over 16000 jobs and take over an hour to run.
#
# The workflow is that, when a developer is working on a feature branch and is
# happy to start testing some of his code they will execute the job
# setup_clusterhq_flocker passing his branch as the parameter.
#
# Their jobs will then be available under the path:
# /ClusterHQ-Flocker/<branch>/
#
# Inside that folder there will be a large number of jobs, where at the top
# she/he can see a job called '_main_multijob'. This job is responsible for
# executing all other jobs in parallel and collecting the produced artifacts
# from each job after its execution.
#
# The artifacts in this case are trial logs, coverage xml reports, subunit
# reports.
#
# Those artifacts are consumed by the _main_multijob to produce an overall
# coverage report, and an aggregated summary of all the executed tests and
# their failures/skips/successes.
#

# The project contains the github owner and repository to build
project: 'ClusterHQ/flocker'

# git_url, contains the full HTTPS url for the repository to build
git_url: 'https://github.com/ClusterHQ/flocker.git'


# We use a set of YAML aliases and anchors to define the different steps
# in our jobs.
# This helps us to keep some of the code DRY, we are not forced to use YAML
# operators, a different approach could be used:
#  - bash functions
#  - python functions
#  - Rust or D code
#
common_cli:
  hashbang: &hashbang |
    #!/bin/bash -l
    # don't leak secrets
    set +x
    set -e


  # add_shell_functions, contains our the bash functions consumed by the
  # the build script.
  # We set the shebang to /bin/bash. Jenkins 'should' respect this.
  # Note:
  # We noticed that our Ubuntu /bin/bash call were being executed as /bin/sh.
  # So we as part of the slave image build process symlinked
  # /bin/sh -> /bin/bash.
  # TODO: https://clusterhq.atlassian.net/browse/FLOC-2986
  add_shell_functions: &add_shell_functions |

    # set default AWS region for S3 operations
    export S3_REGION=us-west-2
    # Docs buckets (clusterhq-staging-docs, doc-dev.clusterhq.com) are hosted in us-east-1
    export S3_DOCS_REGION=us-east-1

    # The long directory names where we build our code cause pip to fail.
    # https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/20
    # http://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel
    # https://github.com/spotify/dh-virtualenv/issues/10
    # so we set our virtualenv dir to live in /tmp/<random number>
    #
    export venv=/tmp/${RANDOM}
    # make sure the virtualenv doesn't already exist
    while [ -e ${venv} ]
    do
      export venv=/tmp/${RANDOM}
    done

    # fix the docker permission issue, the base image doesn't have the correct
    # permissions/owners.
    # This is to be tackled as part of:
    # https://clusterhq.atlassian.net/browse/FLOC-2689
    test -e /var/run/docker.sock && sudo chmod 777 /var/run/docker.sock

    # Returns the ip address for eth0
    # We consume this as part or the vagrant build tests.
    # Those tests run on our Mesos Cluster, the job connects to a local nginx
    # instance running on the Mesos Slave where the job is running.
    export eth0_ip=$( ip -o -4 addr show eth0 |awk '{ print $4 '} | cut -f 1 -d "/")

    # pass the exit code from the previous and return the the aggregated
    # exit status for the whole job.
    function updateExitStatus {
      # if we had previous failures, we just fail here.
      if [ "${JOB_EXIT_STATUS}" != "0" ] && [ "${JOB_EXIT_STATUS}" != "" ]; then
        echo 1
      else
         if [ "$1" !=  "0" ]; then
         # first failure on the job, lets return 1, which will set
         # JOB_EXIT_STATUS to 1 and mark the job as failed.
             echo 1
         fi
      fi
    }

    # Retries a command if it fails, using the exponential backoff algorithm.
    # Usage:
    # retry_n_times_with_timeout <max retries> <timeout in seconds> command
    #
    function retry_n_times_with_timeout {
      set +e
      max_tries=${1}
      seconds=${2}
      shift 2

      let delay=15
      let tries=0
      while [ ${tries} -lt ${max_tries} ]; do
        echo "executing...  ${*}"
        if (timeout --foreground --signal=SIGKILL ${seconds} bash -c "${*}"); then
            break
        fi
        echo Command failed, waiting ${delay}...
        sleep ${delay}
        let delay=delay*2
        let tries=tries+1

        # abort on too many failures
        if [ ${tries} -eq ${max_tries} ]; then
          echo "too many failed retries... aborting"
          exit 1
        fi

      done
    }

    # uploads a file to S3, retrying on failure.
    # usage:
    # s3_upload <bucket> <filename>
    function s3_upload() {
      echo "uploading ${2} to S3 bucket ${1} ..."
      retry_n_times_with_timeout 3 900 \
        aws --region ${S3_REGION} s3 cp "${2}" s3://${1}/
    }

    # creates a metadata file to be used with vagrant up; vagrant box update
    # usage: create_vagrant_metadata_file
    #   <filename> <box_name> <description> <version> <url> <sha1>
    function create_vagrant_metadata_file() {
      cat <<EOF_METADATA > ${1}
      {
        "name": "${2}",
        "description": "${3}",
        "versions": [{
          "version": "${4}",
          "providers": [{
            "name": "virtualbox",
            "url": "${5}",
            "checksum_type": "sha1",
            "checksum": "${6}"
          }]
        }]
      }
    EOF_METADATA
    }

    # destroy the vagrant vm and aborts the build
    # this is used when one the of the intermediate steps has failed
    # usage:
    # <some code> || abort_build
    function abort_build() {
      echo "aborting ..."
      VAGRANT_LOG=info vagrant_destroy
      exit 1
    }

    # attempts a vagrant up, and aborts the build on failure
    # usage:
    # vagrant_up
    function vagrant_up() {
      (
        retry_n_times_with_timeout 3 1200 \
          VAGRANT_LOG=info vagrant up
      ) || abort_build
    }

    # attempts to refresh the local vagrant box, and aborts the build on error
    # make sure we are using the latest available version of this vagrant box
    # usage:
    # vagrant_box_update
    function vagrant_box_update() {
      (
        retry_n_times_with_timeout 2 1800 \
          VAGRANT_LOG=info vagrant box update
      ) || abort_build
    }

    # destroys the vagrant vm
    # usage:
    # vagrant_destroy
    function vagrant_destroy() {
      VAGRANT_LOG=info vagrant destroy -f
    }

  # TODO: do we need to clean up old files on ubuntu and centos or
  # does the pre-scm plugin does this correctly for us ?
  # https://clusterhq.atlassian.net/browse/FLOC-3139
  cleanup: &cleanup |
    export PATH=/usr/local/bin:${PATH}
    # clean up the stuff from previous runs
    # due to the length of the jobname workspace, we are hitting limits in
    # our sheebang path name in pip.
    # https://github.com/spotify/dh-virtualenv/issues/10
    # http://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel
    # https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/20
    # So we will place the virtualenv in /tmp/v instead
    #
    rm -rf ${venv}

  setup_venv: &setup_venv |
    # Set up the new venv.
    virtualenv -p python2.7 --clear ${venv}
    . ${venv}/bin/activate
    # Upgrade Python packaging tools.
    ${venv}/bin/pip install pip==8.1.2 setuptools==23.0.0 tox==2.3.1
    # Report the versions to aid debugging.
    ${venv}/bin/python --version
    ${venv}/bin/pip --version
    ${venv}/bin/python \
        -c 'import setuptools; print "SETUPTOOLS_VERSION:", setuptools.__version__'
    ${venv}/bin/tox --version

  setup_flocker_modules: &setup_flocker_modules |
    # installs all the required python modules as well as the flocker code.
    pip install --requirement dev-requirements.txt
    # install junix for our coverage report
    pip install python-subunit junitxml

  setup_aws_env_vars: &setup_aws_env_vars |
    # set vars and run tests
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE=/tmp/acceptance.yaml
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION=storage-drivers.aws
    AWS_ZONE=$(\
        wget -q -O - \
        http://169.254.169.254/latest/meta-data/placement/availability-zone\
    )
    export FLOCKER_FUNCTIONAL_TEST_CONFIG_OVERRIDE_zone="${AWS_ZONE}"
    # There isn't a separate meta-data endpoint to get the instance region.
    # So assume that region is always ZONE without the trailing character.
    export FLOCKER_FUNCTIONAL_TEST_CONFIG_OVERRIDE_region="${AWS_ZONE:0:-1}"
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_PROVIDER=aws
    # The minimum size of an EBS volume is 1GiB.
    export FLOCKER_FUNCTIONAL_TEST_MINIMUM_ALLOCATABLE_SIZE=$((1024 ** 3 * 1))

  setup_rackspace_env_vars: &setup_rackspace_env_vars |
    # set vars and run tests
    # The /tmp/acceptance.yaml file is deployed to the jenkins slave during
    # bootstrapping. These are copied from the Jenkins Master /etc/slave_config
    # directory.
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE=/tmp/acceptance.yaml
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION=storage-drivers.rackspace
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_PROVIDER=rackspace
    export FLOCKER_FUNCTIONAL_TEST_CONFIG_OVERRIDE_region="DFW"
    # The minimum size of a Rackspace SATA Cinder volume is 75GiB. See:
    # * http://www.rackspace.com/knowledge_center/product-faq/cloud-block-storage
    export FLOCKER_FUNCTIONAL_TEST_MINIMUM_ALLOCATABLE_SIZE=$((1024 ** 3 * 75))

  setup_gce_env_vars: &setup_gce_env_vars |
    # set vars and run tests
    # The /tmp/acceptance.yaml file is deployed to the jenkins slave during
    # bootstrapping. These are copied from the Jenkins Master /etc/slave_config
    # directory.
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE=/tmp/acceptance.yaml
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_SECTION=storage-drivers.gce
    # Segredos has project set to clusterhq-acceptance so override it.
    export FLOCKER_FUNCTIONAL_TEST_CONFIG_OVERRIDE_project="clusterhqjenkins"
    # The minimum size of a GCE PD volume is 10GiB.
    export FLOCKER_FUNCTIONAL_TEST_MINIMUM_ALLOCATABLE_SIZE=$((1024 ** 3 * 10))

  setup_coverage: &setup_coverage |
    # we install the python coverage module so generate coverage.xml files
    # which Jenkins will process through the Jenkins Cobertura plugin.
    # https://wiki.jenkins-ci.org/display/JENKINS/Cobertura+Plugin
    # This plugin allows for producing coverage reports over time for a
    # particular job, or aggregated reports across a set of related jobs.
    # This is achieved by downloading the coverage.xml artifacts from the
    # downstream child jobs to the parent job and processing those files
    # one last time through the cobertura plugin.
    # The resulting report wil contain stats from every single job.
    pip install coverage

  run_coverage: &run_coverage |
    # run coverage and produce a report
    coverage xml --include=flocker*

  convert_results_to_junit: &convert_results_to_junit |
    # pip the trial.log results through subunit and export them as junit in xml
    cat trial.log | subunit-1to2 | subunit2junitxml \
      --no-passthrough --output-to=results.xml

  # flocker artifacts contains the list of files we want to collect from our
  # _main_multijob. These are used to produce the coverage, test reports.
  flocker_artifacts: &flocker_artifacts
    - results.xml
    - _trial_temp/test.log
    - coverage.xml

  # acceptance-test-artifacts contains the list of files we want to collect
  # from our _main_multijob.
  # These are the remote logs from the acceptance tests.
  acceptance_tests_artifacts: &acceptance_tests_artifacts
    - results.xml
    - run-acceptance-tests.log
    - _trial_temp/test.log
    - remote_logs.log

  # Ubuntu acceptance tests do not collect the logs from the remote nodes
  # https://clusterhq.atlassian.net/browse/FLOC-2560
  acceptance_tests_artifacts_ubuntu_special_case: &acceptance_tests_artifacts_ubuntu_special_case
    - results.xml
    - run-acceptance-tests.log
    - remote_logs.log
    - _trial_temp/test.log

  run_trial_directories_to_delete: &run_trial_directories_to_delete
    - ${WORKSPACE}/_trial_temp
    - ${WORKSPACE}/.hypothesis

  run_acceptance_directories_to_delete: &run_acceptance_directories_to_delete
    - ${WORKSPACE}/dist

  run_trial_with_coverage: &run_trial_with_coverage |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # the 'MODULE' parameter.
    # This is how we tell trial which flocker module to call.
    coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      ${MODULE} 2>&1 | tee trial.log

  run_trial_with_coverage_as_root: &run_trial_with_coverage_as_root |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # the 'MODULE' parameter.
    # This is how we tell trial which flocker module to call.
    sudo ${venv}/bin/coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      ${MODULE} 2>&1 | tee trial.log

  run_trial_for_storage_drivers_with_coverage: &run_trial_for_storage_drivers_with_coverage |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # Consume the MODULE parameter set in the job configuration
    sudo -E ${venv}/bin/coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      ${MODULE} 2>&1 | tee trial.log

  setup_authentication: &setup_authentication |
    # acceptance tests rely on this file existing
    touch ${HOME}/.ssh/known_hosts
    chmod 0600 /tmp/id_rsa
    eval `ssh-agent -s`
    ssh-add /tmp/id_rsa

  run_acceptance_tests: &run_acceptance_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    #
    # The admin/run-acceptance-tests will provision a flocker cluster of
    # several nodes. These nodes will install the flocker packages (RPM/DEB)
    # during the provisioning process by that tool. These packages are fetched
    # from a repository on the network through a common apt-get/yum install.
    # The jenkins slave will be the repository host containing those packages
    # which are made available through a webserver running on port 80.
    # We pass the URL of our Jenkins Slave to the acceptance test nodes
    # through the --build-server parameter below.
    #
    EXTERNAL_IP=$(\
        curl \
            --silent \
            --header "Metadata-Flavor: Google" \
            http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip \
    )
    BUILD_SERVER="http://${EXTERNAL_IP}/${BUILD_TAG}/"
    ${venv}/bin/python admin/run-acceptance-tests \
    --distribution ${DISTRIBUTION_NAME} \
    --provider ${ACCEPTANCE_TEST_PROVIDER} \
    --dataset-backend ${ACCEPTANCE_TEST_PROVIDER} \
    --branch ${TRIGGERED_BRANCH} \
    --build-server ${BUILD_SERVER} \
    --config-file /tmp/acceptance.yaml \
    -- --reporter=subunit ${ACCEPTANCE_TEST_MODULE} 2>&1 | tee trial.log
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  run_acceptance_loopback_tests: &run_acceptance_loopback_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    #
    # The admin/run-acceptance-tests will provision a flocker cluster with a
    # single node since the loopback backend doesn't support moving data across
    # hosts.
    EXTERNAL_IP=$(\
        curl \
            --silent \
            --header "Metadata-Flavor: Google" \
            http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip \
    )
    BUILD_SERVER="http://${EXTERNAL_IP}/${BUILD_TAG}/"
    ${venv}/bin/python admin/run-acceptance-tests \
    --distribution ${DISTRIBUTION_NAME} --number-of-nodes 1 \
    --provider ${ACCEPTANCE_TEST_PROVIDER} --dataset-backend loopback \
    --branch ${TRIGGERED_BRANCH} \
    --build-server $BUILD_SERVER \
    --config-file /tmp/acceptance.yaml \
    -- --reporter=subunit ${ACCEPTANCE_TEST_MODULE} 2>&1 | tee trial.log
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  run_client_tests: &run_client_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    EXTERNAL_IP=$(\
        curl \
            --silent \
            --header "Metadata-Flavor: Google" \
            http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip \
    )
    BUILD_SERVER="http://${EXTERNAL_IP}/${BUILD_TAG}/"
    ${venv}/bin/python admin/run-client-tests \
    --distribution ${DISTRIBUTION_NAME} \
    --branch ${TRIGGERED_BRANCH} \
    --build-server $BUILD_SERVER
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  disable_selinux: &disable_selinux |
    sudo /usr/sbin/setenforce 0

  check_version: &check_version |
    export FLOCKER_VERSION=$(${venv}/bin/python setup.py --version)

  build_sdist: &build_sdist  |
    # package the goodies
    ${venv}/bin/python setup.py sdist

  build_package: &build_package  |
    # Build an RPM / DEB package using docker.
    # The packages are served by a local nginx server.
    REPO_BASE=/var/www/html/${BUILD_TAG}
    REPO_PATH=${REPO_BASE}/results/omnibus/${TRIGGERED_BRANCH}/${DISTRIBUTION_NAME}
    # Export for subsequent functions that perform operations on the REPO_BASE
    # and REPO_PATH.
    export REPO_BASE REPO_PATH
    sudo mkdir -p ${REPO_PATH}
    ${venv}/bin/python admin/build-package \
    --destination-path ${REPO_PATH} \
    --distribution ${DISTRIBUTION_NAME_PACKAGE:-$DISTRIBUTION_NAME} \
    /flocker/dist/Flocker-${FLOCKER_VERSION}.tar.gz

  build_repo_metadata: &build_repo_metadata |
    # the acceptance tests look for a package in a yum repository,
    # we provide one by starting a webserver and pointing the tests
    # to look over there
    cd ${REPO_PATH}
    distribution="${DISTRIBUTION_NAME_PACKAGE:-$DISTRIBUTION_NAME}"
    # create a repo on either centos or ubuntu
    case "${distribution}" in
        ubuntu*)
            sudo sh -c 'dpkg-scanpackages --multiversion . | gzip > Packages.gz'
            ;;
        centos*)
            sudo createrepo .
            ;;
        *)
            echo "ERROR: Unsupported distribution '${distribution}'." >&2
            exit 1
            ;;
    esac
    cd -

  clean_packages: &clean_packages |
    # Delete the repo files we created
    sudo rm -rf $REPO_BASE

  exit_with_return_code_from_test: &exit_with_return_code_from_test |
    # this is where we make sure we exit with the correct return code
    # from the tests we executed above.
    exit ${JOB_EXIT_STATUS}

  push_image_to_dockerhub: &push_image_to_dockerhub |
    # the /tmp/dockerhub_creds is copied from the Jenkins Master on
    # /etc/slave_config to the slave during the bootstrap process of the slave.
    # This contains the login details for our dockerhub instance, which is
    # deployed as part of our caching platform.
    #
    export D_USER=$( cat /tmp/dockerhub_creds | cut -f 1 -d ":" )
    export D_PASSWORD=$( cat /tmp/dockerhub_creds | cut -f 2 -d ":" )
    export D_EMAIL=$( cat /tmp/dockerhub_creds | cut -f 3 -d ":" )
    docker login -u ${D_USER} -p ${D_PASSWORD} -e ${D_EMAIL}
    echo y | docker push ${DOCKER_IMAGE}

  build_docker_image: &build_docker_image |
    # we want to make sure we are fetching the latest OS updates every time
    # so we build the docker image using --no-cache, this way apt-get updates
    # are actually executed.
    # See: https://github.com/docker/docker/issues/3313
    docker build --no-cache -t $DOCKER_IMAGE .

  build_docker_image_flocker: &build_docker_image_flocker |
    # Build Flocker Docker image
    BUILD_SERVER_IP="$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)"
    REPOSITORY_PATH="${BUILD_TAG}/results/omnibus/${TRIGGERED_BRANCH}/${DISTRIBUTION_NAME}"
    docker build \
        --rm \
        --tag "${DOCKER_IMAGE}" \
        --label "FLOCKER_VERSION=${FLOCKER_VERSION}" \
        --label "GIT_COMMIT=${GIT_COMMIT}" \
        --label "TRIGGERED_BRANCH=${TRIGGERED_BRANCH}" \
        --label "BUILD_SERVER_IP=${BUILD_SERVER_IP}" \
        --build-arg "FLOCKER_REPOSITORY=http://${BUILD_SERVER_IP}/${REPOSITORY_PATH}" \
        dockerfiles/dataset
    JOB_EXIT_STATUS="$( updateExitStatus $? )"


  build_docker_image_flocker_control: &build_docker_image_flocker_control |
    # Build Flocker-control Docker image
    BUILD_SERVER_IP="$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)"
    REPOSITORY_PATH="${BUILD_TAG}/results/omnibus/${TRIGGERED_BRANCH}/${DISTRIBUTION_NAME}"
    docker build \
        --rm \
        --tag "${DOCKER_IMAGE}" \
        --label "FLOCKER_VERSION=${FLOCKER_VERSION}" \
        --label "GIT_COMMIT=${GIT_COMMIT}" \
        --label "TRIGGERED_BRANCH=${TRIGGERED_BRANCH}" \
        --label "BUILD_SERVER_IP=${BUILD_SERVER_IP}" \
        --build-arg "FLOCKER_REPOSITORY=http://${BUILD_SERVER_IP}/${REPOSITORY_PATH}" \
        dockerfiles/control
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  build_docker_image_flocker_docker_plugin: &build_docker_image_flocker_docker_plugin |
    # Build Flocker-docker-plugin Docker image
    BUILD_SERVER_IP="$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)"
    REPOSITORY_PATH="${BUILD_TAG}/results/omnibus/${TRIGGERED_BRANCH}/${DISTRIBUTION_NAME}"
    docker build \
        --rm \
        --tag "${DOCKER_IMAGE}" \
        --label "FLOCKER_VERSION=${FLOCKER_VERSION}" \
        --label "GIT_COMMIT=${GIT_COMMIT}" \
        --label "TRIGGERED_BRANCH=${TRIGGERED_BRANCH}" \
        --label "BUILD_SERVER_IP=${BUILD_SERVER_IP}" \
        --build-arg "FLOCKER_REPOSITORY=http://${BUILD_SERVER_IP}/${REPOSITORY_PATH}" \
        dockerfiles/docker-plugin
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  # These are the docker images we will be using during our tests.
  # We build them every 24 hours, making sure we have the latest OS updates
  # installed on those images.
  # By doing this we speed up the bootstrapping of our client/acceptance tests.
  #
  build_dockerfile_centos7: &build_dockerfile_centos7 |
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-centos-7:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "# URLGRABBER_DEBUG=1 to log low-level network info \
          - see FLOC-2640" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum groupinstall \
          --assumeyes 'Development Tools'" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum install \
          --assumeyes epel-release" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum install --assumeyes \
          git ruby-devel python-devel libffi-devel openssl-devel enchant-devel \
          rpmlint" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum update --assumeyes" >> Dockerfile
    echo "ADD https://bootstrap.pypa.io/get-pip.py /tmp/" >> Dockerfile
    echo 'RUN ["python", "/tmp/get-pip.py", "pip==8.1.2", "setuptools==23.0.0"]' >> Dockerfile

  build_dockerfile_ubuntu_trusty: &build_dockerfile_ubuntu_trusty |
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-ubuntu-trusty:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "RUN apt-get update" >> Dockerfile
    echo "RUN apt-get install --no-install-recommends -y git ruby-dev \
          libffi-dev libssl-dev libenchant-dev build-essential \
          python2.7-dev lintian python" >> Dockerfile
    echo "ADD https://bootstrap.pypa.io/get-pip.py /tmp/" >> Dockerfile
    echo 'RUN ["python", "/tmp/get-pip.py", "pip==8.1.2", "setuptools==23.0.0"]' >> Dockerfile

  build_dockerfile_ubuntu_xenial: &build_dockerfile_ubuntu_xenial |
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-ubuntu-xenial:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "RUN apt-get update" >> Dockerfile
    echo "RUN apt-get install --no-install-recommends -y \
          git ruby-dev libffi-dev libssl-dev libenchant-dev build-essential \
          python2.7-dev lintian python" >> Dockerfile
    echo "ADD https://bootstrap.pypa.io/get-pip.py /tmp/" >> Dockerfile
    echo 'RUN ["python", "/tmp/get-pip.py", "pip==8.1.2", "setuptools==23.0.0"]' >> Dockerfile

  do_not_abort_on_errors: &do_not_abort_on_errors |
    # make sure we don't abort the job on the first error we find.
    #
    # jenkins will execute our shellscript with -e, which will cause the
    # script to terminate at the first error and mark the job as failed.
    # This is fine for the majority of the build cases, but in some situations
    # we don't want to terminate the job straight away. For example, a job
    # which generates a new Virtual Machine where the tests are executed.
    # With '-e' defined, jenkins would abort the job and leave an orphan VM
    # behind.
    # To avoid it, we set '+e' on this shell.
    set +e

  cleanup_cloud_resources: &cleanup_cloud_resources |
    # Cleanup leaked or stale cloud instances and volumes.
    ${venv}/bin/python setup.py --version
    ${venv}/bin/python admin/cleanup_cloud_resources \
    --config-file /tmp/acceptance.yaml \
    -- 1>cleanup_cloud_resources.stdout 2>cleanup_cloud_resources.stderr

  source_git_commit: &source_git_commit |
    # we use a script 'git-commit.sh' to pass any the git commit sha1
    # to our vagrant machine. This file is copied and sourced as part of our
    # build.sh run inside the virtual machine.
    chmod 755 git-commit.sh
    . git-commit.sh

  set_home_variable_on_mesos: &set_home_variable_on_mesos |
    # mesos is not setting HOME, so we set it here.
    # it needs to be set to /root, so that we can re-use the existing
    # vagrant base images which live under /root/.vagrant.d
    export HOME=/root

  # this is a wrapper that we can use for building a script that will be
  # populated with different cli actions.
  # we copy the resulting build.sh file to the vagrant box for execution.
  begin_build_sh_EOF: &begin_build_sh_EOF cat <<'EOF_BUILD_SH' > build.sh

  end_build_sh_EOF: &end_build_sh_EOF |
     EOF_BUILD_SH
     chmod 755 build.sh

  run_build_sh_script_on_vagrant_box: &run_build_sh_script_on_vagrant_box |
    # copy project files to the vagrant box
    vagrant ssh-config > ssh-config
    rsync -ave 'ssh -F ssh-config' . default:.

    # this sets some variables used by the tests, since the tests run inside
    # virtualbox we store them to a file and copy them to the vagrant box.
    echo "export GIT_COMMIT=${GIT_COMMIT}" > git-commit.sh
    for item in /tmp/pip.sh build.sh git-commit.sh
    do
      vagrant scp ${item} default:${item}
      vagrant ssh -c "chmod 755 ${item}"
    done

    # make sure we don't abort on the first error
    set +e
    # run the build.sh script inside our vagrant box
    VAGRANT_LOG=info vagrant ssh -c 'bash build.sh' || abort_build

    # this removes the secrets files we used during provisioning
    for item in "/tmp/pip.sh build.sh git-commit.sh"
    do
      vagrant ssh -c "rm -f ${item}"
    done

#-----------------------------------------------------------------------------#
# Job Definitions below this point
#-----------------------------------------------------------------------------#
# Job Types:
#
# * run_trial
# * run_trial_for_storage_driver (ebs/cinder)
# * run_acceptance (tests)
# * cronly_jobs (builds docker images every 24 hours)
#
# Toggles:
#   * archive_artifacts: ( define if there are files to be archived)
#   * coverage_report: ( enable if this job produces a coverage report file)
#   * clean_repo: (enable if we need to clean old files owned by root)
#

# run_trial_modules contains a list of all the modules we want to execute
# through trial.
run_trial_modules: &run_trial_modules
  - admin
  - benchmark
  - flocker
run_trial_as_root_modules: &run_trial_as_root_modules
  # journald access requires root
  - flocker.common.functional
  # flocker.volume needs to run as root due to ZFS calls
  # so we run it as part of the run_trial_on_<Cloud>_<OS>_as_root jobs
  - flocker.volume
  # flocker.node.agent needs to run as root due to mounting filesystems
  - flocker.node.agents
run_trial_functional_agent_modules: &run_trial_functional_agent_modules
  # The modules have appropriate skips so only EBS tests run on EBS environment,
  # etc.
  - flocker.node.agents.functional

# run_trial_cli contains a list of all the CLI yaml anchors we want to
# execute as part of our run_trial_tasks
run_trial_cli: &run_trial_cli [
  *hashbang,
  *add_shell_functions,
  *cleanup,
  *setup_venv,
  *setup_flocker_modules,
  *setup_coverage,
  *run_trial_with_coverage,
  *run_coverage,
  *convert_results_to_junit ]

run_trial_cli_as_root: &run_trial_cli_as_root [
  *hashbang,
  *add_shell_functions,
  *cleanup,
  *setup_venv,
  *setup_flocker_modules,
  *setup_coverage,
  *run_trial_with_coverage_as_root,
  *run_coverage,
  *convert_results_to_junit ]

# When we have acceptance tests that run quickly enough (as they do on loopback)
# we just run them all in one builder:
run_full_acceptance_modules: &run_full_acceptance_modules
  - flocker.acceptance

# flocker.node.functional is hanging, so we don't run it
job_type:
  run_trial:
    # http://build.clusterhq.com/builders/flocker-centos-7
    run_trial_on_AWS_CentOS_7:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_modules: *run_trial_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_on_AWS_CentOS_7_as_root:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_modules: *run_trial_as_root_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli_as_root }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    # Ubuntu 14.04 non-root tests are run on Travis-CI container infrastructure.
    # See .travis.yml

    run_trial_on_AWS_Ubuntu_Trusty_as_root:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_modules: *run_trial_as_root_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli_as_root }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_on_AWS_Ubuntu_Xenial:
      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
      with_modules: *run_trial_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_on_AWS_Ubuntu_Xenial_as_root:
      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
      with_modules: *run_trial_as_root_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli_as_root }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_on_AWS_CentOS_7_flocker.node.functional.test_docker:
      # FLOC-3903: docker on centos use loop-devmapper
      # by default. That makes it much slower than Ubuntu
      # with aufs. It leads to timeouts, but seems to do
      # a bit better on the medium instance
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      module: flocker.node.functional.test_docker
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Increase the timeout due to FLOC-3903
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    # Split out just to do the CentOS version above,
    # for the reasons outlined in that section
    run_trial_on_AWS_Ubuntu_Trusty_flocker.node.functional.test_docker:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      module: flocker.node.functional.test_docker
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

    # Split out just to do the CentOS version above,
    # for the reasons outlined in that section
    run_trial_on_AWS_Ubuntu_Xenial_flocker.node.functional.test_docker:
      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
      module: flocker.node.functional.test_docker
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_trial_directories_to_delete

  run_trial_for_storage_driver:
    run_trial_for_ebs_storage_driver_on_CentOS_7:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_aws_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_ebs_storage_driver_on_Ubuntu_trusty:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_aws_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_ebs_storage_driver_on_Ubuntu_xenial:
      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_aws_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_cinder_storage_driver_on_CentOS_7:
      on_nodes_with_labels: 'rackspace-jenkins-slave-centos7-selinux-standard-4-dfw'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_rackspace_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Give this job a slightly longer timeout because it has to interact with
      # the cloud-supplied Cinder which can be a bit sluggish even in the best
      # of times.
      timeout: 90
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_cinder_storage_driver_on_Ubuntu_trusty:
      on_nodes_with_labels: 'rackspace-jenkins-slave-ubuntu14-standard-4-dfw'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_rackspace_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Reasoning for run_trial_for_cinder_storage_driver_on_CentOS_7's timeout
      # applies here as well.
      timeout: 90
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_cinder_storage_driver_on_Ubuntu_xenial:
      on_nodes_with_labels: 'rackspace-jenkins-slave-ubuntu16-standard-4-dfw'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_rackspace_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Reasoning for run_trial_for_cinder_storage_driver_on_CentOS_7's timeout
      # applies here as well.
      timeout: 90
      directories_to_delete: *run_trial_directories_to_delete

  # http://build.clusterhq.com/builders/flocker%2Facceptance%2Faws%2Fcentos-7%2Faws
  run_acceptance:
    run_acceptance_loopback_on_AWS_CentOS_7_for:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_loopback_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      publish_test_results: true
      timeout: 45
      directories_to_delete: *run_acceptance_directories_to_delete

    run_acceptance_loopback_on_AWS_RHEL_7_for:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   # For RHEL7 we build Centos-7 packages but put them in a repo
                   # named RHEL7.
                   # XXX We should have a way to build RHEL7 packages.
                   'export DISTRIBUTION_NAME=rhel-7',
                   'export DISTRIBUTION_NAME_PACKAGE=centos-7',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_loopback_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      publish_test_results: true
      timeout: 45
      directories_to_delete: *run_acceptance_directories_to_delete

    run_acceptance_loopback_on_AWS_Ubuntu_Trusty_for:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_loopback_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      timeout: 45
      directories_to_delete: *run_acceptance_directories_to_delete

    run_acceptance_loopback_on_AWS_Ubuntu_Xenial_for:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_loopback_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      timeout: 45
      directories_to_delete: *run_acceptance_directories_to_delete

  run_client:
    run_client_installation_on_Ubuntu_Trusty:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *run_client_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_acceptance_directories_to_delete

    run_client_installation_on_Ubuntu_Xenial:
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *run_client_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_acceptance_directories_to_delete

  run_docker_build:
    run_docker_build_on_Ubuntu_Xenial:
      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   'export DOCKER_IMAGE=clusterhqci/flocker-dataset-agent:${GIT_COMMIT}',
                   *build_docker_image_flocker,
                   *push_image_to_dockerhub,
                   'export DOCKER_IMAGE=clusterhqci/flocker-control:${GIT_COMMIT}',
                   *build_docker_image_flocker_control,
                   *push_image_to_dockerhub,
                   'export DOCKER_IMAGE=clusterhqci/flocker-docker-plugin:${GIT_COMMIT}',
                   *build_docker_image_flocker_docker_plugin,
                   *push_image_to_dockerhub,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      timeout: 30
      directories_to_delete: *run_acceptance_directories_to_delete

  cronly_jobs:
    run_docker_build_centos7_fpm:
      at: '0 0 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-centos-7',
                   *build_dockerfile_centos7,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_docker_build_ubuntu_trusty_fpm:
      at: '0 1 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-ubuntu-trusty',
                   *build_dockerfile_ubuntu_trusty,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
      directories_to_delete: []

    run_docker_build_ubuntu_xenial_fpm:
      at: '0 3 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-ubuntu-xenial',
                   *build_dockerfile_ubuntu_xenial,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_trial_for_gce_storage_driver_on_Ubuntu_trusty:
      at: '0 5 * * *'
      on_nodes_with_labels: 'gce-ubuntu14'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_gce_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_gce_storage_driver_on_Ubuntu_xenial:
      at: '0 5 * * *'
      on_nodes_with_labels: 'gce-ubuntu16'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_gce_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_trial_for_gce_storage_driver_on_CentOS_7:
      at: '0 5 * * *'
      on_nodes_with_labels: 'gce-centos7'
      with_modules: *run_trial_functional_agent_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_gce_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45
      directories_to_delete: *run_trial_directories_to_delete

    run_acceptance_on_AWS_CentOS_7_with_EBS:
      at: '0 5 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      publish_test_results: true
      # Give the acceptance test suite a nice long time to run.  Give it even
      # longer on CentOS than Ubuntu because Docker configuration on CentOS
      # causes some things to be particularly slow.  This value is just a guess
      # at what a reasonable upper-bound for the runtime of the suite might be
      # as of Dec 2015.
      timeout: 120
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Similar to the reasoning for run_acceptance_on_AWS_CentOS_7_with_EBS
      # but slightly shorter since Ubuntu runs the tests faster.
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_AWS_Ubuntu_Xenial_with_EBS:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=aws',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Similar to the reasoning for run_acceptance_on_AWS_CentOS_7_with_EBS
      # but slightly shorter since Ubuntu runs the tests faster.
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_GCE_CentOS_7_with_GCE:
      at: '0 5 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=gce',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_CentOS_7_with_EBS
      timeout: 120
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_GCE_Ubuntu_Trusty_with_GCE:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=gce',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_GCE_Ubuntu_Xenial_with_GCE:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=gce',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_Rackspace_CentOS_7_with_Cinder:
      at: '0 5 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=rackspace',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_CentOS_7_with_EBS
      timeout: 120
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_Rackspace_Ubuntu_Trusty_with_Cinder:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=rackspace',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    run_acceptance_on_Rackspace_Ubuntu_Xenial_with_Cinder:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # so we can actually run run-acceptance-tests from GCE
      on_nodes_with_labels: 'gce-ubuntu16'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-16.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   'export ACCEPTANCE_TEST_PROVIDER=rackspace',
                   *run_acceptance_tests,
                   *convert_results_to_junit,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      publish_test_results: true
      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
      timeout: 90
      directories_to_delete: []
      notify_slack: '#nightly-builds'

    cleanup_cloud_resources:
      at: '0 8 * * *'
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *cleanup_cloud_resources ]
          }
      timeout: 10
      archive_artifacts: [
          "cleanup_cloud_resources.stdout",
          "cleanup_cloud_resources.stderr"
      ]

    run_admin_test_test_installer:
      at: '0 8 * * *'
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      module: admin.test.test_installer
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang,
                   *add_shell_functions,
                   *cleanup,
                   *setup_venv,
                   *setup_flocker_modules,
                   *setup_coverage,
                   *check_version,
                   *setup_authentication,
                   'export ACCEPTANCE_YAML=/tmp/acceptance.yaml',
                   *run_trial_with_coverage,
                   *run_coverage,
                   *convert_results_to_junit ]
        }
      clean_repo: true
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      # CloudFormation stack creation takes 5-10 minutes and the test itself
      # may take another 10. Give 30 minutes to be sure.
      timeout: 30
      directories_to_delete: []



#-----------------------------------------------------------------------------#
# View definitions below this point
#-----------------------------------------------------------------------------#

# Use the section below to define multiple 'views/tabs' that appear at the top
# of the jenkins build page for every branch.
# These views allow for filtering of particular jobs according to its type,
# cloud, operating system, ...
#
#
# The views are built from :
# * the dictionary 'key', which is the view name.
# * description, containing a description of the jobs in this view
# * regex, containing a regular expression that is applied to build the view.
#
# the site below can help with regular expressions:
# http://pythex.org/

views:
  trial:
    description: 'All jobs that are executed directly using trial'
    regex: 'run_(?i)trial_.*.*'
  acceptance:
    description: 'All Acceptance jobs'
    regex: 'run_(?i)acceptance_.*.*'
  client:
    description: 'All client installation jobs'
    regex: 'run_(?i)client_.*.*'
  cinder:
    description: 'cinder jobs'
    regex: '.*_(?i)cinder_.*.*'
  ebs:
    description: 'ebs jobs'
    regex: '.*_(?i)ebs_.*.*'
  storage_drivers:
    description: 'All Storage Driver Jobs'
    regex: '.*_(?i)storage_(?i)driver_.*.*'
  on_AWS:
    description: 'All AWS Jobs'
    regex: '(.*_(?i)ebs_.*.*|.*_(?i)aws_.*.*)'
  on_Rackspace:
    description: 'All Rackspace Jobs'
    regex: '(.*_(?i)cinder_.*.*|.*_(?i)rackspace_.*.*)'
  CentOS_7:
    description: 'All CentOS jobs'
    regex: '.*_(?i)centos_.*.*'
  Ubuntu_Trusty_14_04_LTS:
    description: 'All Ubuntu Trusty LTS (14.04) jobs'
    regex: '.*_(?i)trusty_.*.*'
  Ubuntu_Xenial_16_04:
    description: 'All Ubuntu Xenial (16.04) jobs'
    regex: '.*_(?i)xenial_.*.*'
  cron:
    description: 'All Nightly Cron jobs'
    regex: '^_[^_](?i)[a-z]*.*'