cloudbuild_CBI.yaml

# Copyright 2018 Google LLC.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# This file instructs cloud container builder on how to build and test
# DeepVariant's docker images. Following docker images are built.
# 1) DeepVariant binaries for CPU usage.
# 2) DeepVariant binaries for GPU usage.
# 3) A runner binary for running DeepVariant at scale using the
#    Google Cloud Platform via the Genomics Pipelines API.

options:
  machineType: N1_HIGHCPU_32

steps:
  - name: 'ubuntu:16.04'
    args: ['bash', './deepvariant/docker/build_docker_bins.sh']
    id: 'build-deepvariant-bins'
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--tag=gcr.io/$PROJECT_ID/deepvariant:${COMMIT_SHA}'
      - '.'
    id: 'build-deepvarint-docker-cpu'
    dir: 'deepvariant/docker'
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--tag=gcr.io/$PROJECT_ID/deepvariant_gpu:${COMMIT_SHA}'
      - '--build-arg=FROM_IMAGE=nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04'
      - '--build-arg=DV_GPU_BUILD=1'
      - '.'
    id: 'build-deepvariant-docker-gpu'
    dir: 'deepvariant/docker'
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'build'
      - '--tag=gcr.io/$PROJECT_ID/deepvariant_runner:${COMMIT_SHA}'
      - '--file=Dockerfile.runner'
      - '.'
    id: 'build-deepvariant-runner-docker'
    dir: 'deepvariant/docker'

  # Push images right away. This is required for testing.
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'push'
      - 'gcr.io/${PROJECT_ID}/deepvariant:${COMMIT_SHA}'
    id: 'push-deepvariant-docker-cpu'
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'push'
      - 'gcr.io/${PROJECT_ID}/deepvariant_gpu:${COMMIT_SHA}'
    id: 'push-deepvariant-docker-gpu'
  - name: 'gcr.io/cloud-builders/docker'
    args:
      - 'push'
      - 'gcr.io/${PROJECT_ID}/deepvariant_runner:${COMMIT_SHA}'
    id: 'push-deepvariant-runner-docker'


  # Run DeepVariant on CPU, and verify the result.
  - name: 'gcr.io/${PROJECT_ID}/deepvariant_runner:${COMMIT_SHA}'
    args:
      - 'bash'
      - '/opt/deepvariant_runner/bin/run_and_verify.sh'
      - '${PROJECT_ID}'
      - '${COMMIT_SHA}'
      - '${_MODEL}'
      - 'cpu'
    id: 'run-and-verify-deepvariant-cpu'

  # Run DeepVariant with GPU, and verify the result.
  - name: 'gcr.io/${PROJECT_ID}/deepvariant_runner:${COMMIT_SHA}'
    args:
      - 'bash'
      - '/opt/deepvariant_runner/bin/run_and_verify.sh'
      - '${PROJECT_ID}'
      - '${COMMIT_SHA}'
      - '${_MODEL}'
      - 'gpu'
    id: 'run-and-verify-deepvariant-gpu'

  # Run DeepVariant with TPU, and verify the result.
  - name: 'gcr.io/${PROJECT_ID}/deepvariant_runner:${COMMIT_SHA}'
    args:
      - 'bash'
      - '/opt/deepvariant_runner/bin/run_and_verify.sh'
      - '${PROJECT_ID}'
      - '${COMMIT_SHA}'
      - '${_MODEL}'
      - 'tpu'
    id: 'run-and-verify-deepvariant-tpu'

substitutions:
    _MODEL: gs://deepvariant/models/DeepVariant/0.7.2/DeepVariant-inception_v3-0.7.2+data-wgs_standard

timeout: 3h
images:
  - 'gcr.io/$PROJECT_ID/deepvariant:${COMMIT_SHA}'
  - 'gcr.io/$PROJECT_ID/deepvariant_gpu:${COMMIT_SHA}'
  - 'gcr.io/$PROJECT_ID/deepvariant_runner:${COMMIT_SHA}'