diff --git a/.ci/build_compose.sh b/.ci/build_compose.sh new file mode 100755 index 000000000..5b0150405 --- /dev/null +++ b/.ci/build_compose.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -o errexit + +pip --version +# --user does not work if you are already in a virtual-env +pip install docker-compose galaxy-parsec --user || pip install docker-compose galaxy-parsec + +export DOCKER_RUN_CONTAINER="galaxy-web" +INSTALL_REPO_ARG="--galaxy-url http://localhost:80" +SAMPLE_TOOLS=/export/config/sample_tool_list.yaml + +pushd $COMPOSE_DIR + +# For build script +export CONTAINER_REGISTRY=quay.io/ +export CONTAINER_USER=bgruening +./build-orchestration-images.sh --no-push --condor --grafana --slurm --k8s + +cat ./tags-for-compose-to-source.sh +source ./tags-for-compose-to-source.sh + +container_size_check quay.io/bgruening/galaxy-base:$TAG 350 +container_size_check quay.io/bgruening/galaxy-web:$TAG 650 +container_size_check quay.io/bgruening/galaxy-htcondor-base:$TAG 280 + +export COMPOSE_PROJECT_NAME=galaxy_compose +docker-compose up -d + +until docker-compose exec galaxy-web ps -fC uwsgi +do + echo "sleeping for 20 seconds" + sleep 20 + docker-compose logs --tail 10 +done + +# back to the root of the repo +popd + diff --git a/.ci/cleanup.sh b/.ci/cleanup.sh new file mode 100644 index 000000000..463b881ba --- /dev/null +++ b/.ci/cleanup.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +sudo rm -rf ./local_folder | true +sudo rm -rf /export | true +docker stop galaxy | true +docker rm galaxy | true diff --git a/.ci/compose_condor_docker.sh b/.ci/compose_condor_docker.sh new file mode 100755 index 000000000..1cb816da7 --- /dev/null +++ b/.ci/compose_condor_docker.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -o errexit + +echo '#### Start compose condor Docker testing ####' + +bash .ci/cleanup.sh + +source .ci/set_env.sh +source .ci/functions.sh + +# The compose file recognises ENV vars to change the defaul behavior +pushd ${COMPOSE_DIR} +ln -sf .env_htcondor_docker .env + +# Galaxy needs to a full path for the the jobs, in- and outputs. +# Do we want to run each job in it's own container and this container uses the host +# container engine (not Docker in Docker) then the path to all files inside and outside +# of the container needs to be the same. +sudo mkdir /export +sudo chmod 777 /export +sudo chown 1450:1450 /export + +# start building this repo +#git submodule update --init --recursive +#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + +popd +source .ci/build_compose.sh +pushd ${COMPOSE_DIR} + +source ./tags-for-compose-to-source.sh + +echo "Stopping SLURM container" +docker-compose stop galaxy-slurm +sleep 30 + +docker-compose logs --tail 50 +docker ps + +sleep 10 +docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" + +popd + +echo '#### Start .ci/testing.sh ####' +source .ci/testing.sh + +echo '#### Run a ton of BioBlend test against our servers ####' +pushd $WORKING_DIR/test/bioblend/ +source ./test.sh 'compose' +popd + +pushd ${COMPOSE_DIR} +docker-compose down +popd + diff --git a/.ci/compose_slurm.sh b/.ci/compose_slurm.sh new file mode 100755 index 000000000..57223ae66 --- /dev/null +++ b/.ci/compose_slurm.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit + +echo '#### Start compose Slurm testing ####' + +bash .ci/cleanup.sh + +source .ci/set_env.sh +source .ci/functions.sh + +# The compose file recognises ENV vars to change the defaul behavior +pushd ${COMPOSE_DIR} +ln -sf .env_slurm .env +popd + +# start building this repo +#git submodule update --init --recursive +#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + +source .ci/build_compose.sh + +echo "Stopping HT-Condor containers" +docker-compose stop galaxy-htcondor galaxy-htcondor-executor galaxy-htcondor-executor-big +sleep 30 + +docker-compose logs --tail 50 +docker ps + + +sleep 10 +docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" + diff --git a/.ci/compose_slurm_singularity.sh b/.ci/compose_slurm_singularity.sh new file mode 100755 index 000000000..be2da64ad --- /dev/null +++ b/.ci/compose_slurm_singularity.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +set -o errexit + +source .ci/set_env.sh +source .ci/functions.sh + +# The compose file recognises ENV vars to change the defaul behavior +pushd ${COMPOSE_DIR} +ln -sf .env_slurm_singularity .env +popd + +# start building this repo +#git submodule update --init --recursive +#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + +bash .ci/build_compose.sh + +# turn down the htcondor services +echo "Stopping HT-Condor containers" +docker-compose stop galaxy-htcondor galaxy-htcondor-executor galaxy-htcondor-executor-big +sleep 30 + + +# docker-compose is already started and has pre-populated the /export dir +# we now turn it down again and copy in an example tool with tool_conf.xml and +# a test singularity image. If we copy this from the beginning, the magic Docker Galax startup +# script will not work as it detects something in /export/ +echo "Downloading Singularity test files and images." +docker-compose exec galaxy-web mkdir -p /export/database/container_images/singularity/mulled/ +docker-compose exec galaxy-web curl -L -o /export/database/container_images/singularity/mulled/samtools:1.4.1--0 https://github.com/bgruening/singularity-galaxy-tests/raw/master/samtools:1.4.1--0 +docker-compose exec galaxy-web curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml +docker-compose exec galaxy-web curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml +docker-compose exec galaxy-web chown 1450:1450 /export/cat* /export/database/container_images/ -R +docker-compose down +sleep 20 +rm .env +ln -sf .env_slurm_singularity2 .env + +docker-compose up -d + +until docker-compose exec galaxy-web ps -fC uwsgi +do + echo "Starting up Singularity test container: sleeping for 40 seconds" + sleep 40 + docker-compose logs +done +docker-compose logs galaxy-web +sleep 30 +docker-compose logs galaxy-web +sleep 30 +docker-compose logs galaxy-web +echo "waiting until Galaxy is up" +docker ps +pip show ephemeris +which galaxy-wait +#galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v +#docker-compose logs galaxy-web +#galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v +docker-compose logs galaxy-web +sleep 500 +docker-compose logs +echo "parsec init" +parsec init --api_key admin --url $BIOBLEND_GALAXY_URL +HISTORY_ID=$(parsec histories create_history | jq .id -r) +DATASET_ID=$(parsec tools paste_content 'asdf' $HISTORY_ID | jq '.outputs[0].id' -r) +OUTPUT_ID=$(parsec tools run_tool $HISTORY_ID cat '{"input1": {"src": "hda", "id": "'$DATASET_ID'"}}' | jq '.outputs | .[0].id' -r) +sleep 10 +echo "run parsec jobs show_job" +parsec jobs show_job --full_details $OUTPUT_ID +# TODO: find a way to get a log trace that this tool actually was running with singularity +#parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity + + + +docker-compose logs --tail 50 +docker ps + +sleep 10 +docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" + diff --git a/.ci/functions.sh b/.ci/functions.sh new file mode 100755 index 000000000..48419bdb2 --- /dev/null +++ b/.ci/functions.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +## define a container size check function, first parameter is the container name, second the max allowed size in MB +container_size_check () { + + # check that the image size is not growing too much between releases + # the 19.05 monolithic image was around 1.500 MB + size=`docker image inspect $1 --format='{{.Size}}'` + size_in_mb=$(($size/(1024*1024))) + if [[ $size_in_mb -ge $2 ]] + then + echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2" + sleep 2 + #exit + fi +} + +# Define start functions +docker_exec() { + cd $WORKING_DIR + docker-compose exec galaxy-web "$@" +} +docker_exec_run() { +cd $WORKING_DIR + docker-compose exec galaxy-web "$@" +} +docker_run() { + cd $WORKING_DIR + docker-compose run "$@" +} diff --git a/.ci/kube.sh b/.ci/kube.sh new file mode 100755 index 000000000..5f57527b8 --- /dev/null +++ b/.ci/kube.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -o errexit + +source .ci/set_env.sh +source .ci/functions.sh + +# setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel +gimme 1.11.1 +source ~/.gimme/envs/go1.11.1.env +sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt +sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go +go version +mkdir ../kubernetes +wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes +cd ../kubernetes +# k8s API port is running by default on 8080 as Galaxy, can this to 8000 +export API_PORT=8000 +./hack/install-etcd.sh +sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd +sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl +# this needs to run in backgroud later, for now try to see the output +./hack/local-up-cluster.sh & +cd ../docker-galaxy-stable + +# Installing kompose to convert the docker-compose YAML file + +# The compose file recognises ENV vars to change the defaul behavior +cd ${COMPOSE_DIR} +ln -sf .env_k8_native .env + +curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose +chmod +x kompose +sudo mv ./kompose /usr/bin/kompose + +# start building this repo +#git submodule update --init --recursive +#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp + + +bash .ci/build_compose.sh + +docker-compose logs --tail 50 +docker ps + + +sleep 10 +docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" + diff --git a/.ci/monolithic.sh b/.ci/monolithic.sh new file mode 100755 index 000000000..4bcec67a5 --- /dev/null +++ b/.ci/monolithic.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -o errexit + +bash .ci/cleanup.sh + +source .ci/set_env.sh +source .ci/functions.sh + +export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" +INSTALL_REPO_ARG="" + +cd "$WORKING_DIR" +docker build -t quay.io/bgruening/galaxy galaxy/ + +container_size_check quay.io/bgruening/galaxy 1550 + +mkdir local_folder +docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \ + --name galaxy \ + --privileged=true \ + -v `pwd`/local_folder:/export/ \ + -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \ + -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \ + -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \ + -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \ + -v /tmp/:/tmp/ \ + quay.io/bgruening/galaxy + +sleep 30 +docker logs galaxy + +# Define start functions +docker_exec() { + cd $WORKING_DIR + docker exec -t -i galaxy "$@" +} +docker_exec_run() { + cd $WORKING_DIR + docker run quay.io/bgruening/galaxy "$@" +} +docker_run() { + cd $WORKING_DIR + docker run "$@" +} + +docker ps + +echo '#### Test submitting jobs to an external slurm cluster ####' +pushd $WORKING_DIR/test/slurm/ +bash test.sh +popd + +# Test submitting jobs to an external gridengine cluster +# This test is not testing compose, thus disabled +# TODO 19.05, need to enable this again! +echo "skip SGE test, fix me" +# cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR + +echo '#### Start .ci/testing.sh ####' +source .ci/testing.sh + +echo '### Run a ton of BioBlend test against our servers ####' +pushd $WORKING_DIR/test/bioblend/ +source ./test.sh 'monolithic' +popd ##cd $WORKING_DIR/ + + +echo '#### Test tool installation ####' +docker_exec_run install-tools "$SAMPLE_TOOLS" + +bash .ci/cleanup.sh diff --git a/.ci/set_env.sh b/.ci/set_env.sh new file mode 100755 index 000000000..9166b842d --- /dev/null +++ b/.ci/set_env.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +export WORKING_DIR="${WORKING_DIR:=`pwd`}" + +export GALAXY_HOME=/home/galaxy +export GALAXY_USER=admin@galaxy.org +export GALAXY_USER_EMAIL=admin@galaxy.org +export GALAXY_USER_PASSWD=admin +export BIOBLEND_GALAXY_API_KEY=admin +export BIOBLEND_GALAXY_URL=http://localhost:8080 +export COMPOSE_DIR="${WORKING_DIR}/compose" +export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml diff --git a/.ci/setup.sh b/.ci/setup.sh new file mode 100755 index 000000000..bce7a5692 --- /dev/null +++ b/.ci/setup.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +sudo apt-get update -qq +sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" +sudo apt-get install sshpass --no-install-recommends -y + +pip install ephemeris + +docker --version +docker info diff --git a/.ci/testing.sh b/.ci/testing.sh new file mode 100755 index 000000000..4fffe01b2 --- /dev/null +++ b/.ci/testing.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit + +echo 'Waiting for Galaxy to come up.' +galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300 + +curl -v --fail $BIOBLEND_GALAXY_URL/api/version + +# Test self-signed HTTPS +docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER +# TODO 19.05 +# - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version +#- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned +docker logs httpstest && docker stop httpstest && docker rm httpstest + +# Test FTP Server upload +date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true + +# Test FTP Server get +curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD + +# Test CVMFS +docker_exec bash -c "service autofs start" +docker_exec bash -c "cvmfs_config chksetup" +docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand" + +# Test SFTP Server +sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt' + +# Test the Conda installation +docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes' + diff --git a/.travis.yml b/.travis.yml index 6e94b192a..545acd0cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ services: env: matrix: - - TOX_ENV=py27 + - MONOLITHIC=True TOX_ENV=py27 - COMPOSE_SLURM=True - COMPOSE_SLURM_SINGULARITY=True # Compose version with Condor submitting Docker jobs @@ -25,349 +25,56 @@ git: submodules: false before_install: - - set -e - - export GALAXY_HOME=/home/galaxy - - export GALAXY_USER=admin@galaxy.org - - export GALAXY_USER_EMAIL=admin@galaxy.org - - export GALAXY_USER_PASSWD=admin - - export BIOBLEND_GALAXY_API_KEY=admin - - export BIOBLEND_GALAXY_URL=http://localhost:8080 - - export COMPOSE_DIR="${TRAVIS_BUILD_DIR}/compose" + - export WORKING_DIR="$TRAVIS_BUILD_DIR" + - bash .ci/setup.sh - - sudo apt-get update -qq - - sudo apt-get install docker-ce --no-install-recommends -y -o Dpkg::Options::="--force-confmiss" -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" - - sudo apt-get install sshpass --no-install-recommends -y - - pip install ephemeris + - git submodule update --init --recursive + - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - - docker --version - - docker info - # Build a k8s cluster + # Build and test the monolithic container + - | + if [ "${MONOLITHIC}" ] + then + bash .ci/monolithic.sh + fi + + # Build and test the k8s compose containers - | if [ "${KUBE}" ] then - # setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel - gimme 1.11.1 - source ~/.gimme/envs/go1.11.1.env - sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt - sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go - go version - mkdir ../kubernetes - wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes - cd ../kubernetes - # k8s API port is running by default on 8080 as Galaxy, can this to 8000 - export API_PORT=8000 - ./hack/install-etcd.sh - sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd - sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl - # this needs to run in backgroud later, for now try to see the output - ./hack/local-up-cluster.sh & - cd ../docker-galaxy-stable + bash .ci/kube.sh fi - # load all configurations needed for SLURM testing + # Build and test the compose SLURM enabled containers - | if [ "${COMPOSE_SLURM}" ] then - # The compose file recognises ENV vars to change the defaul behavior - cd ${COMPOSE_DIR} - ln -sf .env_slurm .env + bash .ci/compose_slurm.sh fi - # load all configurations needed for SLURM testing + # Build and test the compose SLURM enabled containers with Singularity dependency resolver - | if [ "${COMPOSE_SLURM_SINGULARITY}" ] then - # The compose file recognises ENV vars to change the defaul behavior - cd ${COMPOSE_DIR} - ln -sf .env_slurm_singularity .env + bash .ci/compose_slurm_singularity.sh fi - # load all configurations needed for Condor and Docker + # Build and test the compose ht-condor enabled containers with Docker dependency resolver - | if [ "${COMPOSE_CONDOR_DOCKER}" ] then - # The compose file recognises ENV vars to change the defaul behavior - cd ${COMPOSE_DIR} - ln -sf .env_htcondor_docker .env - - # Galaxy needs to a full path for the the jobs, in- and outputs. - # Do we want to run each job in it's own container and this container uses the host - # container engine (not Docker in Docker) then the path to all files inside and outside - # of the container needs to be the same. - sudo mkdir /export - sudo chmod 777 /export - sudo chown 1450:1450 /export - fi - - # Installing kompose to convert the docker-compose YAML file - - | - if [ "${KUBE}" ] - then - # The compose file recognises ENV vars to change the defaul behavior - cd ${COMPOSE_DIR} - ln -sf .env_k8_native .env - - curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose - chmod +x kompose - sudo mv ./kompose /usr/bin/kompose + bash .ci/compose_condor_docker.sh fi - # start building this repo - - git submodule update --init --recursive - - sudo chown 1450 /tmp && sudo chmod a=rwx /tmp - - | - ## define a container size check function, first parameter is the container name, second the max allowed size in MB - container_size_check () { - - # check that the image size is not growing too much between releases - # the 19.05 monolithic image was around 1.500 MB - size=`docker image inspect $1 --format='{{.Size}}'` - size_in_mb=$(($size/(1024*1024))) - if [[ $size_in_mb -ge $2 ]] - then - echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2" - sleep 2 - #exit - fi - } - - if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${KUBE}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] - then - pip install docker-compose galaxy-parsec - export WORKING_DIR="$TRAVIS_BUILD_DIR/compose" - export DOCKER_RUN_CONTAINER="galaxy-web" - INSTALL_REPO_ARG="--galaxy-url http://localhost:80" - SAMPLE_TOOLS=/export/config/sample_tool_list.yaml - cd "$WORKING_DIR" - # For build script - export CONTAINER_REGISTRY=quay.io/ - export CONTAINER_USER=bgruening - ./build-orchestration-images.sh --no-push --condor --grafana --slurm --k8s - source ./tags-for-compose-to-source.sh - - container_size_check quay.io/bgruening/galaxy-base:$TAG 350 - container_size_check quay.io/bgruening/galaxy-web:$TAG 650 - container_size_check quay.io/bgruening/galaxy-htcondor-base:$TAG 280 - - export COMPOSE_PROJECT_NAME=galaxy_compose - docker-compose up -d - - until docker-compose exec galaxy-web ps -fC uwsgi - do - echo "sleeping for 20 seconds" - sleep 20 - docker-compose logs --tail 10 - done - - if [ "${COMPOSE_CONDOR_DOCKER}" ] - then - # turn down the slurm service - echo "Stopping SLURM container" - docker-compose stop galaxy-slurm - sleep 30 - fi - - if [ "${COMPOSE_SLURM}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] - then - # turn down the htcondor services - echo "Stopping HT-Condor containers" - docker-compose stop galaxy-htcondor galaxy-htcondor-executor galaxy-htcondor-executor-big - sleep 30 - fi - - if [ "${COMPOSE_SLURM_SINGULARITY}" ] - then - # docker-compose is already started and has pre-populated the /export dir - # we now turn it down again and copy in an example tool with tool_conf.xml and - # a test singularity image. If we copy this from the beginning, the magic Docker Galax startup - # script will not work as it detects something in /export/ - echo "Downloading Singularity test files and images." - docker-compose exec galaxy-web mkdir -p /export/database/container_images/singularity/mulled/ - docker-compose exec galaxy-web curl -L -o /export/database/container_images/singularity/mulled/samtools:1.4.1--0 https://github.com/bgruening/singularity-galaxy-tests/raw/master/samtools:1.4.1--0 - docker-compose exec galaxy-web curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml - docker-compose exec galaxy-web curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml - docker-compose exec galaxy-web chown 1450:1450 /export/cat* /export/database/container_images/ -R - docker-compose down - sleep 20 - rm .env - ln -sf .env_slurm_singularity2 .env - - docker-compose up -d - until docker-compose exec galaxy-web ps -fC uwsgi - do - echo "Starting up Singularity test container: sleeping for 40 seconds" - sleep 40 - docker-compose logs - done - docker-compose logs galaxy-web - sleep 30 - docker-compose logs galaxy-web - sleep 30 - docker-compose logs galaxy-web - echo "waiting until Galaxy is up" - docker ps - pip show ephemeris - which galaxy-wait - #galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v - #docker-compose logs galaxy-web - #galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v - docker-compose logs galaxy-web - sleep 500 - docker-compose logs - echo "parsec init" - parsec init --api_key admin --url $BIOBLEND_GALAXY_URL - HISTORY_ID=$(parsec histories create_history | jq .id -r) - DATASET_ID=$(parsec tools paste_content 'asdf' $HISTORY_ID | jq '.outputs[0].id' -r) - OUTPUT_ID=$(parsec tools run_tool $HISTORY_ID cat '{"input1": {"src": "hda", "id": "'$DATASET_ID'"}}' | jq '.outputs | .[0].id' -r) - sleep 10 - echo "run parsec jobs show_job" - parsec jobs show_job --full_details $OUTPUT_ID - # TODO: find a way to get a log trace that this tool actually was running with singularity - #parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity - fi - docker-compose logs --tail 50 - # Define start functions - docker_exec() { - cd $WORKING_DIR - docker-compose exec galaxy-web "$@" - } - docker_exec_run() { - cd $WORKING_DIR - docker-compose exec galaxy-web "$@" - } - docker_run() { - cd $WORKING_DIR - docker-compose run "$@" - } - else - export WORKING_DIR="$TRAVIS_BUILD_DIR" - export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy" - INSTALL_REPO_ARG="" - SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml - cd "$WORKING_DIR" - docker build -t quay.io/bgruening/galaxy galaxy/ - container_size_check quay.io/bgruening/galaxy 1500 - - mkdir local_folder - docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \ - --name galaxy \ - --privileged=true \ - -v `pwd`/local_folder:/export/ \ - -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \ - -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \ - -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \ - -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \ - -v /tmp/:/tmp/ \ - quay.io/bgruening/galaxy - - sleep 30 - docker logs galaxy - # Define start functions - docker_exec() { - cd $WORKING_DIR - docker exec -t -i galaxy "$@" - } - docker_exec_run() { - cd $WORKING_DIR - docker run quay.io/bgruening/galaxy "$@" - } - docker_run() { - cd $WORKING_DIR - docker run "$@" - } - - fi - - docker ps script: - - set -e - # Test submitting jobs to an external slurm cluster - - | - if [ ! "${COMPOSE_SLURM}" ] && [ ! "${KUBE}" ] && [ ! "${COMPOSE_CONDOR_DOCKER}" ] && [ ! "${COMPOSE_SLURM_SINGULARITY}" ] - then - # For compose slurm is already included and thus tested - cd $TRAVIS_BUILD_DIR/test/slurm/ && bash test.sh && cd $WORKING_DIR - fi - # Test submitting jobs to an external gridengine cluster - - | - if [ ! "${COMPOSE_SLURM}" ] && [ ! "${KUBE}" ] && [ ! "${COMPOSE_CONDOR_DOCKER}" ] && [ ! "${COMPOSE_SLURM_SINGULARITY}" ] - then - # This test is not testing compose, thus disabled - # TODO 19.05, need to enable this again! - echo "skip SGE test, fix me" - # cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR - fi - # Test Web api - - | - if [ "${COMPOSE_CONDOR_DOCKER}" ] - then - docker-compose logs --tail 50 - fi - - - echo 'Waiting for Galaxy to come up.' - - galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 300 - - - curl -v --fail $BIOBLEND_GALAXY_URL/api/version - - # Test self-signed HTTPS - - docker_run -d --name httpstest -p 443:443 -e "USE_HTTPS=True" $DOCKER_RUN_CONTAINER - # TODO 19.05 - # - sleep 90s && curl -v -k --fail https://127.0.0.1:443/api/version - #- echo | openssl s_client -connect 127.0.0.1:443 2>/dev/null | openssl x509 -issuer -noout| grep selfsigned - - docker logs httpstest && docker stop httpstest && docker rm httpstest - - # Test FTP Server upload - - date > time.txt && curl -v --fail -T time.txt ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD || true - # Test FTP Server get - - curl -v --fail ftp://localhost:8021 --user $GALAXY_USER:$GALAXY_USER_PASSWD - - # Test CVMFS - - docker_exec bash -c "service autofs start" - - docker_exec bash -c "cvmfs_config chksetup" - - docker_exec bash -c "ls /cvmfs/data.galaxyproject.org/byhand" - # Test SFTP Server - - sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt' - # Run a ton of BioBlend test against our servers. - - cd $TRAVIS_BUILD_DIR/test/bioblend/ && . ./test.sh && cd $WORKING_DIR/ - - # not working anymore in 18.01 - # executing: /galaxy_venv/bin/uwsgi --yaml /etc/galaxy/galaxy.yml --master --daemonize2 galaxy.log --pidfile2 galaxy.pid --log-file=galaxy_install.log --pid-file=galaxy_install.pid - # [uWSGI] getting YAML configuration from /etc/galaxy/galaxy.yml - # /galaxy_venv/bin/python: unrecognized option '--log-file=galaxy_install.log' - # getopt_long() error - # cat: galaxy_install.pid: No such file or directory - # tail: cannot open ‘galaxy_install.log’ for reading: No such file or directory - #- | - # if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] - # then - # # Test without install-repository wrapper - # sleep 10 - # docker_exec_run bash -c 'cd $GALAXY_ROOT && python ./scripts/api/install_tool_shed_repositories.py --api admin -l http://localhost:80 --url https://toolshed.g2.bx.psu.edu -o devteam --name cut_columns --panel-section-name BEDTools' - # fi - - - # Test the 'new' tool installation script -# - docker_exec bash -c "install-tools $GALAXY_HOME/ephemeris/sample_tool_list.yaml" - - | - if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] - then - # Compose uses the online installer (uses the running instance) - sleep 10 - docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS" - else - docker_exec_run install-tools "$SAMPLE_TOOLS" - fi - # Test the Conda installation - - docker_exec_run bash -c 'export PATH=$GALAXY_CONFIG_TOOL_DEPENDENCY_DIR/_conda/bin/:$PATH && conda --version && conda install samtools -c bioconda --yes' - # Test Docker in Docker, used by Interactive Environments; This needs to be at the end as Docker takes some time to start. - #- docker_exec docker info # Check if the database image matches the current galaxy version - | if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] diff --git a/README.md b/README.md index c3d3b929e..7f30a8001 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ The Image is based on [Ubuntu 14.04 LTS](http://releases.ubuntu.com/14.04/) and - [Integrating non-Tool Shed tools into the container](#Integrating-non-Tool-Shed-tools-into-the-container) - [Users & Passwords](#Users-Passwords) - [Development](#Development) + - [Testing](#Testing) - [Requirements](#Requirements) - [History](#History) - [Support & Bug Reports](#Support-Bug-Reports) @@ -774,6 +775,18 @@ If you simply want to change the Galaxy repository and/or the Galaxy branch, fro --build-arg GALAXY_REPO=https://github.com/manabuishii/galaxy ``` +## Testing the images + +You can run our test against the newly build images with our test scripts. Please note that for a few tests we currently need to run commands with sudo. +We also assume you have Docker installed. + +Testing the monolithic container: + +```bash +.ci/monolithic.sh +``` + + # Requirements [[toc]](#toc) - [Docker](https://www.docker.io/gettingstarted/#h_installation) diff --git a/galaxy/Dockerfile b/galaxy/Dockerfile index 66ec6aa22..8d13f122f 100644 --- a/galaxy/Dockerfile +++ b/galaxy/Dockerfile @@ -99,7 +99,7 @@ RUN groupadd -r postgres -g $GALAXY_POSTGRES_GID \ && apt-get install git --no-install-recommends -y \ && apt-get install gridengine-common gridengine-drmaa1.0 --no-install-recommends -y \ && apt-get install rabbitmq-server --no-install-recommends -y \ - && apt-get install --no-install-recommends -y libswitch-perl supervisor \ + && apt-get install --no-install-recommends -y uuid-runtime libswitch-perl supervisor \ && apt-get purge -y software-properties-common gpg-agent apt-transport-https python3-minimal \ && apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ \ && mkdir -p /etc/supervisor/conf.d/ /var/log/supervisor/ \ @@ -197,7 +197,9 @@ RUN mkdir -p /shed_tools $EXPORT_DIR/ftp/ \ --extra-vars galaxy_config_dir=$GALAXY_CONFIG_DIR \ --extra-vars galaxy_job_conf_path=$GALAXY_CONFIG_JOB_CONFIG_FILE \ --extra-vars galaxy_job_metrics_conf_path=$GALAXY_CONFIG_JOB_METRICS_CONFIG_FILE \ - --extra-vars supervisor_manage_slurm="" \ + --extra-vars supervisor_manage_slurm=True \ + --extra-vars supervisor_slurm_autostart=False \ + --extra-vars supervisor_condor_autostart=False \ --extra-vars galaxy_extras_config_condor=True \ --extra-vars galaxy_extras_config_condor_docker=True \ --extra-vars galaxy_extras_config_rabbitmq=True \ diff --git a/galaxy/roles/galaxyprojectdotorg.galaxyextras b/galaxy/roles/galaxyprojectdotorg.galaxyextras index 9aa0286fc..9677970ee 160000 --- a/galaxy/roles/galaxyprojectdotorg.galaxyextras +++ b/galaxy/roles/galaxyprojectdotorg.galaxyextras @@ -1 +1 @@ -Subproject commit 9aa0286fc5e50c321950fec1285b7e10929aa25d +Subproject commit 9677970eee69660a30e73bed251d2482959aab61 diff --git a/test/bioblend/test.sh b/test/bioblend/test.sh index 1f84d7add..3d54d7680 100644 --- a/test/bioblend/test.sh +++ b/test/bioblend/test.sh @@ -1,5 +1,6 @@ #!/bin/bash -if [ "${COMPOSE_SLURM}" ] || [ "${KUBE}" ] || [ "${COMPOSE_CONDOR_DOCKER}" ] || [ "${COMPOSE_SLURM_SINGULARITY}" ] + +if [ $1 -eq "compose" ] then docker_exec bash -c 'cd /home/galaxy ; . /galaxy_venv/bin/activate ; diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile index 76495a16d..53c74bbe5 100644 --- a/test/slurm/Dockerfile +++ b/test/slurm/Dockerfile @@ -11,7 +11,7 @@ RUN apt-get update -qq && apt-get install -y --no-install-recommends \ apt install -y docker-ce && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && rm -rf ~/.cache/ && \ adduser galaxy &&\ - /usr/sbin/create-munge-key &&\ + yes | /usr/sbin/create-munge-key &&\ touch /var/log/slurm-llnl/slurmctld.log /var/log/slurm-llnl/slurmd.log &&\ mkdir /tmp/slurm diff --git a/test/slurm/startup.sh b/test/slurm/startup.sh index 9b6cb4439..09c22ed91 100644 --- a/test/slurm/startup.sh +++ b/test/slurm/startup.sh @@ -18,7 +18,6 @@ if [ ! -f "$SLURM_CONF_PATH" ] then python /usr/local/bin/configure_slurm.py cp /etc/slurm-llnl/slurm.conf "$SLURM_CONF_PATH" - rm /etc/slurm-llnl/slurm.conf fi if [ ! -f "$GALAXY_DIR"/.venv ] then diff --git a/test/slurm/test.sh b/test/slurm/test.sh index d966f6789..122376ccc 100644 --- a/test/slurm/test.sh +++ b/test/slurm/test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -x +set -o errexit # Test that jobs run successfully on an external slurm cluster # We use a temporary directory as an export dir that will hold the shared data between @@ -36,4 +36,5 @@ sleep 60s docker exec galaxy-slurm-test su - galaxy -c 'srun hostname' | grep slurm docker stop galaxy-slurm-test slurm docker rm galaxy-slurm-test slurm + # TODO: Run a galaxy tool and check it runs on the cluster