WIP: More modular testing infrastructure

.ci/build_compose.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+pip --version
+# --user does not work if you are already in a virtual-env
+pip install docker-compose galaxy-parsec --user || pip install docker-compose galaxy-parsec
+
+export DOCKER_RUN_CONTAINER="galaxy-web"
+INSTALL_REPO_ARG="--galaxy-url http://localhost:80"
+SAMPLE_TOOLS=/export/config/sample_tool_list.yaml
+
+pushd $COMPOSE_DIR
+
+# For build script
+export CONTAINER_REGISTRY=quay.io/
+export CONTAINER_USER=bgruening
+./build-orchestration-images.sh --no-push --condor --grafana --slurm --k8s
+
+cat ./tags-for-compose-to-source.sh
+source ./tags-for-compose-to-source.sh
+
+container_size_check   quay.io/bgruening/galaxy-base:$TAG               350 
+container_size_check   quay.io/bgruening/galaxy-web:$TAG                650
+container_size_check   quay.io/bgruening/galaxy-htcondor-base:$TAG      280
+
+export COMPOSE_PROJECT_NAME=galaxy_compose
+docker-compose up -d
+
+until docker-compose exec galaxy-web ps -fC uwsgi
+do
+  echo "sleeping for 20 seconds"
+  sleep 20
+  docker-compose logs --tail 10
+done
+
+# back to the root of the repo
+popd
+

.ci/cleanup.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+sudo rm -rf ./local_folder | true
+sudo rm -rf /export | true
+docker stop galaxy | true
+docker rm galaxy | true

.ci/compose_condor_docker.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+echo '#### Start compose condor Docker testing ####'
+
+bash .ci/cleanup.sh
+
+source .ci/set_env.sh
+source .ci/functions.sh
+
+# The compose file recognises ENV vars to change the defaul behavior
+pushd ${COMPOSE_DIR}
+ln -sf .env_htcondor_docker .env
+
+# Galaxy needs to a full path for the the jobs, in- and outputs.
+# Do we want to run each job in it's own container and this container uses the host
+# container engine (not Docker in Docker) then the path to all files inside and outside
+# of the container needs to be the same.
+sudo mkdir /export
+sudo chmod 777 /export
+sudo chown 1450:1450 /export
+
+# start building this repo
+#git submodule update --init --recursive
+#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+popd
+source .ci/build_compose.sh
+pushd ${COMPOSE_DIR}
+
+source ./tags-for-compose-to-source.sh
+
+echo "Stopping SLURM container"
+docker-compose stop galaxy-slurm
+sleep 30
+
+docker-compose logs --tail 50
+docker ps
+
+sleep 10
+docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
+
+popd
+
+echo '#### Start .ci/testing.sh ####'
+source .ci/testing.sh
+
+echo '#### Run a ton of BioBlend test against our servers ####'
+pushd $WORKING_DIR/test/bioblend/
+source ./test.sh 'compose'
+popd
+
+pushd ${COMPOSE_DIR}
+docker-compose down
+popd
+

.ci/compose_slurm.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+echo '#### Start compose Slurm testing ####'
+
+bash .ci/cleanup.sh
+
+source .ci/set_env.sh
+source .ci/functions.sh
+
+# The compose file recognises ENV vars to change the defaul behavior
+pushd ${COMPOSE_DIR}
+ln -sf .env_slurm .env
+popd
+
+# start building this repo
+#git submodule update --init --recursive
+#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+source .ci/build_compose.sh
+
+echo "Stopping HT-Condor containers"
+docker-compose stop galaxy-htcondor galaxy-htcondor-executor galaxy-htcondor-executor-big
+sleep 30
+
+docker-compose logs --tail 50
+docker ps
+
+
+sleep 10
+docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
+

.ci/compose_slurm_singularity.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+source .ci/set_env.sh
+source .ci/functions.sh
+
+# The compose file recognises ENV vars to change the defaul behavior
+pushd ${COMPOSE_DIR}
+ln -sf .env_slurm_singularity .env
+popd
+
+# start building this repo
+#git submodule update --init --recursive
+#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+bash .ci/build_compose.sh
+
+# turn down the htcondor services
+echo "Stopping HT-Condor containers"
+docker-compose stop galaxy-htcondor galaxy-htcondor-executor galaxy-htcondor-executor-big
+sleep 30
+
+
+# docker-compose is already started and has pre-populated the /export dir
+# we now turn it down again and copy in an example tool with tool_conf.xml and
+# a test singularity image. If we copy this from the beginning, the magic Docker Galax startup
+# script will not work as it detects something in /export/
+echo "Downloading Singularity test files and images."
+docker-compose exec galaxy-web    mkdir -p /export/database/container_images/singularity/mulled/
+docker-compose exec galaxy-web    curl -L -o /export/database/container_images/singularity/mulled/samtools:1.4.1--0 https://github.com/bgruening/singularity-galaxy-tests/raw/master/samtools:1.4.1--0
+docker-compose exec galaxy-web    curl -L -o /export/cat_tool_conf.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat_tool_conf.xml
+docker-compose exec galaxy-web    curl -L -o /export/cat.xml https://github.com/bgruening/singularity-galaxy-tests/raw/master/cat.xml
+docker-compose exec galaxy-web    chown 1450:1450 /export/cat* /export/database/container_images/ -R
+docker-compose down
+sleep 20
+rm .env
+ln -sf .env_slurm_singularity2 .env
+
+docker-compose up -d
+
+until docker-compose exec galaxy-web ps -fC uwsgi
+do
+  echo "Starting up Singularity test container: sleeping for 40 seconds"
+  sleep 40
+  docker-compose logs
+done
+docker-compose logs galaxy-web
+sleep 30
+docker-compose logs galaxy-web
+sleep 30
+docker-compose logs galaxy-web
+echo "waiting until Galaxy is up"
+docker ps
+pip show ephemeris
+which galaxy-wait
+#galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v
+#docker-compose logs galaxy-web
+#galaxy-wait -g $BIOBLEND_GALAXY_URL --timeout 60 -v
+docker-compose logs galaxy-web
+sleep 500
+docker-compose logs
+echo "parsec init"
+parsec init --api_key admin --url $BIOBLEND_GALAXY_URL
+HISTORY_ID=$(parsec histories create_history | jq .id -r)
+DATASET_ID=$(parsec tools paste_content 'asdf' $HISTORY_ID | jq '.outputs[0].id' -r)
+OUTPUT_ID=$(parsec tools run_tool $HISTORY_ID cat '{"input1": {"src": "hda", "id": "'$DATASET_ID'"}}' | jq '.outputs | .[0].id' -r)
+sleep 10
+echo "run parsec jobs show_job"
+parsec jobs show_job --full_details $OUTPUT_ID
+# TODO: find a way to get a log trace that this tool actually was running with singularity
+#parsec jobs show_job --full_details $OUTPUT_ID | jq .stderr | grep singularity
+
+
+
+docker-compose logs --tail 50
+docker ps
+
+sleep 10
+docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
+

.ci/functions.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+## define a container size check function, first parameter is the container name, second the max allowed size in MB
+container_size_check () {
+
+  # check that the image size is not growing too much between releases
+  # the 19.05 monolithic image was around 1.500 MB
+  size=`docker image inspect $1 --format='{{.Size}}'`
+  size_in_mb=$(($size/(1024*1024)))
+  if [[ $size_in_mb -ge $2 ]]
+  then
+      echo "The new compiled image ($1) is larger than allowed. $size_in_mb vs. $2"
+      sleep 2
+      #exit
+  fi
+}
+
+# Define start functions
+docker_exec() {
+    cd $WORKING_DIR
+    docker-compose exec galaxy-web "$@"
+}
+docker_exec_run() {
+cd $WORKING_DIR
+    docker-compose exec galaxy-web "$@"
+}
+docker_run() {
+    cd $WORKING_DIR
+    docker-compose run "$@"
+}

.ci/kube.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+source .ci/set_env.sh
+source .ci/functions.sh
+
+# setup k8s, we will do this before building Galaxy because it takes some time and hopefully we can do both in prallel
+gimme 1.11.1
+source ~/.gimme/envs/go1.11.1.env
+sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/gofmt /usr/bin/gofmt
+sudo ln -s /home/travis/.gimme/versions/go1.11.1.linux.amd64/bin/go /usr/bin/go
+go version
+mkdir ../kubernetes
+wget -q -O - https://github.com/kubernetes/kubernetes/archive/master.tar.gz | tar xzf - --strip-components=1 -C ../kubernetes
+cd ../kubernetes
+# k8s API port is running by default on 8080 as Galaxy, can this to 8000
+export API_PORT=8000
+./hack/install-etcd.sh
+sudo ln -s `pwd`/third_party/etcd/etcd /usr/bin/etcd
+sudo ln -s `pwd`/third_party/etcd/etcdctl /usr/bin/etcdctl
+# this needs to run in backgroud later, for now try to see the output
+./hack/local-up-cluster.sh &
+cd ../docker-galaxy-stable
+
+# Installing kompose to convert the docker-compose YAML file
+
+# The compose file recognises ENV vars to change the defaul behavior
+cd ${COMPOSE_DIR}
+ln -sf .env_k8_native .env
+
+curl -L https://github.com/kubernetes-incubator/kompose/releases/download/v1.17.1/kompose-linux-amd64 -o kompose
+chmod +x kompose
+sudo mv ./kompose /usr/bin/kompose
+
+# start building this repo
+#git submodule update --init --recursive
+#sudo chown 1450 /tmp && sudo chmod a=rwx /tmp
+
+
+bash .ci/build_compose.sh
+
+docker-compose logs --tail 50
+docker ps
+
+
+sleep 10
+docker_exec_run shed-tools install -g "http://localhost:80" -a admin -t "$SAMPLE_TOOLS"
+

.ci/monolithic.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+bash .ci/cleanup.sh
+
+source .ci/set_env.sh
+source .ci/functions.sh
+
+export DOCKER_RUN_CONTAINER="quay.io/bgruening/galaxy"
+INSTALL_REPO_ARG=""
+
+cd "$WORKING_DIR"
+docker build -t quay.io/bgruening/galaxy galaxy/
+
+container_size_check   quay.io/bgruening/galaxy  1550
+
+mkdir local_folder
+docker run -d -p 8080:80 -p 8021:21 -p 8022:22 \
+    --name galaxy \
+    --privileged=true \
+    -v `pwd`/local_folder:/export/ \
+    -e GALAXY_CONFIG_ALLOW_USER_DATASET_PURGE=True \
+    -e GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE=True \
+    -e GALAXY_CONFIG_ENABLE_USER_DELETION=True \
+    -e GALAXY_CONFIG_ENABLE_BETA_WORKFLOW_MODULES=True \
+    -v /tmp/:/tmp/ \
+    quay.io/bgruening/galaxy
+
+sleep 30
+docker logs galaxy
+
+# Define start functions
+docker_exec() {
+  cd $WORKING_DIR
+  docker exec -t -i galaxy "$@"
+}
+docker_exec_run() {
+  cd $WORKING_DIR
+  docker run quay.io/bgruening/galaxy "$@"
+}
+docker_run() {
+  cd $WORKING_DIR
+  docker run "$@"
+}
+
+docker ps
+
+echo '#### Test submitting jobs to an external slurm cluster ####'
+pushd $WORKING_DIR/test/slurm/
+bash test.sh
+popd
+
+# Test submitting jobs to an external gridengine cluster
+# This test is not testing compose, thus disabled
+# TODO 19.05, need to enable this again!
+echo "skip SGE test, fix me"
+# cd $TRAVIS_BUILD_DIR/test/gridengine/ && bash test.sh && cd $WORKING_DIR
+
+echo '#### Start .ci/testing.sh ####'
+source .ci/testing.sh
+
+echo '### Run a ton of BioBlend test against our servers ####'
+pushd $WORKING_DIR/test/bioblend/
+source ./test.sh 'monolithic'
+popd ##cd $WORKING_DIR/
+
+
+echo '#### Test tool installation ####'
+docker_exec_run install-tools "$SAMPLE_TOOLS"
+
+bash .ci/cleanup.sh

.ci/set_env.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+export WORKING_DIR="${WORKING_DIR:=`pwd`}"
+
+export GALAXY_HOME=/home/galaxy
+export [email protected]
+export [email protected]
+export GALAXY_USER_PASSWD=admin
+export BIOBLEND_GALAXY_API_KEY=admin
+export BIOBLEND_GALAXY_URL=http://localhost:8080
+export COMPOSE_DIR="${WORKING_DIR}/compose"
+export SAMPLE_TOOLS=$GALAXY_HOME/ephemeris/sample_tool_list.yaml