Merge pull request #112 from ci-for-research/84-singularity

84-add singularity guide

README.md

@@ -32,6 +32,7 @@ Linux Ubuntu.
 | Status | Client OS | Server hardware | Runner |
 | --- | --- | --- | --- |
 | :heavy_check_mark: Completed | Linux Ubuntu | local machine via Docker           | [link](/ubuntu-docker/README.md)          |
+| :heavy_check_mark: Completed | Linux Ubuntu | local machine via Singularity      | [link](/ubuntu-singularity/README.md)     |
 | :heavy_check_mark: Completed | Linux Ubuntu | local machine via Vagrant          | [link](/ubuntu-vagrant/README.md)         |
 | :heavy_check_mark: Completed | Linux Ubuntu | local machine via VirtualBox       | [link](/ubuntu-virtualbox/README.md)      |
 | :heavy_check_mark: Completed | Linux Ubuntu | remote machine at [SURF HPC Cloud] | [link](/ubuntu-surf-hpc-cloud/README.md)  |

ubuntu-docker/Dockerfile

@@ -8,16 +8,16 @@ ENV RUNNER_VERSION=2.267.1
 ENV AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends ca-certificates \
-                                               curl \
-                                               jq \
-                                               libcurl4-openssl-dev \
-                                               sudo \
-                                               tar && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        jq \
+        libcurl4-openssl-dev \
+        tar && \
     rm -rf /var/lib/apt/lists/*
 
 RUN printf "\n\033[0;44m---> Creating SSH user.\033[0m\n"
-RUN useradd -rm -d /home/${DOCKER_USER} -s /bin/bash -G sudo -u 1001 ${DOCKER_USER}
+RUN useradd -rm -d /home/${DOCKER_USER} -s /bin/bash -G ${DOCKER_USER} -u 1001 ${DOCKER_USER}
 RUN echo "${DOCKER_USER}:${DOCKER_PASS}" | chpasswd
 WORKDIR /home/${DOCKER_USER}/actions-runner
 

ubuntu-singularity/README.md

@@ -0,0 +1,228 @@
+# Setting up a CI server for a GitHub Action runner Singularity from Linux Ubuntu
+
+After following this guide, you'll have a simple GitHub action workflow on a GitHub repository of your choice. When new commits are made to your repository, the workflow delegates work to a server which runs in a [Singlularity](https://sylabs.io/singularity/) container. You can use this Singularity container to on a HPC cluster as a regular user and you will not need root permissions.
+
+This guide distinguishes between the _client_ and the _server_; the client is your own machine; the server is whichever
+machine will run the tests. This document describes the case where the server is a Singularity container running on your own machine.
+
+For guides on how to configure other features in addition to just the runner, go [here](/README.md).
+
+## Prerequisites
+
+1. Install Singularity: https://sylabs.io/guides/3.5/user-guide/quick_start.html#quick-installation-steps
+
+2. Follow a [short tutorial](https://sylabs.io/guides/3.5/user-guide/quick_start.html#overview-of-the-singularity-interface) (Optional)
+
+### Testing your Singularity setup
+
+Check Singularity version
+```shell
+> singularity version
+
+3.5.3
+```
+
+Pull an example image
+
+```shell
+singularity pull library://sylabsed/examples/lolcow
+```
+
+Start a shell in the Singlarity container
+
+```shell
+singularity shell lolcow_latest.sif
+```
+
+Run some test commands
+
+```shell
+Singularity> id
+
+uid=1000(fdiblen) gid=985(users) groups=985(users),98(power),108(vboxusers),972(docker),988(storage),998(wheel)
+
+Singularity> hostname
+
+archlinux
+```
+
+## Server side configuration
+
+### Build image
+
+Now we are ready to build our Singularity image. The following command will use [Definition file](github-actions-runner-singularity.def) to build the image. It will create a system install necessary system packages and dependencies for the runner. In order to create a Singularity image, you will need root permission (or sudo) on your system.
+
+```shell
+sudo singularity build github-actions-runner-singularity.sif github-actions-runner-singularity.def
+```
+
+This command will generate ``github-actions-runner-singularity.sif`` (SIF stands for Singularity Image Format) image which we will use to set up the runner.
+
+## Client side configuration
+
+### Generate an OAuth token
+
+We're almost ready to use our Docker image to set up a GitHub Runner, but first we need to
+generate an OAuth token, as follows:
+
+1. Go to [https://github.com/settings/tokens](https://github.com/settings/tokens) and click the ``Generate new token`` button.
+2. Provide your GitHub password when prompted
+3. Fill in a description for the token, for example _GitHub runner for github.com/<your organization>/<your repository>_
+4. Enable the ``repo`` scope and all of its checkboxes, like so:
+
+    ![Token permissions](/images/token_permissions.png)
+
+5. Click ``Generate`` at the bottom. Make sure to copy its value because we'll need it in the next step
+
+### Run the server
+
+#### Preperation
+Before using the Singularity image we need to set some environment variables. The Singularity container will use these environment variables to set up the runner.
+
+```shell
+export SINGULARITYENV_PERSONAL_ACCESS_TOKEN="<Github OAuth token>"
+export SINGULARITYENV_RUNNER_NAME="<runner name to appear on Github>"
+export SINGULARITYENV_RUNNER_WORKDIR="/tmp/actions-runner-repo"
+export SINGULARITYENV_GITHUB_ORG="<organization or username>"
+export SINGULARITYENV_GITHUB_REPO="<name of the repository>"
+```
+
+Create an envionment file which will be user by the runner to save some variables.
+
+```shell
+cp env.template env
+```
+
+#### Instance mode
+
+Alternatively, you can start it as an instance (service).
+
+```shell
+singularity instance start github-actions-runner-singularity.sif github-actions-runner --writable-tmpfs --bind ./env:/opt/actions-runner/.env
+```
+
+For more information about Singularity services see [this link](https://sylabs.io/guides/3.5/user-guide/running_services.html).
+
+
+To list the running instances:
+
+```shell
+singularity instance list
+```
+
+To stop the running Singularity instance:
+
+```shell
+singularity instance stop github-actions-runner
+```
+
+To start the Singularity instance again:
+
+```shell
+singularity instance start github-actions-runner
+```
+
+#### Temporary mode
+
+Now we can run Singularity container with the following command.
+
+```shell
+singularity run \
+    --writable-tmpfs \
+    --bind ./env:/opt/actions-runner/.env \
+    github-actions-runner-singularity.sif
+```
+
+Singularity containers by-default starts in ``read-only`` mode so you cannot make changes. While setting up the runner, some scripts needs to create a few files so we need a write access. This is achieved by adding ``--writable-tmpfs`` argument.
+
+If you stop the running container or interrupt it by pressing to ``CTRL+C``, the Github actions runner will stop and it will be unregistered from your Github repository.
+
+#### Accessing the logs
+
+The singularity instances save the logs in
+`~/.singularity/instances/logs/INSTANCE_NAME` folder.
+
+## Using on a HPC Cluster
+
+In most of the cases, the HPC user does not have root persmissions s it wont be possible to build the Singularity image. The Singularity image can be built locally and copied to
+the cluster. We will use `scp` command to copy the singularity image.
+
+```shell
+scp github-actions-runner-singularity.sif USERNAME@CLUSTER_IP_ADDRESS:$REMOTE_FOLDER
+```
+
+The `$REMOTE_FOLDER` is typically your home folder which you can figure out using the command below.
+
+```shell
+echo $HOME
+```
+
+In oder to use the Singularity image on a HPC cluster, we first need to create a jobscript. The job script will be handled by the scheduler and eventually it will set up the runner when it is executed.
+
+
+Example:
+```
+#!/bin/bash
+#SBATCH -t 1:00:00
+#SBATCH -n 480
+
+cd $HOME/work
+
+export SINGULARITYENV_PERSONAL_ACCESS_TOKEN="<Github OAuth token>"
+export SINGULARITYENV_RUNNER_NAME="<runner name to appear on Github>"
+export SINGULARITYENV_RUNNER_WORKDIR="/tmp/actions-runner-repo"
+export SINGULARITYENV_GITHUB_ORG="<organization or username>"
+export SINGULARITYENV_GITHUB_REPO="<name of the repository>"
+
+srun singularity run \
+    --writable-tmpfs \
+    github-actions-runner-singularity.sif
+```
+
+To submit the job script using ``sbatch``:
+
+```shell
+sbatch jobscript
+```
+
+## GPU support
+
+See [Singularity documentation](https://sylabs.io/guides/3.5/user-guide/gpu.html)
+
+## Limitations
+
+GitHub owned servers have some pre-installed software (see [here](https://docs.github.com/en/actions/reference/software-installed-on-github-hosted-runners)). Self hosted runners are able to download and use these software. However, this is only possible if self-hosted runner is being run on one of the supported platforms such as Ubuntu. Otherwise, users have to install the required software by themselves.
+Also actions based on Docker images will not work as running Docker containers inside a Singularity container does not work.
+
+### Extras
+
+#### Get Singularity image details
+
+Use `singularity inspect` to display details of the image
+
+```shell
+> singularity inspect github-actions-runner-singularity.sif*
+WARNING: No SIF metadata partition, searching in container...
+org.label-schema.build-date: Monday_6_July_2020_16:55:58_CEST
+org.label-schema.schema-version: 1.0
+org.label-schema.usage.singularity.deffile.bootstrap: library
+org.label-schema.usage.singularity.deffile.from: ubuntu:19.10
+org.label-schema.usage.singularity.deffile.mirrorurl: http://us.archive.ubuntu.com/ubuntu/
+org.label-schema.usage.singularity.deffile.osversion: eoan
+org.label-schema.usage.singularity.deffile.stage: build
+org.label-schema.usage.singularity.version: 3.5.3
+```
+
+#### Accessing Singularity container
+
+If you need access to a shell on the running Singularity container:
+
+```shell
+singularity shell \
+    --writable-tmpfs \
+    github-actions-runner-singularity.sif
+```
+
+### What's next
+
+Find instructions for provisioning additional functionality [here](../README.md).

ubuntu-singularity/entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+export RUNNER_USERNAME=$(id -un)
+export RUNNER_USERGROUP=$(id -gn)
+
+export AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
+
+cd /opt/actions-runner
+
+if [[ -z "${RUNNER_NAME}" ]]; then
+    RUNNER_NAME="singularity-$(hostname)"
+fi
+
+ACTIONS_URL="https://api.github.com/repos/${GITHUB_ORG}/${GITHUB_REPO}/actions/runners/registration-token"
+echo "Requesting registration URL at '${ACTIONS_URL}'"
+
+PAYLOAD=$(curl -sX POST -H "Authorization: token ${PERSONAL_ACCESS_TOKEN}" ${ACTIONS_URL})
+export RUNNER_TOKEN=$(echo $PAYLOAD | jq .token --raw-output)
+
+printf "\n\033[0;44m---> Configuring the runner.\033[0m\n"
+./config.sh \
+    --name ${RUNNER_NAME} \
+    --token ${RUNNER_TOKEN} \
+    --url https://github.com/${GITHUB_ORG}/${GITHUB_REPO} \
+    --work ${RUNNER_WORKDIR} \
+    --labels "singularity,github" \
+    --unattended \
+    --replace
+
+remove_runner() {
+    printf "\n\033[0;44m---> Removing the runner.\033[0m\n"
+    ./config.sh remove --unattended --token "${RUNNER_TOKEN}"
+}
+
+# run remove_runner function if "./run.sh" script is interrupted
+trap "remove_runner" EXIT SIGINT SIGTERM KILL
+
+printf "\n\033[0;44m---> Starting the runner.\033[0m\n"
+./run.sh "$*" &
+wait $!

ubuntu-singularity/env.template

@@ -0,0 +1 @@
+AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache

ubuntu-slurm/ga-runner-singularity.def → ubuntu-singularity/github-actions-runner-singularity.def

@@ -1,21 +1,21 @@
 Bootstrap: library
-From: ubuntu:19.10
+From: ubuntu:20.04
 Stage: build
 
-# %setup
-#     touch /file1
-#     touch ${SINGULARITY_ROOTFS}/file2
 
 %files
     ./entrypoint.sh /
-    # /entrypoint.sh /
 
 %environment
     export LC_ALL=C
     export DEBIAN_FRONTEND=noninteractive
     export RUNNER_VERSION=2.267.1
+    export AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache
 
 %post
+    export RUNNER_VERSION=2.267.1
+    chmod +x /entrypoint.sh
+
     apt-get install -y --no-install-recommends software-properties-common
     add-apt-repository universe
     add-apt-repository multiverse
@@ -24,30 +24,20 @@ Stage: build
     ca-certificates \
     curl \
     jq \
+    git \
     libcurl4-openssl-dev
+
+    echo ${RUNNER_VERSION}
+    mkdir /opt/actions-runner && cd /opt/actions-runner
+    curl -L "https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz" > actions_runner.tar.gz && \
+    tar -zxf actions_runner.tar.gz && \
+    rm -f actions_runner.tar.gz && \
+    ./bin/installdependencies.sh
+
     NOW=`date`
     echo "export NOW=\"${NOW}\"" >> $SINGULARITY_ENVIRONMENT
 
 %runscript
     echo "Container was created $NOW"
     echo "Arguments received: $*"
     exec /bin/bash /entrypoint.sh "$@"
-
-# %startscript
-#     nc -lp $LISTEN_PORT
-
-# %test
-#     grep -q NAME=\"Ubuntu\" /etc/os-release
-#     if [ $? -eq 0 ]; then
-#         echo "Container base is Ubuntu as expected."
-#     else
-#         echo "Container base is not Ubuntu."
-#     fi
-
-# %labels
-#     Author [email protected]
-#     Version v0.0.1
-
-# %help
-#     This is a demo container used to illustrate a def file that uses all
-#     supported sections.