Retry jobs 3 times

Silex · May 16, 2020 · c804553 · c804553
1 parent ffedfc7
commit c804553
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 44 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -12,181 +12,181 @@ before_script:
 - apk add --no-cache bash
 img-master:
   stage: std
-  script: bin/build "master/ubuntu/18.04" "silex/emacs:master" "-t silex/emacs:master"
+  script: bin/retry bin/build "master/ubuntu/18.04" "silex/emacs:master" "-t silex/emacs:master"
   needs:
   - img-master-dev
 img-master-dev:
   stage: dev
-  script: bin/build "master/ubuntu/18.04/dev" "silex/emacs:master-dev" "-t silex/emacs:master-dev"
+  script: bin/retry bin/build "master/ubuntu/18.04/dev" "silex/emacs:master-dev" "-t silex/emacs:master-dev"
 img-master-alpine:
   stage: std
-  script: bin/build "master/alpine/3.9" "silex/emacs:master-alpine" "-t silex/emacs:master-alpine"
+  script: bin/retry bin/build "master/alpine/3.9" "silex/emacs:master-alpine" "-t silex/emacs:master-alpine"
   needs:
   - img-master-alpine-dev
 img-master-alpine-dev:
   stage: dev
-  script: bin/build "master/alpine/3.9/dev" "silex/emacs:master-alpine-dev" "-t silex/emacs:master-alpine-dev"
+  script: bin/retry bin/build "master/alpine/3.9/dev" "silex/emacs:master-alpine-dev" "-t silex/emacs:master-alpine-dev"
 img-27.0:
   stage: std
-  script: bin/build "27.0/ubuntu/18.04" "silex/emacs:27.0" "-t silex/emacs:27.0 -t silex/emacs:27"
+  script: bin/retry bin/build "27.0/ubuntu/18.04" "silex/emacs:27.0" "-t silex/emacs:27.0 -t silex/emacs:27"
   needs:
   - img-27.0-dev
 img-27.0-dev:
   stage: dev
-  script: bin/build "27.0/ubuntu/18.04/dev" "silex/emacs:27.0-dev" "-t silex/emacs:27.0-dev -t silex/emacs:27-dev"
+  script: bin/retry bin/build "27.0/ubuntu/18.04/dev" "silex/emacs:27.0-dev" "-t silex/emacs:27.0-dev -t silex/emacs:27-dev"
 img-27.0-alpine:
   stage: std
-  script: bin/build "27.0/alpine/3.9" "silex/emacs:27.0-alpine" "-t silex/emacs:27.0-alpine -t silex/emacs:27-alpine"
+  script: bin/retry bin/build "27.0/alpine/3.9" "silex/emacs:27.0-alpine" "-t silex/emacs:27.0-alpine -t silex/emacs:27-alpine"
   needs:
   - img-27.0-alpine-dev
 img-27.0-alpine-dev:
   stage: dev
-  script: bin/build "27.0/alpine/3.9/dev" "silex/emacs:27.0-alpine-dev" "-t silex/emacs:27.0-alpine-dev -t silex/emacs:27-alpine-dev"
+  script: bin/retry bin/build "27.0/alpine/3.9/dev" "silex/emacs:27.0-alpine-dev" "-t silex/emacs:27.0-alpine-dev -t silex/emacs:27-alpine-dev"
 img-26.3:
   stage: std
-  script: bin/build "26.3/ubuntu/18.04" "silex/emacs:26.3" "-t silex/emacs:26.3 -t silex/emacs:26 -t silex/emacs:latest"
+  script: bin/retry bin/build "26.3/ubuntu/18.04" "silex/emacs:26.3" "-t silex/emacs:26.3 -t silex/emacs:26 -t silex/emacs:latest"
   needs:
   - img-26.3-dev
 img-26.3-dev:
   stage: dev
-  script: bin/build "26.3/ubuntu/18.04/dev" "silex/emacs:26.3-dev" "-t silex/emacs:26.3-dev -t silex/emacs:26-dev -t silex/emacs:dev"
+  script: bin/retry bin/build "26.3/ubuntu/18.04/dev" "silex/emacs:26.3-dev" "-t silex/emacs:26.3-dev -t silex/emacs:26-dev -t silex/emacs:dev"
 img-26.3-alpine:
   stage: std
-  script: bin/build "26.3/alpine/3.9" "silex/emacs:26.3-alpine" "-t silex/emacs:26.3-alpine -t silex/emacs:26-alpine -t silex/emacs:alpine"
+  script: bin/retry bin/build "26.3/alpine/3.9" "silex/emacs:26.3-alpine" "-t silex/emacs:26.3-alpine -t silex/emacs:26-alpine -t silex/emacs:alpine"
   needs:
   - img-26.3-alpine-dev
 img-26.3-alpine-dev:
   stage: dev
-  script: bin/build "26.3/alpine/3.9/dev" "silex/emacs:26.3-alpine-dev" "-t silex/emacs:26.3-alpine-dev -t silex/emacs:26-alpine-dev -t silex/emacs:alpine-dev"
+  script: bin/retry bin/build "26.3/alpine/3.9/dev" "silex/emacs:26.3-alpine-dev" "-t silex/emacs:26.3-alpine-dev -t silex/emacs:26-alpine-dev -t silex/emacs:alpine-dev"
 img-26.2:
   stage: std
-  script: bin/build "26.2/ubuntu/18.04" "silex/emacs:26.2" "-t silex/emacs:26.2"
+  script: bin/retry bin/build "26.2/ubuntu/18.04" "silex/emacs:26.2" "-t silex/emacs:26.2"
   needs:
   - img-26.2-dev
 img-26.2-dev:
   stage: dev
-  script: bin/build "26.2/ubuntu/18.04/dev" "silex/emacs:26.2-dev" "-t silex/emacs:26.2-dev"
+  script: bin/retry bin/build "26.2/ubuntu/18.04/dev" "silex/emacs:26.2-dev" "-t silex/emacs:26.2-dev"
 img-26.2-alpine:
   stage: std
-  script: bin/build "26.2/alpine/3.9" "silex/emacs:26.2-alpine" "-t silex/emacs:26.2-alpine"
+  script: bin/retry bin/build "26.2/alpine/3.9" "silex/emacs:26.2-alpine" "-t silex/emacs:26.2-alpine"
   needs:
   - img-26.2-alpine-dev
 img-26.2-alpine-dev:
   stage: dev
-  script: bin/build "26.2/alpine/3.9/dev" "silex/emacs:26.2-alpine-dev" "-t silex/emacs:26.2-alpine-dev"
+  script: bin/retry bin/build "26.2/alpine/3.9/dev" "silex/emacs:26.2-alpine-dev" "-t silex/emacs:26.2-alpine-dev"
 img-26.1:
   stage: std
-  script: bin/build "26.1/ubuntu/18.04" "silex/emacs:26.1" "-t silex/emacs:26.1"
+  script: bin/retry bin/build "26.1/ubuntu/18.04" "silex/emacs:26.1" "-t silex/emacs:26.1"
   needs:
   - img-26.1-dev
 img-26.1-dev:
   stage: dev
-  script: bin/build "26.1/ubuntu/18.04/dev" "silex/emacs:26.1-dev" "-t silex/emacs:26.1-dev"
+  script: bin/retry bin/build "26.1/ubuntu/18.04/dev" "silex/emacs:26.1-dev" "-t silex/emacs:26.1-dev"
 img-26.1-alpine:
   stage: std
-  script: bin/build "26.1/alpine/3.9" "silex/emacs:26.1-alpine" "-t silex/emacs:26.1-alpine"
+  script: bin/retry bin/build "26.1/alpine/3.9" "silex/emacs:26.1-alpine" "-t silex/emacs:26.1-alpine"
   needs:
   - img-26.1-alpine-dev
 img-26.1-alpine-dev:
   stage: dev
-  script: bin/build "26.1/alpine/3.9/dev" "silex/emacs:26.1-alpine-dev" "-t silex/emacs:26.1-alpine-dev"
+  script: bin/retry bin/build "26.1/alpine/3.9/dev" "silex/emacs:26.1-alpine-dev" "-t silex/emacs:26.1-alpine-dev"
 img-25.3:
   stage: std
-  script: bin/build "25.3/ubuntu/18.04" "silex/emacs:25.3" "-t silex/emacs:25.3 -t silex/emacs:25"
+  script: bin/retry bin/build "25.3/ubuntu/18.04" "silex/emacs:25.3" "-t silex/emacs:25.3 -t silex/emacs:25"
   needs:
   - img-25.3-dev
 img-25.3-dev:
   stage: dev
-  script: bin/build "25.3/ubuntu/18.04/dev" "silex/emacs:25.3-dev" "-t silex/emacs:25.3-dev -t silex/emacs:25-dev"
+  script: bin/retry bin/build "25.3/ubuntu/18.04/dev" "silex/emacs:25.3-dev" "-t silex/emacs:25.3-dev -t silex/emacs:25-dev"
 img-25.3-alpine:
   stage: std
-  script: bin/build "25.3/alpine/3.9" "silex/emacs:25.3-alpine" "-t silex/emacs:25.3-alpine -t silex/emacs:25-alpine"
+  script: bin/retry bin/build "25.3/alpine/3.9" "silex/emacs:25.3-alpine" "-t silex/emacs:25.3-alpine -t silex/emacs:25-alpine"
   needs:
   - img-25.3-alpine-dev
   variables:
     DOCKER_PLATFORMS: linux/amd64,linux/386
 img-25.3-alpine-dev:
   stage: dev
-  script: bin/build "25.3/alpine/3.9/dev" "silex/emacs:25.3-alpine-dev" "-t silex/emacs:25.3-alpine-dev -t silex/emacs:25-alpine-dev"
+  script: bin/retry bin/build "25.3/alpine/3.9/dev" "silex/emacs:25.3-alpine-dev" "-t silex/emacs:25.3-alpine-dev -t silex/emacs:25-alpine-dev"
   variables:
     DOCKER_PLATFORMS: linux/amd64,linux/386
 img-25.2:
   stage: std
-  script: bin/build "25.2/ubuntu/18.04" "silex/emacs:25.2" "-t silex/emacs:25.2"
+  script: bin/retry bin/build "25.2/ubuntu/18.04" "silex/emacs:25.2" "-t silex/emacs:25.2"
   needs:
   - img-25.2-dev
 img-25.2-dev:
   stage: dev
-  script: bin/build "25.2/ubuntu/18.04/dev" "silex/emacs:25.2-dev" "-t silex/emacs:25.2-dev"
+  script: bin/retry bin/build "25.2/ubuntu/18.04/dev" "silex/emacs:25.2-dev" "-t silex/emacs:25.2-dev"
 img-25.2-alpine:
   stage: std
-  script: bin/build "25.2/alpine/3.9" "silex/emacs:25.2-alpine" "-t silex/emacs:25.2-alpine"
+  script: bin/retry bin/build "25.2/alpine/3.9" "silex/emacs:25.2-alpine" "-t silex/emacs:25.2-alpine"
   needs:
   - img-25.2-alpine-dev
   variables:
     DOCKER_PLATFORMS: linux/amd64,linux/386
 img-25.2-alpine-dev:
   stage: dev
-  script: bin/build "25.2/alpine/3.9/dev" "silex/emacs:25.2-alpine-dev" "-t silex/emacs:25.2-alpine-dev"
+  script: bin/retry bin/build "25.2/alpine/3.9/dev" "silex/emacs:25.2-alpine-dev" "-t silex/emacs:25.2-alpine-dev"
   variables:
     DOCKER_PLATFORMS: linux/amd64,linux/386
 img-25.1:
   stage: std
-  script: bin/build "25.1/ubuntu/18.04" "silex/emacs:25.1" "-t silex/emacs:25.1"
+  script: bin/retry bin/build "25.1/ubuntu/18.04" "silex/emacs:25.1" "-t silex/emacs:25.1"
   needs:
   - img-25.1-dev
 img-25.1-dev:
   stage: dev
-  script: bin/build "25.1/ubuntu/18.04/dev" "silex/emacs:25.1-dev" "-t silex/emacs:25.1-dev"
+  script: bin/retry bin/build "25.1/ubuntu/18.04/dev" "silex/emacs:25.1-dev" "-t silex/emacs:25.1-dev"
 img-24.5:
   stage: std
-  script: bin/build "24.5/ubuntu/18.04" "silex/emacs:24.5" "-t silex/emacs:24.5 -t silex/emacs:24"
+  script: bin/retry bin/build "24.5/ubuntu/18.04" "silex/emacs:24.5" "-t silex/emacs:24.5 -t silex/emacs:24"
   needs:
   - img-24.5-dev
 img-24.5-dev:
   stage: dev
-  script: bin/build "24.5/ubuntu/18.04/dev" "silex/emacs:24.5-dev" "-t silex/emacs:24.5-dev -t silex/emacs:24-dev"
+  script: bin/retry bin/build "24.5/ubuntu/18.04/dev" "silex/emacs:24.5-dev" "-t silex/emacs:24.5-dev -t silex/emacs:24-dev"
 img-24.4:
   stage: std
-  script: bin/build "24.4/ubuntu/14.04" "silex/emacs:24.4" "-t silex/emacs:24.4"
+  script: bin/retry bin/build "24.4/ubuntu/14.04" "silex/emacs:24.4" "-t silex/emacs:24.4"
   needs:
   - img-24.4-dev
 img-24.4-dev:
   stage: dev
-  script: bin/build "24.4/ubuntu/14.04/dev" "silex/emacs:24.4-dev" "-t silex/emacs:24.4-dev"
+  script: bin/retry bin/build "24.4/ubuntu/14.04/dev" "silex/emacs:24.4-dev" "-t silex/emacs:24.4-dev"
 img-24.3:
   stage: std
-  script: bin/build "24.3/ubuntu/14.04" "silex/emacs:24.3" "-t silex/emacs:24.3"
+  script: bin/retry bin/build "24.3/ubuntu/14.04" "silex/emacs:24.3" "-t silex/emacs:24.3"
   needs:
   - img-24.3-dev
 img-24.3-dev:
   stage: dev
-  script: bin/build "24.3/ubuntu/14.04/dev" "silex/emacs:24.3-dev" "-t silex/emacs:24.3-dev"
+  script: bin/retry bin/build "24.3/ubuntu/14.04/dev" "silex/emacs:24.3-dev" "-t silex/emacs:24.3-dev"
 img-24.2:
   stage: std
-  script: bin/build "24.2/ubuntu/14.04" "silex/emacs:24.2" "-t silex/emacs:24.2"
+  script: bin/retry bin/build "24.2/ubuntu/14.04" "silex/emacs:24.2" "-t silex/emacs:24.2"
   needs:
   - img-24.2-dev
 img-24.2-dev:
   stage: dev
-  script: bin/build "24.2/ubuntu/14.04/dev" "silex/emacs:24.2-dev" "-t silex/emacs:24.2-dev"
+  script: bin/retry bin/build "24.2/ubuntu/14.04/dev" "silex/emacs:24.2-dev" "-t silex/emacs:24.2-dev"
 img-24.1:
   stage: std
-  script: bin/build "24.1/ubuntu/14.04" "silex/emacs:24.1" "-t silex/emacs:24.1"
+  script: bin/retry bin/build "24.1/ubuntu/14.04" "silex/emacs:24.1" "-t silex/emacs:24.1"
   needs:
   - img-24.1-dev
 img-24.1-dev:
   stage: dev
-  script: bin/build "24.1/ubuntu/14.04/dev" "silex/emacs:24.1-dev" "-t silex/emacs:24.1-dev"
+  script: bin/retry bin/build "24.1/ubuntu/14.04/dev" "silex/emacs:24.1-dev" "-t silex/emacs:24.1-dev"
 img-23.4:
   stage: std
-  script: bin/build "23.4/ubuntu/14.04/bootstrap" "silex/emacs:23.4" "-t silex/emacs:23.4 -t silex/emacs:23"
+  script: bin/retry bin/build "23.4/ubuntu/14.04/bootstrap" "silex/emacs:23.4" "-t silex/emacs:23.4 -t silex/emacs:23"
   needs:
   - img-23.4-dev
   variables:
     DOCKER_PLATFORMS: linux/amd64
 img-23.4-dev:
   stage: dev
-  script: bin/build "23.4/ubuntu/14.04/bootstrap/dev" "silex/emacs:23.4-dev" "-t silex/emacs:23.4-dev -t silex/emacs:23-dev"
+  script: bin/retry bin/build "23.4/ubuntu/14.04/bootstrap/dev" "silex/emacs:23.4-dev" "-t silex/emacs:23.4-dev -t silex/emacs:23-dev"
   variables:
     DOCKER_PLATFORMS: linux/amd64
diff --git a/bin/environment b/bin/environment
@@ -2,11 +2,12 @@
 
 set -euo pipefail
 
+# Variables
 export DOCKER_CLI_EXPERIMENTAL="enabled"
 export DOCKER_BUILDX_BUILDER="emacs-$CI_JOB_NAME-builder"
-
 for i in $(env | egrep ^FILE64 | cut -d= -f1); do
   export "$(echo $i | sed 's/FILE64_//')=$(base64 -d $(eval echo \$$i))"
 done
 
+# SSH agent
 eval $(ssh-agent -s)
diff --git a/bin/retry b/bin/retry
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_retry.bash
+
+retry() {
+  local ANSI_RED="\033[31;1m"
+  local ANSI_GREEN="\033[32;1m"
+  local ANSI_RESET="\033[0m"
+  local result=0
+  local count=1
+  while [ $count -le 3 ]; do
+    [ $result -ne 0 ] && {
+      echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2
+    }
+    "$@"
+    result=$?
+    [ $result -eq 0 ] && break
+    count=$(($count + 1))
+    sleep 1
+  done
+
+  [ $count -gt 3 ] && {
+    echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2
+  }
+
+  return $result
+}
+
+retry "$@"
diff --git a/templates/.gitlab-ci.yml b/templates/.gitlab-ci.yml
@@ -17,5 +17,5 @@ before_script:
 
 template:
   stage: {{STAGE}}
-  script: bin/build "{{DIRECTORY}}" "{{CACHE}}" "{{TAGS}}"
+  script: bin/retry bin/build "{{DIRECTORY}}" "{{CACHE}}" "{{TAGS}}"
   needs: {{NEEDS}}