Drop external Docker builder (#4523)

* Use a vg-like system toa void needing a Kubernetes docker builder

* Specify inline caching correctly

* Drop extra import

* Fix the wandering typedef two ways

* Fix it only one way

MyPy can't handle a Pythonic try-except import: python/mypy#1393
We could put # type: ignore[attr-defined] except then whichever branch
actually passed on the currently installed version of the module would
fail due to having an apparently-unneeded ignore.

* Manually force out busted stubs package

.gitignore

@@ -22,3 +22,4 @@ tmp/
 .coverage*
 .mypy_cache
 /contrib/admin/.issue_cache/
+.docker_cache/

.gitlab-ci.yml

@@ -17,6 +17,8 @@ before_script:
         echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
         export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
     fi
+  # Restart or start the Docker daemon
+  - stopdocker || true
   - startdocker || true
   - docker info
   - cat /etc/hosts
@@ -26,9 +28,9 @@ before_script:
   # a service account bearer token for auth and triggers https://github.com/docker/buildx/issues/267
   # where buildx can't use a bearer token from a kube config and falls back to anonymous instead
   # of using the system's service account.
-  - KUBECONFIG=/dev/null docker buildx create --use --name toilbuilder --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
-  # Dump the builder info, and make sure it exists.
-  - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found in our Kubernetes namespace! Are we on the right Gitlab runner?" && exit 1)
+  - if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then KUBECONFIG=/dev/null docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container ; fi
+  # Report on the builders, and make sure they exist.
+  - docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
   # This will hang if we can't talk to the builder
   - (echo "y" | docker buildx prune --keep-storage 80G) || true
 

Makefile

@@ -166,24 +166,28 @@ pre_pull_docker:
 	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull sscaling/mtail && break || sleep 60; done
 
 toil_docker: pre_pull_docker docker/Dockerfile
+	mkdir -p .docker_cache
 	@set -ex \
 	; cd docker \
-	; docker buildx build --platform=$(arch) --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	; docker buildx build --platform=$(arch) --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(docker_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(docker_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../.docker-cache/toil --cache-to type=local,dest=../.docker-cache/toil -f Dockerfile .
 
 prometheus_docker: pre_pull_docker
+	mkdir -p .docker_cache
 	@set -ex \
 	; cd dashboard/prometheus \
-	; docker buildx build --platform=$(arch) --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	; docker buildx build --platform=$(arch) --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(prometheus_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(prometheus_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/prometheus --cache-to type=local,dest=../../.docker-cache/prometheus -f Dockerfile .
 
 grafana_docker: pre_pull_docker
+	mkdir -p .docker_cache
 	@set -ex \
 	; cd dashboard/grafana \
-	; docker buildx build --platform=$(arch) --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	; docker buildx build --platform=$(arch) --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(grafana_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(grafana_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/grafana --cache-to type=local,dest=../../.docker-cache/grafana -f Dockerfile .
 
 mtail_docker: pre_pull_docker
+	mkdir -p .docker_cache
 	@set -ex \
 	; cd dashboard/mtail \
-	; docker buildx build --platform=$(arch) --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	; docker buildx build --platform=$(arch) --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(mtail_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(mtail_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/mtail --cache-to type=local,dest=../../.docker-cache/mtail -f Dockerfile .
 
 docker/$(sdist_name): dist/$(sdist_name)
 	cp $< $@
@@ -196,17 +200,42 @@ clean_docker:
 	-docker rmi $(docker_image):$(TOIL_DOCKER_TAG)
 
 push_docker: docker
-	# Weird if logic is so we fail if all the pushes fail
-	cd docker ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
-	cd dashboard/prometheus ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
-	cd dashboard/grafana ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
-	cd dashboard/mtail ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
+	# Weird if logic is so we fail if all the pushes fail.
+	# We need to build from the local cache to the cache tag and again from the local cache to the real tag.
+	cd docker ; \
+	for i in $$(seq 1 6); do \
+		if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
+		docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../.docker-cache/toil --cache-to type=inline -f Dockerfile . && \
+			docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile . && \
+			break || sleep 60; \
+	done
+	cd dashboard/prometheus ; \
+	for i in $$(seq 1 6); do \
+		if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
+		docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/prometheus --cache-to type=inline -f Dockerfile . && \
+			docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile . && \
+			break || sleep 60; \
+	done
+	cd dashboard/grafana ; \
+	for i in $$(seq 1 6); do \
+		if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
+		docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/grafana --cache-to type=inline -f Dockerfile . && \
+			docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile . && \
+			break || sleep 60; \
+	done
+	cd dashboard/mtail ; \
+	for i in $$(seq 1 6); do \
+		if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
+		docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/mtail --cache-to type=inline -f Dockerfile . && \
+			docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile . && \
+			break || sleep 60; \
+	done
 
 load_docker: docker
-	cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
-	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile . 
-	cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
-	cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile .
+	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile . 
+	cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile .
+	cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile .
 
 else
 

common.mk

@@ -15,6 +15,8 @@ SHELL=bash
 export TOIL_DOCKER_REGISTRY?=quay.io/ucsc_cgl
 export TOIL_DOCKER_NAME?=toil
 export TOIL_DOCKER_TAG?=$(shell python version_template.py dockerTag)
+export TOIL_DOCKER_CACHE_TAG?=$(shell python version_template.py cacheTag)
+export TOIL_DOCKER_MAIN_CACHE_TAG?=$(shell python version_template.py mainCacheTag)
 export TOIL_APPLIANCE_SELF?=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME):$(TOIL_DOCKER_TAG)
 
 # TOIL_CHECK_ENV=''  # Determines whether toil refers to the same virtualenv paths it spawned from (across machines)

requirements-aws.txt

@@ -1,3 +1,4 @@
 boto>=2.48.0, <3
-boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3, <2
-moto>=4.1.11, <5
+boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2
+mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
+moto>=4.1.11, <5

src/toil/lib/aws/iam.py

@@ -7,7 +7,7 @@
 
 import boto3
 from mypy_boto3_iam import IAMClient
-from mypy_boto3_iam.type_defs import AttachedPolicyOutputTypeDef
+from mypy_boto3_iam.type_defs import AttachedPolicyTypeDef
 from mypy_boto3_sts import STSClient
 
 from toil.lib.aws import zone_to_region
@@ -145,7 +145,7 @@ def get_actions_from_policy_document(policy_doc: Dict[str, Any]) -> AllowedActio
                             allowed_actions[resource][key].append(statement[key])
 
     return allowed_actions
-def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyOutputTypeDef]) -> AllowedActionCollection:
+def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyTypeDef]) -> AllowedActionCollection:
     """
     Go through all attached policy documents and create an AllowedActionCollection representing granted permissions.
 

version_template.py

@@ -39,6 +39,28 @@ def version():
     """
     return '-'.join(filter(None, [distVersion(), currentCommit(), ('dirty' if dirty() else None)]))
 
+def cacheTag():
+    """
+    A Docker tag that we should use to cache Docker image build layers for this commit.
+    """
+
+    import os
+    return ''.join([
+        "cache-",
+        # Pick up branch or tag from Gitlagb CI, or just use "local" for everyone.
+        ((os.getenv('CI_COMMIT_BRANCH', '') + os.getenv('CI_COMMIT_TAG', '')) or 'local').replace('/', '-'),
+        _pythonVersionSuffix()
+    ])
+
+def mainCacheTag():
+    """
+    A Docker tag where the Toil mainline builds cache their layers.
+    """
+
+    return ''.join([
+        "cache-master",
+        _pythonVersionSuffix()
+    ])
 
 def distVersion():
     """The distribution version identifying a published release on PyPI."""