Merge pull request adobe#66 from shanejbrown/main

Add inject dockerfile support for multiplatform images and test for mp image reuse in other steps.

buildrunner/docker/__init__.py

@@ -8,6 +8,8 @@
 
 import os
 import ssl
+import tempfile
+from typing import Tuple
 import urllib.parse
 import docker
 
@@ -105,3 +107,28 @@ def force_remove_container(docker_client, container):
         force=True,
         v=True,
     )
+
+
+def get_dockerfile(dockerfile: str, temp_dir: str = None) -> Tuple[str, bool]:
+    """
+    Check if the dockerfile exists, if not create a temporary file and write the dockerfile to it.
+    :param dockerfile: the dockerfile
+    :param temp_dir: the temporary directory
+    :return: the dockerfile and a boolean indicating if the dockerfile was created
+    """
+    cleanup_dockerfile = False
+    curr_dockerfile = None
+
+    if dockerfile:
+        if os.path.exists(dockerfile):
+            curr_dockerfile = dockerfile
+        else:
+            # pylint: disable=consider-using-with
+            df_file = tempfile.NamedTemporaryFile(delete=False, dir=temp_dir)
+            try:
+                df_file.write(dockerfile.encode('utf-8'))
+                cleanup_dockerfile = True
+                curr_dockerfile = df_file.name
+            finally:
+                df_file.close()
+    return curr_dockerfile, cleanup_dockerfile

buildrunner/docker/builder.py

@@ -15,7 +15,7 @@
 import docker
 import docker.errors
 
-from buildrunner.docker import new_client, force_remove_container
+from buildrunner.docker import get_dockerfile, new_client, force_remove_container
 
 
 class DockerBuilder:  # pylint: disable=too-many-instance-attributes
@@ -39,18 +39,8 @@ def __init__(
         self.temp_dir = temp_dir
         self.dockerfile = None
         self.cleanup_dockerfile = False
-        if dockerfile:
-            if os.path.exists(dockerfile):
-                self.dockerfile = dockerfile
-            else:
-                # pylint: disable=consider-using-with
-                df_file = tempfile.NamedTemporaryFile(delete=False, dir=self.temp_dir)
-                try:
-                    df_file.write(dockerfile.encode('utf-8'))
-                    self.cleanup_dockerfile = True
-                    self.dockerfile = df_file.name
-                finally:
-                    df_file.close()
+
+        self.dockerfile, self.cleanup_dockerfile = get_dockerfile(dockerfile, self.temp_dir)
 
         self.docker_client = new_client(
             dockerd_url=dockerd_url,

buildrunner/docker/multiplatform_image_builder.py

@@ -14,6 +14,8 @@
 import python_on_whales
 from python_on_whales import docker
 
+from buildrunner.docker import get_dockerfile
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -308,7 +310,9 @@ def build_multiple_images(self,
             build_args = {}
         build_args['DOCKER_REGISTRY'] = docker_registry
 
-        LOGGER.debug(f"Building {name}:{tags} for platforms {platforms} from {file}")
+        dockerfile, cleanup_dockerfile = get_dockerfile(file)
+
+        LOGGER.debug(f"Building {name}:{tags} for platforms {platforms} from {dockerfile}")
 
         if self._use_local_registry and not self._local_registry_is_running:
             # Starts local registry container to do ephemeral image storage
@@ -335,7 +339,7 @@ def build_multiple_images(self,
                                                platform,
                                                push,
                                                path,
-                                               file,
+                                               dockerfile,
                                                tags,
                                                build_args,
                                                self._intermediate_built_images[name])))
@@ -344,7 +348,7 @@ def build_multiple_images(self,
                                          platform,
                                          push,
                                          path,
-                                         file,
+                                         dockerfile,
                                          tags,
                                          build_args,
                                          self._intermediate_built_images[name])
@@ -355,6 +359,10 @@ def build_multiple_images(self,
         for proc in processes:
             proc.join()
 
+        if cleanup_dockerfile:
+            if dockerfile and os.path.exists(dockerfile):
+                os.remove(dockerfile)
+
         return self._intermediate_built_images[name]
 
     def push(self, name: str, dest_names: List[str] = None) -> None:

buildrunner/steprunner/tasks/push.py

@@ -98,6 +98,9 @@ def run(self, context):  # pylint: disable=too-many-branches
                     tags=repo.tags,
                     dest_name=repo.repository)
 
+                for tag in repo.tags:
+                    self.step_runner.build_runner.committed_images.add(f'{repo.repository}:{tag}')
+
                 # add image as artifact
                 if not self._commit_only:
                     images = self.step_runner.multi_platform.get_built_images(self.get_unique_build_name())

tests/test-files/test-docker-pull-failure.yaml

@@ -0,0 +1,6 @@
+# Tests that a failure to pull an image results in a failure.
+steps:
+  use-bogus-image:
+    run:
+      image: user1/buildrunner-test-multi-platform:bogus
+      cmd: echo "Hello World"

tests/test-files/test-mp-image-reuse.yaml

@@ -0,0 +1,20 @@
+# Test to ensure that subsequent steps can use the recently built image
+steps:
+  build-container-multi-platform:
+    build:
+      dockerfile: |
+        FROM {{ DOCKER_REGISTRY }}/busybox
+      platforms:
+        - linux/amd64
+        - linux/arm64/v8
+    push:
+      repository: user1/buildrunner-test-multi-platform
+      tags: [ 'latest', '0.0.1' ]
+    run:
+      image: user1/buildrunner-test-multi-platform:0.0.1
+      cmd: echo "Hello World"
+
+  use-built-image:
+    run:
+      image: user1/buildrunner-test-multi-platform:0.0.1
+      cmd: echo "Hello World"

tests/test_buildrunner_files.py

@@ -35,6 +35,9 @@ def _get_exit_code(file_name: str) -> int:
     if file_name.startswith('test-inject-nonexistent-dir'):
         return os.EX_CONFIG
 
+    if file_name.startswith('test-docker-pull-failure'):
+        return os.EX_CONFIG
+
     return os.EX_OK
 
 

tests/test_runner.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import docker
 
 sys.path.insert(
     0,
@@ -58,5 +59,8 @@ def run_tests(argv, master_config_file=None, global_config_files=None):
     except BuildRunnerConfigurationError as brce:
         print(str(brce))
         return os.EX_CONFIG
+    except docker.errors.ImageNotFound as inf:
+        print(str(inf))
+        return os.EX_CONFIG
     return os.EX_OK