Merge pull request #158 from shanejbrown/get_ssh_agent_image_update

Add buildx builder support to get_ssh_agent_image

.github/workflows/build.yaml

@@ -39,6 +39,8 @@ jobs:
         # this key is rather worthless and can be stored safely in code here. This *could* be used as a secret,
         # but since secrets are not available to forks, we cannot test SSH functionality in PRs which defeats
         # the purpose somewhat.
+        #
+        # Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa
         DEPLOY_SSH_KEY: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtwAAAJiLXobki16G\n5AAAAAtzc2gtZWQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtw\nAAAEBcRwB1PEnUHF5aK6q3JYyuOlT+adQ0mcRrIxsmJiiq1Vqy9ShHMAB0dnmcVa0NH/kN\nBT4Va0Ng1cPTE8ZJ2Mq3AAAAEWJ1aWxkcnVubmVyQGFkb2JlAQIDBA==\n-----END OPENSSH PRIVATE KEY-----"
         DEPLOY_SSH_KEY_PUB: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFqy9ShHMAB0dnmcVa0NH/kNBT4Va0Ng1cPTE8ZJ2Mq3 buildrunner@adobe"
       run: |

buildrunner/__init__.py

@@ -299,7 +299,10 @@ def get_source_image(self):
                         pull=False,
                     )
                 )
-                assert len(built_images_info.built_images) == 1
+                if len(built_images_info.built_images) != 1:
+                    raise BuildRunnerProcessingError(
+                        "Failed to build source image. Retrying the build may resolve the issue."
+                    )
                 self._source_image = built_images_info.built_images[0].trunc_digest
 
         return self._source_image

buildrunner/sshagent/__init__.py

@@ -28,11 +28,12 @@
 from paramiko.util import asbytes
 from paramiko.message import Message
 
+import buildrunner.config
+import buildrunner.docker.builder as legacy_builder
 from buildrunner.errors import (
     BuildRunnerConfigurationError,
     BuildRunnerProcessingError,
 )
-import buildrunner.docker.builder as legacy_builder
 
 SSH_AGENT_PROXY_BUILD_CONTEXT = os.path.join(
     os.path.dirname(__file__), "SSHAgentProxyImage"
@@ -88,7 +89,9 @@ class DockerSSHAgentProxy:
     implementation that is managed by this class.
     """
 
-    def __init__(self, docker_client, log, docker_registry):
+    def __init__(
+        self, docker_client, log, docker_registry, multiplatform_image_builder
+    ):
         """ """
         self.docker_client = docker_client
         self.log = log
@@ -97,6 +100,7 @@ def __init__(self, docker_client, log, docker_registry):
         self._ssh_agent_container = None
         self._ssh_client = None
         self._ssh_channel = None
+        self._multiplatform_image_builder = multiplatform_image_builder
 
     def get_info(self):
         """
@@ -117,6 +121,7 @@ def start(self, keys):
           - keys a dict with the key being the file path and the value being a
             password (or null if not required)
         """
+
         # load the keys
         if not keys:
             raise BuildRunnerConfigurationError("Invalid private keys")
@@ -249,15 +254,37 @@ def get_ssh_agent_image(self):
         """
         Get and/or create the image used to proxy the ssh agent to a container.
         """
+        buildrunner_config = buildrunner.config.BuildRunnerConfig.get_instance()
         if not self._ssh_agent_image:
-            self.log.write("Creating ssh-agent image\n")
-            image = legacy_builder.build_image(
-                path=SSH_AGENT_PROXY_BUILD_CONTEXT,
-                docker_registry=self.docker_registry,
-                nocache=False,
-                pull=False,
-            )
-            self._ssh_agent_image = image
+            if buildrunner_config.run_config.use_legacy_builder:
+                self.log.write("Creating ssh-agent image\n")
+                image = legacy_builder.build_image(
+                    path=SSH_AGENT_PROXY_BUILD_CONTEXT,
+                    docker_registry=self.docker_registry,
+                    nocache=False,
+                    pull=False,
+                )
+                self._ssh_agent_image = image
+            else:
+                native_platform = (
+                    self._multiplatform_image_builder.get_native_platform()
+                )
+                platforms = [native_platform]
+                built_images_info = (
+                    self._multiplatform_image_builder.build_multiple_images(
+                        platforms=platforms,
+                        path=SSH_AGENT_PROXY_BUILD_CONTEXT,
+                        file=f"{SSH_AGENT_PROXY_BUILD_CONTEXT}/Dockerfile",
+                        cache=True,
+                        pull=False,
+                        use_threading=False,
+                    )
+                )
+                if len(built_images_info.built_images) != 1:
+                    raise BuildRunnerProcessingError(
+                        "Failed to build ssh-agent image. Retrying the build may resolve the issue."
+                    )
+                self._ssh_agent_image = built_images_info.built_images[0].trunc_digest
         return self._ssh_agent_image
 
 

buildrunner/steprunner/__init__.py

@@ -112,6 +112,9 @@ def run(self):
                         f'Step "{self.name}" failed with exception: {err}\n    '
                         f"Ignoring due to XFAIL\n"
                     )
+                except Exception as err:  # pylint: disable=broad-except
+                    self.log.write(f'Step "{self.name}" failed with exception: {err}\n')
+                    raise err
         finally:
             for _task in _tasks:
                 try:

buildrunner/steprunner/tasks/run.py

@@ -825,6 +825,7 @@ def run(self, context: dict):  # pylint: disable=too-many-statements,too-many-br
                 self._docker_client,
                 self.step_runner.log,
                 buildrunner_config.global_config.docker_registry,
+                self.step_runner.multi_platform,
             )
             self._sshagent.start(
                 buildrunner_config.get_ssh_keys_from_aliases(

tests/test-files/test-ssh-buildx.yaml

@@ -0,0 +1,19 @@
+# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa
+# Look in the .github/workflows/build.yaml file for the SSH private key
+use-legacy-builder: False
+steps:
+  clone:
+    build:
+      dockerfile: |
+        FROM {{ DOCKER_REGISTRY }}/rockylinux:8.5
+        RUN yum install -y git-core openssh-clients && yum clean all
+    run:
+      ssh-keys: ['buildrunner-deploy']
+      cmds:
+      - mkdir ~/.ssh
+      - ssh-keyscan github.com > ~/.ssh/known_hosts
+      - chmod 700 ~/.ssh
+      - chmod 600 ~/.ssh/known_hosts
+      # Clone into temp directory since the "buildrunner" directory may already exist
+      - rm -rf /tmp/test-clone
+      - git clone [email protected]:adobe/buildrunner.git /tmp/test-clone

tests/test-files/test-ssh.yaml

@@ -1,3 +1,5 @@
+# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa.
+# Look in the .github/workflows/build.yaml file for the SSH private key.
 steps:
   clone:
     build:

tests/test_builders.py

@@ -56,10 +56,10 @@ def fixture_set_env():
 
 
 @pytest.mark.parametrize(
-    "use_legacy_builder, config,",
+    "description, use_legacy_builder, config,",
     [
-        # Use default builder
         (
+            "Use buildx builder with platform",
             False,
             """
             use-legacy-builder: false
@@ -74,6 +74,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Use buildx builder",
             False,
             """
             use-legacy-builder: false
@@ -87,6 +88,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Overwrite use-legacy-builder with platforms",
             False,
             """
             use-legacy-builder: true
@@ -103,6 +105,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Use buildx builder with platforms",
             False,
             """
             use-legacy-builder: false
@@ -119,6 +122,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Default builder with platforms",
             False,
             """
             steps:
@@ -133,8 +137,8 @@ def fixture_set_env():
                             - linux/arm64
             """,
         ),
-        # Use legacy builder
         (
+            "Default builder",
             True,
             """
             steps:
@@ -147,6 +151,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Use legacy builder with platform",
             True,
             """
             use-legacy-builder: true
@@ -161,6 +166,7 @@ def fixture_set_env():
             """,
         ),
         (
+            "Use legacy builder with use-legacy-builder",
             True,
             """
             use-legacy-builder: true
@@ -184,9 +190,11 @@ def fixture_set_env():
 def test_builders(
     mock_buildx_builder,
     mock_legacy_build,
+    description,
     use_legacy_builder,
     config,
 ):
+    _ = description
     with tempfile.TemporaryDirectory() as tmpdirname:
         tmp_filename = f"{tmpdirname}/config.yaml"
         with open(tmp_filename, "w") as f: