diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 98aead9c..4fefb8eb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -39,6 +39,8 @@ jobs: # this key is rather worthless and can be stored safely in code here. This *could* be used as a secret, # but since secrets are not available to forks, we cannot test SSH functionality in PRs which defeats # the purpose somewhat. + # + # Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa DEPLOY_SSH_KEY: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtwAAAJiLXobki16G\n5AAAAAtzc2gtZWQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtw\nAAAEBcRwB1PEnUHF5aK6q3JYyuOlT+adQ0mcRrIxsmJiiq1Vqy9ShHMAB0dnmcVa0NH/kN\nBT4Va0Ng1cPTE8ZJ2Mq3AAAAEWJ1aWxkcnVubmVyQGFkb2JlAQIDBA==\n-----END OPENSSH PRIVATE KEY-----" DEPLOY_SSH_KEY_PUB: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFqy9ShHMAB0dnmcVa0NH/kNBT4Va0Ng1cPTE8ZJ2Mq3 buildrunner@adobe" run: | diff --git a/buildrunner/__init__.py b/buildrunner/__init__.py index 97945bb6..b38cf27f 100644 --- a/buildrunner/__init__.py +++ b/buildrunner/__init__.py @@ -299,7 +299,10 @@ def get_source_image(self): pull=False, ) ) - assert len(built_images_info.built_images) == 1 + if len(built_images_info.built_images) != 1: + raise BuildRunnerProcessingError( + "Failed to build source image. Retrying the build may resolve the issue." + ) self._source_image = built_images_info.built_images[0].trunc_digest return self._source_image diff --git a/buildrunner/sshagent/__init__.py b/buildrunner/sshagent/__init__.py index 7c04977e..17a787b5 100644 --- a/buildrunner/sshagent/__init__.py +++ b/buildrunner/sshagent/__init__.py @@ -28,11 +28,12 @@ from paramiko.util import asbytes from paramiko.message import Message +import buildrunner.config +import buildrunner.docker.builder as legacy_builder from buildrunner.errors import ( BuildRunnerConfigurationError, BuildRunnerProcessingError, ) -import buildrunner.docker.builder as legacy_builder SSH_AGENT_PROXY_BUILD_CONTEXT = os.path.join( os.path.dirname(__file__), "SSHAgentProxyImage" @@ -88,7 +89,9 @@ class DockerSSHAgentProxy: implementation that is managed by this class. """ - def __init__(self, docker_client, log, docker_registry): + def __init__( + self, docker_client, log, docker_registry, multiplatform_image_builder + ): """ """ self.docker_client = docker_client self.log = log @@ -97,6 +100,7 @@ def __init__(self, docker_client, log, docker_registry): self._ssh_agent_container = None self._ssh_client = None self._ssh_channel = None + self._multiplatform_image_builder = multiplatform_image_builder def get_info(self): """ @@ -117,6 +121,7 @@ def start(self, keys): - keys a dict with the key being the file path and the value being a password (or null if not required) """ + # load the keys if not keys: raise BuildRunnerConfigurationError("Invalid private keys") @@ -249,15 +254,37 @@ def get_ssh_agent_image(self): """ Get and/or create the image used to proxy the ssh agent to a container. """ + buildrunner_config = buildrunner.config.BuildRunnerConfig.get_instance() if not self._ssh_agent_image: - self.log.write("Creating ssh-agent image\n") - image = legacy_builder.build_image( - path=SSH_AGENT_PROXY_BUILD_CONTEXT, - docker_registry=self.docker_registry, - nocache=False, - pull=False, - ) - self._ssh_agent_image = image + if buildrunner_config.run_config.use_legacy_builder: + self.log.write("Creating ssh-agent image\n") + image = legacy_builder.build_image( + path=SSH_AGENT_PROXY_BUILD_CONTEXT, + docker_registry=self.docker_registry, + nocache=False, + pull=False, + ) + self._ssh_agent_image = image + else: + native_platform = ( + self._multiplatform_image_builder.get_native_platform() + ) + platforms = [native_platform] + built_images_info = ( + self._multiplatform_image_builder.build_multiple_images( + platforms=platforms, + path=SSH_AGENT_PROXY_BUILD_CONTEXT, + file=f"{SSH_AGENT_PROXY_BUILD_CONTEXT}/Dockerfile", + cache=True, + pull=False, + use_threading=False, + ) + ) + if len(built_images_info.built_images) != 1: + raise BuildRunnerProcessingError( + "Failed to build ssh-agent image. Retrying the build may resolve the issue." + ) + self._ssh_agent_image = built_images_info.built_images[0].trunc_digest return self._ssh_agent_image diff --git a/buildrunner/steprunner/__init__.py b/buildrunner/steprunner/__init__.py index ebe0bd65..0c4094ed 100644 --- a/buildrunner/steprunner/__init__.py +++ b/buildrunner/steprunner/__init__.py @@ -112,6 +112,9 @@ def run(self): f'Step "{self.name}" failed with exception: {err}\n ' f"Ignoring due to XFAIL\n" ) + except Exception as err: # pylint: disable=broad-except + self.log.write(f'Step "{self.name}" failed with exception: {err}\n') + raise err finally: for _task in _tasks: try: diff --git a/buildrunner/steprunner/tasks/run.py b/buildrunner/steprunner/tasks/run.py index 618627ad..c677b85a 100644 --- a/buildrunner/steprunner/tasks/run.py +++ b/buildrunner/steprunner/tasks/run.py @@ -825,6 +825,7 @@ def run(self, context: dict): # pylint: disable=too-many-statements,too-many-br self._docker_client, self.step_runner.log, buildrunner_config.global_config.docker_registry, + self.step_runner.multi_platform, ) self._sshagent.start( buildrunner_config.get_ssh_keys_from_aliases( diff --git a/tests/test-files/test-ssh-buildx.yaml b/tests/test-files/test-ssh-buildx.yaml new file mode 100644 index 00000000..15b285df --- /dev/null +++ b/tests/test-files/test-ssh-buildx.yaml @@ -0,0 +1,19 @@ +# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa +# Look in the .github/workflows/build.yaml file for the SSH private key +use-legacy-builder: False +steps: + clone: + build: + dockerfile: | + FROM {{ DOCKER_REGISTRY }}/rockylinux:8.5 + RUN yum install -y git-core openssh-clients && yum clean all + run: + ssh-keys: ['buildrunner-deploy'] + cmds: + - mkdir ~/.ssh + - ssh-keyscan github.com > ~/.ssh/known_hosts + - chmod 700 ~/.ssh + - chmod 600 ~/.ssh/known_hosts + # Clone into temp directory since the "buildrunner" directory may already exist + - rm -rf /tmp/test-clone + - git clone git@github.com:adobe/buildrunner.git /tmp/test-clone diff --git a/tests/test-files/test-ssh.yaml b/tests/test-files/test-ssh.yaml index b64b4aee..b9f1e6b0 100644 --- a/tests/test-files/test-ssh.yaml +++ b/tests/test-files/test-ssh.yaml @@ -1,3 +1,5 @@ +# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa. +# Look in the .github/workflows/build.yaml file for the SSH private key. steps: clone: build: diff --git a/tests/test_builders.py b/tests/test_builders.py index a258d481..605ce493 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -56,10 +56,10 @@ def fixture_set_env(): @pytest.mark.parametrize( - "use_legacy_builder, config,", + "description, use_legacy_builder, config,", [ - # Use default builder ( + "Use buildx builder with platform", False, """ use-legacy-builder: false @@ -74,6 +74,7 @@ def fixture_set_env(): """, ), ( + "Use buildx builder", False, """ use-legacy-builder: false @@ -87,6 +88,7 @@ def fixture_set_env(): """, ), ( + "Overwrite use-legacy-builder with platforms", False, """ use-legacy-builder: true @@ -103,6 +105,7 @@ def fixture_set_env(): """, ), ( + "Use buildx builder with platforms", False, """ use-legacy-builder: false @@ -119,6 +122,7 @@ def fixture_set_env(): """, ), ( + "Default builder with platforms", False, """ steps: @@ -133,8 +137,8 @@ def fixture_set_env(): - linux/arm64 """, ), - # Use legacy builder ( + "Default builder", True, """ steps: @@ -147,6 +151,7 @@ def fixture_set_env(): """, ), ( + "Use legacy builder with platform", True, """ use-legacy-builder: true @@ -161,6 +166,7 @@ def fixture_set_env(): """, ), ( + "Use legacy builder with use-legacy-builder", True, """ use-legacy-builder: true @@ -184,9 +190,11 @@ def fixture_set_env(): def test_builders( mock_buildx_builder, mock_legacy_build, + description, use_legacy_builder, config, ): + _ = description with tempfile.TemporaryDirectory() as tmpdirname: tmp_filename = f"{tmpdirname}/config.yaml" with open(tmp_filename, "w") as f: