Merge pull request #149 from shanejbrown/default-builder

Add use-legacy-builder flag to config. Set legacy builder to be the default.

README.rst

@@ -86,7 +86,26 @@ step name.
 
 .. note:: Artifacts from previous steps are not available within remote builds
 
-Jinja Templating
+There are two image builders in ``buildrunner``. The default builder is
+equivalent to ``docker build`` and only supports single-platform images,
+this is referenced as the legacy image builder.
+The new image builder is equivalent to ``docker buildx`` and is used for for both
+single and multi-platform images. To use the ``docker buildx`` builder,
+set ``use-legacy-builder: false`` in the configuration file or use ``platforms``
+in the ``build`` section. The legacy builder will be removed in a future release.
+
+.. code:: yaml
+
+  use-legacy-builder: false
+  steps:
+    step1:
+      build: <build config>
+      run: <run config>
+      push: <push config>
+      # or
+      remote: <remote config>
+
+Jinja 
 ================
 
 The 'buildrunner.yaml' file is processed as a 

buildrunner/__init__.py

@@ -30,13 +30,13 @@
     BuildRunnerConfig,
 )
 from buildrunner.config.models import DEFAULT_CACHES_ROOT
-from buildrunner.docker.builder import DockerBuilder
 from buildrunner.errors import (
     BuildRunnerConfigurationError,
     BuildRunnerProcessingError,
 )
 from buildrunner.steprunner import BuildStepRunner
 from buildrunner.docker.multiplatform_image_builder import MultiplatformImageBuilder
+import buildrunner.docker.builder as legacy_builder
 
 
 LOGGER = logging.getLogger(__name__)
@@ -276,22 +276,16 @@ def get_source_image(self):
                 SOURCE_DOCKERFILE: "Dockerfile",
             }
             buildrunner_config = BuildRunnerConfig.get_instance()
-            source_builder = DockerBuilder(
+            image = legacy_builder.build_image(
                 temp_dir=buildrunner_config.global_config.temp_dir,
                 inject=inject,
                 timeout=self.docker_timeout,
                 docker_registry=buildrunner_config.global_config.docker_registry,
-            )
-            exit_code = source_builder.build(
                 nocache=True,
                 pull=False,
             )
-            if exit_code != 0 or not source_builder.image:
-                raise BuildRunnerProcessingError(
-                    f"Error building source image ({exit_code}), this may be a transient docker"
-                    " error if no output is available above"
-                )
-            self._source_image = source_builder.image
+            self._source_image = image
+
         return self._source_image
 
     def _write_artifact_manifest(self):

buildrunner/config/models.py

@@ -142,6 +142,7 @@ class Config(BaseModel, extra="forbid"):
     """Top level config model"""
 
     version: Optional[float] = None
+    use_legacy_builder: Optional[bool] = Field(alias="use-legacy-builder", default=True)
     steps: Dict[str, Step]
 
     @field_validator("steps")

buildrunner/docker/builder.py

@@ -16,10 +16,86 @@
 import docker
 import docker.errors
 
+from buildrunner.errors import (
+    BuildRunnerProcessingError,
+)
+
 from buildrunner.docker import get_dockerfile, new_client, force_remove_container
 
 logger = logging.getLogger(__name__)
 
+NO_CACHE_DEFAULT = False
+RM_DEFAULT = True
+PULL_DEFAULT = True
+
+
+def build_image(
+    path=None,
+    inject=None,
+    dockerfile=None,
+    dockerd_url=None,
+    timeout=None,
+    docker_registry=None,
+    temp_dir=None,
+    console=None,
+    nocache=NO_CACHE_DEFAULT,
+    cache_from=None,
+    rm=RM_DEFAULT,
+    pull=PULL_DEFAULT,
+    buildargs=None,
+    platform=None,
+    target=None,
+):
+    """
+    Build a Docker image using the DockerBuilder class.
+    Args:
+        path (str): Path to the build context.
+        inject (dict): Dictionary of files to inject into the build context.
+        dockerfile (str): Path to the Dockerfile to use.
+        dockerd_url (str): URL to the Docker daemon.
+        timeout (int): Timeout for the Docker client.
+        docker_registry (str): Docker registry to use.
+        temp_dir (str): Temporary directory to use.
+        console (object): Console object to write output to.
+        nocache (bool): Whether to use the Docker cache.
+        cache_from (list): List of images to use as cache sources.
+        rm (bool): Whether to remove intermediate containers.
+        pull (bool): Whether to pull images from the registry.
+        buildargs (dict): Build arguments to pass to the Docker client.
+        platform (str): Platform to build the image for.
+        target (str): Target stage to build.
+    Returns:
+        str: The ID of the built image.
+    """
+    logger.info("Using legacy builder")
+    image = None
+    builder = DockerBuilder(
+        path,
+        inject=inject,
+        dockerfile=dockerfile,
+        dockerd_url=dockerd_url,
+        timeout=timeout,
+        docker_registry=docker_registry,
+        temp_dir=temp_dir,
+    )
+    try:
+        exit_code = builder.build(
+            console=console,
+            nocache=nocache,
+            cache_from=cache_from,
+            rm=rm,
+            pull=pull,
+            buildargs=buildargs,
+            platform=platform,
+            target=target,
+        )
+        if exit_code != 0 or not builder.image:
+            raise BuildRunnerProcessingError("Error building image")
+        image = builder.image
+    finally:
+        builder.cleanup()
+    return image
+
 
 class DockerBuilder:  # pylint: disable=too-many-instance-attributes
     """
@@ -52,7 +128,7 @@ def __init__(
             timeout=timeout,
         )
         self.docker_registry = docker_registry
-        self.image = None
+        self._image = None
         self.intermediate_containers = []
 
     @staticmethod
@@ -77,10 +153,10 @@ def _sanitize_buildargs(buildargs=None):
     def build(
         self,
         console=None,
-        nocache=False,
+        nocache=NO_CACHE_DEFAULT,
         cache_from=None,
-        rm=True,
-        pull=True,
+        rm=RM_DEFAULT,
+        pull=PULL_DEFAULT,
         buildargs=None,
         platform=None,
         target=None,
@@ -89,6 +165,7 @@ def build(
         Run a docker build using the configured context, constructing the
         context tar file if necessary.
         """
+        logger.info("Using legacy builder")
         if cache_from is None:
             cache_from = []
         if buildargs is None:

buildrunner/sshagent/__init__.py

@@ -32,7 +32,7 @@
     BuildRunnerConfigurationError,
     BuildRunnerProcessingError,
 )
-from buildrunner.docker.builder import DockerBuilder
+import buildrunner.docker.builder as legacy_builder
 
 SSH_AGENT_PROXY_BUILD_CONTEXT = os.path.join(
     os.path.dirname(__file__), "SSHAgentProxyImage"
@@ -251,17 +251,13 @@ def get_ssh_agent_image(self):
         """
         if not self._ssh_agent_image:
             self.log.write("Creating ssh-agent image\n")
-            ssh_agent_builder = DockerBuilder(
+            image = legacy_builder.build_image(
                 path=SSH_AGENT_PROXY_BUILD_CONTEXT,
                 docker_registry=self.docker_registry,
-            )
-            exit_code = ssh_agent_builder.build(
                 nocache=False,
                 pull=False,
             )
-            if exit_code != 0 or not ssh_agent_builder.image:
-                raise BuildRunnerProcessingError("Error building ssh agent image")
-            self._ssh_agent_image = ssh_agent_builder.image
+            self._ssh_agent_image = image
         return self._ssh_agent_image
 
 

buildrunner/steprunner/tasks/build.py

@@ -10,17 +10,20 @@
 import os
 import re
 
+
+from buildrunner import loggers
 import buildrunner.docker
 from buildrunner.config import BuildRunnerConfig
 from buildrunner.config.models_step import StepBuild
 from buildrunner.docker.importer import DockerImporter
-from buildrunner.docker.builder import DockerBuilder
+import buildrunner.docker.builder as legacy_builder
 from buildrunner.errors import (
     BuildRunnerConfigurationError,
-    BuildRunnerProcessingError,
 )
 from buildrunner.steprunner.tasks import BuildStepRunnerTask
 
+LOGGER = loggers.ConsoleLogger(__name__)
+
 
 class BuildBuildStepRunnerTask(BuildStepRunnerTask):  # pylint: disable=too-many-instance-attributes
     """
@@ -204,15 +207,29 @@ def run(self, context):
         buildrunner_config = BuildRunnerConfig.get_instance()
         docker_registry = buildrunner_config.global_config.docker_registry
         self.step_runner.log.write("Running docker build\n")
-        builder = DockerBuilder(
-            self.path,
-            inject=self.to_inject,
-            dockerfile=self.dockerfile,
-            docker_registry=docker_registry,
-        )
+
         try:
-            if self.platforms:
-                built_image = self.step_runner.multi_platform.build_multiple_images(
+            if self.platforms or not buildrunner_config.run_config.use_legacy_builder:
+                if self.platforms and buildrunner_config.run_config.use_legacy_builder:
+                    LOGGER.warning(
+                        f"Ignoring use-legacy-builder. Using the legacy builder for multiplatform images {self.platforms} is not supported. "
+                        "If you are want to use the legacy builder and specify the platform, please use 'platform' in the configuration file."
+                    )
+
+                # Setting the platform if it is set in the step
+                if self.platform:
+                    LOGGER.info(f"Setting platform to {self.platform}")
+                    self.platforms = [self.platform]
+
+                if not self.platforms:
+                    native_platform = (
+                        self.step_runner.multi_platform.get_native_platform()
+                    )
+                    LOGGER.info(f"Setting platforms to [{native_platform}]")
+                    self.platforms = [native_platform]
+
+                # Build the image
+                built_images = self.step_runner.multi_platform.build_multiple_images(
                     platforms=self.platforms,
                     path=self.path,
                     file=self.dockerfile,
@@ -223,21 +240,33 @@ def run(self, context):
                     pull=self.pull,
                 )
 
-                num_platforms = len(self.platforms)
-
-                if buildrunner_config.global_config.disable_multi_platform:
-                    num_platforms = 1
+                # Set expected number of platforms
+                expected_num_platforms = 1
+                if (
+                    self.platforms
+                    and not buildrunner_config.global_config.disable_multi_platform
+                ):
+                    expected_num_platforms = len(self.platforms)
 
-                num_built_platforms = len(built_image.platforms)
+                # Set the number of built platforms
+                num_built_platforms = len(built_images.platforms)
 
-                assert num_built_platforms == num_platforms, (
+                # Compare the number of built platforms with the expected number of platforms
+                assert num_built_platforms == expected_num_platforms, (
                     f"Number of built images ({num_built_platforms}) does not match "
-                    f"the number of platforms ({num_platforms})"
+                    f"the number of expected platforms ({expected_num_platforms})"
                 )
-                context["mp_built_image"] = built_image
+                context["mp_built_image"] = built_images
+                if num_built_platforms > 0:
+                    context["image"] = built_images.native_platform_image.trunc_digest
 
             else:
-                exit_code = builder.build(
+                # Use the legacy builder
+                image = legacy_builder.build_image(
+                    path=self.path,
+                    inject=self.to_inject,
+                    dockerfile=self.dockerfile,
+                    docker_registry=docker_registry,
                     console=self.step_runner.log,
                     nocache=self.nocache,
                     cache_from=self.cache_from,
@@ -246,12 +275,8 @@ def run(self, context):
                     platform=self.platform,
                     target=self.target,
                 )
-                if exit_code != 0 or not builder.image:
-                    raise BuildRunnerProcessingError("Error building image")
-                context["image"] = builder.image
-                self.step_runner.build_runner.generated_images.append(builder.image)
+                context["image"] = image
+                self.step_runner.build_runner.generated_images.append(image)
         except Exception as exc:
             self.step_runner.log.write(f"ERROR: {exc}\n")
             raise
-        finally:
-            builder.cleanup()

setup.py

@@ -13,7 +13,7 @@
 from setuptools import setup, find_packages
 
 
-BASE_VERSION = "3.10"
+BASE_VERSION = "3.11"
 
 SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
 BUILDRUNNER_DIR = os.path.join(SOURCE_DIR, "buildrunner")