diff --git a/Dockerfile.test b/Dockerfile.test index de53f21..b7ec292 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -4,10 +4,7 @@ FROM registry.access.redhat.com/ubi8/python-39@sha256:ad1e728e0ebeffae9159c29d5a ENV CI=true USER root -# Install docker-ce for RHEL8 -RUN dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo \ - && dnf install -y docker-ce \ - && dnf clean all +RUN dnf install -y podman && dnf clean all WORKDIR /managedtenant-cli diff --git a/Makefile b/Makefile index 9cd57e0..6abab10 100644 --- a/Makefile +++ b/Makefile @@ -57,11 +57,11 @@ check: pylint MINIMAL_IMAGE := registry.access.redhat.com/ubi8/ubi-minimal@sha256:e83a3146aa8d34dccfb99097aa79a3914327942337890aa6f73911996a80ebb8 test: - docker container create --name storageContainer -v sharedCertsVol:/certs $(MINIMAL_IMAGE) - docker cp ./managedtenants/bundles/certs/. storageContainer:/certs - docker rm storageContainer + podman container create --name storageContainer -v sharedCertsVol:/certs $(MINIMAL_IMAGE) + podman cp ./managedtenants/bundles/certs/. storageContainer:/certs + podman rm storageContainer pipenv run pytest --cache-clear -v tests/ - docker volume rm sharedCertsVol + podman volume rm sharedCertsVol release: python -m pip install twine wheel @@ -85,11 +85,11 @@ pre-commit-autoupdate: develop DEV_IMAGE := managedtenants_cli:dev docker-build: - docker build -t $(DEV_IMAGE) -f Dockerfile.test . + podman build -t $(DEV_IMAGE) -f Dockerfile.test . CMD := check docker-run: docker-build - docker run --rm -it --name managedtenants_cli-dev -v "/var/run/docker.sock:/var/run/docker.sock" $(DEV_IMAGE) $(CMD) + podman run --rm -it --name managedtenants_cli-dev -v "/var/run/docker.sock:/var/run/docker.sock" $(DEV_IMAGE) $(CMD) clean: pipenv --rm || true diff --git a/Pipfile b/Pipfile index 638cf0e..3a8d8b4 100644 --- a/Pipfile +++ b/Pipfile @@ -19,6 +19,7 @@ black = "22.12.0" yamllint = "1.29.0" # vscode LSP jedi = "*" +podman = "5.0.0" [packages] managedtenants_cli = {editable = true,path = "."} diff --git a/build_tag.sh b/build_tag.sh index 11ded30..727f488 100755 --- a/build_tag.sh +++ b/build_tag.sh @@ -5,8 +5,8 @@ set -exvo pipefail -o nounset IMAGE_TEST=managedtenants-cli # Run build in sandbox and inject secrets from gh-build-tag integration -docker build -t ${IMAGE_TEST} -f Dockerfile.test . -docker run --rm \ +podman build -t ${IMAGE_TEST} -f Dockerfile.test . +podman run --rm \ -e "TWINE_USERNAME=${TWINE_USERNAME}" \ -e "TWINE_PASSWORD=${TWINE_PASSWORD}" \ -e "GITHUB_TOKEN=${GITHUB_TOKEN}" \ diff --git a/managedtenants/bundles/cli.py b/managedtenants/bundles/cli.py index c9792a4..d6f7f4e 100644 --- a/managedtenants/bundles/cli.py +++ b/managedtenants/bundles/cli.py @@ -8,7 +8,7 @@ from managedtenants.bundles.addon_bundles import AddonBundles from managedtenants.bundles.addon_package import AddonPackage from managedtenants.bundles.bundle_builder import BundleBuilder -from managedtenants.bundles.docker_api import DockerAPI +from managedtenants.bundles.docker_api import ContainerRuntime from managedtenants.bundles.exceptions import MtbundlesCLIError from managedtenants.bundles.imageset_creator import ImageSetCreator from managedtenants.bundles.index_builder import IndexBuilder @@ -99,7 +99,7 @@ def _get_target_addons(self): return list(self.addons_dir.iterdir()) def _init_docker_api(self): - return DockerAPI( + return ContainerRuntime.from_env( registry=f"quay.io/{self.args.quay_org}", quay_org=self.args.quay_org, dockercfg_path=os.environ.get("DOCKER_CONF"), diff --git a/managedtenants/bundles/docker_api.py b/managedtenants/bundles/docker_api.py index a9bc9fa..d114f6f 100644 --- a/managedtenants/bundles/docker_api.py +++ b/managedtenants/bundles/docker_api.py @@ -1,3 +1,4 @@ +import abc import io import logging import os @@ -5,6 +6,9 @@ import docker import docker.api.build +import podman +import podman.errors +from podman import PodmanClient from requests.exceptions import HTTPError from sretoolbox.utils.logger import get_text_logger @@ -19,32 +23,18 @@ ) -class DockerAPI: - """ - Class to build and push docker images. - - :param quay_api: (optional) QuayApi object used for pushing images. Default - to osd-addons, reading token from QUAY_APIKEY env var. - :param registry: docker registry to use. Default: quay.io/{quay_api.org}. - :param dockercfg_path: (optional) Custom path for the Docker config file. - If not present, the client does not login. - :param force_push: overwrite an existing remote image. - :param debug: Enable debug logging. - :raise ValueError: If an invalid empty username is provided. - """ - +class ContainerRuntime(abc.ABC): # pylint: disable=too-many-arguments def __init__( self, registry, dockercfg_path, quay_org, - debug=False, - force_push=False, + debug, + force_push, ): self.registry = registry self.dockercfg_path = dockercfg_path - self.client = docker.from_env() self.force_push = force_push self.log = get_text_logger( "managedtenants-docker", @@ -53,6 +43,31 @@ def __init__( if self._is_quay_registry(): self.quay_api = QuayAPI(org=quay_org, debug=debug) + @staticmethod + def from_env( + registry, + dockercfg_path, + quay_org, + debug=False, + force_push=False, + ): + if os.getenv("CONTAINER_RUNTIME") == "podman": + return PodmanAPI( + registry=registry, + dockercfg_path=dockercfg_path, + quay_org=quay_org, + debug=debug, + force_push=force_push, + ) + + return DockerAPI( + registry=registry, + dockercfg_path=dockercfg_path, + quay_org=quay_org, + debug=debug, + force_push=force_push, + ) + def build_bundle(self, bundle): dockerfile = """ FROM scratch @@ -77,6 +92,67 @@ def build_package(self, addon_package): tag=addon_package.image.url_tag, ) + @abc.abstractmethod + def _build(self, path, dockerfile, tag, labels=None): + pass + + @abc.abstractmethod + def push(self, image, ensure_repo=True): + pass + + @abc.abstractmethod + def check_image_size_non_zero(self, tag): + pass + + @abc.abstractmethod + def extract_file_from_container(self, tag, path): + pass + + def _image_exists(self, image): + # The Image(...) sretoolbox library requires a valid quay registry. + if not self._is_quay_registry(): + return False + + try: + self.log.info( + f"Skipping pushing {image.url_digest} as it already exists." + ) + return True + except HTTPError: + # image.url_digest, calls digest which raises HTTPError + # https://github.com/app-sre/sretoolbox/blob/master/sretoolbox/container/image.py#L119 + return False + + def _is_quay_registry(self): + return self.registry.startswith("quay.io") + + +class DockerAPI(ContainerRuntime): + """ + Class to build and push docker images. + + :param quay_api: (optional) QuayApi object used for pushing images. Default + to osd-addons, reading token from QUAY_APIKEY env var. + :param registry: docker registry to use. Default: quay.io/{quay_api.org}. + :param dockercfg_path: (optional) Custom path for the Docker config file. + If not present, the client does not login. + :param force_push: overwrite an existing remote image. + :param debug: Enable debug logging. + :raise ValueError: If an invalid empty username is provided. + """ + + # pylint: disable=too-many-arguments + def __init__( + self, + registry, + dockercfg_path, + quay_org, + debug, + force_push, + ): + super().__init__(registry, dockercfg_path, quay_org, debug, force_push) + self.client = docker.from_env() + def _build(self, path, dockerfile, tag, labels=None): """ Build a docker image using in-memory dockerfile. @@ -138,23 +214,123 @@ def push(self, image, ensure_repo=True): except docker.errors.APIError as e: raise DockerError(f"Failed to push {image.url_tag}, got {e}.") - def _is_quay_registry(self): - return self.registry.startswith("quay.io") + def check_image_size_non_zero(self, tag): + """ + Bundle containers only contain data, so detecting a 0 size is enough + to validate we successfully inserted data in the bundle. + """ + image = self.client.images.get(tag) + self.log.debug(image.attrs) + if image.attrs.get("Size", -1) == 0: + raise DockerError(f"Built an empty image: {tag}.") - def _image_exists(self, image): - # The Image(...) sretoolbox library requires a valid quay registry. - if not self._is_quay_registry(): - return False + def extract_file_from_container(self, tag, path): + """ + Creates a temporary container and returns the given file as an + io.BytesIO object. + :tag str: image from which to create the container + :path str: path of the file to be extracted from container + :returns: in-memory fileobj of the extracted file + """ try: - self.log.info( - f"Skipping pushing {image.url_digest} as it already exists." + in_memory_tar = io.BytesIO() + container = self.client.containers.create(tag) + tar_bytes, _ = container.get_archive(path) + + for tb in tar_bytes: + in_memory_tar.write(tb) + + container.remove(force=True) + + in_memory_tar.seek(0) + with tarfile.open(fileobj=in_memory_tar) as tf: + return tf.extractfile(os.path.basename(path)) + + except docker.errors.ImageNotFound as e: + raise DockerError( + f"Could not find local index image {tag} got {e}." ) - return True - except HTTPError: - # image.url_digest, calls digest which raises HTTPError - # https://github.com/app-sre/sretoolbox/blob/master/sretoolbox/container/image.py#L119 - return False + + except docker.errors.APIError as e: + raise DockerError( + f"Failed to read file from index image {tag} got {e}" + ) + + +class PodmanAPI(ContainerRuntime): + # pylint: disable=too-many-arguments + def __init__( + self, + registry, + dockercfg_path, + quay_org, + debug, + force_push, + ): + super().__init__(registry, dockercfg_path, quay_org, debug, force_push) + self.client = PodmanClient.from_env() + + def _build(self, path, dockerfile, tag, labels=None): + """ + Build a docker image using in-memory dockerfile. + + :param path: Path to the context directory. + :param dockerfile: A file object to use as the Dockerfile. + :param tag: Tag for the built image. + :param labels: (Optional) image labels. + + :raise DockerError: on a build error or if the built image has Size 0. + """ + try: + out_image, log_generator = self.client.images.build( + path=str(path), # Path(..) obj are not allowed + dockerfile=dockerfile, + labels=labels if labels is not None else {}, + tag=tag, + ) + for log in log_generator: + self.log.debug(log) + + self.check_image_size_non_zero(tag) + return out_image + + except podman.errors.BuildError as e: + raise DockerError( + f"Failed to build image for path {path}, got {e}." + ) + + def push(self, image, ensure_repo=True): + """ + Push an image to a remote repository. + + :param image: Sretoolbox Image(..) to be pushed. + + :raise DockerError: failed to push image. + """ + try: + # docker-py add auth headers from cred store + # https://github.com/docker/docker-py/blob/a48a5a9647761406d66e8271f19fab7fa0c5f582/docker/utils/config.py#L33-L38 + os.environ["DOCKER_CONFIG"] = str(self.dockercfg_path) + + if self._is_quay_registry() and ensure_repo: + self.log.info(f"Ensuring quay repo: {image.image}.") + self.quay_api.ensure_repo(image.image) + + if not self._image_exists(image) or self.force_push: + response = self.client.images.push( + image.url_tag, stream=True, decode=True + ) + for log in response: + self.log.debug(log) + + except QuayAPIError as e: + raise DockerError( + f"Failed to ensure quay repo {image.repository} got {e}." + ) + + except podman.errors.APIError as e: + raise DockerError(f"Failed to push {image.url_tag}, got {e}.") def check_image_size_non_zero(self, tag): """ @@ -189,12 +365,12 @@ def extract_file_from_container(self, tag, path): with tarfile.open(fileobj=in_memory_tar) as tf: return tf.extractfile(os.path.basename(path)) - except docker.errors.ImageNotFound as e: + except podman.errors.ImageNotFound as e: raise DockerError( f"Could not find local index image {tag} got {e}." ) - except docker.errors.APIError as e: + except podman.errors.APIError as e: raise DockerError( f"Failed to read file from index image {tag} got {e}" ) diff --git a/pr_check.sh b/pr_check.sh index 2bfea0f..3802006 100755 --- a/pr_check.sh +++ b/pr_check.sh @@ -4,13 +4,17 @@ set -exvo pipefail -o nounset IMAGE_TEST=managedtenants-cli -docker build -t ${IMAGE_TEST} -f Dockerfile.test . +podman build -t ${IMAGE_TEST} -f Dockerfile.test . -docker_run_args=( +systemctl --user start podman.socket + +podman_run_args=( --rm - -v "/var/run/docker.sock:/var/run/docker.sock" + -e "CONTAINER_HOST=unix:///var/run/podman.sock" + -e "CONTAINER_RUNTIME=podman" + --security-opt label=disable + -v "${XDG_RUNTIME_DIR}/podman/podman.sock:/var/run/podman.sock" --net "host" ) - -docker run "${docker_run_args[@]}" "${IMAGE_TEST}" check test +podman run "${podman_run_args[@]}" "${IMAGE_TEST}" check test diff --git a/setup.py b/setup.py index ef57f37..bded8f6 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ def get_version(): "python-gitlab~=2.6", "checksumdir~=1.2", "docker ~=5.0.3", + "podman~=5.0.0", ], entry_points={ "console_scripts": [