Working tests

buildrunner/docker/multiplatform_images.py

@@ -42,11 +42,13 @@ def __init__(self, use_local_registry:bool=True):
         # key is destination image name, value is list of built images
         self._built_images = {}
 
+    def __enter__(self):
         # Starts local registry container to do ephemeral image storage
         if self._use_local_registry:
             self._start_local_registry()
+        return self
 
-    def __del__(self):
+    def __exit__(self, exc_type, exc_value, traceback):
         if self._use_local_registry:
             self._stop_local_registry()
 
@@ -67,6 +69,12 @@ def registry_ip(self) -> int:
     def registry_port(self) -> int:
         return self._reg_port
 
+    def registry_address(self) -> str:
+        return f"{self._reg_ip}:{self._reg_port}"
+
+    def santize_image_name(self, name:str) -> str:
+        pass
+
     def _start_local_registry(self):
         """
         Starts a local docker registry container
@@ -121,7 +129,9 @@ def build(self,
         print(f"Building {name}:{tags} for platforms {platforms} from {file}")
 
         push=True
-        base_image_name = f"{self._reg_ip}:{self._reg_port}/{name}"
+        # Updates name to be compatible with docker
+        santized_name = name.replace('/','-').replace(':','-')
+        base_image_name = f"{self._reg_ip}:{self._reg_port}/{santized_name}"
 
         # Keeps track of the built images {name: [ImageInfo(image_names)]]}
         self._built_images[name]=[]
@@ -145,41 +155,56 @@ def build(self,
             proc.join()
 
         return self._built_images[name]
- 
-    def push(self, src_key:str, dest_names:str) -> None:
+
+    def push(self, name:str) -> None:
         # TODO change dest name to be list of strings
-        print(f"Creating manifest list {dest_names}")
+        print(f"Creating manifest list for {name}")
 
         initial_timeout_seconds = 60
         timeout_step_seconds = 60
         timeout_max_seconds = 600
         retries = 5
 
-        src_names = self._built_images[src_key]
+        src_images = self._built_images[name]
+        tagged_names = []
+        src_names = []
+        # only need get tags for one image, since they should be identical
+        assert len(src_images) > 0
+
+        for tag in src_images[0].tags:
+            tagged_names.append(f'{name}:{tag}')
+
+        for image in src_images:
+            for tag in image.tags:
+                src_names.append(f'{image.repo}:{tag}')
 
         timeout_seconds = initial_timeout_seconds
-        # TODO handle tags ':'
         while retries > 0:
             retries -= 1
-            print(f"Creating manifest list {dest_names} with timeout {timeout_seconds} seconds")
-            p = Process(target=docker.buildx.imagetools.create, kwargs={"sources": src_names, "tags": [dest_names],})
-            p.start()
-            p.join(timeout_seconds)
-            if p.is_alive():
-                print(f"Timeout after {timeout_seconds} seconds, killing process")
-                p.kill()
-                print(f"Process killed and retries remaining {retries}")
-                if retries == 0:
-                    raise Exception(f"Timeout after {retries} retries and {timeout_seconds} seconds each try")
-            else:
-                # Process finished within timeout
-                print(f"Process finished")
-                break
+            print(f"Creating manifest list {name} with timeout {timeout_seconds} seconds")
+
+            docker.buildx.imagetools.create(sources=src_names, tags=tagged_names)
+            break
+            # p = Process(target=docker.buildx.imagetools.create, kwargs={"sources": src_names, "tags": tagged_names,})
+            # p.start()
+            # p.join(timeout_seconds)
+            # print(f"p.status: {")
+            # if p.is_alive():
+            #     print(f"Timeout after {timeout_seconds} seconds, killing process")
+            #     p.kill()
+            #     print(f"Process killed and retries remaining {retries}")
+            #     if retries == 0:
+            #         raise Exception(f"Timeout after {retries} retries and {timeout_seconds} seconds each try")
+            # else:
+            #     # Process finished within timeout
+            #     print(f"Process finished")
+            #     break
             timeout_seconds += timeout_step_seconds
             if timeout_seconds > timeout_max_seconds:
                 timeout_seconds = timeout_max_seconds
 
-        print(f"Finished creating manifest list {dest_names}")
+        print(f"Finished creating manifest list {name}")
+        return tagged_names
 
     def _find_native_platform(self, name:str) -> str:
         """

tests/test_multiplatform.py

@@ -1,8 +1,7 @@
 import os
 import platform
-from time import sleep
 from unittest.mock import MagicMock
-from python_on_whales import Volume, docker
+from python_on_whales import docker
 
 import pytest
 
@@ -11,33 +10,33 @@
 
 TEST_DIR = os.path.basename(os.path.dirname(__file__))
 
+# TODO make sure all created created/images are removed after tests
 
 def test_use_local_registry():
     registry_name = None
     volume_name = None
 
-    mp = MultiplatformImages(use_local_registry=True)
-    registry_name = mp._reg_name
+    with MultiplatformImages(use_local_registry=True) as mp:
+        registry_name = mp._reg_name
 
-    # Check that the registry is running and only one is found with that name
-    registry_container = docker.ps(filters={"name": registry_name})
-    assert len(registry_container) == 1
-    registry_container = registry_container[0]
+        # Check that the registry is running and only one is found with that name
+        registry_container = docker.ps(filters={"name": registry_name})
+        assert len(registry_container) == 1
+        registry_container = registry_container[0]
 
-    # Check that the registry only has one mount
-    mounts = registry_container.mounts
-    assert len(mounts) == 1
-    mount = mounts[0]
-    assert mount.type == 'volume'
-    volume_name = mount.name
-    assert docker.volume.exists(volume_name)
+        # Check that the registry only has one mount
+        mounts = registry_container.mounts
+        assert len(mounts) == 1
+        mount = mounts[0]
+        assert mount.type == 'volume'
+        volume_name = mount.name
+        assert docker.volume.exists(volume_name)
 
-    # Check that the running container is the registry
-    assert registry_container.config.image == 'registry'
-    assert registry_container.state.running
+        # Check that the running container is the registry
+        assert registry_container.config.image == 'registry'
+        assert registry_container.state.running
 
     # Check that the registry is stopped and cleaned up
-    del mp
     registry_container = docker.ps(filters={"name": registry_name})
     assert len(registry_container) == 0
     assert not docker.volume.exists(volume_name)
@@ -110,33 +109,41 @@ def test_find_native_platform(name,
     )])
 def test_tag_single_platform(name, platforms, expected_image_names):
     tag='latest'
-    mp = MultiplatformImages()
-    built_images = mp.build(name=name,
-                            platforms=platforms,
-                            path=f'{TEST_DIR}/test-files/multiplatform',
-                            file='Dockerfile',
-                            do_multiprocessing=False)
-    print(built_images[0])
-    mp.tag_single_platform(name)
-    found_image = docker.image.list(filters={'reference': f'{name}*'})
-    assert len(found_image) == 1
-    assert f'{name}:{tag}' in found_image[0].repo_tags
+    with MultiplatformImages() as mp:
+        built_images = mp.build(name=name,
+                                platforms=platforms,
+                                path=f'{TEST_DIR}/test-files/multiplatform',
+                                file='Dockerfile',
+                                do_multiprocessing=False)
+        print(built_images[0])
+        mp.tag_single_platform(name)
+        found_image = docker.image.list(filters={'reference': f'{name}*'})
+        assert len(found_image) == 1
+        assert f'{name}:{tag}' in found_image[0].repo_tags
 
 
 def test_push():
-    registry = 'shanejbrown'
-    dest_name = {f'{registry}/test-image-2001': ['latest', '0.1.0']}
-
-    build_name = 'test-image-2001'
-    platforms = ['linux/arm64','linux/amd64']
-    mp = MultiplatformImages()
-    built_images = mp.build(name=build_name,
-                            platforms=platforms,
-                            path=f'{TEST_DIR}/test-files/multiplatform',
-                            file='Dockerfile',
-                            do_multiprocessing=False)
-
-    mp.push(src_key=build_name, dest_names=dest_name)
+    with MultiplatformImages() as remote_mp:
+        reg_add = remote_mp.registry_address()
+        assert reg_add is not None
+
+        tags=['latest', '0.1.0']
+        build_name = f'{reg_add}/test-image-2001'
+        platforms = ['linux/arm64','linux/amd64']
+        with MultiplatformImages() as mp:
+            built_images = mp.build(name=build_name,
+                                    platforms=platforms,
+                                    path=f'{TEST_DIR}/test-files/multiplatform',
+                                    file='Dockerfile',
+                                    do_multiprocessing=False,
+                                    tags=tags)
+
+            mp.push(build_name)
+            docker.image.remove(build_name, force=True)
+            docker.pull(build_name)
+            found_image = docker.image.list(filters={'reference': f'{build_name}'})
+            assert len(found_image) == 1
+            docker.image.remove(build_name, force=True)
 
 
 @pytest.mark.parametrize("name, platforms, expected_image_names",[
@@ -174,19 +181,21 @@ def test_build_multiple_builds():
     platforms1 = ['linux/amd64', 'linux/arm64']
     expected_image_names1 = ['test-image-2001-linux-amd64', 'test-image-2001-linux-arm64']
 
-
     name2 = 'test-image-2002'
     platforms2 = ['linux/amd64', 'linux/arm64']
     expected_image_names2 = ['test-image-2002-linux-amd64', 'test-image-2002-linux-arm64']
 
     MultiplatformImages.build_image = MagicMock()
     mp = MultiplatformImages()
+
+    # Build set 1
     built_images1 = mp.build(name=name1,
                             platforms=platforms1,
                             path=f'{TEST_DIR}/test-files/multiplatform',
                             file='Dockerfile',
                             do_multiprocessing=False)
 
+    # Build set 2
     built_images2 = mp.build(name=name2,
                             platforms=platforms2,
                             path=f'{TEST_DIR}/test-files/multiplatform',
@@ -244,5 +253,8 @@ def test_build_with_tags(name, tags, platforms, expected_image_names):
         for built_image in built_images:
             if built_image.repo.endswith(image):
                 found = True
+                assert len(built_image.tags) == len(tags)
+                for tag in built_image.tags:
+                    assert tag in tags
                 break
-        assert found == True, f"Could not find {image} in {built_images}"
+        assert found == True, f"Could not find {image} in {built_images}"