Add architecture to manifest model

closes: #1767

CHANGES/1767.feature

@@ -0,0 +1,2 @@
+The Manifest model has been enhanced with a new `architecture` field,
+which specifies the CPU architecture for which the binaries in the image are designed to run.

pulp_container/app/management/commands/container-repair-manifest-metadatas.py

@@ -0,0 +1,73 @@
+import json
+from json.decoder import JSONDecodeError
+
+from gettext import gettext as _
+
+from contextlib import suppress
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.management import BaseCommand
+
+from pulpcore.plugin.cache import SyncContentCache
+
+from pulp_container.app.models import ContainerDistribution, Manifest
+
+
+class Command(BaseCommand):
+    """
+    A management command to handle the initialization of empty architecture fields for
+    container images.
+
+    This command retrieves a list of manifests that have a null architecture field and
+    populates them with the appropriate architecture definitions sourced from the corresponding
+    ConfigBlob.
+    """
+
+    help = _(__doc__)
+
+    def handle(self, *args, **options):
+        manifests_updated_count = 0
+
+        manifests_v1 = Manifest.objects.filter(architecture__isnull=True)
+        manifests_updated_count += self.update_manifests(manifests_v1)
+
+        self.stdout.write(
+            self.style.SUCCESS("Successfully updated %d manifests." % manifests_updated_count)
+        )
+
+        if settings.CACHE_ENABLED and manifests_updated_count != 0:
+            base_paths = ContainerDistribution.objects.values_list("base_path", flat=True)
+            if base_paths:
+                SyncContentCache().delete(base_key=base_paths)
+
+            self.stdout.write(self.style.SUCCESS("Successfully flushed the cache."))
+
+    def update_manifests(self, manifests_qs):
+        manifests_updated_count = 0
+        manifests_to_update = []
+        for manifest in manifests_qs.iterator():
+            # suppress non-existing/already migrated artifacts and corrupted JSON files
+            with suppress(ObjectDoesNotExist, JSONDecodeError):
+                manifest_data = json.loads(manifest.data)
+                manifest.init_metadata(manifest_data)
+                manifests_to_update.append(manifest)
+
+            if len(manifests_to_update) > 1000:
+                fields_to_update = ["architecture"]
+                manifests_qs.model.objects.bulk_update(
+                    manifests_to_update,
+                    fields_to_update,
+                )
+                manifests_updated_count += len(manifests_to_update)
+                manifests_to_update.clear()
+
+        if manifests_to_update:
+            fields_to_update = ["architecture"]
+            manifests_qs.model.objects.bulk_update(
+                manifests_to_update,
+                fields_to_update,
+            )
+            manifests_updated_count += len(manifests_to_update)
+
+        return manifests_updated_count

pulp_container/app/migrations/0042_add_manifest_architecture.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-09-20 17:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0041_add_pull_through_pull_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='architecture',
+            field=models.TextField(null=True),
+        ),
+    ]

pulp_container/app/models.py

@@ -77,6 +77,8 @@ class Manifest(Content):
         labels (models.JSONField): Metadata stored inside the image configuration.
         is_bootable (models.BooleanField): Indicates whether the image is bootable or not.
         is_flatpak (models.BooleanField): Indicates whether the image is a flatpak package or not.
+        architecture (models.TextField): CPU architecture for which the binaries in the image are
+            designed to run.
 
     Relations:
         blobs (models.ManyToManyField): Many-to-many relationship with Blob.
@@ -103,6 +105,7 @@ class Manifest(Content):
 
     annotations = models.JSONField(default=dict)
     labels = models.JSONField(default=dict)
+    architecture = models.TextField(null=True)
 
     is_bootable = models.BooleanField(default=False)
     is_flatpak = models.BooleanField(default=False)
@@ -124,6 +127,7 @@ def init_metadata(self, manifest_data=None):
         has_annotations = self.init_annotations(manifest_data)
         has_labels = self.init_labels()
         has_image_nature = self.init_image_nature()
+        self.init_architecture(manifest_data)
         return has_annotations or has_labels or has_image_nature
 
     def init_annotations(self, manifest_data=None):
@@ -176,6 +180,18 @@ def init_manifest_nature(self):
         else:
             return False
 
+    def init_architecture(self, manifest_data):
+        # manifestv2 schema1 has the architecture definition in the Manifest (not in the ConfigBlob)
+        if architecture := manifest_data.get("architecture", None):
+            self.architecture = architecture
+            return
+
+        manifest_config = manifest_data.get("config", None)
+        config_blob_sha256 = manifest_config.get("digest", None)
+        blob_artifact = Artifact.objects.get(sha256=config_blob_sha256.removeprefix("sha256:"))
+        config_blob, _ = get_content_data(blob_artifact)
+        self.architecture = config_blob.get("architecture", None)
+
     def is_bootable_image(self):
         if (
             self.annotations.get("containers.bootc") == "1"

pulp_container/app/registry.py

@@ -455,9 +455,11 @@ async def init_pending_content(self, digest, manifest_data, media_type, raw_text
             data=raw_text_data,
         )
 
-        # skip if media_type of schema1
+        # if media_type of schema1 configure only manifest architecture
         if media_type in (MEDIA_TYPE.MANIFEST_V2, MEDIA_TYPE.MANIFEST_OCI):
             await sync_to_async(manifest.init_metadata)(manifest_data=manifest_data)
+        else:
+            await sync_to_async(manifest.init_architecture)(manifest_data=manifest_data)
 
         try:
             await manifest.asave()

pulp_container/app/serializers.py

@@ -103,6 +103,11 @@ class ManifestSerializer(NoArtifactContentSerializer):
         default=False,
         help_text=_("A boolean determining whether the image bundles a Flatpak application"),
     )
+    architecture = serializers.CharField(
+        help_text="The CPU architecture which the binaries in this image are built to run on.",
+        required=False,
+        default=None,
+    )
 
     class Meta:
         fields = NoArtifactContentSerializer.Meta.fields + (
@@ -116,6 +121,7 @@ class Meta:
             "labels",
             "is_bootable",
             "is_flatpak",
+            "architecture",
         )
         model = models.Manifest
 

pulp_container/app/tasks/builder.py

@@ -13,7 +13,7 @@
     Tag,
 )
 from pulp_container.constants import MEDIA_TYPE
-from pulp_container.app.utils import calculate_digest
+from pulp_container.app.utils import calculate_digest, get_content_data
 from pulpcore.plugin.models import Artifact, ContentArtifact, Content
 
 
@@ -83,6 +83,9 @@ def add_image_from_directory_to_repository(path, repository, tag):
 
         config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
         manifest.config_blob = config_blob
+        blob_artifact = Artifact.objects.get(sha256=config_blob.digest.removeprefix("sha256:"))
+        config_blob_dict, _ = get_content_data(blob_artifact)
+        manifest.architecture = config_blob_dict.get("architecture", None)
         manifest.save()
 
         pks_to_add = []

pulp_container/app/tasks/sync_stages.py

@@ -390,8 +390,8 @@ def create_manifest(self, manifest_data, raw_text_data, media_type, digest=None)
             media_type=media_type,
             data=raw_text_data,
             annotations=manifest_data.get("annotations", {}),
+            architecture=manifest_data.get("architecture", None),
         )
-
         manifest_dc = DeclarativeContent(content=manifest)
         return manifest_dc
 
@@ -639,6 +639,16 @@ def _post_save(self, batch):
                             os_features=platform.get("os.features"),
                         )
                     )
+            if "config_blob_dc" in dc.extra_data:
+                manifest_dc = dc.content
+                config_blob_sha256 = dc.extra_data["config_blob_dc"].content.digest
+                blob_artifact = Artifact.objects.get(
+                    sha256=config_blob_sha256.removeprefix("sha256:")
+                )
+                config_blob, _ = get_content_data(blob_artifact)
+                manifest_dc.architecture = config_blob.get("architecture", None)
+                manifest_dc.save()
+
         if blob_manifests:
             BlobManifest.objects.bulk_create(blob_manifests, ignore_conflicts=True)
         if manifest_list_manifests:

pulp_container/tests/functional/api/test_build_images.py

@@ -31,6 +31,7 @@ def containerfile_name():
 
 def test_build_image(
     pulpcore_bindings,
+    container_manifest_api,
     container_repository_api,
     container_distribution_api,
     gen_object_with_cleanup,
@@ -61,3 +62,7 @@ def test_build_image(
     local_registry.pull(distribution.base_path)
     image = local_registry.inspect(distribution.base_path)
     assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"]
+
+    manifest = container_manifest_api.list(digest=image[0]["Digest"])
+    manifest = manifest.to_dict()["results"][0]
+    assert manifest["architecture"] == "amd64"

pulp_container/tests/functional/api/test_pull_through_cache.py

@@ -38,6 +38,7 @@ def _add_pull_through_entities_to_cleanup(path):
 def pull_and_verify(
     anonymous_user,
     add_pull_through_entities_to_cleanup,
+    container_manifest_api,
     container_pull_through_distribution_api,
     container_distribution_api,
     container_repository_api,
@@ -59,6 +60,11 @@ def _pull_and_verify(images, pull_through_distribution):
             local_registry.pull(local_image_path)
             local_image = local_registry.inspect(local_image_path)
 
+            # 1.1. check pulp manifest model fields
+            manifest = container_manifest_api.list(digest=local_image[0]["Digest"])
+            manifest = manifest.to_dict()["results"][0]
+            assert manifest["architecture"] == "amd64"
+
             path, tag = local_image_path.split(":")
             tags_to_verify.append(tag)
 

pulp_container/tests/functional/api/test_push_content.py

@@ -39,6 +39,7 @@ def test_push_using_registry_client_admin(
     add_to_cleanup,
     registry_client,
     local_registry,
+    container_manifest_api,
     container_namespace_api,
 ):
     """Test push with official registry client and logged in as admin."""
@@ -48,6 +49,13 @@ def test_push_using_registry_client_admin(
     registry_client.pull(image_path)
     local_registry.tag_and_push(image_path, local_url)
     local_registry.pull(local_url)
+
+    # check pulp manifest model fields
+    local_image = local_registry.inspect(local_url)
+    manifest = container_manifest_api.list(digest=local_image[0]["Digest"])
+    manifest = manifest.to_dict()["results"][0]
+    assert manifest["architecture"] == "amd64"
+
     # ensure that same content can be pushed twice without permission errors
     local_registry.tag_and_push(image_path, local_url)
 

pulp_container/tests/functional/api/test_sync.py

@@ -3,7 +3,12 @@
 import pytest
 from pulpcore.tests.functional.utils import PulpTaskError
 
-from pulp_container.tests.functional.constants import PULP_FIXTURE_1, PULP_LABELED_FIXTURE
+from pulp_container.constants import MEDIA_TYPE
+from pulp_container.tests.functional.constants import (
+    PULP_FIXTURE_1,
+    PULP_LABELED_FIXTURE,
+    PULP_HELLO_WORLD_LINUX_AMD64_DIGEST,
+)
 
 from pulp_container.tests.functional.constants import (
     REGISTRY_V2_FEED_URL,
@@ -39,8 +44,14 @@ def _synced_container_repository_factory(
 
 
 @pytest.mark.parallel
-def test_basic_sync(container_repo, container_remote, container_repository_api, container_sync):
-    container_sync(container_repo, container_remote)
+def test_basic_sync(
+    container_repo,
+    container_remote,
+    container_repository_api,
+    container_sync,
+    container_manifest_api,
+):
+    repo_version = container_sync(container_repo, container_remote).created_resources[0]
     repository = container_repository_api.read(container_repo.pulp_href)
 
     assert "versions/1/" in repository.latest_version_href
@@ -53,6 +64,14 @@ def test_basic_sync(container_repo, container_remote, container_repository_api,
 
     assert repository.latest_version_href == latest_version_href
 
+    manifest = container_manifest_api.list(
+        repository_version=repo_version,
+        media_type=[MEDIA_TYPE.MANIFEST_V2],
+        digest=PULP_HELLO_WORLD_LINUX_AMD64_DIGEST,
+    )
+    manifest = manifest.to_dict()["results"][0]
+    assert manifest["architecture"] == "amd64"
+
 
 @pytest.mark.parallel
 def test_sync_labelled_image(