Support for Flatpak index endpoints

closes #1315

.github/workflows/scripts/install.sh

@@ -68,7 +68,7 @@ services:
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
-pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]}
+pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true}
 pulp_scheme: https
 
 pulp_container_tag: https
@@ -107,7 +107,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: null\
+pulp_scenario_settings: {"flatpak_index": true}\
 ' vars/main.yaml
 fi
 

.github/workflows/scripts/post_before_script.sh

@@ -0,0 +1,4 @@
+if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
+  # Needed by pulp_container/tests/functional/api/test_flatpak.py:
+  cmd_prefix dnf install -yq dbus-daemon flatpak
+fi

CHANGES/1315.feature

@@ -0,0 +1 @@
+Added support for Flatpak index endpoints.

docs/index.rst

@@ -28,6 +28,7 @@ Features
 * Host content either `locally or on S3 <https://docs.pulpproject.org/installation/storage.html>`_
 * De-duplication of all saved content
 * Support disconnected and air-gapped environments with the Pulp Import/Export facility for container repositories
+* Support for :ref:`hosting Flatpak content in OCI format <flatpak-workflow>`
 
 Tech Preview
 ------------

docs/tech-preview.rst

@@ -4,3 +4,4 @@ Tech previews
 The following features are currently being released as part of a tech preview:
 
 * Build an OCI image from a Containerfile
+* Support for hosting Flatpak content in OCI format.

docs/workflows/flatpak-support.rst

@@ -0,0 +1,64 @@
+.. _flatpak-workflow:
+
+Hosting Flatpak Content in OCI Format
+=====================================
+
+Pulp can host Flatpak application and runtime images that are distributed in OCI format.  To make
+such content discoverable, it can provide ``/index/dynamic`` and ``/index/static`` endpoints as
+specified by `the Flatpak registry index protocol
+<https://github.com/flatpak/flatpak-oci-specs/blob/main/registry-index.md>`_.  This is not enabled
+by default.  To enable it, define ``FLATPAK_INDEX = True`` in the settings file.
+
+Clients like the ``flatpak`` command-line tool or the GNOME Software application will typically
+query the ``/index/static`` endpoint, which is intended to be called repeatedly with identical query
+parameters, and whose responses are meant to be cached.  The ``/index/dynamic`` endpoint serves
+exactly the same content, but is intended for one-off requests that should not be cached.  These
+endpoints can be accessed without authentication.  They only provide information about public
+repositories.
+
+The two endpoints support a number of query parameters (``architecture``, ``tag``, ``label``, etc.),
+see the protocol specification for details.  Two notes:
+
+* Every request must include a ``label:org.flatpak.ref:exists=1`` query parameter.  This acts as a
+  marker to only report Flatpak content, and to exclude other container content that may also be
+  provided by the Pulp instance.
+
+* This implementation does not support annotations.  Including any ``annotation`` query parameters
+  will result in a 400 failure response.  Use ``label`` query parameters instead.  (Existing clients
+  like the ``flatpak`` command-line tool never issue requests including any ``annotation`` query
+  parameters.)
+
+Install a Flatpak image from Pulp
+---------------------------------
+
+This section assumes that you have created at least one public distribution in your Pulp instance
+that serves a repository containing Flatpak content.  To do this, see the general :doc:`host`
+documentation.
+
+You can for example use the ``flatpak`` `command-line tool
+<https://docs.flatpak.org/en/latest/using-flatpak.html#the-flatpak-command>`_ to set up a Flatpak
+remote (named ``pulp`` here) that references your Pulp instance:
+
+.. code:: shell
+
+  flatpak remote-add pulp oci+"$BASE_ADDR"
+
+Then, use
+
+.. code:: shell
+
+  flatpak remote-ls pulp
+
+to retrieve a list of all Flatpak applications and runtimes that your Pulp instance serves.  (This
+queries the ``/index/static`` endpoint, as explained above.)  Finally, if your Pulp instance serves
+e.g. the ``org.gnome.gedit`` application, use
+
+.. code:: shell
+
+  flatpak install pulp org.gnome.gedit
+
+to install it and run it with
+
+.. code:: shell
+
+  flatpak run org.gnome.gedit

docs/workflows/index.rst

@@ -78,3 +78,4 @@ OCI artifact support
    cosign-support
    helm-support
    oci-artifacts
+   flatpak-support

pulp_container/app/cache.py

@@ -6,6 +6,7 @@
 from pulp_container.app.exceptions import RepositoryNotFound
 
 ACCEPT_HEADER_KEY = "accept_header"
+QUERY_KEY = "query"
 
 
 class RegistryCache:
@@ -72,3 +73,19 @@ def find_base_path_cached(request, cached):
         except ObjectDoesNotExist:
             raise RepositoryNotFound(name=path)
         return distro.base_path
+
+
+class FlatpakIndexStaticCache(SyncContentCache):
+    def __init__(self, expires_ttl=None, auth=None):
+        updated_keys = (QUERY_KEY,)
+        super().__init__(
+            base_key="/index/static", expires_ttl=expires_ttl, keys=updated_keys, auth=auth
+        )
+
+    def make_key(self, request):
+        """Make a key composed of the request's query."""
+        all_keys = {
+            QUERY_KEY: request.query_params.urlencode(),
+        }
+        key = ":".join(all_keys[k] for k in self.keys)
+        return key

pulp_container/app/models.py

@@ -11,7 +11,9 @@
 from django.conf import settings
 from django.contrib.postgres import fields
 from django.shortcuts import redirect
+from django_lifecycle import hook, AFTER_CREATE, AFTER_DELETE, AFTER_UPDATE
 
+from pulpcore.plugin.cache import SyncContentCache
 from pulpcore.plugin.download import DownloaderFactory
 from pulpcore.plugin.models import (
     Artifact,
@@ -612,6 +614,23 @@ def redirect_to_content_app(self, url):
             url = self.content_guard.cast().preauthenticate_url(url)
         return redirect(url)
 
+    @hook(AFTER_CREATE)
+    @hook(AFTER_DELETE)
+    @hook(
+        AFTER_UPDATE,
+        when_any=[
+            "base_path",
+            "private",
+            "repository",
+            "repository_version",
+        ],
+        has_changed=True,
+    )
+    def invalidate_flatpak_index_cache(self):
+        """Invalidates the cache for /index/static."""
+        if settings.CACHE_ENABLED:
+            SyncContentCache().delete(base_key="/index/static")
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         permissions = [

pulp_container/app/registry_api.py

@@ -46,7 +46,11 @@
 
 from pulp_container.app import models, serializers
 from pulp_container.app.authorization import AuthorizationService
-from pulp_container.app.cache import find_base_path_cached, RegistryApiCache
+from pulp_container.app.cache import (
+    find_base_path_cached,
+    FlatpakIndexStaticCache,
+    RegistryApiCache,
+)
 from pulp_container.app.exceptions import (
     unauthorized_exception_handler,
     InvalidRequest,
@@ -473,6 +477,134 @@ def get_queryset(self, *args, **kwargs):
         return (public_repositories | accessible_repositories).distinct()
 
 
+class FlatpakIndexDynamicView(APIView):
+    """
+    Handles requests to the /index/dynamic endpoint
+    """
+
+    authentication_classes = []
+    permission_classes = []
+
+    def recurse_through_manifest_lists(self, tag, manifest, oss, architectures, manifests):
+        if manifest.media_type in (models.MEDIA_TYPE.MANIFEST_V2, models.MEDIA_TYPE.MANIFEST_OCI):
+            manifests.setdefault(manifest, set()).add(tag)
+        elif manifest.media_type in (models.MEDIA_TYPE.MANIFEST_LIST, models.MEDIA_TYPE.INDEX_OCI):
+            mlms = manifest.listed_manifests.through.objects.filter(image_manifest__pk=manifest.pk)
+            if oss:
+                mlms.filter(os__in=oss)
+            if architectures:
+                mlms.filter(architecture__in=architectures)
+            for mlm in mlms:
+                self.recurse_through_manifest_lists(
+                    tag, mlm.manifest_list, oss, architectures, manifests
+                )
+
+    def get(self, request):
+        req_repositories = None
+        req_tags = None
+        req_oss = None
+        req_architectures = None
+        req_label_exists = set()
+        req_label_values = {}
+        for key, values in request.query_params.lists():
+            if key == "repository":
+                req_repositories = values
+            elif key == "tag":
+                req_tags = values
+            elif key == "os":
+                req_oss = values
+            elif key == "architecture":
+                req_architectures = values
+            elif key.startswith("label:"):
+                if key.endswith(":exists"):
+                    if any(v != "1" for v in values):
+                        raise ParseError(detail=f"{key} must have value 1.")
+                    label = key[len("label:") : len(key) - len(":exists")]
+                    req_label_exists.add(label)
+                else:
+                    label = key[len("label:") :]
+                    req_label_values[label] = values
+            else:
+                # In particularly, this covers any annotation:... parameters, which this
+                # implementation does not support:
+                raise ParseError(detail=f"Unsupported {key}.")
+
+        if "org.flatpak.ref" not in req_label_exists:
+            raise ParseError(detail="Missing label:org.flatpak.ref:exists=1.")
+
+        distributions = models.ContainerDistribution.objects.filter(private=False).only("base_path")
+
+        if req_repositories:
+            distributions = distributions.filter(base_path__in=req_repositories)
+
+        results = []
+        for distribution in distributions:
+            images = []
+            if distribution.repository:
+                repository_version = distribution.repository.latest_version()
+            else:
+                repository_version = distribution.repository_version
+            tags = models.Tag.objects.select_related("tagged_manifest").filter(
+                pk__in=repository_version.content
+            )
+            if req_tags:
+                tags = tags.filter(name__in=req_tags)
+            manifests = {}  # mapping manifests to sets of encountered tag names
+            for tag in tags:
+                self.recurse_through_manifest_lists(
+                    tag.name, tag.tagged_manifest, req_oss, req_architectures, manifests
+                )
+            for manifest, tagged in manifests.items():
+                with storage.open(manifest.config_blob._artifacts.get().file.name) as file:
+                    raw_data = file.read()
+                config_data = json.loads(raw_data)
+                labels = config_data.get("config", {}).get("Labels")
+                if not labels:
+                    continue
+                if any(label not in labels.keys() for label in req_label_exists):
+                    continue
+                os = config_data["os"]
+                if req_oss and os not in req_oss:
+                    continue
+                architecture = config_data["architecture"]
+                if req_architectures and architecture not in req_architectures:
+                    continue
+                if any(
+                    labels.get(label) not in values for label, values in req_label_values.items()
+                ):
+                    continue
+                images.append(
+                    {
+                        "Tags": tagged,
+                        "Digest": manifest.digest,
+                        "MediaType": manifest.media_type,
+                        "OS": os,
+                        "Architecture": architecture,
+                        "Labels": labels,
+                    }
+                )
+            if images:
+                results.append({"Name": distribution.base_path, "Images": images})
+
+        return Response(data={"Registry": settings.CONTENT_ORIGIN, "Results": results})
+
+
+class FlatpakIndexStaticView(FlatpakIndexDynamicView):
+    """
+    Handles requests to the /index/static endpoint
+    """
+
+    @FlatpakIndexStaticCache()
+    def get(self, request):
+        response = super().get(request)
+        # Avoid django.template.response.ContentNotRenderedError:
+        response.accepted_renderer = JSONRenderer()
+        response.accepted_media_type = JSONRenderer.media_type
+        response.renderer_context = {}
+        response.render()
+        return response
+
+
 class ContainerTagListSerializer(ModelSerializer):
     """
     Serializer for Tags in the tags list endpoint of the registry.

pulp_container/app/settings.py

@@ -46,3 +46,5 @@
         "application/vnd.wasm.content.layer.v1+wasm",
     ],
 }
+
+FLATPAK_INDEX = False

pulp_container/app/urls.py

@@ -1,10 +1,13 @@
+from django.conf import settings
 from django.urls import include, path
 from rest_framework.routers import Route, SimpleRouter
 from pulp_container.app.registry_api import (
     BearerTokenView,
     Blobs,
     BlobUploads,
     CatalogView,
+    FlatpakIndexDynamicView,
+    FlatpakIndexStaticView,
     Manifests,
     Signatures,
     TagsListView,
@@ -35,3 +38,10 @@
     path("v2/<path:path>/tags/list", TagsListView.as_view()),
     path("", include(router.urls)),
 ]
+if settings.FLATPAK_INDEX:
+    urlpatterns.extend(
+        [
+            path("index/dynamic", FlatpakIndexDynamicView.as_view()),
+            path("index/static", FlatpakIndexStaticView.as_view()),
+        ]
+    )