diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b3cadd2a7..2d18a2d01 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -68,7 +68,7 @@ services: VARSYAML cat >> vars/main.yaml << VARSYAML -pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} +pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true} pulp_scheme: https pulp_container_tag: https @@ -82,7 +82,7 @@ if [ "$TEST" == 'stream' ]; then - ./ssh/id_ed25519.pub:/home/foo/.ssh/keys/id_ed25519.pub\ command: "foo::::storage"' vars/main.yaml sed -i -e '$a stream_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"flatpak_index": false}\ ' vars/main.yaml fi @@ -99,7 +99,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"flatpak_index": false}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" fi @@ -118,7 +118,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: null\ +pulp_scenario_settings: {"flatpak_index": true}\ ' vars/main.yaml fi diff --git a/.github/workflows/scripts/post_before_script.sh b/.github/workflows/scripts/post_before_script.sh new file mode 100644 index 000000000..867f3eae3 --- /dev/null +++ b/.github/workflows/scripts/post_before_script.sh @@ -0,0 +1,4 @@ +if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then + # Needed by pulp_container/tests/functional/api/test_flatpak.py: + cmd_prefix dnf install -yq dbus-daemon flatpak +fi diff --git a/CHANGES/1315.feature b/CHANGES/1315.feature new file mode 100644 index 000000000..45c198433 --- /dev/null +++ b/CHANGES/1315.feature @@ -0,0 +1 @@ +Added support for Flatpak index endpoints. diff --git a/docs/index.rst b/docs/index.rst index c4c19ca8b..5ad31f7af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,6 +28,7 @@ Features * Host content either `locally or on S3 `_ * De-duplication of all saved content * Support disconnected and air-gapped environments with the Pulp Import/Export facility for container repositories +* Support for :ref:`hosting Flatpak content in OCI format ` Tech Preview ------------ diff --git a/docs/tech-preview.rst b/docs/tech-preview.rst index 146f4c3ad..f25f86048 100644 --- a/docs/tech-preview.rst +++ b/docs/tech-preview.rst @@ -4,3 +4,4 @@ Tech previews The following features are currently being released as part of a tech preview: * Build an OCI image from a Containerfile +* Support for hosting Flatpak content in OCI format. diff --git a/docs/workflows/flatpak-support.rst b/docs/workflows/flatpak-support.rst new file mode 100644 index 000000000..258b5b5fd --- /dev/null +++ b/docs/workflows/flatpak-support.rst @@ -0,0 +1,64 @@ +.. _flatpak-workflow: + +Hosting Flatpak Content in OCI Format +===================================== + +Pulp can host Flatpak application and runtime images that are distributed in OCI format. To make +such content discoverable, it can provide ``/index/dynamic`` and ``/index/static`` endpoints as +specified by `the Flatpak registry index protocol +`_. This is not enabled +by default. To enable it, define ``FLATPAK_INDEX = True`` in the settings file. + +Clients like the ``flatpak`` command-line tool or the GNOME Software application will typically +query the ``/index/static`` endpoint, which is intended to be called repeatedly with identical query +parameters, and whose responses are meant to be cached. The ``/index/dynamic`` endpoint serves +exactly the same content, but is intended for one-off requests that should not be cached. These +endpoints can be accessed without authentication. They only provide information about public +repositories. + +The two endpoints support a number of query parameters (``architecture``, ``tag``, ``label``, etc.), +see the protocol specification for details. Two notes: + +* Every request must include a ``label:org.flatpak.ref:exists=1`` query parameter. This acts as a + marker to only report Flatpak content, and to exclude other container content that may also be + provided by the Pulp instance. + +* This implementation does not support annotations. Including any ``annotation`` query parameters + will result in a 400 failure response. Use ``label`` query parameters instead. (Existing clients + like the ``flatpak`` command-line tool never issue requests including any ``annotation`` query + parameters.) + +Install a Flatpak image from Pulp +--------------------------------- + +This section assumes that you have created at least one public distribution in your Pulp instance +that serves a repository containing Flatpak content. To do this, see the general :doc:`host` +documentation. + +You can for example use the ``flatpak`` `command-line tool +`_ to set up a Flatpak +remote (named ``pulp`` here) that references your Pulp instance: + +.. code:: shell + + flatpak remote-add pulp oci+"$BASE_ADDR" + +Then, use + +.. code:: shell + + flatpak remote-ls pulp + +to retrieve a list of all Flatpak applications and runtimes that your Pulp instance serves. (This +queries the ``/index/static`` endpoint, as explained above.) Finally, if your Pulp instance serves +e.g. the ``org.gnome.gedit`` application, use + +.. code:: shell + + flatpak install pulp org.gnome.gedit + +to install it and run it with + +.. code:: shell + + flatpak run org.gnome.gedit diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst index 35fc20da5..dbb9101eb 100644 --- a/docs/workflows/index.rst +++ b/docs/workflows/index.rst @@ -78,3 +78,4 @@ OCI artifact support cosign-support helm-support oci-artifacts + flatpak-support diff --git a/pulp_container/app/cache.py b/pulp_container/app/cache.py index 31e7c09f9..4b9151027 100644 --- a/pulp_container/app/cache.py +++ b/pulp_container/app/cache.py @@ -6,6 +6,7 @@ from pulp_container.app.exceptions import RepositoryNotFound ACCEPT_HEADER_KEY = "accept_header" +QUERY_KEY = "query" class RegistryCache: @@ -72,3 +73,19 @@ def find_base_path_cached(request, cached): except ObjectDoesNotExist: raise RepositoryNotFound(name=path) return distro.base_path + + +class FlatpakIndexStaticCache(SyncContentCache): + def __init__(self, expires_ttl=None, auth=None): + updated_keys = (QUERY_KEY,) + super().__init__( + base_key="/index/static", expires_ttl=expires_ttl, keys=updated_keys, auth=auth + ) + + def make_key(self, request): + """Make a key composed of the request's query.""" + all_keys = { + QUERY_KEY: request.query_params.urlencode(), + } + key = ":".join(all_keys[k] for k in self.keys) + return key diff --git a/pulp_container/app/models.py b/pulp_container/app/models.py index d82fd9cf4..beef5f984 100644 --- a/pulp_container/app/models.py +++ b/pulp_container/app/models.py @@ -11,7 +11,9 @@ from django.conf import settings from django.contrib.postgres import fields from django.shortcuts import redirect +from django_lifecycle import hook, AFTER_CREATE, AFTER_DELETE, AFTER_UPDATE +from pulpcore.plugin.cache import SyncContentCache from pulpcore.plugin.download import DownloaderFactory from pulpcore.plugin.models import ( Artifact, @@ -612,6 +614,23 @@ def redirect_to_content_app(self, url): url = self.content_guard.cast().preauthenticate_url(url) return redirect(url) + @hook(AFTER_CREATE) + @hook(AFTER_DELETE) + @hook( + AFTER_UPDATE, + when_any=[ + "base_path", + "private", + "repository", + "repository_version", + ], + has_changed=True, + ) + def invalidate_flatpak_index_cache(self): + """Invalidates the cache for /index/static.""" + if settings.CACHE_ENABLED: + SyncContentCache().delete(base_key="/index/static") + class Meta: default_related_name = "%(app_label)s_%(model_name)s" permissions = [ diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 6bb21a9ac..cda1cbcf9 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -46,7 +46,11 @@ from pulp_container.app import models, serializers from pulp_container.app.authorization import AuthorizationService -from pulp_container.app.cache import find_base_path_cached, RegistryApiCache +from pulp_container.app.cache import ( + find_base_path_cached, + FlatpakIndexStaticCache, + RegistryApiCache, +) from pulp_container.app.exceptions import ( unauthorized_exception_handler, InvalidRequest, @@ -473,6 +477,134 @@ def get_queryset(self, *args, **kwargs): return (public_repositories | accessible_repositories).distinct() +class FlatpakIndexDynamicView(APIView): + """ + Handles requests to the /index/dynamic endpoint + """ + + authentication_classes = [] + permission_classes = [] + + def recurse_through_manifest_lists(self, tag, manifest, oss, architectures, manifests): + if manifest.media_type in (models.MEDIA_TYPE.MANIFEST_V2, models.MEDIA_TYPE.MANIFEST_OCI): + manifests.setdefault(manifest, set()).add(tag) + elif manifest.media_type in (models.MEDIA_TYPE.MANIFEST_LIST, models.MEDIA_TYPE.INDEX_OCI): + mlms = manifest.listed_manifests.through.objects.filter(image_manifest__pk=manifest.pk) + if oss: + mlms.filter(os__in=oss) + if architectures: + mlms.filter(architecture__in=architectures) + for mlm in mlms: + self.recurse_through_manifest_lists( + tag, mlm.manifest_list, oss, architectures, manifests + ) + + def get(self, request): + req_repositories = None + req_tags = None + req_oss = None + req_architectures = None + req_label_exists = set() + req_label_values = {} + for key, values in request.query_params.lists(): + if key == "repository": + req_repositories = values + elif key == "tag": + req_tags = values + elif key == "os": + req_oss = values + elif key == "architecture": + req_architectures = values + elif key.startswith("label:"): + if key.endswith(":exists"): + if any(v != "1" for v in values): + raise ParseError(detail=f"{key} must have value 1.") + label = key[len("label:") : len(key) - len(":exists")] + req_label_exists.add(label) + else: + label = key[len("label:") :] + req_label_values[label] = values + else: + # In particularly, this covers any annotation:... parameters, which this + # implementation does not support: + raise ParseError(detail=f"Unsupported {key}.") + + if "org.flatpak.ref" not in req_label_exists: + raise ParseError(detail="Missing label:org.flatpak.ref:exists=1.") + + distributions = models.ContainerDistribution.objects.filter(private=False).only("base_path") + + if req_repositories: + distributions = distributions.filter(base_path__in=req_repositories) + + results = [] + for distribution in distributions: + images = [] + if distribution.repository: + repository_version = distribution.repository.latest_version() + else: + repository_version = distribution.repository_version + tags = models.Tag.objects.select_related("tagged_manifest").filter( + pk__in=repository_version.content + ) + if req_tags: + tags = tags.filter(name__in=req_tags) + manifests = {} # mapping manifests to sets of encountered tag names + for tag in tags: + self.recurse_through_manifest_lists( + tag.name, tag.tagged_manifest, req_oss, req_architectures, manifests + ) + for manifest, tagged in manifests.items(): + with storage.open(manifest.config_blob._artifacts.get().file.name) as file: + raw_data = file.read() + config_data = json.loads(raw_data) + labels = config_data.get("config", {}).get("Labels") + if not labels: + continue + if any(label not in labels.keys() for label in req_label_exists): + continue + os = config_data["os"] + if req_oss and os not in req_oss: + continue + architecture = config_data["architecture"] + if req_architectures and architecture not in req_architectures: + continue + if any( + labels.get(label) not in values for label, values in req_label_values.items() + ): + continue + images.append( + { + "Tags": tagged, + "Digest": manifest.digest, + "MediaType": manifest.media_type, + "OS": os, + "Architecture": architecture, + "Labels": labels, + } + ) + if images: + results.append({"Name": distribution.base_path, "Images": images}) + + return Response(data={"Registry": settings.CONTENT_ORIGIN, "Results": results}) + + +class FlatpakIndexStaticView(FlatpakIndexDynamicView): + """ + Handles requests to the /index/static endpoint + """ + + @FlatpakIndexStaticCache() + def get(self, request): + response = super().get(request) + # Avoid django.template.response.ContentNotRenderedError: + response.accepted_renderer = JSONRenderer() + response.accepted_media_type = JSONRenderer.media_type + response.renderer_context = {} + response.render() + return response + + class ContainerTagListSerializer(ModelSerializer): """ Serializer for Tags in the tags list endpoint of the registry. diff --git a/pulp_container/app/settings.py b/pulp_container/app/settings.py index 3060606d6..f01a50938 100644 --- a/pulp_container/app/settings.py +++ b/pulp_container/app/settings.py @@ -46,3 +46,5 @@ "application/vnd.wasm.content.layer.v1+wasm", ], } + +FLATPAK_INDEX = False diff --git a/pulp_container/app/urls.py b/pulp_container/app/urls.py index 0b7786561..864f0b2ed 100644 --- a/pulp_container/app/urls.py +++ b/pulp_container/app/urls.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.urls import include, path from rest_framework.routers import Route, SimpleRouter from pulp_container.app.registry_api import ( @@ -5,6 +6,8 @@ Blobs, BlobUploads, CatalogView, + FlatpakIndexDynamicView, + FlatpakIndexStaticView, Manifests, Signatures, TagsListView, @@ -35,3 +38,10 @@ path("v2//tags/list", TagsListView.as_view()), path("", include(router.urls)), ] +if settings.FLATPAK_INDEX: + urlpatterns.extend( + [ + path("index/dynamic", FlatpakIndexDynamicView.as_view()), + path("index/static", FlatpakIndexStaticView.as_view()), + ] + ) diff --git a/pulp_container/tests/functional/api/test_flatpak.py b/pulp_container/tests/functional/api/test_flatpak.py new file mode 100644 index 000000000..0aa241bc3 --- /dev/null +++ b/pulp_container/tests/functional/api/test_flatpak.py @@ -0,0 +1,72 @@ +"""Tests that verify Flatpak support""" +import pytest +import subprocess + +from django.conf import settings + +from pulp_container.tests.functional.constants import REGISTRY_V2 + + +def test_flatpak_install( + add_to_cleanup, + registry_client, + local_registry, + container_namespace_api, +): + if not settings.FLATPAK_INDEX: + pytest.skip("This test requires FLATPAK_INDEX to be enabled") + + image_path1 = f"{REGISTRY_V2}/pulp/oci-net.fishsoup.busyboxplatform:latest" + registry_client.pull(image_path1) + local_registry.tag_and_push(image_path1, "pulptest/oci-net.fishsoup.busyboxplatform:latest") + image_path2 = f"{REGISTRY_V2}/pulp/oci-net.fishsoup.hello:latest" + registry_client.pull(image_path2) + local_registry.tag_and_push(image_path2, "pulptest/oci-net.fishsoup.hello:latest") + namespace = container_namespace_api.list(name="pulptest").results[0] + add_to_cleanup(container_namespace_api, namespace.pulp_href) + + # Install flatpak: + subprocess.check_call( + [ + "flatpak", + "--user", + "remote-add", + "pulptest", + "oci+" + settings.CONTENT_ORIGIN, + ] + ) + # See + # "lorax-embed-flatpaks.tmpl: Run the flatpak-install under dbus-run-session" for the need for + # dbus-run-session to avoid "error: Cannot autolaunch D-Bus without X11 $DISPLAY": + subprocess.check_call( + [ + "dbus-run-session", + "flatpak", + "--user", + "install", + "--noninteractive", + "pulptest", + "net.fishsoup.Hello", + ] + ) + + # Clean up flatpak: + subprocess.run( + [ + "flatpak", + "--user", + "uninstall", + "--noninteractive", + "net.fishsoup.Hello", + ] + ) + subprocess.run( + [ + "flatpak", + "--user", + "uninstall", + "--noninteractive", + "net.fishsoup.BusyBoxPlatform", + ] + ) + subprocess.run(["flatpak", "--user", "remote-delete", "pulptest"]) diff --git a/template_config.yml b/template_config.yml index c398ee1ab..d68b78549 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,7 +1,7 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-225-gbfce9ef +# generated with plugin_template@2021.08.26-237-g4bbee21 additional_repos: [] api_root: /pulp/ @@ -62,10 +62,18 @@ pulp_settings: - /tmp allowed_import_paths: - /tmp -pulp_settings_azure: null + flatpak_index: true +pulp_settings_azure: + flatpak_index: true pulp_settings_gcp: null -pulp_settings_s3: null -pulp_settings_stream: null +pulp_settings_s3: + # Do not enable flatpak_index for s3 for now, as it would cause + # pulp_container/tests/functional/api/test_flatpak.py to fail due to + # "Odd 400 response with https GET 302 + # redirecting to http": + flatpak_index: false +pulp_settings_stream: + flatpak_index: false pulpprojectdotorg_key_id: aa499d7938ed pydocstyle: true pypi_username: pulp