Support for Flatpak index endpoints

closes #1315

.flake8

@@ -6,6 +6,8 @@
 # For more info visit https://github.com/pulp/plugin_template
 [flake8]
 exclude = ./docs/*,*/migrations/*
+per-file-ignores = */__init__.py: F401
+
 ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402
 max-line-length = 100
 

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-228-g9949044
+2021.08.26-232-g10acbaa

.github/workflows/scripts/install.sh

@@ -118,7 +118,7 @@ if [ "$TEST" = "azure" ]; then
       - ./azurite:/etc/pulp\
     command: "azurite-blob --blobHost 0.0.0.0 --cert /etc/pulp/azcert.pem --key /etc/pulp/azkey.pem"' vars/main.yaml
   sed -i -e '$a azure_test: true\
-pulp_scenario_settings: null\
+pulp_scenario_settings: {"flatpak_index": true}\
 ' vars/main.yaml
 fi
 

.github/workflows/scripts/post_before_script.sh

@@ -0,0 +1,4 @@
+if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
+  # Needed by pulp_container/tests/functional/api/test_flatpak.py:
+  cmd_prefix dnf install -yq dbus-daemon flatpak
+fi

.github/workflows/scripts/update_backport_labels.py

@@ -0,0 +1,55 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_container' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import requests
+import yaml
+import random
+import os
+
+
+def random_color():
+    """Generates a random 24-bit number in hex"""
+    color = random.randrange(0, 2**24)
+    return format(color, "06x")
+
+
+session = requests.Session()
+token = os.getenv("GITHUB_TOKEN")
+
+headers = {
+    "Authorization": f"token {token}",
+    "Accept": "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+}
+session.headers.update(headers)
+
+# get all labels from the repository's current state
+response = session.get("https://api.github.com/repos/pulp/pulp_container/labels", headers=headers)
+assert response.status_code == 200
+old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")])
+
+# get ci_update_branches from template_config.yml
+with open("./template_config.yml", "r") as f:
+    plugin_template = yaml.safe_load(f)
+new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]])
+
+# delete old labels that are not in new labels
+for label in old_labels.difference(new_labels):
+    response = session.delete(
+        f"https://api.github.com/repos/pulp/pulp_container/labels/{label}", headers=headers
+    )
+    assert response.status_code == 204
+
+# create new labels that are not in old labels
+for label in new_labels.difference(old_labels):
+    color = random_color()
+    response = session.post(
+        "https://api.github.com/repos/pulp/pulp_container/labels",
+        headers=headers,
+        json={"name": label, "color": color},
+    )
+    assert response.status_code == 201

.github/workflows/update-labels.yml

@@ -0,0 +1,40 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_container' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+
+---
+name: Update Labels
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'template_config.yml'
+
+jobs:
+  update_backport_labels:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+      - name: Configure Git with pulpbot name and email
+        run: |
+          git config --global user.name 'pulpbot'
+          git config --global user.email '[email protected]'
+      - name: Install python dependencies
+        run: |
+          echo ::group::PYDEPS
+          pip install requests pyyaml
+          echo ::endgroup::
+      - uses: actions/checkout@v3
+      - name: Update labels
+        run: |
+          python3 .github/workflows/scripts/update_backport_labels.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+

.github/workflows/update_ci.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Run update
         working-directory: pulp_container
         run: |
-          .github/workflows/scripts/update_ci.sh
+          ../plugin_template/scripts/update_ci.sh
 
       - name: Create Pull Request for CI files
         uses: peter-evans/create-pull-request@v4
@@ -80,7 +80,7 @@ jobs:
       - name: Run update
         working-directory: pulp_container
         run: |
-          .github/workflows/scripts/update_ci.sh
+          ../plugin_template/scripts/update_ci.sh
 
       - name: Create Pull Request for CI files
         uses: peter-evans/create-pull-request@v4
@@ -107,7 +107,7 @@ jobs:
       - name: Run update
         working-directory: pulp_container
         run: |
-          .github/workflows/scripts/update_ci.sh
+          ../plugin_template/scripts/update_ci.sh
 
       - name: Create Pull Request for CI files
         uses: peter-evans/create-pull-request@v4
@@ -134,7 +134,7 @@ jobs:
       - name: Run update
         working-directory: pulp_container
         run: |
-          .github/workflows/scripts/update_ci.sh
+          ../plugin_template/scripts/update_ci.sh
 
       - name: Create Pull Request for CI files
         uses: peter-evans/create-pull-request@v4

CHANGES/1315.feature

@@ -0,0 +1 @@
+Added support for Flatpak index endpoints.

docs/index.rst

@@ -28,6 +28,7 @@ Features
 * Host content either `locally or on S3 <https://docs.pulpproject.org/installation/storage.html>`_
 * De-duplication of all saved content
 * Support disconnected and air-gapped environments with the Pulp Import/Export facility for container repositories
+* Support for :ref:`hosting Flatpak content in OCI format <flatpak-workflow>`
 
 Tech Preview
 ------------

docs/tech-preview.rst

@@ -4,3 +4,4 @@ Tech previews
 The following features are currently being released as part of a tech preview:
 
 * Build an OCI image from a Containerfile
+* Support for hosting Flatpak content in OCI format.

docs/workflows/flatpak-support.rst

@@ -0,0 +1,64 @@
+.. _flatpak-workflow:
+
+Hosting Flatpak Content in OCI Format
+=====================================
+
+Pulp can host Flatpak application and runtime images that are distributed in OCI format.  To make
+such content discoverable, it can provide ``/index/dynamic`` and ``/index/static`` endpoints as
+specified by `the Flatpak registry index protocol
+<https://github.com/flatpak/flatpak-oci-specs/blob/main/registry-index.md>`_.  This is not enabled
+by default.  To enable it, define ``FLATPAK_INDEX = True`` in the settings file.
+
+Clients like the ``flatpak`` command-line tool or the GNOME Software application will typically
+query the ``/index/static`` endpoint, which is intended to be called repeatedly with identical query
+parameters, and whose responses are meant to be cached.  The ``/index/dynamic`` endpoint serves
+exactly the same content, but is intended for one-off requests that should not be cached.  These
+endpoints can be accessed without authentication.  They only provide information about public
+repositories.
+
+The two endpoints support a number of query parameters (``architecture``, ``tag``, ``label``, etc.),
+see the protocol specification for details.  Two notes:
+
+* Every request must include a ``label:org.flatpak.ref:exists=1`` query parameter.  This acts as a
+  marker to only report Flatpak content, and to exclude other container content that may also be
+  provided by the Pulp instance.
+
+* This implementation does not support annotations.  Including any ``annotation`` query parameters
+  will result in a 400 failure response.  Use ``label`` query parameters instead.  (Existing clients
+  like the ``flatpak`` command-line tool never issue requests including any ``annotation`` query
+  parameters.)
+
+Install a Flatpak image from Pulp
+---------------------------------
+
+This section assumes that you have created at least one public distribution in your Pulp instance
+that serves a repository containing Flatpak content.  To do this, see the general :doc:`host`
+documentation.
+
+You can for example use the ``flatpak`` `command-line tool
+<https://docs.flatpak.org/en/latest/using-flatpak.html#the-flatpak-command>`_ to set up a Flatpak
+remote (named ``pulp`` here) that references your Pulp instance:
+
+.. code:: shell
+
+  flatpak remote-add pulp oci+"$BASE_ADDR"
+
+Then, use
+
+.. code:: shell
+
+  flatpak remote-ls pulp
+
+to retrieve a list of all Flatpak applications and runtimes that your Pulp instance serves.  (This
+queries the ``/index/static`` endpoint, as explained above.)  Finally, if your Pulp instance serves
+e.g. the ``org.gnome.gedit`` application, use
+
+.. code:: shell
+
+  flatpak install pulp org.gnome.gedit
+
+to install it and run it with
+
+.. code:: shell
+
+  flatpak run org.gnome.gedit

docs/workflows/index.rst

@@ -78,3 +78,4 @@ OCI artifact support
    cosign-support
    helm-support
    oci-artifacts
+   flatpak-support

pulp_container/app/cache.py

@@ -6,6 +6,7 @@
 from pulp_container.app.exceptions import RepositoryNotFound
 
 ACCEPT_HEADER_KEY = "accept_header"
+QUERY_KEY = "query"
 
 
 class RegistryCache:
@@ -72,3 +73,19 @@ def find_base_path_cached(request, cached):
         except ObjectDoesNotExist:
             raise RepositoryNotFound(name=path)
         return distro.base_path
+
+
+class FlatpakIndexStaticCache(SyncContentCache):
+    def __init__(self, expires_ttl=None, auth=None):
+        updated_keys = (QUERY_KEY,)
+        super().__init__(
+            base_key="/index/static", expires_ttl=expires_ttl, keys=updated_keys, auth=auth
+        )
+
+    def make_key(self, request):
+        """Make a key composed of the request's query."""
+        all_keys = {
+            QUERY_KEY: request.query_params.urlencode(),
+        }
+        key = ":".join(all_keys[k] for k in self.keys)
+        return key

pulp_container/app/models.py

@@ -11,7 +11,9 @@
 from django.conf import settings
 from django.contrib.postgres import fields
 from django.shortcuts import redirect
+from django_lifecycle import hook, AFTER_CREATE, AFTER_DELETE, AFTER_UPDATE
 
+from pulpcore.plugin.cache import SyncContentCache
 from pulpcore.plugin.download import DownloaderFactory
 from pulpcore.plugin.models import (
     Artifact,
@@ -612,6 +614,23 @@ def redirect_to_content_app(self, url):
             url = self.content_guard.cast().preauthenticate_url(url)
         return redirect(url)
 
+    @hook(AFTER_CREATE)
+    @hook(AFTER_DELETE)
+    @hook(
+        AFTER_UPDATE,
+        when_any=[
+            "base_path",
+            "private",
+            "repository",
+            "repository_version",
+        ],
+        has_changed=True,
+    )
+    def invalidate_flatpak_index_cache(self):
+        """Invalidates the cache for /index/static."""
+        if settings.CACHE_ENABLED:
+            SyncContentCache().delete(base_key="/index/static")
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         permissions = [