From d7b03bb050b089cdcbd412d3663024211d33b36e Mon Sep 17 00:00:00 2001 From: Dennis Kliban Date: Fri, 24 Feb 2023 16:16:14 -0500 Subject: [PATCH] Added upload support closes: #52 --- CHANGES/52.feature | 1 + docs/workflows/index.rst | 1 + docs/workflows/upload.rst | 49 +++++++++++ pulp_maven/app/serializers.py | 70 ++++++++++------ pulp_maven/app/viewsets.py | 50 +++++++---- .../tests/functional/api/test_upload.py | 83 +++++++++++++++++++ pulp_maven/tests/functional/conftest.py | 20 +++++ pulp_maven/tests/functional/utils.py | 12 +++ 8 files changed, 245 insertions(+), 41 deletions(-) create mode 100644 CHANGES/52.feature create mode 100644 docs/workflows/upload.rst create mode 100644 pulp_maven/tests/functional/api/test_upload.py diff --git a/CHANGES/52.feature b/CHANGES/52.feature new file mode 100644 index 0000000..c209727 --- /dev/null +++ b/CHANGES/52.feature @@ -0,0 +1 @@ +Added ability to upload Maven Artifacts to repositories. diff --git a/docs/workflows/index.rst b/docs/workflows/index.rst index e9f8759..bd2d8a2 100644 --- a/docs/workflows/index.rst +++ b/docs/workflows/index.rst @@ -29,3 +29,4 @@ set is the hostname and port:: :maxdepth: 2 cache + upload \ No newline at end of file diff --git a/docs/workflows/upload.rst b/docs/workflows/upload.rst new file mode 100644 index 0000000..bc660bb --- /dev/null +++ b/docs/workflows/upload.rst @@ -0,0 +1,49 @@ +Upload a Jar to a Maven Repository +================================== + +Create a maven Repository for the + +Create a Maven Repository +------------------------- + +``$ http POST http://localhost:24817/pulp/api/v3/repositories/maven/maven/ name='my-snapshot-repository'`` + +Create a Maven Distribution for the Maven Repository +---------------------------------------------------- + +``$ http POST http://localhost:24817/pulp/api/v3/distributions/maven/maven/ name='my-snapshot-repository' base_path='my/local/snapshots' repository=$REPO_HREF`` + + +.. code:: json + + { + "pulp_href": "/pulp/api/v3/distributions/67baa17e-0a9f-4302-b04a-dbf324d139de/" + } + +Upload a Jar to the Repository +------------------------------ + +``$ http --form POST http://localhost:24817/pulp/api/v3/content/maven/artifact/ group_id='org.openapitools' artifact_id='openapi-generator-cli' version='6.4.0-SNAPSHOT' filename='openapi-generator-cli-6.4.0.jar' file@./openapi-generator-cli.jar repository=$REPO_HREF`` + + +.. code:: json + + { + "task": "/pulp/api/v3/tasks/03d5a40b-4bda-4ee7-96cb-f0639b6c5d6a/" + } + +Add Pulp as mirror for Maven +---------------------------- + +.. code:: xml + + + + + pulp-maven-central + Local Maven Central mirror + http://localhost:24816/pulp/content/my/local/my/local/snapshots + central + + + diff --git a/pulp_maven/app/serializers.py b/pulp_maven/app/serializers.py index 6e2ca2c..34cc76c 100644 --- a/pulp_maven/app/serializers.py +++ b/pulp_maven/app/serializers.py @@ -2,48 +2,70 @@ from rest_framework import serializers -from pulpcore.plugin import serializers as platform +from pulpcore.plugin.serializers import ( + ContentChecksumSerializer, + DetailRelatedField, + DistributionSerializer, + RemoteSerializer, + RepositorySerializer, + SingleArtifactContentUploadSerializer, +) from . import models -class MavenRepositorySerializer(platform.RepositorySerializer): +class MavenRepositorySerializer(RepositorySerializer): """ Serializer for Maven Repositories. """ class Meta: - fields = platform.RepositorySerializer.Meta.fields + fields = RepositorySerializer.Meta.fields model = models.MavenRepository -class MavenArtifactSerializer(platform.SingleArtifactContentSerializer): +class MavenArtifactSerializer(SingleArtifactContentUploadSerializer, ContentChecksumSerializer): """ A Serializer for MavenArtifact. """ - group_id = serializers.CharField( - help_text=_("Group Id of the artifact's package."), read_only=True - ) - artifact_id = serializers.CharField( - help_text=_("Artifact Id of the artifact's package."), read_only=True - ) - version = serializers.CharField( - help_text=_("Version of the artifact's package."), read_only=True - ) - filename = serializers.CharField(help_text=_("Filename of the artifact."), read_only=True) + group_id = serializers.CharField(help_text=_("Group Id of the artifact's package.")) + artifact_id = serializers.CharField(help_text=_("Artifact Id of the artifact's package.")) + version = serializers.CharField(help_text=_("Version of the artifact's package.")) + filename = serializers.CharField(help_text=_("Filename of the artifact.")) + + def deferred_validate(self, data): + """Validate the FileContent data.""" + data = super().deferred_validate(data) + data["relative_path"] = ( + f"{data['group_id'].replace('.', '/')}/{data['artifact_id']}/{data['version']}/" + "{data['filename']}" + ) + return data + + def retrieve(self, validated_data): + content = models.MavenArtifact.objects.filter( + group_id=validated_data["group_id"], + artifact_id=validated_data["artifact_id"], + version=validated_data["version"], + filename=validated_data["filename"], + ) + return content.first() class Meta: - fields = platform.SingleArtifactContentSerializer.Meta.fields + ( - "group_id", - "artifact_id", - "version", - "filename", + fields = ( + SingleArtifactContentUploadSerializer.Meta.fields + + ContentChecksumSerializer.Meta.fields + + ("group_id", "artifact_id", "version", "filename") ) + # Remove relative_path + fields = tuple(field for field in fields if field != "relative_path") model = models.MavenArtifact + # Validation occurs in the task. + validators = [] -class MavenRemoteSerializer(platform.RemoteSerializer): +class MavenRemoteSerializer(RemoteSerializer): """ A Serializer for MavenRemote. @@ -58,16 +80,16 @@ class Meta: """ class Meta: - fields = platform.RemoteSerializer.Meta.fields + fields = RemoteSerializer.Meta.fields model = models.MavenRemote -class MavenDistributionSerializer(platform.DistributionSerializer): +class MavenDistributionSerializer(DistributionSerializer): """ Serializer for Maven Distributions. """ - remote = platform.DetailRelatedField( + remote = DetailRelatedField( required=False, help_text=_("Remote that can be used to fetch content when using pull-through caching."), queryset=models.MavenRemote.objects.all(), @@ -76,5 +98,5 @@ class MavenDistributionSerializer(platform.DistributionSerializer): ) class Meta: - fields = platform.DistributionSerializer.Meta.fields + ("remote",) + fields = DistributionSerializer.Meta.fields + ("remote",) model = models.MavenDistribution diff --git a/pulp_maven/app/viewsets.py b/pulp_maven/app/viewsets.py index fd4bd7a..f15a9b4 100644 --- a/pulp_maven/app/viewsets.py +++ b/pulp_maven/app/viewsets.py @@ -1,50 +1,66 @@ -from pulpcore.plugin import viewsets as core +from pulpcore.plugin.viewsets import ( + ContentFilter, + DistributionViewSet, + RemoteViewSet, + RepositoryVersionViewSet, + RepositoryViewSet, + SingleArtifactContentUploadViewSet, +) +from pulpcore.plugin.actions import ModifyRepositoryActionMixin -from . import models, serializers +from pulp_maven.app.models import MavenArtifact, MavenRemote, MavenRepository, MavenDistribution -class MavenArtifactFilter(core.ContentFilter): +from pulp_maven.app.serializers import ( + MavenArtifactSerializer, + MavenRemoteSerializer, + MavenRepositorySerializer, + MavenDistributionSerializer, +) + + +class MavenArtifactFilter(ContentFilter): """ FilterSet for MavenArtifact. """ class Meta: - model = models.MavenArtifact + model = MavenArtifact fields = ["group_id", "artifact_id", "version", "filename"] -class MavenArtifactViewSet(core.ContentViewSet): +class MavenArtifactViewSet(SingleArtifactContentUploadViewSet): """ A ViewSet for MavenArtifact. """ endpoint_name = "artifact" - queryset = models.MavenArtifact.objects.all() - serializer_class = serializers.MavenArtifactSerializer + queryset = MavenArtifact.objects.prefetch_related("_artifacts").all() + serializer_class = MavenArtifactSerializer filterset_class = MavenArtifactFilter -class MavenRemoteViewSet(core.RemoteViewSet): +class MavenRemoteViewSet(RemoteViewSet): """ A ViewSet for MavenRemote. """ endpoint_name = "maven" - queryset = models.MavenRemote.objects.all() - serializer_class = serializers.MavenRemoteSerializer + queryset = MavenRemote.objects.all() + serializer_class = MavenRemoteSerializer -class MavenRepositoryViewSet(core.RepositoryViewSet): +class MavenRepositoryViewSet(RepositoryViewSet, ModifyRepositoryActionMixin): """ A ViewSet for MavenRemote. """ endpoint_name = "maven" - queryset = models.MavenRepository.objects.all() - serializer_class = serializers.MavenRepositorySerializer + queryset = MavenRepository.objects.all() + serializer_class = MavenRepositorySerializer -class MavenRepositoryVersionViewSet(core.RepositoryVersionViewSet): +class MavenRepositoryVersionViewSet(RepositoryVersionViewSet): """ MavenRepositoryVersion represents a single Maven repository version. """ @@ -52,11 +68,11 @@ class MavenRepositoryVersionViewSet(core.RepositoryVersionViewSet): parent_viewset = MavenRepositoryViewSet -class MavenDistributionViewSet(core.DistributionViewSet): +class MavenDistributionViewSet(DistributionViewSet): """ ViewSet for Maven Distributions. """ endpoint_name = "maven" - queryset = models.MavenDistribution.objects.all() - serializer_class = serializers.MavenDistributionSerializer + queryset = MavenDistribution.objects.all() + serializer_class = MavenDistributionSerializer diff --git a/pulp_maven/tests/functional/api/test_upload.py b/pulp_maven/tests/functional/api/test_upload.py new file mode 100644 index 0000000..c323fdf --- /dev/null +++ b/pulp_maven/tests/functional/api/test_upload.py @@ -0,0 +1,83 @@ +import hashlib +from urllib.parse import urljoin + +from pulp_maven.tests.functional.utils import download_file + + +def test_upload_workflow( + maven_repo_api_client, + maven_repo_factory, + random_maven_artifact_factory, + maven_artifact_api_client, + maven_distribution_factory, + gen_object_with_cleanup, +): + # Create a repository and assert that the latest version is 0 + repo = maven_repo_factory() + assert repo.latest_version_href.endswith("/versions/0/") + + # Create a random jar + jar_file = random_maven_artifact_factory() + + # Upload the jar into the repository + artifact_kwargs = dict( + group_id=jar_file["group_id"], + artifact_id=jar_file["artifact_id"], + version=jar_file["version"], + filename=jar_file["filename"], + file=jar_file["full_path"], + repository=repo.pulp_href, + ) + maven_artifact = gen_object_with_cleanup(maven_artifact_api_client, **artifact_kwargs) + + # Assert that a Maven Artifact was created + assert maven_artifact.group_id == jar_file["group_id"] + assert maven_artifact.artifact_id == jar_file["artifact_id"] + assert maven_artifact.version == jar_file["version"] + assert maven_artifact.filename == jar_file["filename"] + + # Assert that a repository version was created + repo = maven_repo_api_client.read(repo.pulp_href) + assert repo.latest_version_href.endswith("/versions/1/") + + # Assert that this Maven Artifact is in the repository version + content_in_repo_version = maven_artifact_api_client.list( + repository_version=repo.latest_version_href + ) + assert content_in_repo_version.results[0].pulp_href == maven_artifact.pulp_href + + # Create a second repository and assert that latest version is 0 + repo2 = maven_repo_factory() + assert repo2.latest_version_href.endswith("/versions/0/") + + # Assert that the same content unit can be uploaded again + artifact_kwargs["repository"] = repo2.pulp_href + maven_artifact2 = gen_object_with_cleanup(maven_artifact_api_client, **artifact_kwargs) + + # Assert that the existing artifact was identified by the upload API. + assert maven_artifact.pulp_href == maven_artifact2.pulp_href + + # Assert that a new repository version was created. + repo2 = maven_repo_api_client.read(repo2.pulp_href) + assert repo2.latest_version_href.endswith("/versions/1/") + + # Assert that this Maven Artifact is in the repository version + content_in_repo2_version = maven_artifact_api_client.list( + repository_version=repo2.latest_version_href + ) + assert content_in_repo2_version.results[0].pulp_href == maven_artifact2.pulp_href + + # Create a distribution and serve repo + distribution = maven_distribution_factory(repository=repo.pulp_href) + + # Download the jar from the distribution + unit_path = ( + f"{jar_file['group_id'].replace('.', '/')}/{jar_file['artifact_id']}/" + "{jar_file['version']}/{jar_file['filename']}" + ) + pulp_unit_url = urljoin(distribution.base_url, unit_path) + downloaded_file = download_file(pulp_unit_url) + downloaded_file_checksum = hashlib.sha256(downloaded_file.body).hexdigest() + + # Assert that the downloaded file's checksum matches the original + assert jar_file["sha256"] == downloaded_file_checksum diff --git a/pulp_maven/tests/functional/conftest.py b/pulp_maven/tests/functional/conftest.py index f52afe0..c69e928 100644 --- a/pulp_maven/tests/functional/conftest.py +++ b/pulp_maven/tests/functional/conftest.py @@ -10,6 +10,8 @@ RepositoriesMavenApi, ) +from pulp_maven.tests.functional.utils import generate_jar + @pytest.fixture(scope="session") def maven_client(_api_client_set, bindings_cfg): @@ -69,3 +71,21 @@ def _maven_remote_factory(**kwargs): return gen_object_with_cleanup(maven_remote_api_client, kwargs) yield _maven_remote_factory + + +@pytest.fixture +def random_maven_artifact_factory(tmp_path): + """A factory to generate a random maven artifact.""" + + def _random_maven_artifact_factory(**kwargs): + kwargs.setdefault("group_id", f"org.{str(uuid.uuid4())}") + kwargs.setdefault("artifact_id", str(uuid.uuid4())) + kwargs.setdefault("version", str(uuid.uuid4())) + kwargs.setdefault("filename", f"{str(uuid.uuid4())}.jar") + full_path = tmp_path / kwargs["filename"] + _, checksum = generate_jar(full_path) + kwargs["full_path"] = full_path + kwargs["sha256"] = checksum + return kwargs + + return _random_maven_artifact_factory diff --git a/pulp_maven/tests/functional/utils.py b/pulp_maven/tests/functional/utils.py index 1130a04..2000f58 100644 --- a/pulp_maven/tests/functional/utils.py +++ b/pulp_maven/tests/functional/utils.py @@ -3,6 +3,8 @@ import aiohttp import asyncio from dataclasses import dataclass +import os +import hashlib @dataclass @@ -32,3 +34,13 @@ async def _download_file(url, auth=None, headers=None): async with aiohttp.ClientSession(auth=auth, raise_for_status=True) as session: async with session.get(url, verify_ssl=False, headers=headers) as response: return Download(body=await response.read(), response_obj=response) + + +def generate_jar(full_path, size=1024, relative_path=None): + """Generate a random file.""" + with open(full_path, "wb") as fout: + contents = os.urandom(size) + fout.write(contents) + fout.flush() + digest = hashlib.sha256(contents).hexdigest() + return full_path, digest