Merge pull request #25 from wtsi-hgi/feature/upload-endpoint

Add upload endpoint for builder to pass artifacts to be uploaded to repo.

README.md

@@ -28,7 +28,9 @@ SoftPack Core - GraphQL backend service
 
 ### External dependencies
 
-SoftPack Core relies on Spack. Install that first:
+SoftPack Core requires Python version 3.11 or greater.
+
+This project also relies on Spack. Install that first:
 
 ``` console
 $ git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git

pyproject.toml

@@ -13,8 +13,6 @@ classifiers=[
     'License :: OSI Approved :: MIT License',
     'Natural Language :: English',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.9',
-    'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
 ]
 packages = [

softpack_core/artifacts.py

@@ -9,11 +9,12 @@
 from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
-from typing import Iterable, Iterator, Optional, Tuple
+from typing import Iterable, Iterator, List, Optional, Tuple, Union
 
 import pygit2
 import strawberry
 from box import Box
+from fastapi import UploadFile
 
 from softpack_core.spack import Spack
 
@@ -400,7 +401,7 @@ def create_file(
     def create_files(
         self,
         folder_path: Path,
-        files: list[Tuple[str, str]],
+        files: List[Tuple[str, Union[str, UploadFile]]],
         new_folder: bool = False,
         overwrite: bool = False,
     ) -> pygit2.Oid:
@@ -431,7 +432,10 @@ def create_files(
             new_treebuilder = self.repo.TreeBuilder(folder)
 
         for file_name, contents in files:
-            file_oid = self.repo.create_blob(contents.encode())
+            if isinstance(contents, str):
+                file_oid = self.repo.create_blob(contents)
+            else:
+                file_oid = self.repo.create_blob_fromiobase(contents.file)
             new_treebuilder.insert(
                 file_name, file_oid, pygit2.GIT_FILEMODE_BLOB
             )

softpack_core/schemas/environment.py

@@ -10,8 +10,9 @@
 from typing import Iterable, List, Optional, Tuple, Union, cast
 
 import httpx
+import starlette.datastructures
 import strawberry
-from starlette.datastructures import UploadFile
+from fastapi import UploadFile
 from strawberry.file_uploads import Upload
 
 from softpack_core.artifacts import Artifacts, Package, State
@@ -264,8 +265,8 @@ def create_new_env(
         already exists.
 
         Args:
-            env (EnvironmentInput): Details of the new environment. env_type
-            (str): One of Artifacts.built_by_softpack_file or
+            env (EnvironmentInput): Details of the new environment.
+            env_type (str): One of Artifacts.built_by_softpack_file or
             Artifacts.generated_from_module_file that denotes how the
             environment was made.
 
@@ -471,9 +472,15 @@ async def convert_module_file_to_artifacts(
         yml = ToSoftpackYML(env_name, contents)
         readme = GenerateEnvReadme(module_path)
 
-        module_file = UploadFile(file=io.BytesIO(contents))
-        softpack_file = UploadFile(file=io.BytesIO(yml))
-        readme_file = UploadFile(file=io.BytesIO(readme))
+        module_file = UploadFile(
+            filename=cls.artifacts.module_file, file=io.BytesIO(contents)
+        )
+        softpack_file = UploadFile(
+            filename=cls.artifacts.environments_file, file=io.BytesIO(yml)
+        )
+        readme_file = UploadFile(
+            filename=cls.artifacts.readme_file, file=io.BytesIO(readme)
+        )
 
         return await cls.write_module_artifacts(
             module_file=module_file,
@@ -531,7 +538,7 @@ async def write_artifact(
 
     @classmethod
     async def write_artifacts(
-        cls, folder_path: str, files: list[Upload]
+        cls, folder_path: str, files: list[Union[Upload, UploadFile]]
     ) -> WriteArtifactResponse:  # type: ignore
         """Add one or more files to the Artifacts repo.
 
@@ -540,10 +547,16 @@ async def write_artifacts(
             files: the files to add to the repo.
         """
         try:
-            new_files: List[Tuple[str, str]] = []
+            new_files: List[Tuple[str, Union[str, UploadFile]]] = []
             for file in files:
-                contents = cast(str, (await file.read()).decode())
-                new_files.append((file.name, contents))
+                if isinstance(file, starlette.datastructures.UploadFile):
+                    new_files.append(
+                        (file.filename or "", cast(UploadFile, file))
+                    )
+                else:
+                    new_files.append(
+                        (file.name, cast(str, (await file.read()).decode()))
+                    )
 
             tree_oid = cls.artifacts.create_files(
                 Path(folder_path), new_files, overwrite=True
@@ -621,12 +634,12 @@ class Mutation:
         createEnvironment: CreateResponse = Environment.create  # type: ignore
         updateEnvironment: UpdateResponse = Environment.update  # type: ignore
         deleteEnvironment: DeleteResponse = Environment.delete  # type: ignore
-        writeArtifact: WriteArtifactResponse = (  # type: ignore
-            Environment.write_artifact
-        )
-        writeArtifacts: WriteArtifactResponse = (  # type: ignore
-            Environment.write_artifacts
-        )
+        # writeArtifact: WriteArtifactResponse = (  # type: ignore
+        #     Environment.write_artifact
+        # )
+        # writeArtifacts: WriteArtifactResponse = (  # type: ignore
+        #     Environment.write_artifacts
+        # )
         createFromModule: CreateResponse = (  # type: ignore
             Environment.create_from_module
         )

softpack_core/service.py

@@ -5,11 +5,22 @@
 """
 
 
+import urllib.parse
+from pathlib import Path
+
 import typer
 import uvicorn
+from fastapi import APIRouter, Request, UploadFile
 from typer import Typer
 from typing_extensions import Annotated
 
+from softpack_core.schemas.environment import (
+    CreateEnvironmentSuccess,
+    Environment,
+    EnvironmentInput,
+    WriteArtifactSuccess,
+)
+
 from .api import API
 from .app import app
 
@@ -19,6 +30,7 @@ class ServiceAPI(API):
 
     prefix = "/service"
     commands = Typer(help="Commands for managing core service.")
+    router = APIRouter()
 
     @staticmethod
     @commands.command(help="Start the SoftPack Core API service.")
@@ -46,3 +58,37 @@ def run(
             reload=reload,
             log_level="debug",
         )
+
+    @staticmethod
+    @router.post("/upload")
+    async def upload_artifacts(  # type: ignore[no-untyped-def]
+        request: Request,
+        file: list[UploadFile],
+    ):
+        """upload_artifacts is a POST fn that adds files to an environment.
+
+          The environment does not need to exist already.
+
+        Args:
+            file (List[UploadFile]): The files to be uploaded.
+            request (Request): The POST request which contains the environment
+            path in the query.
+
+        Returns:
+            WriteArtifactResponse
+        """
+        env_path = urllib.parse.unquote(request.url.query)
+        if Environment.check_env_exists(Path(env_path)) is not None:
+            create_response = Environment.create_new_env(
+                EnvironmentInput.from_path(env_path),
+                Environment.artifacts.built_by_softpack_file,
+            )
+
+            if not isinstance(create_response, CreateEnvironmentSuccess):
+                return create_response
+
+        resp = await Environment.write_artifacts(env_path, file)
+        if not isinstance(resp, WriteArtifactSuccess):
+            raise Exception(resp)
+
+        return resp

tests/integration/conftest.py

@@ -7,9 +7,13 @@
 import os
 
 import pytest
-from starlette.datastructures import UploadFile
 
-from softpack_core.artifacts import app
+from softpack_core.artifacts import Artifacts, Package, app
+from softpack_core.schemas.environment import Environment, EnvironmentInput
+from tests.integration.utils import (
+    get_user_path_without_environments,
+    new_test_artifacts,
+)
 
 
 @pytest.fixture(scope="package", autouse=True)
@@ -38,6 +42,19 @@ def httpx_post(mocker):
     return post_mock
 
 
-@pytest.fixture()
-def upload(mocker):
-    return mocker.Mock(spec=UploadFile)
+@pytest.fixture
+def testable_env_input(mocker) -> EnvironmentInput:
+    ad = new_test_artifacts()
+    artifacts: Artifacts = ad["artifacts"]
+    user = ad["test_user"]
+
+    mocker.patch.object(Environment, 'artifacts', new=artifacts)
+
+    testable_env_input = EnvironmentInput(
+        name="test_env_create",
+        path=str(get_user_path_without_environments(artifacts, user)),
+        description="description",
+        packages=[Package(name="pkg_test")],
+    )
+
+    yield testable_env_input

tests/integration/test_builderupload.py

@@ -0,0 +1,58 @@
+"""Copyright (c) 2023 Genome Research Ltd.
+
+This source code is licensed under the MIT license found in the
+LICENSE file in the root directory of this source tree.
+"""
+
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from softpack_core.app import app
+from softpack_core.schemas.environment import Environment
+from softpack_core.service import ServiceAPI
+from tests.integration.utils import file_in_repo
+
+pytestmark = pytest.mark.repo
+
+
+def test_builder_upload(testable_env_input):
+    ServiceAPI.register()
+    client = TestClient(app.router)
+
+    env_parent = "groups/hgi"
+    env_name = "unknown-env"
+    env_path = env_parent + "/" + env_name
+
+    softpackYaml = "softpack.yaml"
+    softpackYamlContents = b"softpack yaml file"
+
+    spackLock = "spack.lock"
+    spackLockContents = b"spack lock file"
+
+    assert Environment.check_env_exists(Path(env_path)) is not None
+    resp = client.post(
+        url="/upload?" + env_path,
+        files=[
+            ("file", (softpackYaml, softpackYamlContents)),
+            ("file", (spackLock, spackLockContents)),
+        ],
+    )
+    assert resp.status_code == 200
+    assert resp.json().get("message") == "Successfully written artifact(s)"
+    assert Environment.check_env_exists(Path(env_path)) is None
+    assert file_in_repo(
+        Environment.artifacts,
+        Path(Environment.artifacts.environments_root, env_path, softpackYaml),
+    )
+    assert file_in_repo(
+        Environment.artifacts,
+        Path(Environment.artifacts.environments_root, env_path, spackLock),
+    )
+
+    tree = Environment.artifacts.get(env_parent, env_name)
+    assert tree is not None
+
+    assert tree[softpackYaml].data == softpackYamlContents
+    assert tree[spackLock].data == spackLockContents