Merge pull request #278 from helxplatform/develop

Develop to master - GPU, ephemeral storage

.github/workflows/build-push-dev-image.yml

@@ -44,25 +44,25 @@ jobs:
       run: echo ${{ steps.vars.outputs.short_sha }}
 
     # https://github.com/marketplace/actions/git-semantic-version
-    - name: Semver Check
-      uses: paulhatch/[email protected]
-      id: version
-      with:
-        # The prefix to use to identify tags
-        tag_prefix: "v"
-        # A string which, if present in a git commit, indicates that a change represents a
-        # major (breaking) change, supports regular expressions wrapped with '/'
-        major_pattern: "/(breaking)|(major)/"
-        # A string which indicates the flags used by the `major_pattern` regular expression. Supported flags: idgs
-        major_regexp_flags: "ig"
-        # Same as above except indicating a minor change, supports regular expressions wrapped with '/'
-        minor_pattern: "/(feat)|(feature)|(minor)/"
-        # A string which indicates the flags used by the `minor_pattern` regular expression. Supported flags: idgs
-        minor_regexp_flags: "ig"
-        # A string to determine the format of the version output
-        # version_format: "${major}.${minor}.${patch}-prerelease${increment}"
-        version_format: "${major}.${minor}.${patch}-prerelease${increment}"
-        search_commit_body: false
+    # - name: Semver Check
+    #   uses: paulhatch/[email protected]
+    #   id: version
+    #   with:
+    #     # The prefix to use to identify tags
+    #     tag_prefix: "v"
+    #     # A string which, if present in a git commit, indicates that a change represents a
+    #     # major (breaking) change, supports regular expressions wrapped with '/'
+    #     major_pattern: "/(breaking)|(major)/"
+    #     # A string which indicates the flags used by the `major_pattern` regular expression. Supported flags: idgs
+    #     major_regexp_flags: "ig"
+    #     # Same as above except indicating a minor change, supports regular expressions wrapped with '/'
+    #     minor_pattern: "/(feat)|(feature)|(minor)/"
+    #     # A string which indicates the flags used by the `minor_pattern` regular expression. Supported flags: idgs
+    #     minor_regexp_flags: "ig"
+    #     # A string to determine the format of the version output
+    #     # version_format: "${major}.${minor}.${patch}-prerelease${increment}"
+    #     version_format: "${major}.${minor}.${patch}-prerelease${increment}"
+    #     search_commit_body: false
 
     # Docker Buildx is important to caching in the Build And Push Container
     # step
@@ -97,10 +97,8 @@ jobs:
         # Push to renci-registry and dockerhub here.
         # cache comes from dockerhub.
         tags: |
-          ${{ github.repository }}:v${{ steps.version.outputs.version }}
           ${{ github.repository }}:develop
           ${{ github.repository }}:${{ steps.vars.outputs.short_sha }}
-          containers.renci.org/${{ github.repository }}:v${{ steps.version.outputs.version }}
           containers.renci.org/${{ github.repository }}:develop
           containers.renci.org/${{ github.repository }}:${{ steps.vars.outputs.short_sha }}
         cache-from: type=registry,ref=${{ github.repository }}:buildcache-dev

README.md

@@ -50,6 +50,8 @@ Prerelease version is the expected next release version based on current commits
 
 [Releases](https://github.com/helxplatform/appstore/releases) are also automated following the semver specification ('major', 'minor','patch') for the project when merging `develop` branch to `master`. 
 
+For an automatic increase of the major version add a comment with "breaking|major" in it.  For an increase in the minor version add "feat|feature|minor" in a comment.  Make sure it is in the first part of the comment so it goes into the header.
+
 During a new release workflow, a new tag solidifying the semver version of the new release is first added to the master branch, then a github release object is created with release notes highlighting major, minor, patch changes and contributors for each release.
 
 The template for the release notes may be found [here](https://github.com/helxplatform/appstore/blob/develop/.github/release.yml)

appstore/api/v1/models.py

@@ -10,6 +10,7 @@ class Resources:
     cpus: float
     gpus: int
     memory: str
+    ephemeralStorage: str
 
 
 @dataclass
@@ -41,6 +42,7 @@ class Instance:
     cpus: float
     gpus: int
     memory: float
+    ephemeralStorage: str
     host: InitVar[str]
     username: InitVar[str]
     url: str = field(init=False)
@@ -73,6 +75,7 @@ class ResourceRequest:
     cpus: float
     gpus: int
     memory: str
+    ephemeralStorage: str = ""
     resources: dict = None
 
     def __post_init__(self):
@@ -83,11 +86,13 @@ def __post_init__(self):
                         "cpus": self.cpus,
                         "memory": self.memory,
                         "gpus": self.gpus,
+                        "ephemeralStorage": self.ephemeralStorage,
                     },
                     "reservations": {
                         "cpus": self.cpus,
                         "memory": self.memory,
                         "gpus": self.gpus,
+                        "ephemeralStorage": self.ephemeralStorage,
                     },
                 }
             }

appstore/api/v1/serializers.py

@@ -31,6 +31,7 @@ class InstanceSerializer(serializers.Serializer):
     gpus = serializers.IntegerField(default=0)
     # TODO switch to Float potentially, or validator
     memory = serializers.CharField()
+    ephemeralStorage = serializers.CharField()
     url = serializers.CharField()
     status = serializers.CharField()
 
@@ -56,6 +57,9 @@ class ResourceSerializer(serializers.Serializer):
     cpus = serializers.FloatField()
     gpus = serializers.IntegerField(default=0)
     memory = serializers.CharField(validators=[memory_format_validator])
+    # Can do something like this if we add an ephemeral storage selector
+    # in the web UI.
+    # ephemeralStorage = serializers.CharField(validators=[memory_format_validator])
 
     def create(self, validated_data):
         return ResourceRequest(**validated_data)

appstore/api/v1/views.py

@@ -111,8 +111,8 @@ def search_for_gpu_reservation(reservations):
     https://github.com/compose-spec/compose-spec/blob/master/deploy.md#capabilities
     for more details.
     """
-    for d in reservations.get("devices", {}).items():
-        if d.get("capabilities") == "gpu":
+    for d in reservations.get("devices", {}):
+        if "gpu" in d.get("capabilities"):
             # Returning 0 for now if a device id is specified, gpu spec needs to be
             # further defined for app-prototypes and tycho.
             # https://github.com/compose-spec/compose-spec/blob/master/deploy.md
@@ -184,7 +184,8 @@ def list(self, request):
                 # https://github.com/compose-spec/compose-spec/blob/master/deploy.md
                 # #capabilities
                 # https://github.com/helxplatform/tycho/search?q=gpu
-                gpu = search_for_gpu_reservation(reservations)
+                gpu_reservations = search_for_gpu_reservation(reservations)
+                gpu_limits = search_for_gpu_reservation(limits)
                 spec = App(
                     app_data["name"],
                     app_id,
@@ -196,12 +197,18 @@ def list(self, request):
                     asdict(
                         Resources(
                             reservations.get("cpus", 0),
-                            gpu,
+                            gpu_reservations,
                             reservations.get("memory", 0),
+                            reservations.get("ephemeralStorage", 0),
                         )
                     ),
                     asdict(
-                        Resources(limits.get("cpus", 0), gpu, limits.get("memory", 0))
+                        Resources(
+                            limits.get("cpus", 0),
+                            gpu_limits,
+                            limits.get("memory", 0),
+                            limits.get("ephemeralStorage", 0),
+                        )
                     ),
                 )
 
@@ -211,6 +218,7 @@ def list(self, request):
                 continue
 
         apps = {key: value for key, value in sorted(apps.items())}
+        logging.debug(f"apps:\n${apps}")
         serializer = self.get_serializer(data=apps)
         serializer.is_valid()
         if serializer.errors:
@@ -229,7 +237,8 @@ def retrieve(self, request, app_id=None):
         spec = tycho.get_definition(app_id)
         limits, reservations = parse_spec_resources(app_id, spec, app_data)
 
-        gpu = search_for_gpu_reservation(reservations)
+        gpu_reservations = search_for_gpu_reservation(reservations)
+        gpu_limits = search_for_gpu_reservation(limits)
 
         app = App(
             app_data["name"],
@@ -241,11 +250,20 @@ def retrieve(self, request, app_id=None):
             app_data["count"],
             asdict(
                 Resources(
-                    reservations.get("cpus", 0), gpu, reservations.get("memory", 0)
+                    reservations.get("cpus", 0),
+                    gpu_reservations,
+                    reservations.get("memory", 0),
+                    reservations.get("ephemeralStorage", 0)
                 )
             ),
-            asdict(Resources(limits.get("cpus", 0), gpu, limits.get("memory", 0))),
+            asdict(
+                Resources(
+                    limits.get("cpus", 0),
+                    gpu_limits,
+                    limits.get("memory", 0))),
+                    limits.get("ephemeralStorage", 0)
         )
+        logging.debug(f"app:\n${app}")
 
         serializer = self.get_serializer(data=asdict(app))
         serializer.is_valid()
@@ -321,6 +339,7 @@ def list(self, request):
                         instance.total_util["cpu"],
                         instance.total_util["gpu"],
                         instance.total_util["memory"],
+                        instance.total_util["ephemeralStorage"],
                         host,
                         username,
                     )
@@ -339,8 +358,11 @@ def create(self, request):
         """
 
         serializer = self.get_serializer(data=request.data)
+        logging.debug("checking if request is valid")
         serializer.is_valid(raise_exception=True)
+        logging.debug("creating resource_request")
         resource_request = serializer.create(serializer.validated_data)
+        logging.debug(f"resource_request: {resource_request}")
         irods_enabled = os.environ.get("IROD_HOST",'').strip()
         # TODO update social query to fetch user.
         tokens = get_social_tokens(request.user)
@@ -409,6 +431,7 @@ def retrieve(self, request, sid=None):
                     instance.total_util["cpu"],
                     instance.total_util["gpu"],
                     instance.total_util["memory"],
+                    instance.total_util["ephemeralStorage"],
                     app.get("app_id"),
                     host,
                     username,

appstore/appstore/settings/base.py

@@ -251,6 +251,10 @@
             "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s",
             "datefmt": "%d/%b/%Y %H:%M:%S",
         },
+        "verbose2": {
+            "format": "[%(asctime)s %(levelname)s %(filename)s->%(funcName)s():%(lineno)s]: %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
         "simple": {
             "format": "[%(asctime)s] %(levelname)s %(message)s",
             "datefmt": "%d/%b/%Y %H:%M:%S",
@@ -271,7 +275,7 @@
         "console": {
             "level": LOG_LEVEL,
             "class": "logging.StreamHandler",
-            "formatter": "simple",
+            "formatter": "verbose2",
         },
         "djangoLog": {
             "level": LOG_LEVEL,

appstore/core/urls.py

@@ -1,7 +1,8 @@
-from django.urls import path
-from .views import auth,index
+from django.urls import path, re_path
+from .views import auth, index, HandlePrivateURL404s
 
 urlpatterns = [
     path('default/', index, name='index'),
     path("auth/", auth, name="auth"),
+    re_path(r"^private/*", HandlePrivateURL404s, name="private"),
 ]

appstore/core/views.py

@@ -105,6 +105,11 @@ def auth(request):
         )
     return response
 
+def HandlePrivateURL404s(request):
+    response = HttpResponse("App service not ready.", content_type="text/plain", status=404)
+    logger.debug(f"Ambassador app resource may not be mapped to the app service yet. Redirection to app UI happens when it is ready.")
+    return response
+
 def index(request):
     # Any additional logic to prepare data for the template can be added here
     # return render(request,'index.html')
@@ -123,4 +128,4 @@ def custom404(request, exception):
     else:
         template_name = "404.html"
     context = {"req_path": request.path}
-    return render(request, template_name, context=context, status=404)
+    return render(request, template_name, context=context, status=404)

bin/appstore

@@ -1,7 +1,7 @@
 #!/bin/bash
 #########################################################################################################
 ##
-## Call out to the makefile, this is here to help with migrating to make
+## Call out to the makefile, this is here to help with migrating to make 
 ##
 #########################################################################################################
 set -ex

requirements.txt

@@ -15,7 +15,7 @@ python3-openid==3.1.0
 requests==2.27.1
 requests-oauthlib==1.3.1
 selenium==3.141.0
-tycho-api==1.13.0
+tycho-api==1.14.0
 webdriver-manager==3.2.1
 sqlparse==0.4.2
 asgiref==3.4.1

setup.cfg

@@ -33,7 +33,7 @@ install_requires =
     django-saml2-auth==2.2.1
     djangorestframework==3.12.2
     drf-spectacular==0.15.1
-    tycho-api==1.13.0
+    tycho-api==1.14.0
     pysaml2==6.3.1
     python3-openid==3.1.0
     requests==2.20.1