Merge pull request #837 from OpenSourceBrain/develop

v0.7.4

.github/workflows/docker-jupyterlab.yml

@@ -0,0 +1,25 @@
+name: Docker Image Build Jupyterlab
+
+on:
+  push:
+    branches: [ master, dev*, feature/docker* ]
+  pull_request:
+    branches: [ master, dev*, feature/docker* ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build the Docker image
+      run: |
+        cd applications/jupyterlab
+        docker build -t myjlab -f Dockerfile --no-cache .
+
+    - name: Info on Docker image sizes
+      run: |
+        docker images

.github/workflows/docker-netpyne.yml

@@ -0,0 +1,25 @@
+name: Docker Image Build NetPyNE-UI
+
+on:
+  push:
+    branches: [ master, dev*, feature/docker* ]
+  pull_request:
+    branches: [ master, dev*, feature/docker* ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build the Docker image
+      run: |
+        cd applications/netpyne
+        docker build -t mynpui -f Dockerfile --no-cache .
+        
+    - name: Info on Docker image sizes
+      run: |
+        docker images

.github/workflows/docker-nwbe.yml

@@ -0,0 +1,27 @@
+name: Docker Image Build NWB Explorer
+
+on:
+  push:
+    branches: [ master, dev*, feature/docker* ]
+  pull_request:
+    branches: [ master, dev*, feature/docker* ]
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build the Docker image
+      run: |
+        cd applications/nwb-explorer
+        docker build -t mynwbosb -f Dockerfile --no-cache .
+        
+    - name: Info on Docker image sizes
+      run: |
+        docker images
+
+        

applications/accounts-api/backend/accounts_api/services/user_service.py

@@ -4,13 +4,11 @@
 from cloudharness.auth.quota import get_user_quotas
 from keycloak.exceptions import KeycloakGetError, KeycloakError
 from accounts_api.models import User
-from cloudharness.auth import AuthClient
+from cloudharness.auth import AuthClient, UserNotFound
 from cloudharness import log
 import typing
 # from cloudharness.models import User as CHUser # Cloudharness 2.0.0
 
-class UserNotFound(Exception): pass
-
 
 class UserNotAuthorized(Exception): pass
 
@@ -20,7 +18,8 @@ def get_user(username_or_id: str) -> User:
     try:
 
         kc_user = client.get_user(username_or_id)
-
+    except UserNotFound:
+        raise
     except KeycloakGetError as e:
         if e.response_code == 404:
             raise UserNotFound(username_or_id)

applications/accounts/themes/custom/login/resources/css/login.css

@@ -19,7 +19,7 @@ body {
 }
 
 .login-pf-logo {
-  background: url(../img/osblogo.png) no-repeat;
+  background: url(../img/osb-logo.png) no-repeat;
   background-size: contain;
   width: 132px;
   height: 20px;

applications/accounts/themes/custom/login/template.ftl

@@ -13,7 +13,7 @@
         </#list>
     </#if>
     <title>${msg("loginTitle",(realm.displayName!''))}</title>
-    <link rel="icon" href="${url.resourcesPath}/img/favicon.ico" />
+    <link rel="icon" href="${url.resourcesPath}/img/favicon.svg" />
     <#if properties.stylesCommon?has_content>
         <#list properties.stylesCommon?split(' ') as style>
             <link href="${url.resourcesCommonPath}/${style}" rel="stylesheet" />

applications/jupyterhub/.dockerignore

@@ -0,0 +1 @@
+/deploy

applications/jupyterhub/Dockerfile

@@ -36,8 +36,6 @@ COPY --chown=1000:1000 theming/spawn_pending.html /usr/local/share/jupyterhub/te
 COPY --chown=1000:1000 theming/hot_fix_for_eventsource.js /usr/local/share/jupyterhub/static/hot_fix_for_eventsource.js
 COPY --chown=1000:1000 theming/arrow-dropdown.svg /usr/local/share/jupyterhub/static/arrow-dropdown.svg
 
-COPY hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py
-
 RUN chmod 777 /usr/src/app -R
 USER jovyan
 

applications/jupyterhub/hub/jupyter_notebook_config.py → applications/jupyterhub/deploy/resources/applications/jupyter_notebook_config.py

@@ -20,7 +20,7 @@
 print(c.NotebookApp.tornado_settings)
 c.NotebookApp.tornado_settings = {
     'headers': {
-        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost *.osb.local *.opensourcebrain.org *.metacell.us",
+        'Content-Security-Policy': "frame-ancestors 'self' localhost:3000 localhost:* localhost:3000 *.osb.local  *.opensourcebrain.org *.metacell.us",
     }
 }
 print(c.NotebookApp.tornado_settings)

applications/jupyterhub/deploy/templates/configmap.yaml

@@ -0,0 +1,9 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: "jupyterhub-notebook-config"
+  labels:
+    app: jupytehub
+data:
+  {{- (.Files.Glob "resources/jupyterhub/applications/*").AsConfig | nindent 2 }}
+---

applications/jupyterhub/deploy/values-minimal.yaml

@@ -2,3 +2,4 @@ harness:
   dependencies:
     soft: [nfsserver, accounts]
     hard: []
+nfs_volumes: false

applications/jupyterhub/deploy/values.yaml

@@ -28,7 +28,7 @@ harness:
     quota-ws-maxmem: 1
     # sets the storage dedicated to the user data in Gb units (float)
     quota-storage-max : 1.25
-
+nfs_volumes: false
 hub:
   config:
     JupyterHub:

applications/jupyterhub/src/chauthenticator/chauthenticator/auth.py

@@ -27,8 +27,19 @@ def initialize(self, force_new_server, process_user):
 
     @gen.coroutine
     def get(self):
+        if 'open=' in self.request.uri:
+            url = self.request.uri.split('open=').pop()
+            self.request.cookies.set("loadurl", bytes(
+                url, 'utf-8'), encrypted=False, httponly=False)
+
+        # legacy nwb explorer support
+        elif 'nwbfile=' in self.request.uri:
+            print("Nwb file found")
+            url = self.request.uri.split('nwbfile=').pop().split("&")[0]
+            print("NWB URL", url)
+            self._set_cookie("nwbloadurl", bytes(
+                url, 'utf-8'), encrypted=False, httponly=False)
         self.clear_login_cookie()
-
         try:
 
             accessToken = self.request.cookies.get(
@@ -50,7 +61,9 @@ def get(self):
             self.request.cookies.clear()
             raw_user = self.get_anonymous_user()
 
-        print("JH user: ", raw_user.__dict__)
+
+
+        # print("JH user: ", raw_user.__dict__)
         self.set_login_cookie(raw_user)
         user = yield gen.maybe_future(self.process_user(raw_user, self))
         self.redirect(self.get_next_url(user))
@@ -99,7 +112,10 @@ def get_handlers(self, app):
             'process_user': self.process_user
         }
         return [
-            ('/chkclogin', CloudHarnessAuthenticateHandler, extra_settings)
+            ('/chkclogin', CloudHarnessAuthenticateHandler, extra_settings),
+            ('/nwbfile=.*', CloudHarnessAuthenticateHandler, extra_settings),
+            ('/open=.*', CloudHarnessAuthenticateHandler, extra_settings),
+            ('/query?.*', CloudHarnessAuthenticateHandler, extra_settings),
         ]
 
     def login_url(self, base_url):

applications/jupyterhub/src/osb_jupyter/osb_jupyter/jupyterhub.py

@@ -1,9 +1,15 @@
+import re
 from jupyterhub.user import User
 from kubespawner.spawner import KubeSpawner
 
 from cloudharness.auth import AuthClient
 from cloudharness import log
 from cloudharness import applications
+from cloudharness.auth.exceptions import UserNotFound
+from urllib.parse import parse_qs, urlparse
+
+allowed_chars = set(
+    "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
 
 def affinity_spec(key, value):
@@ -22,9 +28,11 @@ def affinity_spec(key, value):
         'topologyKey': 'kubernetes.io/hostname'
     }
 
+
 class CookieNotFound(Exception):
     pass
 
+
 def change_pod_manifest(self: KubeSpawner):
     """
     Application Hook to change the manifest of the notebook image
@@ -47,7 +55,8 @@ def get_from_cookie(cookie_name):
         return cookie.value
 
     def user_volume_is_legacy(user_id):
-        print("User id", user_id, "max", self.config['apps']['jupyterhub'].get('legacyusermax', 0))
+        print("User id", user_id, "max",
+              self.config['apps']['jupyterhub'].get('legacyusermax', 0))
         return int(user_id) < self.config['apps']['jupyterhub'].get('legacyusermax', 0)
 
     def workspace_volume_is_legacy(workspace_id):
@@ -75,21 +84,20 @@ def workspace_volume_is_legacy(workspace_id):
             self.volumes.append(ws_pvc)
 
         app_user = get_app_user(self.user)
-
 
         # Add labels to use for affinity
-        clean_username = "".join(c for c in app_user.username if c.isalnum())
+        clean_username = "".join(
+            c for c in app_user.username if c in allowed_chars)
         labels = {
             'workspace': str(workspace_id),
             'username': clean_username,
             'user': str(self.user.id),
         }
 
-
-
         self.common_labels = labels
         self.extra_labels = labels
-        self.storage_class = f'{self.config["namespace"]}-nfs-client'
+        if self.config['apps']['jupyterhub'].get('nfs_volumes', False):
+            self.storage_class = f'{self.config["namespace"]}-nfs-client'
 
         if not user_volume_is_legacy(self.user.id):
             # User pod affinity is by default added by cloudharness
@@ -98,10 +106,11 @@ def workspace_volume_is_legacy(workspace_id):
         workspace = get_workspace(workspace_id, get_from_cookie("accessToken"))
         write_access = has_user_write_access(
             workspace, self.user, app_user=app_user)
-        
+
         if workspace_volume_is_legacy(workspace_id):
             # Pods with write access must be on the same node
-            self.pod_affinity_required.append(affinity_spec('workspace', workspace_id))
+            self.pod_affinity_required.append(
+                affinity_spec('workspace', workspace_id))
         from pprint import pprint
         pprint(self.volumes)
         self.pod_name = f'ws-{clean_username}-{workspace_id}-{appname}'
@@ -111,7 +120,16 @@ def workspace_volume_is_legacy(workspace_id):
                 'mountPath': '/opt/workspace',
                 'readOnly': not write_access
             })
-    except CookieNotFound:
+        if "image=" in self.handler.request.uri and is_user_trusted(self.user):
+            print("Image override")
+            image = parse_qs(urlparse(self.handler.request.uri).query)[
+                'image'][0]
+            print("Image is", image)
+            self.image = image
+            self.pod_name = f"{self.pod_name}--{re.sub('[^0-9a-zA-Z]+', '-', image)}"
+            # open external resources
+
+    except (CookieNotFound, UserNotFound):
         # Setup a readonly default session
         self.pod_name = f'anonymous-{self.user.name}-{appname}'
         self.storage_pvc_ensure = False
@@ -122,29 +140,55 @@ def workspace_volume_is_legacy(workspace_id):
     except Exception as e:
         log.error('Change pod manifest failed due to an error.', exc_info=True)
 
+    # Add customized config map for jupyter notebook config
+    self.volumes.append({
+        'name': 'jupyterhub-notebook-config',
+        'configMap': {'name': 'jupyterhub-notebook-config'}
+    })
+    self.volume_mounts.append({
+        'name': 'jupyterhub-notebook-config',
+        'subPath': 'jupyter_notebook_config.py',
+        'mountPath': '/etc/jupyter/jupyter_notebook_config.py',
+        'readOnly': True
+    })
+    self.volume_mounts.append({
+        'name': 'jupyterhub-notebook-config',
+        'subPath': 'jupyter_notebook_config.py',
+        'mountPath': '/opt/conda/etc/jupyter/nbconfig/jupyter_notebook_config.py',
+        'readOnly': True
+    })
+
+
+
 
 def get_app_user(user: User):
     auth_client = AuthClient()
     kc_user = auth_client.get_user(user.name)
     return kc_user
 
+
 def has_user_write_access(workspace, user: User, app_user=None):
     print('Checking access, name:', user.name, "workspace:", workspace["id"])
-
-
     workspace_owner = workspace["user"]["id"]
-    print("Workspace owner", workspace_owner, "-", workspace["user"]["username"])
-
-    if workspace_owner == user.name:
-        return True
+    print("Workspace owner", workspace_owner,
+          "-", workspace["user"]["username"])
+    return workspace_owner == user.name or is_user_trusted(user)
+
+
+def is_user_trusted(user: User):
     auth_client = AuthClient()
-    return auth_client.user_has_realm_role(app_user.id, 'administrator')
+    kc_user = auth_client.get_user(user.name)
+    return auth_client.user_has_realm_role(kc_user.id, 'administrator') or\
+        auth_client.user_has_realm_role(kc_user.id, 'trusted')
+
 
 def get_workspace(workspace_id, token, workspace_base_url=None):
     if workspace_base_url is None:
-        workspace_conf: applications.ApplicationConfiguration = applications.get_configuration('workspaces')
+        workspace_conf: applications.ApplicationConfiguration = applications.get_configuration(
+            'workspaces')
         workspace_base_url = workspace_conf.get_service_address()
     import requests
-    workspace = requests.get(f"{workspace_base_url}/api/workspace/{workspace_id}", headers={"Authorization": f"Bearer {token}"}).json()
-
-    return workspace
+    workspace = requests.get(f"{workspace_base_url}/api/workspace/{workspace_id}",
+                             headers={"Authorization": f"Bearer {token}"}).json()
+
+    return workspace