Merge pull request #730 from OpenSourceBrain/release/0.7.1

Release/0.7.1

applications/jupyterhub/src/chauthenticator/README.rst

@@ -0,0 +1,21 @@
+cloudharness keycloak authenticator
+===================================
+
+Authenticator to use Jupyterhub with the keycloak gatekeeper.
+
+
+Running Tests:
+--------------
+
+.. code-block:: bash
+
+    pip install -r test-requirements.txt
+    pip install -e .
+    pytest
+
+PyTest does a lot of caching and may run into troubles if old files linger around.
+Use one ore more of the following commands to clean up before running pytest.
+
+.. code-block:: bash
+
+    find ./ -name '*.pyc' -delete

applications/jupyterhub/src/chauthenticator/chauthenticator/auth.py

@@ -0,0 +1,96 @@
+import random
+import sys
+import logging
+
+from jupyterhub.auth import Authenticator
+from jupyterhub.handlers import BaseHandler
+from tornado import gen
+from traitlets import Bool
+from jupyterhub.utils import url_path_join
+from cloudharness.auth import AuthClient
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.DEBUG)
+logging.getLogger().addHandler(handler)
+
+class CloudHarnessAuthenticateHandler(BaseHandler):
+    """
+    Handler for /chkclogin
+    Creates a new user based on the keycloak user, and auto starts their server
+    """
+    def initialize(self, force_new_server, process_user):
+        super().initialize()
+        self.force_new_server = force_new_server
+        self.process_user = process_user
+
+    @gen.coroutine
+    def get(self):
+        self.clear_login_cookie()
+
+        try:
+
+                accessToken = self.request.cookies.get(
+                    'kc-access', None) or self.request.cookies.get('accessToken', None)
+                print("Token", accessToken)
+                if accessToken == '-1' or not accessToken:
+                    import socket
+                    raw_user = self.user_from_username("a-%s-%0.5x" % (socket.inet_aton(self.request.remote_ip).hex(), random.randint(0, 99999)))
+                else:
+                    accessToken = accessToken.value
+                    user_data = AuthClient.decode_token(accessToken)
+                    username = user_data['sub']
+                    print("Username", username, "-",user_data['preferred_username'])
+                    raw_user = self.user_from_username(username)
+                print("JH user: ", raw_user.__dict__)
+                self.set_login_cookie(raw_user)
+        except Exception as e:
+                logging.error("Error getting user from session", exc_info=True)
+                raise
+
+        user = yield gen.maybe_future(self.process_user(raw_user, self))
+        self.redirect(self.get_next_url(user))
+
+
+class CloudHarnessAuthenticator(Authenticator):
+    """
+    JupyterHub Authenticator for use with Cloud Harness
+    When JupyterHub is configured to use this authenticator, the client 
+    needs to set the accessToken domain cookie
+    """
+
+    auto_login = True
+    login_service = 'chkc'
+
+    force_new_server = Bool(
+        True,
+        help="""
+        Stop the user's server and start a new one when visiting /hub/chlogin
+        When set to True, users going to /hub/chlogin will *always* get a
+        new single-user server. When set to False, they'll be
+        redirected to their current session if one exists.
+        """,
+        config=True
+    )
+
+    def process_user(self, user, handler):
+        """
+        Do additional arbitrary things to the created user before spawn.
+        user is a user object, and handler is a CloudHarnessAuthenticateHandler 
+        object. Should return the new user object.
+        This method can be a @tornado.gen.coroutine.
+        Note: This is primarily for overriding in subclasses
+        """
+        return user
+
+    def get_handlers(self, app):
+        # FIXME: How to do this better?
+        extra_settings = {
+            'force_new_server': self.force_new_server,
+            'process_user': self.process_user
+        }
+        return [
+            ('/chkclogin', CloudHarnessAuthenticateHandler, extra_settings)
+        ]
+
+    def login_url(self, base_url):
+        return url_path_join(base_url, 'chkclogin')

applications/jupyterhub/src/osb_jupyter/osb_jupyter/jupyterhub.py

@@ -22,6 +22,8 @@ def affinity_spec(key, value):
         'topologyKey': 'kubernetes.io/hostname'
     }
 
+class CookieNotFound(Exception):
+    pass
 
 def change_pod_manifest(self: KubeSpawner):
     """
@@ -40,7 +42,7 @@ def change_pod_manifest(self: KubeSpawner):
     def get_from_cookie(cookie_name):
         cookie = self.handler.request.cookies.get(cookie_name, None)
         if cookie is None:
-            raise Exception(
+            raise CookieNotFound(
                 "Required cookie not found. Check that the cookie named '%s' is set." % cookie_name)
         return cookie.value
 
@@ -51,6 +53,8 @@ def user_volume_is_legacy(user_id):
     def workspace_volume_is_legacy(workspace_id):
         return int(workspace_id) < self.config['apps']['jupyterhub'].get('legacyworkspacemax', 0)
 
+    appname = self.image.split('/')[-1].split(':')[0]
+
     try:
         workspace_id = get_from_cookie('workspaceId')
         volume_name = f'workspace-{workspace_id}'
@@ -81,7 +85,7 @@ def workspace_volume_is_legacy(workspace_id):
             'user': str(self.user.id),
         }
 
-        appname = self.image.split('/')[-1].split(':')[0]
+
 
         self.common_labels = labels
         self.extra_labels = labels
@@ -107,6 +111,17 @@ def workspace_volume_is_legacy(workspace_id):
                 'mountPath': '/opt/workspace',
                 'readOnly': not write_access
             })
+    except CookieNotFound:
+        # Setup a readonly default session
+        self.pod_name = f'anonymous-{self.user.username}-{appname}'
+        from pprint import pprint
+        pprint(self.volumes)
+
+        self.volumes = []
+        pprint(self.volume_mounts)
+        self.volume_mounts = []
+        self.maxAge
+
     except Exception as e:
         log.error('Change pod manifest failed due to an error.', exc_info=True)
 

applications/jupyterlab-minimal/Dockerfile

@@ -1,3 +1,7 @@
 FROM jupyter/base-notebook:hub-1.4.2
 
-COPY hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py
+COPY hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py
+USER root
+RUN mkdir /opt/workspace
+RUN chown -R jovyan:users /opt/workspace
+USER jovyan

applications/jupyterlab/Dockerfile

@@ -37,7 +37,13 @@ RUN jupyter labextension install plotlywidget
 USER root
 
 ### Some aliases
-RUN echo -e '\n\nalias cd..="cd .." \nalias h=history \nalias ll="ls -alt" \nalias jnml="java -classpath /opt/conda/lib/python3.9/site-packages/pyneuroml/lib/jNeuroML-*-jar-with-dependencies.jar  org.neuroml.JNeuroML"\n' >> ~/.bashrc
+RUN echo -e '\n\nalias cd..="cd .." \nalias h=history \nalias ll="ls -alt" \n' >> ~/.bashrc
+
+### Set up jnml, reusing pynml jar
+RUN echo -e '#!/bin/bash\n#Reusing the jNeuroML jar from the pip installed pyNeuroML for the jnml command\n\njava -classpath /opt/conda/lib/python3.9/site-packages/pyneuroml/lib/jNeuroML-*-jar-with-dependencies.jar  org.neuroml.JNeuroML $@' >> /opt/conda/bin/jnml
+RUN chmod +x /opt/conda/bin/jnml
+ENV JNML_HOME=/opt/conda/bin
+
 RUN cat ~/.bashrc
 
 

applications/jupyterlab/requirements.txt

@@ -3,22 +3,25 @@ pyelectro
 git+https://github.com/NeuralEnsemble/neurotune.git
 
 ####  NEURON & NetPyNE
-neuron
+neuron==8.1
 
 # Install specific version of NetPyNE
 git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne
 
 ####  Other simulators
 
 # Arbor
-arbor
+arbor==0.6.0
+
+# EDEN
+eden-simulator==0.2.1
 
 # Brian
 brian2
 brian2tools
 
 # PyNN
-pynn==0.10.0
+pynn==0.10.1
 
 # elephant
 elephant
@@ -42,12 +45,7 @@ seaborn
 torch==1.11.0+cpu
 
 # For MDF
-git+https://github.com/SheffieldML/GPy.git@devel
-dask==2.30.0
-distributed==2.30.1
-protobuf==3.17.0
-git+https://github.com/SheffieldML/GPy.git@devel
-#modeci_mdf==0.3.3 # big jump in size of image...
+modeci_mdf==0.4.5 # big jump in size of image...
 
 scikit-learn # Required for some Neuromatch Academy material
 fasttext # Required for some Neuromatch Academy material
@@ -63,4 +61,4 @@ ipympl
 plotly
 
 ####  Final updates
-numpy  # Removes some issues with LFPy...
+numpy  # Removes some issues with LFPy...

applications/netpyne/Dockerfile

@@ -1,8 +1,8 @@
 FROM node:13.14 as jsbuild
 ENV REPO=https://github.com/MetaCell/NetPyNE-UI.git
-ENV BRANCH_TAG=osb2-dev
+ENV BRANCH_TAG=release/1.0.0
 ENV FOLDER=netpyne
-RUN echo "no-cache 2023-3-15b"
+RUN echo "no-cache 2023-4-14"
 RUN git clone $REPO -b $BRANCH_TAG $FOLDER
 RUN rm -Rf .git
 
@@ -20,9 +20,7 @@ RUN mv node_modules/@metacell . && rm -Rf node_modules/* && mv @metacell node_mo
 FROM jupyter/base-notebook:hub-1.4.2
 ENV NB_UID=jovyan
 ENV FOLDER=netpyne
-
-# branch with latest neuroml updates
-ARG NETPYNE_CORE_BRANCH_TAG=osbv2-dev
+ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
 
 USER root
 
@@ -35,32 +33,11 @@ RUN conda install python=3.7 -y
 
 WORKDIR $FOLDER
 COPY --from=jsbuild --chown=1000:1000 $FOLDER/requirements.txt requirements.txt
-RUN pip install -r requirements.txt --no-cache-dir --prefer-binary
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+    pip install -r requirements.txt --prefer-binary
 
 COPY --from=jsbuild --chown=1000:1000 $FOLDER .
 
-
-
-# Temporary fix for deprecated api usage on some requirement
-# RUN pip install setuptools==45
-
-#RUN conda install pandas=0.23.4 -y
-
-RUN jupyter nbextension install --py --symlink --sys-prefix jupyter_geppetto
-RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
-RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
-RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
-
-RUN python utilities/install.py --npm-skip --netpyne $NETPYNE_CORE_BRANCH_TAG
-
-
-
-
-RUN jupyter labextension disable @jupyterlab/hub-extension
-
-
-
-USER root
 COPY overrides/hub/jupyter_notebook_config.py /etc/jupyter/jupyter_notebook_config.py
 
 
@@ -70,18 +47,46 @@ RUN rm -f ~/.jupyter/*.json
 RUN chown $NB_UID .
 RUN chown $NB_UID /opt
 RUN rm -Rf workspace
-# RUN wget -P `pip show LFPykit | grep "Location:" | awk '{print $2"/lfpykit"}'` https://www.parralab.org/nyhead/sa_nyhead.mat
+
 USER $NB_UID
 
 # sym link workspace pvc to $FOLDER
 RUN mkdir -p /opt/workspace
 RUN mkdir -p /opt/user
 
-RUN ln -s /opt/workspace workspace
+
+
 
 ENV NEURON_HOME=/opt/conda
 
+
+USER root
+
+RUN jupyter nbextension install --py --symlink --sys-prefix jupyter_geppetto
+RUN jupyter nbextension enable --py --sys-prefix jupyter_geppetto
+RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN jupyter serverextension enable --py --sys-prefix jupyter_geppetto
+
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ 
+  python utilities/install.py --npm-skip --no-test
+
+COPY overrides/requirements.txt overrides/requirements.txt
+RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\
+    pip install -r overrides/requirements.txt
+
+RUN mv workspace /opt/workspace/tutorials
+RUN ln -s /opt/workspace workspace
+
+RUN jupyter labextension disable @jupyterlab/hub-extension
+
+
+USER $NB_UID
+
+
 EXPOSE 8888
 
 ENTRYPOINT ["tini", "-g", "--"]
+
+
+
 CMD ./NetPyNE-UI

applications/netpyne/overrides/requirements.txt

@@ -0,0 +1,2 @@
+lfpykit==0.5.1
+git+https://github.com/Neurosim-lab/netpyne.git@osbv2