Integrate Python Language Server to Monaco Editor (Closes #1908, #571)

.deployment/README.md

@@ -5,9 +5,13 @@ The script `deploy-deepforge` is used for standard deployment of deepforge using
 
 Additionally, this contains a file with customizations to the standard docker-compose.yml file which allows us to modify the entrypoint and install a version of tensorflow [compatible with the CPU of the deployment machine](https://github.com/deepforge-dev/deepforge/issues/1561).
 
-The deployment is updated by first creating the custom docker compose file using [yaml-merge](https://github.com/alexlafroscia/yaml-merge):
+Moreover, we also a proxy server with that spins up different language servers that we provide for
+ intelligent syntax
+ highlighting in deepforge's browser. For more information checkout the language server's docker [file](../docker/Dockerfile.langservers)
+
+The deployment is updated by first creating the custom docker compose file using [yq](https://github.com/mikefarah/yq):
 ```
-yaml-merge docker-compose.yml .deployment/docker-compose-overrides.yml > custom-docker-compose.yml
+yq m -a docker/docker-compose.yml "$DEEPFORGE_DEPLOYMENT_DIR"/docker-compose-overrides.yml > custom-docker-compose.yml
 ```
 Next, the generated file can be used with docker-compose:
 ```

.deployment/deploy-deepforge

@@ -13,8 +13,9 @@ fi;
 < config/components.json  jq '.Compute.backends=(.Compute.backends | map(select(. != "local")))' \
 > config/components2.json
 < config/components2.json jq '.Storage.backends=(.Storage.backends | map(select(. != "gme")))' \
-> config/components.json
-rm config/components2.json
+> config/components3.json
+jq '.LanguageServers = input' config/components3.json utils/languageServers.json > config/components.json
+rm config/components3.json config/components2.json
 
 # Merging the custom override yml file
 yq m -a docker/docker-compose.yml "$DEEPFORGE_DEPLOYMENT_DIR"/docker-compose-overrides.yml > custom-docker-compose.yml
@@ -25,4 +26,12 @@ docker-compose --file custom-docker-compose.yml -p deepforge stop $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge rm -f $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge up -d $SERVER_NAME
 
+if [[ $SERVER_NAME = "server_stable" ]]; then
+  docker-compose --file custom-docker-compose.yml -p deepforge pull language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge stop language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge rm -f language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge up -d language_servers
+fi;
+
+
 docker image prune -f

.deployment/docker-compose-overrides.yml

@@ -4,6 +4,7 @@ services:
     environment:
       - "DEEPFORGE_HOST=https://dev.deepforge.org"
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=https://dev-compute.deepforge.org"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
     volumes:
       - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
@@ -16,6 +17,7 @@ services:
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=https://compute.deepforge.org"
       - "DEEPFORGE_PUBLIC_KEY=/token_keys/public_key"
       - "DEEPFORGE_PRIVATE_KEY=/token_keys/private_key"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
     image: deepforge/kitchen-sink:stable
     ports:
       - "9000:8888"
@@ -26,3 +28,16 @@ services:
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
     depends_on:
       - mongo
+
+  language_servers:
+    image: deepforge/language-servers:latest
+    ports:
+      - "5000:5000"
+    volumes:
+      -  "${DEEPFORGE_DEPLOYMENT_DIR}/../language-servers.yml:/root/language-servers.yml"
+      - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
+      - "/tmp/python-models:/tmp/python-models"
+    entrypoint: /.deployment/langserver-entrypoint.sh
+    depends_on:
+      - server
+      - server_stable

.deployment/langserver-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Remove pypi tensorflow in favor of conda installation
+source activate base
+pip uninstall tensorflow -y
+conda install tensorflow==1.14 -y
+
+node $(npm root -g)/jsonrpc-ws-proxy/dist/server.js --port $PORT --languageServers ~/language-servers.yml
+

.github/workflows/docker-build.yml

@@ -49,6 +49,19 @@ jobs:
           dockerfile: docker/Dockerfile.kitchensink
           buildargs: "TAG=${{ env.TAG }}"
 
+      - name: Publish to docker hub (language-servers)
+        uses: elgohr/Publish-Docker-Github-Action@master
+
+        with:
+          name: deepforge/language-servers
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          # ToDo: When should this image be updated? On every push to master??
+          tags: latest
+          dockerfile: dockerfile/Dockerfile.langservers
+        if: github.event_name != 'release'
+
+
       - name: Deploy (dev.deepforge.org)
         uses: appleboy/ssh-action@master
         with:

config/config.extensions.js

@@ -1,6 +1,18 @@
+const components = require('./components.json');
+
+function getLanguageServersConfig () {
+    if(components.LanguageServers) {
+        return {
+            hostName: process.env.DEEPFORGE_LANGUAGE_SERVER_HOST || components.LanguageServers.hostName,
+            servers: components.LanguageServers.servers
+        };
+    }
+}
+
 
 module.exports = config => {
     config.extensions = {};
     config.extensions.InteractiveComputeHost = process.env.DEEPFORGE_INTERACTIVE_COMPUTE_HOST;
+    config.extensions.LanguageServers = getLanguageServersConfig();
     return config;
 };

docker/Dockerfile.langservers

@@ -0,0 +1,33 @@
+FROM continuumio/anaconda3
+
+LABEL maintainer1.name="Brian Broll"\
+      maintainer1.email="[email protected]"
+
+LABEL maintainer2.name="Umesh Timalsina"\
+      maintainer2.email="[email protected]"
+
+EXPOSE 5000
+
+ENV PORT 5000
+
+RUN apt-get update && curl -sL https://deb.nodesource.com/setup_12.x | bash - &&\
+ apt-get install -y nodejs && apt-get install unzip
+
+RUN npm install -g npm && npm -g config set user root && npm install -g jsonrpc-ws-proxy js-yaml
+
+RUN conda update conda -yq && conda install python=3.7
+
+COPY src/plugins/GenerateJob/templates/environment.worker.yml .
+
+COPY utils/install-microsoft-python-langserver.sh .
+
+COPY docker/create-servers-yml.js .
+
+COPY utils/languageServers.json utils/
+
+RUN ./install-microsoft-python-langserver.sh && npm link js-yaml &&\
+ node ./create-servers-yml.js && rm create-servers-yml.js
+
+RUN conda env update -n base --file environment.worker.yml && rm environment.worker.yml && conda clean -afy
+
+ENTRYPOINT node $(npm root -g)/jsonrpc-ws-proxy/dist/server.js --port $PORT --languageServers ~/languageServers.yml

docker/create-servers-yml.js

@@ -0,0 +1,22 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const path = require('path');
+const langServers = require(path.resolve(__dirname, '..', 'utils', 'languageServers'));
+
+function dumpLangServerYaml(fileName) {
+    const serversYml = {
+        langservers: {}
+    };
+
+    Object.keys(langServers.servers).forEach(server => {
+        const command = langServers.servers[server].command;
+        serversYml.langservers[server] = Array.isArray(command) ? command : [command];
+    });
+
+    fs.writeFileSync(fileName, yaml.safeDump(serversYml));
+}
+
+
+if (require.main === module) {
+    dumpLangServerYaml('languageServers.yml');
+}

environment.server.yml

@@ -1,3 +1,4 @@
 name: deepforge
 dependencies:
   - python=3.7
+  - python-language-server