Integrate Python Language Server to Monaco Editor (Closes #1908, #571)

.deployment/README.md

@@ -5,9 +5,13 @@ The script `deploy-deepforge` is used for standard deployment of deepforge using
 
 Additionally, this contains a file with customizations to the standard docker-compose.yml file which allows us to modify the entrypoint and install a version of tensorflow [compatible with the CPU of the deployment machine](https://github.com/deepforge-dev/deepforge/issues/1561).
 
-The deployment is updated by first creating the custom docker compose file using [yaml-merge](https://github.com/alexlafroscia/yaml-merge):
+Moreover, we also a proxy server with that spins up different language servers that we provide for
+ intelligent syntax
+ highlighting in deepforge's browser. For more information checkout the language server's docker [file](../docker/Dockerfile.langservers)
+
+The deployment is updated by first creating the custom docker compose file using [yq](https://github.com/mikefarah/yq):
 ```
-yaml-merge docker-compose.yml .deployment/docker-compose-overrides.yml > custom-docker-compose.yml
+yq m -a docker/docker-compose.yml "$DEEPFORGE_DEPLOYMENT_DIR"/docker-compose-overrides.yml > custom-docker-compose.yml
 ```
 Next, the generated file can be used with docker-compose:
 ```

.deployment/deploy-deepforge

@@ -25,4 +25,12 @@ docker-compose --file custom-docker-compose.yml -p deepforge stop $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge rm -f $SERVER_NAME
 docker-compose --file custom-docker-compose.yml -p deepforge up -d $SERVER_NAME
 
+if [[ $SERVER_NAME = "server_stable" ]]; then
+  docker-compose --file custom-docker-compose.yml -p deepforge pull language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge stop language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge rm -f language_servers
+  docker-compose --file custom-docker-compose.yml -p deepforge up -d language_servers
+fi;
+
+
 docker image prune -f

.deployment/docker-compose-overrides.yml

@@ -4,6 +4,7 @@ services:
     environment:
       - "DEEPFORGE_HOST=https://dev.deepforge.org"
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=https://dev-compute.deepforge.org"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
     volumes:
       - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
@@ -16,6 +17,7 @@ services:
       - "DEEPFORGE_INTERACTIVE_COMPUTE_HOST=https://compute.deepforge.org"
       - "DEEPFORGE_PUBLIC_KEY=/token_keys/public_key"
       - "DEEPFORGE_PRIVATE_KEY=/token_keys/private_key"
+      - "DEEPFORGE_LANGUAGE_SERVER_HOST=https://langservers.deepforge.org"
     image: deepforge/kitchen-sink:stable
     ports:
       - "9000:8888"
@@ -26,3 +28,16 @@ services:
       - "${DEEPFORGE_DEPLOYMENT_DIR}/../config/components.json:/deepforge/config/components.json"
     depends_on:
       - mongo
+
+  language_servers:
+    image: deepforge/language-servers:latest
+    ports:
+      - "5000:5000"
+    volumes:
+      -  "${DEEPFORGE_DEPLOYMENT_DIR}/../language-servers.yml:/root/language-servers.yml"
+      - "${DEEPFORGE_DEPLOYMENT_DIR}:/.deployment"
+      - "/tmp/python-models:/tmp/python-models"
+    entrypoint: /.deployment/langserver-entrypoint.sh
+    depends_on:
+      - server
+      - server_stable

.deployment/langserver-entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Remove pypi tensorflow in favor of conda installation
+source activate base
+pip uninstall tensorflow -y
+conda install tensorflow==1.14 -y
+
+node $(npm root -g)/jsonrpc-ws-proxy/dist/server.js --port $PORT --languageServers ~/language-servers.yml
+

.github/workflows/docker-build.yml

@@ -49,6 +49,19 @@ jobs:
           dockerfile: docker/Dockerfile.kitchensink
           buildargs: "TAG=${{ env.TAG }}"
 
+      - name: Publish to docker hub (language-servers)
+        uses: elgohr/Publish-Docker-Github-Action@master
+
+        with:
+          name: deepforge/language-servers
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          # ToDo: When should this image be updated? On every push to master??
+          tags: latest
+          dockerfile: dockerfile/Dockerfile.langservers
+        if: github.event_name != 'release'
+
+
       - name: Deploy (dev.deepforge.org)
         uses: appleboy/ssh-action@master
         with:

config/config.extensions.js

@@ -1,6 +1,30 @@
+const yaml = require('js-yaml');
+const path = require('path');
+const fs = require('fs');
+
+const SERVERS_YML = path.join(__dirname, '..', 'language-servers.yml');
+
+function getAvailableLanguageServers () {
+    const parsed = yaml.safeLoad(fs.readFileSync(SERVERS_YML));
+    return parsed.langservers ? Object.keys(parsed.langservers): [];
+}
+
+function getWorkspaceURIs() {
+    const availableServers = getAvailableLanguageServers();
+    const workspaces = {};
+    availableServers.forEach(server => {
+        workspaces[server] = `file:///tmp/${server}-models/`;
+    });
+    return workspaces;
+}
 
 module.exports = config => {
     config.extensions = {};
     config.extensions.InteractiveComputeHost = process.env.DEEPFORGE_INTERACTIVE_COMPUTE_HOST;
+    config.extensions.languageServers = {
+        host: process.env.DEEPFORGE_LANGUAGE_SERVER_HOST,
+        servers: getAvailableLanguageServers(),
+        workspaceURIs: getWorkspaceURIs(),
+    };
     return config;
 };

docker/Dockerfile.langservers

@@ -0,0 +1,17 @@
+FROM continuumio/anaconda3
+
+EXPOSE 5000
+
+ENV PORT 5000
+
+RUN apt-get update && curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt-get install -y nodejs
+
+RUN npm install -g npm && npm install -g jsonrpc-ws-proxy
+
+RUN conda update conda -yq && conda install python=3.7
+
+COPY src/plugins/GenerateJob/templates/environment.worker.yml .
+
+RUN conda env update -n base --file environment.worker.yml && rm environment.worker.yml && conda clean -afy
+
+ENTRYPOINT node $(npm root -g)/jsonrpc-ws-proxy/dist/server.js --port $PORT --languageServers ~/language-servers.yml

environment.server.yml

@@ -1,3 +1,4 @@
 name: deepforge
 dependencies:
   - python=3.7
+  - python-language-server

environment.worker.yml

@@ -6,3 +6,4 @@ dependencies:
       - matplotlib==3.2.2
       - simplejson
       - plotly
+      - python-language-server==0.25.0

language-servers.yml

@@ -0,0 +1,5 @@
+langservers:
+  python:
+    - pyls
+    - --log-file
+    - pyls.log

package.json

@@ -37,6 +37,7 @@
     "express": "^4.14.0",
     "graceful-fs": "^4.1.10",
     "js-yaml": "^3.13.1",
+    "jsonrpc-ws-proxy": "0.0.5",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.6.2",
     "lodash.template": "^4.4.0",

src/visualizers/widgets/TextEditor/DeepForgeLanguageClient.js

@@ -0,0 +1,103 @@
+/* globals define */
+
+define([
+    './lib/vscode-ws-jsonrpc.min',
+    './lib/monaco-languageclient.min',
+    './lib/reconnecting-websocket.min',
+], function (
+    vscodeWSJSONRpc,
+    LangaugeClient,
+    RS
+) {
+    const {ReconnectingWebSocket} = RS,
+        {listen} = vscodeWSJSONRpc;
+    const {
+        MonacoLanguageClient,
+        CloseAction,
+        ErrorAction,
+        Services,
+        MonacoCommands,
+        MonacoWorkspace,
+        ConsoleWindow,
+        MonacoLanguages,
+        MonacoToProtocolConverter,
+        ProtocolToMonacoConverter,
+        createConnection
+    } = LangaugeClient;
+
+    class DeepForgeLanguageClient {
+        constructor(editor, serverURL, opts) {
+            const services = createServices(editor, opts);
+            Services.install(services);
+            this.serverURL = serverURL;
+            this.socket = createReconnectingWebSocket(this.serverURL, opts);
+            this._initializeClient(opts);
+        }
+
+        _initializeClient(opts) {
+            listen({
+                webSocket: this.socket,
+                onConnection: connection => {
+                    const languageClient = createLanguageClient(connection, opts);
+                    const disposable = languageClient.start();
+                    connection.onClose(() => disposable.dispose());
+                }
+            });
+        }
+    }
+
+    const createLanguageClient = function (connection, opts) {
+        return new MonacoLanguageClient(
+            {
+                name: opts.name || 'DeepForge Language Client',
+                clientOptions: {
+                    documentSelector: [opts.language || 'python'],
+                    errorHandler: {
+                        error: () => ErrorAction.Continue,
+                        closed: () => CloseAction.DoNotRestart
+                    }
+                },
+                connectionProvider: {
+                    get(errorHandler, closeHandler, /*outputChannel*/) {
+                        return Promise.resolve(
+                            createConnection(connection, errorHandler, closeHandler)
+                        );
+                    }
+                }
+            }
+        );
+    };
+
+    const createReconnectingWebSocket = function (url, opts = {}) {
+        const socketOpts = {
+            maxReconnectionDelay: opts.socket.maxReconnectionDelay || 10000,
+            minReconnectionDelay: opts.socket.minReconnectionDelay || 1000,
+            reconnectionDelayGrowFactor: opts.socket.reconnectionDelayGrowFactor || 1.3,
+            connectionTimeout: 10000,
+            maxRetries: Infinity,
+            debug: opts.socket.debug || false
+        };
+
+        return new ReconnectingWebSocket(
+            url,
+            [],
+            socketOpts
+        );
+    };
+
+    const createServices = function (editor, opts) {
+        const m2p = new MonacoToProtocolConverter();
+        const p2m = new ProtocolToMonacoConverter();
+        let services = {
+            commands: new MonacoCommands(editor),
+            languages: new MonacoLanguages(p2m, m2p),
+            workspace: new MonacoWorkspace(p2m, m2p, opts.rootUri)
+        };
+        if (opts.debug) {
+            services.window = new ConsoleWindow();
+        }
+        return services;
+    };
+
+    return DeepForgeLanguageClient;
+});