Merge pull request #71 from InfuseAI/feature/sc-22600/upload-files-to…

…-phfs-via-sdk

[Feature] sc-22600 Implement the upload feature in SDK

.editorconfig

@@ -0,0 +1,27 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[{Makefile,Makefile.*}]
+indent_style = tab
+
+[*.{py,rst}]
+indent_style = space
+indent_size = 4
+
+[*.{md,markdown}]
+trim_trailing_whitespace = false
+
+[*.json]
+indent_size = 2
+insert_final_newline = ignore
+
+[**.min.js]
+indent_style = ignore
+insert_final_newline = ignore

docs/CLI/files.md

@@ -10,6 +10,7 @@ List and download shared files
 Available Commands:
   download             Download shared files
   list                 List shared files
+  upload               Upload shared files
 
 Options:
   -h, --help           Show the help
@@ -54,10 +55,20 @@ primehub files list <path>
 * path: The path to list
 
 
+### Upload
 
+Upload shared files
 
+```
+primehub files upload <src> <path>
+```
+
+* src: The local path to upload artifacts
+* path: The path of file or folder
 
 
+* *(optional)* recursive
+
 ## Examples
 
 We could use `list` to watch files or directory remotely.
@@ -90,4 +101,10 @@ $ primehub files download /jobArtifacts/job-202107290838-aoq173 ./my-download --
 $ tree ./my-download
 ./my-download
 `-- job-202107290838-aoq173
-```
+```
+
+Uses `upload` to upload a file or a directory with `--recursive` options.
+
+```
+$ primehub files upload ./my-download /jobArtifacts/job-202107290838-aoq173 --recursive
+```

docs/notebook/files.ipynb

@@ -110,16 +110,51 @@
     "# You can see the file is downloaded in the sidebar"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "8c2ffb2a",
+   "metadata": {},
+   "source": [
+    "### Upload a file or directory "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "decec17a",
+   "id": "13d0be70",
    "metadata": {},
    "outputs": [],
    "source": [
-    "##\n",
-    "## TODO: add download recursive\n",
-    "##"
+    "# Upload a file to shared files\n",
+    "!touch test.txt\n",
+    "!echo \"Test for Python SDK\" > test.txt\n",
+    "ph.files.upload('test.txt', '/upload')\n",
+    "\n",
+    "# Or upload a folder to share files\n",
+    "!mkdir -p dev\n",
+    "!touch dev/a.out\n",
+    "ph.files.upload('dev', '/upload-dir', recursive=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "81049e29",
+   "metadata": {},
+   "source": [
+    "### Download a directory"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "bff42f30",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Download the share files from a directory\n",
+    "ph.files.download('/upload-dir', 'download-dir', recursive=True)\n",
+    "\n",
+    "# You can see the directory `download-dir` is downloaded in the sidebar"
    ]
   }
  ],
@@ -139,7 +174,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.10"
+   "version": "3.9.5"
   }
  },
  "nbformat": 4,

primehub/__init__.py

@@ -247,6 +247,9 @@ def request_logs(self, endpint: str, follow: bool, tail: int):
     def request_file(self, endpint: str, dest: str):
         return Client(self.primehub_config).request_file(endpint, dest)
 
+    def upload_file(self, endpoint: str, src: str):
+        return Client(self.primehub_config).upload_file(endpoint, src)
+
     def _find_command_class(self, command_class, module_name):
         # create command instance
         if isinstance(command_class, str):
@@ -346,6 +349,7 @@ def __init__(self, primehub: PrimeHub, **kwargs):
         self.request = primehub.request
         self.request_logs = primehub.request_logs
         self.request_file = primehub.request_file
+        self.upload_file = primehub.upload_file
 
     @property
     def current_group(self) -> dict:

primehub/files.py

@@ -1,6 +1,7 @@
 from primehub import Helpful, cmd, Module
 from urllib.parse import urlparse
 import os
+import sys
 
 from primehub.utils.optionals import toggle_flag
 from primehub.utils import create_logger, SharedFileException
@@ -144,6 +145,17 @@ def _execute_list(self, path, **kwargs):
         items = results['data']['files']['items']
         return items
 
+    def _primehub_store_endpoint(self):
+        def to_group_path(group_name: str):
+            if not group_name:
+                return group_name
+            return group_name.replace('_', '-').lower()
+
+        u = urlparse(self.endpoint)
+        endpoint = u._replace(path=f'/api/files/groups/{to_group_path(self.group_name)}').geturl()
+
+        return endpoint
+
     @cmd(name='download', description='Download shared files', optionals=[('recursive', toggle_flag)])
     def download(self, path, dest, **kwargs):
         """
@@ -159,13 +171,7 @@ def download(self, path, dest, **kwargs):
         :param recusive: Copy recursively, it works when a path is a directory.
         """
 
-        def to_group_path(group_name: str):
-            if not group_name:
-                return group_name
-            return group_name.replace('_', '-').lower()
-
-        u = urlparse(self.endpoint)
-        endpoint = u._replace(path=f'/api/files/groups/{to_group_path(self.group_name)}').geturl()
+        endpoint = self._primehub_store_endpoint()
 
         # start download
         src_dst_list = self._generate_download_list(path, dest, **kwargs)
@@ -257,5 +263,71 @@ def _generate_download_list(self, path, dest, **kwargs):
 
         return src_dst_list
 
+    @cmd(name='upload', description='Upload shared files', optionals=[('recursive', toggle_flag)])
+    def upload(self, src, path, **kwargs):
+        """
+        Upload files
+
+        :type path: str
+        :param path: The path of file or folder
+
+        :type src: str
+        :param src: The local path to save artifacts
+
+        :type recusive: bool
+        :param recusive: Upload recursively, it works when a src is a directory.
+        """
+        path = _normalize_user_input_path(path)
+        recursive = kwargs.get('recursive', False)
+
+        # check src
+        if not os.path.exists(src):
+            invalid(f'No such file or directory: {src}')
+            return []
+
+        file_paths = []
+        if recursive is True:
+            if os.path.isfile(src):
+                file_paths.append(os.path.abspath(src))
+            else:
+                for (dirpath, dirnames, filenames) in os.walk(src):
+                    for filename in filenames:
+                        file_paths.append(os.path.join(dirpath, filename))
+            pass
+        else:
+            if os.path.isfile(src):
+                filename = os.path.abspath(src)
+                file_paths.append(filename)
+            else:
+                invalid(f'{src} is not a file')
+                return []
+            pass
+
+        endpoint = self._primehub_store_endpoint()
+        result = []
+        for filepath in file_paths:
+            try:
+                phfs_path = path
+                if os.path.isfile(src):
+                    phfs_path = os.path.join(path, os.path.basename(src))
+                else:
+                    phfs_path = os.path.join(path, os.path.relpath(filepath, os.path.dirname(src)))
+                    pass
+                print('[Uploading] ' + filepath + ' -> phfs://' + phfs_path)
+                response = self._execute_upload(endpoint, filepath, phfs_path)
+                response['phfs'] = phfs_path
+                response['file'] = filepath
+                result.append(response)
+            except Exception as e:
+                print(e, file=sys.stderr)
+                result.append({
+                    'success': False,
+                    'message': e
+                })
+        return result
+
+    def _execute_upload(self, endpoint, src, path):
+        return self.upload_file(endpoint + path, src)
+
     def help_description(self):
         return "List and download shared files"

primehub/utils/http_client.py

@@ -56,6 +56,12 @@ def request_file(self, endpoint, dest):
                 f.write(r.content)
         return
 
+    def upload_file(self, endpoint, src):
+        headers = {'authorization': 'Bearer {}'.format(self.primehub_config.api_token)}
+        with open(src, 'rb') as f:
+            r = requests.post(endpoint, headers=headers, data=f)
+        return r.json()
+
 
 if __name__ == '__main__':
     print(Client.__module__)