diff --git a/siaskynet/__init__.py b/siaskynet/__init__.py index fcac25f..2bc753e 100644 --- a/siaskynet/__init__.py +++ b/siaskynet/__init__.py @@ -24,7 +24,9 @@ class SkynetClient(): get_skykeys ) from ._upload import ( - upload_file, upload_file_request, upload_file_request_with_chunks, + upload, upload_request, + upload_file, upload_file_request, + upload_file_with_chunks, upload_file_request_with_chunks, upload_directory, upload_directory_request ) # pylint: enable=import-outside-toplevel diff --git a/siaskynet/_upload.py b/siaskynet/_upload.py index 947b268..c91834f 100644 --- a/siaskynet/_upload.py +++ b/siaskynet/_upload.py @@ -18,6 +18,72 @@ def default_upload_options(): return obj +def upload(self, upload_data, custom_opts=None): + """Uploads the given generic data and returns the skylink.""" + + response = self.upload_request(upload_data, custom_opts) + sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] + response.close() + return sia_url + + +def upload_request(self, upload_data, custom_opts=None): + """Uploads the given generic data and returns the response object.""" + + opts = default_upload_options() + opts.update(self.custom_opts) + if custom_opts is not None: + opts.update(custom_opts) + + # Upload as a directory if the dirname is set, even if there is only 1 + # file. + issinglefile = len(upload_data) == 1 and not opts['custom_dirname'] + + filename = '' + if issinglefile: + fieldname = opts['portal_file_fieldname'] + else: + if not opts['custom_dirname']: + raise ValueError("custom_dirname must be set when " + "uploading multiple files") + fieldname = opts['portal_directory_file_fieldname'] + filename = opts['custom_dirname'] + + params = { + # 'skykeyname': opts['skykey_name'], + # 'skykeyid': opts['skyket_id'], + } + if filename: + params['filename'] = filename + + ftuples = [] + for filename, data in upload_data.items(): + ftuples.append((fieldname, + (filename, data))) + + if issinglefile: + data = ftuples[0][1][1] + if hasattr(data, '__iter__') and ( + not isinstance(data, bytes) and + not isinstance(data, str) and + not hasattr(data, 'read')): + # an iterator for chunked uploading + params['filename'] = ftuples[0][1][0] + return self.execute_request( + "POST", + opts, + data=data, + headers={'Content-Type': 'application/octet-stream'}, + params=params + ) + return self.execute_request( + "POST", + opts, + files=ftuples, + params=params + ) + + def upload_file(self, path, custom_opts=None): """Uploads file at path with the given options.""" @@ -48,41 +114,57 @@ def upload_file_request(self, path, custom_opts=None): return None with open(path, 'rb') as file_h: - filename = opts['custom_filename'] if opts['custom_filename'] \ - else os.path.basename(file_h.name) - files = {opts['portal_file_fieldname']: (filename, file_h)} + filename = os.path.basename(file_h.name) + if opts['custom_filename']: + filename = opts['custom_filename'] - return self.execute_request( - "POST", - opts, - files=files, - ) + upload_data = {filename: file_h} + return self.upload_request(upload_data, opts) -def upload_file_request_with_chunks(self, path, custom_opts=None): - """Posts request to upload file with chunks.""" + +def upload_file_with_chunks(self, chunks, custom_opts=None): + """ + Uploads a chunked or streaming file with the given options. + For more information on chunked uploading, see: + https://requests.readthedocs.io/en/stable/user/advanced/#chunk-encoded-requests + + :param iter data: An iterator (for chunked encoding) or file-like object + :param dict custom_opts: Custom options. See upload_file. + """ + + response = self.upload_file_request_with_chunks(chunks, custom_opts) + sia_url = utils.uri_skynet_prefix() + response.json()["skylink"] + response.close() + return sia_url + + +def upload_file_request_with_chunks(self, chunks, custom_opts=None): + """ + Posts request for chunked or streaming upload of a single file. + For more information on chunked uploading, see: + https://requests.readthedocs.io/en/stable/user/advanced/#chunk-encoded-requests + + :param iter chunks: An iterator (for chunked encoding) or file-like object + :param dict custom_opts: Custom options. See upload_file. + :return: the full response + :rtype: dict + """ opts = default_upload_options() opts.update(self.custom_opts) if custom_opts is not None: opts.update(custom_opts) - path = os.path.normpath(path) - if not os.path.isfile(path): - print("Given path is not a file") - return None + if opts['custom_filename']: + filename = opts['custom_filename'] + else: + # this is the legacy behavior + filename = str(chunks) - filename = opts['custom_filename'] if opts['custom_filename'] else path - params = {filename: filename} - headers = {'Content-Type': 'application/octet-stream'} + upload_data = {filename: chunks} - return self.execute_request( - "POST", - opts, - data=path, - headers=headers, - params=params, - ) + return self.upload_request(upload_data, opts) def upload_directory(self, path, custom_opts=None): @@ -107,21 +189,13 @@ def upload_directory_request(self, path, custom_opts=None): print("Given path is not a directory") return None - ftuples = [] + upload_data = {} basepath = path if path == '/' else path + '/' - files = list(utils.walk_directory(path).keys()) - for filepath in files: + for filepath in utils.walk_directory(path): assert filepath.startswith(basepath) - ftuples.append((opts['portal_directory_file_fieldname'], - (filepath[len(basepath):], open(filepath, 'rb')))) + upload_data[filepath[len(basepath):]] = open(filepath, 'rb') - dirname = opts['custom_dirname'] if opts['custom_dirname'] else path + if not opts['custom_dirname']: + opts['custom_dirname'] = path - params = {"filename": dirname} - - return self.execute_request( - "POST", - opts, - files=ftuples, - params=params, - ) + return self.upload_request(upload_data, opts) diff --git a/tests/test_integration_upload.py b/tests/test_integration_upload.py index 20f8409..ba19239 100644 --- a/tests/test_integration_upload.py +++ b/tests/test_integration_upload.py @@ -246,3 +246,46 @@ def test_upload_directory_custom_dirname(): filename="file0"') == -1 assert len(responses.calls) == 1 + + +@responses.activate +def test_upload_file_chunks(): + """Test uploading a file with chunks.""" + + src_file = "./testdata/file1" + + # upload a file + + responses.add( + responses.POST, + 'https://siasky.net/skynet/skyfile', + json={'skylink': SKYLINK}, + status=200 + ) + + print("Uploading file "+src_file) + + def chunker(filename): + with open(filename, 'rb') as file: + while True: + data = file.read(3) + if not data: + break + yield data + chunks = chunker(src_file) + sialink2 = client.upload_file_with_chunks(chunks, + {'custom_filename': src_file}) + if SIALINK != sialink2: + sys.exit("ERROR: expected returned sialink "+SIALINK + + ", received "+sialink2) + print("File upload successful, sialink: " + sialink2) + + headers = responses.calls[0].request.headers + assert headers["Transfer-Encoding"] == "chunked" + assert headers["User-Agent"] == "python-requests/2.24.0" + assert "Authorization" not in headers + + body = responses.calls[0].request.body + assert body is chunks + + assert len(responses.calls) == 1