From 060dc10f8ff0a8b5314ebdf305b6de8ea44872ef Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 25 Mar 2021 03:28:05 +0100 Subject: [PATCH 1/4] Fix copying files larger than 5GB --- b2sdk/raw_api.py | 9 ++++++--- b2sdk/raw_simulator.py | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 19d4fabf7..25645b487 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -1100,10 +1100,14 @@ def test_raw_api_helper(raw_api): io.BytesIO(part_contents) ) + # b2_copy_part + print('b2_copy_part') + raw_api.copy_part(api_url, account_auth_token, file_id, large_file_id, 2, (0, 5), sse_b2_aes) + # b2_list_parts print('b2_list_parts') parts_response = raw_api.list_parts(api_url, account_auth_token, large_file_id, 1, 100) - assert [1] == [part['partNumber'] for part in parts_response['parts']] + assert [1, 2] == [part['partNumber'] for part in parts_response['parts']] # b2_list_unfinished_large_files unfinished_list = raw_api.list_unfinished_large_files(api_url, account_auth_token, bucket_id) @@ -1111,14 +1115,13 @@ def test_raw_api_helper(raw_api): assert file_info == unfinished_list['files'][0]['fileInfo'] # b2_finish_large_file - # We don't upload enough data to actually finish on, so we'll just - # check that the right error is returned. print('b2_finish_large_file') try: raw_api.finish_large_file(api_url, account_auth_token, large_file_id, [part_sha1]) raise Exception('finish should have failed') except Exception as e: assert 'large files must have at least 2 parts' in str(e) + # TODO: make another attempt to finish but this time successfully # b2_update_bucket print('b2_update_bucket') diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 746712a5a..4ffda1b01 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -1126,6 +1126,10 @@ def copy_part( bytes_range=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, ): + if destination_server_side_encryption is not None and destination_server_side_encryption.mode == EncryptionMode.SSE_B2: + raise ValueError( + 'unsupported sse mode for copy_part!' + ) # SSE-B2 is only to be marked in b2_start_large_file src_bucket_id = self.file_id_to_bucket_id[source_file_id] src_bucket = self._get_bucket_by_id(src_bucket_id) dest_bucket_id = self.file_id_to_bucket_id[large_file_id] From 89c57da8bfba8c15324ada4e298699ce0479b5b0 Mon Sep 17 00:00:00 2001 From: Maciej Lech Date: Thu, 25 Mar 2021 11:25:49 +0100 Subject: [PATCH 2/4] Fix raw API tests --- b2sdk/raw_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 25645b487..5de0a79d3 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -1102,7 +1102,7 @@ def test_raw_api_helper(raw_api): # b2_copy_part print('b2_copy_part') - raw_api.copy_part(api_url, account_auth_token, file_id, large_file_id, 2, (0, 5), sse_b2_aes) + raw_api.copy_part(api_url, account_auth_token, file_id, large_file_id, 2, (0, 5)) # b2_list_parts print('b2_list_parts') From 0cb5344dc2518bbd1802b138373c79e8c6179f2b Mon Sep 17 00:00:00 2001 From: Pawel Polewicz Date: Thu, 25 Mar 2021 18:46:16 +0100 Subject: [PATCH 3/4] Fix encrypted large file upload --- b2sdk/transfer/emerge/executor.py | 3 +-- b2sdk/transfer/outbound/copy_manager.py | 4 ++++ b2sdk/transfer/outbound/upload_manager.py | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/b2sdk/transfer/emerge/executor.py b/b2sdk/transfer/emerge/executor.py index 3f30f66d2..082552bca 100644 --- a/b2sdk/transfer/emerge/executor.py +++ b/b2sdk/transfer/emerge/executor.py @@ -395,7 +395,6 @@ def __init__( large_file_id, large_file_upload_state, finished_parts=None, - destination_encryption: Optional[EncryptionSetting] = None, ): super(LargeFileEmergeExecutionStepFactory, self).__init__(emerge_execution, emerge_part) self.part_number = part_number @@ -421,7 +420,7 @@ def create_upload_execution_step(self, stream_opener, stream_length=None, stream self.large_file_upload_state, stream_length=stream_length, stream_sha1=stream_sha1, - finished_parts=self.finished_parts + finished_parts=self.finished_parts, ) diff --git a/b2sdk/transfer/outbound/copy_manager.py b/b2sdk/transfer/outbound/copy_manager.py index a6fabed39..d1eb4f4e8 100644 --- a/b2sdk/transfer/outbound/copy_manager.py +++ b/b2sdk/transfer/outbound/copy_manager.py @@ -131,6 +131,10 @@ def _copy_part( EncryptionMode.SSE_B2, ) + # b2_copy_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. + if destination_encryption is not None and destination_encryption.mode == EncryptionMode.SSE_B2: + destination_encryption = None + # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished diff --git a/b2sdk/transfer/outbound/upload_manager.py b/b2sdk/transfer/outbound/upload_manager.py index 496bcb1d1..5e67e803c 100644 --- a/b2sdk/transfer/outbound/upload_manager.py +++ b/b2sdk/transfer/outbound/upload_manager.py @@ -137,6 +137,11 @@ def _upload_part( :param b2sdk.v1.EncryptionSetting encryption: encryption setting (``None`` if unknown) """ assert encryption is None or encryption.mode in (EncryptionMode.SSE_B2,) + + # b2_upload_part doesn't need SSE-B2. Large file encryption is decided on b2_start_large_file. + if encryption is not None and encryption.mode == EncryptionMode.SSE_B2: + encryption = None + # Check if this part was uploaded before if finished_parts is not None and part_number in finished_parts: # Report this part finished From 20327ab6934fbeb578d77b35418fa02eed57c914 Mon Sep 17 00:00:00 2001 From: Maciej Lech Date: Thu, 25 Mar 2021 21:06:41 +0100 Subject: [PATCH 4/4] Add assertion in copy_part --- b2sdk/raw_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 5de0a79d3..08afaeb2c 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -830,6 +830,7 @@ def copy_part( _add_range_header(range_dict, bytes_range) kwargs['range'] = range_dict['Range'] if destination_server_side_encryption is not None: + assert destination_server_side_encryption.mode != EncryptionMode.SSE_B2 kwargs['destinationServerSideEncryption' ] = destination_server_side_encryption.as_value_dict() return self._post_json(