From fed7b78ad9081a14c9e3c0f9509259c045f69ad9 Mon Sep 17 00:00:00 2001 From: jbarragan-bridge <68870834+jbarragan-bridge@users.noreply.github.com> Date: Fri, 4 Oct 2024 06:57:31 -0600 Subject: [PATCH] Not able to upload a compressed S3 file using close() (#838) * Call inner close() call during outer close() call in FileLikeProxy class * Return close() result in FileLikeProxy class --- smart_open/tests/test_s3.py | 37 +++++++++++++++++++++++++++++++++++++ smart_open/utils.py | 7 +++++++ 2 files changed, 44 insertions(+) diff --git a/smart_open/tests/test_s3.py b/smart_open/tests/test_s3.py index 61aec986..78d32c0f 100644 --- a/smart_open/tests/test_s3.py +++ b/smart_open/tests/test_s3.py @@ -634,6 +634,43 @@ def test_writebuffer(self): assert actual == contents + def test_write_gz_using_context_manager(self): + """Does s3 multipart upload create a compressed file using context manager?""" + contents = b'get ready for a surprise' + with smart_open.open( + f's3://{BUCKET_NAME}/{WRITE_KEY_NAME}.gz', + mode="wb", + transport_params={ + "multipart_upload": True, + "min_part_size": 10, + } + ) as fout: + fout.write(contents) + + with smart_open.open(f's3://{BUCKET_NAME}/{WRITE_KEY_NAME}.gz', 'rb') as fin: + actual = fin.read() + + assert actual == contents + + def test_write_gz_not_using_context_manager(self): + """Does s3 multipart upload create a compressed file not using context manager but close()?""" + contents = b'get ready for a surprise' + fout = smart_open.open( + f's3://{BUCKET_NAME}/{WRITE_KEY_NAME}.gz', + mode="wb", + transport_params={ + "multipart_upload": True, + "min_part_size": 10, + } + ) + fout.write(contents) + fout.close() + + with smart_open.open(f's3://{BUCKET_NAME}/{WRITE_KEY_NAME}.gz', 'rb') as fin: + actual = fin.read() + + assert actual == contents + def test_write_gz_with_error(self): """Does s3 multipart upload abort for a failed compressed file upload?""" with self.assertRaises(ValueError): diff --git a/smart_open/utils.py b/smart_open/utils.py index c06e2cf0..efbb9374 100644 --- a/smart_open/utils.py +++ b/smart_open/utils.py @@ -223,3 +223,10 @@ def __exit__(self, *args, **kwargs): def __next__(self): return self.__wrapped__.__next__() + + def close(self): + try: + return self.__wrapped__.close() + finally: + if self.__inner != self.__wrapped__: # Don't close again if inner and wrapped are the same + self.__inner.close()