From 07eb247a6ebb4be7ac6b2442e5bf09cf5d988ef1 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Wed, 29 Aug 2018 15:31:23 +0800 Subject: [PATCH 1/6] add http2.0 feature --- examples/object_http20.py | 38 ++++++++++++++++++++++++++++++++++++++ oss2/api.py | 28 +++++++++++++++++----------- oss2/http.py | 12 ++++++++---- setup.py | 1 + 4 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 examples/object_http20.py diff --git a/examples/object_http20.py b/examples/object_http20.py new file mode 100644 index 00000000..bdd185b7 --- /dev/null +++ b/examples/object_http20.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- + +import os +import shutil + +import oss2 + + +# 以下代码展示了如何启用Http2.0来发送请求。 + + +# 首先初始化AccessKeyId、AccessKeySecret、Endpoint等信息。 +# 通过环境变量获取，或者把诸如“<你的AccessKeyId>”替换成真实的AccessKeyId等。 +# +# 以杭州区域为例，Endpoint可以是： +# https://oss-cn-hangzhou.aliyuncs.com +# 目前Http2.0只支持HTTPS协议访问。 +access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '<你的AccessKeyId>') +access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET', '<你的AccessKeySecret>') +bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>') +endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>') + + +# 确认上面的参数都填写正确了 +for param in (access_key_id, access_key_secret, bucket_name, endpoint): + assert '<' not in param, '请设置参数：' + param + +# 创建Bucket对象，所有Object相关的接口都可以通过Bucket对象来进行 +bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, enable_http20=True) + +# 上传一段字符串。Object名是motto.txt，内容是一段名言。 +bucket.put_object('motto.txt', 'Never give up. - Jack Ma') + +# 下载到本地文件 +bucket.get_object_to_file('motto.txt', '本地文件名.txt') + +# 清除本地文件 +os.remove(u'本地文件名.txt') diff --git a/oss2/api.py b/oss2/api.py index 4ae39db6..12b33d44 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -186,10 +186,10 @@ def progress_callback(bytes_consumed, total_bytes): class _Base(object): def __init__(self, auth, endpoint, is_cname, session, connect_timeout, - app_name='', enable_crc=True): + app_name='', enable_crc=True, enable_http20=False): self.auth = auth - self.endpoint = _normalize_endpoint(endpoint.strip()) - self.session = session or http.Session() + self.endpoint = _normalize_endpoint(endpoint.strip(), enable_http20) + self.session = session or http.Session(enable_http20=enable_http20) self.timeout = defaults.get(connect_timeout, defaults.connect_timeout) self.app_name = app_name self.enable_crc = enable_crc @@ -268,11 +268,12 @@ class Service(_Base): def __init__(self, auth, endpoint, session=None, connect_timeout=None, - app_name=''): + app_name='', + enable_http20=False): logger.info("Init oss service, endpoint: {0}, connect_timeout: {1}, app_name: {2}".format( endpoint, connect_timeout, app_name)) super(Service, self).__init__(auth, endpoint, False, session, connect_timeout, - app_name=app_name) + app_name=app_name, enable_http20=enable_http20) def list_buckets(self, prefix='', marker='', max_keys=100): """根据前缀罗列用户的Bucket。 @@ -341,11 +342,12 @@ def __init__(self, auth, endpoint, bucket_name, session=None, connect_timeout=None, app_name='', - enable_crc=True): + enable_crc=True, + enable_http20=False): logger.info("Init oss bucket, endpoint: {0}, isCname: {1}, connect_timeout: {2}, app_name: {3}, enabled_crc: " "{4}".format(endpoint, is_cname, connect_timeout, app_name, enable_crc)) super(Bucket, self).__init__(auth, endpoint, is_cname, session, connect_timeout, - app_name, enable_crc) + app_name, enable_crc, enable_http20=enable_http20) self.bucket_name = bucket_name.strip() @@ -1625,7 +1627,8 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, session=None, connect_timeout=None, app_name='', - enable_crc=True): + enable_crc=True, + enable_http20=False): if not isinstance(crypto_provider, BaseCryptoProvider): raise ClientError('Crypto bucket must provide a valid crypto_provider') @@ -1634,7 +1637,7 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, self.bucket_name = bucket_name.strip() self.enable_crc = enable_crc self.bucket = Bucket(auth, endpoint, bucket_name, is_cname, session, connect_timeout, - app_name, enable_crc=False) + app_name, enable_crc=False, enable_http20=enable_http20) def put_object(self, key, data, headers=None, @@ -1758,9 +1761,12 @@ def get_object_to_file(self, key, filename, return result -def _normalize_endpoint(endpoint): +def _normalize_endpoint(endpoint, default_https): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): - return 'http://' + endpoint + if default_https: + return 'https://' + endpoint + else: + return 'http://' + endpoint else: return endpoint diff --git a/oss2/http.py b/oss2/http.py index 8e8c2ca9..ac7bcb18 100644 --- a/oss2/http.py +++ b/oss2/http.py @@ -19,6 +19,7 @@ from .utils import file_object_remaining_bytes, SizedFileAdapter import logging +from hyper.contrib import HTTP20Adapter _USER_AGENT = 'aliyun-sdk-python/{0}({1}/{2}/{3};{4})'.format( __version__, platform.system(), platform.release(), platform.machine(), platform.python_version()) @@ -27,12 +28,15 @@ class Session(object): """属于同一个Session的请求共享一组连接池，如有可能也会重用HTTP连接。""" - def __init__(self): + def __init__(self, enable_http20=False): self.session = requests.Session() - psize = defaults.connection_pool_size - self.session.mount('http://', requests.adapters.HTTPAdapter(pool_connections=psize, pool_maxsize=psize)) - self.session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=psize, pool_maxsize=psize)) + if enable_http20: + self.session.mount('https://', HTTP20Adapter()) + else: + psize = defaults.connection_pool_size + self.session.mount('http://', requests.adapters.HTTPAdapter(pool_connections=psize, pool_maxsize=psize)) + self.session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=psize, pool_maxsize=psize)) def do_request(self, req, timeout): try: diff --git a/setup.py b/setup.py index 6cfe286a..a5badd76 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ packages=['oss2'], install_requires=['requests!=2.9.0', 'crcmod>=1.7', + 'hyper>=0.7.0', 'pycryptodome>=3.4.7', 'aliyun-python-sdk-kms>=2.4.1', 'aliyun-python-sdk-core>=2.6.2' if sys.version_info[0] == 2 else 'aliyun-python-sdk-core-v3>=2.5.5'], From 60e7fc63dee610128d3f90f762c4120b80672478 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Sat, 29 Sep 2018 14:47:37 +0800 Subject: [PATCH 2/6] add scripts to fix hyper and h2 code --- scripts/patch_hyper.py | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 scripts/patch_hyper.py diff --git a/scripts/patch_hyper.py b/scripts/patch_hyper.py new file mode 100755 index 00000000..6f8f51fe --- /dev/null +++ b/scripts/patch_hyper.py @@ -0,0 +1,55 @@ +#!/usr/bin/python + +import os +import hyper +import h2 + +def fix_hyper(): + if hyper.__version__ == "0.7.0": + hyper_dir = os.path.dirname(hyper.__file__) + fix_file_path = hyper_dir + "/common/headers.py" + + f_read = open(fix_file_path,'r+') + flist = f_read.readlines() + if flist[244] == """ SPECIAL_SNOWFLAKES = set([b'set-cookie', b'set-cookie2'])\n""": + flist[244] = """ SPECIAL_SNOWFLAKES = set([b'set-cookie', b'set-cookie2', b'date', b'if-modified-since', b'if-unmodified-since', b'authorization'])\n""" + + print " =====================================================================================" + print " # OSS already patch to fix hyper library " + print " # fixed file name: ", fix_file_path + print " # fixed line number: 244" + print " # More detail to see: https://github.com/Lukasa/hyper/issues/314 " + print " =====================================================================================" + f_read.close() + + f_wte = open(fix_file_path, 'w+') + f_wte.writelines(flist) + f_wte.close() + +def fix_h2(): + if h2.__version__ == "2.6.2": + h2_dir = os.path.dirname(h2.__file__) + fix_file_path = h2_dir + "/stream.py" + + f_read = open(fix_file_path, 'r+') + flist = f_read.readlines() + if flist[337] == """ raise StreamClosedError(self.stream_id)\n""": + flist[337] = """ #raise StreamClosedError(self.stream_id)\n return []\n""" + print " =====================================================================================" + print " # OSS already patch to fix h2 library " + print " # fixed file name: ", fix_file_path + print " # fixed line number: 337" + print " =====================================================================================" + f_read.close() + + f_wte = open(fix_file_path, 'w+') + f_wte.writelines(flist) + f_wte.close() + +def main(): + fix_hyper() + fix_h2() + + +if __name__ == "__main__": + main() From ebcd02c8f0e599ff0c9e4fe91c16052e289a19cd Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Sat, 29 Sep 2018 15:41:18 +0800 Subject: [PATCH 3/6] modify enable_http20 to http_version --- examples/object_http20.py | 2 +- oss2/__init__.py | 2 +- oss2/api.py | 22 +++++++++++----------- oss2/http.py | 7 +++++-- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/examples/object_http20.py b/examples/object_http20.py index bdd185b7..ad075022 100644 --- a/examples/object_http20.py +++ b/examples/object_http20.py @@ -26,7 +26,7 @@ assert '<' not in param, '请设置参数：' + param # 创建Bucket对象，所有Object相关的接口都可以通过Bucket对象来进行 -bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, enable_http20=True) +bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name, http_version=oss2.HTTP_VERSION_20) # 上传一段字符串。Object名是motto.txt，内容是一段名言。 bucket.put_object('motto.txt', 'Never give up. - Jack Ma') diff --git a/oss2/__init__.py b/oss2/__init__.py index 66b60b4b..3f05e208 100644 --- a/oss2/__init__.py +++ b/oss2/__init__.py @@ -4,7 +4,7 @@ from .api import Service, Bucket, CryptoBucket from .auth import Auth, AuthV2, AnonymousAuth, StsAuth, AUTH_VERSION_1, AUTH_VERSION_2, make_auth -from .http import Session, CaseInsensitiveDict +from .http import Session, CaseInsensitiveDict, HTTP_VERSION_11, HTTP_VERSION_20 from .iterators import (BucketIterator, ObjectIterator, diff --git a/oss2/api.py b/oss2/api.py index 12b33d44..bdb3178d 100644 --- a/oss2/api.py +++ b/oss2/api.py @@ -186,10 +186,10 @@ def progress_callback(bytes_consumed, total_bytes): class _Base(object): def __init__(self, auth, endpoint, is_cname, session, connect_timeout, - app_name='', enable_crc=True, enable_http20=False): + app_name='', enable_crc=True, http_version=http.HTTP_VERSION_11): self.auth = auth - self.endpoint = _normalize_endpoint(endpoint.strip(), enable_http20) - self.session = session or http.Session(enable_http20=enable_http20) + self.endpoint = _normalize_endpoint(endpoint.strip(), http_version) + self.session = session or http.Session(http_version=http_version) self.timeout = defaults.get(connect_timeout, defaults.connect_timeout) self.app_name = app_name self.enable_crc = enable_crc @@ -269,11 +269,11 @@ def __init__(self, auth, endpoint, session=None, connect_timeout=None, app_name='', - enable_http20=False): + http_version=http.HTTP_VERSION_11): logger.info("Init oss service, endpoint: {0}, connect_timeout: {1}, app_name: {2}".format( endpoint, connect_timeout, app_name)) super(Service, self).__init__(auth, endpoint, False, session, connect_timeout, - app_name=app_name, enable_http20=enable_http20) + app_name=app_name, http_version=http_version) def list_buckets(self, prefix='', marker='', max_keys=100): """根据前缀罗列用户的Bucket。 @@ -343,11 +343,11 @@ def __init__(self, auth, endpoint, bucket_name, connect_timeout=None, app_name='', enable_crc=True, - enable_http20=False): + http_version=http.HTTP_VERSION_11): logger.info("Init oss bucket, endpoint: {0}, isCname: {1}, connect_timeout: {2}, app_name: {3}, enabled_crc: " "{4}".format(endpoint, is_cname, connect_timeout, app_name, enable_crc)) super(Bucket, self).__init__(auth, endpoint, is_cname, session, connect_timeout, - app_name, enable_crc, enable_http20=enable_http20) + app_name, enable_crc, http_version=http_version) self.bucket_name = bucket_name.strip() @@ -1628,7 +1628,7 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, connect_timeout=None, app_name='', enable_crc=True, - enable_http20=False): + http_version=http.HTTP_VERSION_11): if not isinstance(crypto_provider, BaseCryptoProvider): raise ClientError('Crypto bucket must provide a valid crypto_provider') @@ -1637,7 +1637,7 @@ def __init__(self, auth, endpoint, bucket_name, crypto_provider, self.bucket_name = bucket_name.strip() self.enable_crc = enable_crc self.bucket = Bucket(auth, endpoint, bucket_name, is_cname, session, connect_timeout, - app_name, enable_crc=False, enable_http20=enable_http20) + app_name, enable_crc=False, http_version=http_version) def put_object(self, key, data, headers=None, @@ -1761,9 +1761,9 @@ def get_object_to_file(self, key, filename, return result -def _normalize_endpoint(endpoint, default_https): +def _normalize_endpoint(endpoint, http_version): if not endpoint.startswith('http://') and not endpoint.startswith('https://'): - if default_https: + if http_version is http.HTTP_VERSION_20: return 'https://' + endpoint else: return 'http://' + endpoint diff --git a/oss2/http.py b/oss2/http.py index ac7bcb18..c7be4bf4 100644 --- a/oss2/http.py +++ b/oss2/http.py @@ -26,12 +26,15 @@ logger = logging.getLogger(__name__) +HTTP_VERSION_11 = 'http11' +HTTP_VERSION_20 = 'http20' + class Session(object): """属于同一个Session的请求共享一组连接池，如有可能也会重用HTTP连接。""" - def __init__(self, enable_http20=False): + def __init__(self, http_version=HTTP_VERSION_11): self.session = requests.Session() - if enable_http20: + if http_version is HTTP_VERSION_20: self.session.mount('https://', HTTP20Adapter()) else: psize = defaults.connection_pool_size From ee1c0acad4e125b42c3c906649ea0a396522a0e6 Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Sat, 29 Sep 2018 16:04:09 +0800 Subject: [PATCH 4/6] add lock to Session avoid hyper http20 multi-threads bug --- oss2/http.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/oss2/http.py b/oss2/http.py index c7be4bf4..097ec52d 100644 --- a/oss2/http.py +++ b/oss2/http.py @@ -20,6 +20,7 @@ import logging from hyper.contrib import HTTP20Adapter +import threading _USER_AGENT = 'aliyun-sdk-python/{0}({1}/{2}/{3};{4})'.format( __version__, platform.system(), platform.release(), platform.machine(), platform.python_version()) @@ -34,6 +35,9 @@ class Session(object): def __init__(self, http_version=HTTP_VERSION_11): self.session = requests.Session() + self.http_version = http_version + self.__lock = threading.Lock() + if http_version is HTTP_VERSION_20: self.session.mount('https://', HTTP20Adapter()) else: @@ -43,9 +47,19 @@ def __init__(self, http_version=HTTP_VERSION_11): def do_request(self, req, timeout): try: - logger.debug("Send request, method: {0}, url: {1}, params: {2}, headers: {3}, timeout: {4}".format( - req.method, req.url, req.params, req.headers, timeout)) - return Response(self.session.request(req.method, req.url, + logger.debug("Send request, method: {0}, url: {1}, params: {2}, headers: {3}, timeout: {4}, http_version: {5}".format( + req.method, req.url, req.params, req.headers, timeout, self.http_version)) + if self.http_version is HTTP_VERSION_20: + with self.__lock: + resp = Response(self.session.request(req.method, req.url, + data=req.data, + params=req.params, + headers=req.headers, + stream=True, + timeout=timeout)) + return resp + else: + return Response(self.session.request(req.method, req.url, data=req.data, params=req.params, headers=req.headers, From b131359c8925614b907a0e9049f394acb5e4bd8c Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Sat, 29 Sep 2018 20:32:01 +0800 Subject: [PATCH 5/6] fix unittest for http20 --- tests/common.py | 11 +++-- tests/test_bucket.py | 20 +++++++++ tests/test_chinese.py | 24 +++++++++- tests/test_download.py | 21 +++++++++ tests/test_image.py | 29 +++++++++++- tests/test_iterator.py | 22 ++++++++- tests/test_live_channel.py | 25 ++++++++++- tests/test_multipart.py | 21 +++++++++ tests/test_object.py | 92 ++++++++++++++++++++++++++++++++++---- tests/test_sts.py | 50 ++++++++++++++++++++- tests/test_upload.py | 21 +++++++++ 11 files changed, 315 insertions(+), 21 deletions(-) diff --git a/tests/common.py b/tests/common.py index eb39edee..d88cf626 100644 --- a/tests/common.py +++ b/tests/common.py @@ -26,6 +26,7 @@ OSS_STS_ARN = os.getenv("OSS_TEST_STS_ARN") OSS_AUTH_VERSION = None +OSS_HTTP_VERSION = None def random_string(n): return ''.join(random.choice(string.ascii_lowercase) for i in range(n)) @@ -80,10 +81,11 @@ def setUp(self): oss2.defaults.multiget_part_size = self.default_multiget_part_size oss2.defaults.multiget_num_threads = random.randint(1, 5) - global OSS_AUTH_VERSION + global OSS_AUTH_VERSION, OSS_HTTP_VERSION OSS_AUTH_VERSION = os.getenv('OSS_TEST_AUTH_VERSION') + OSS_HTTP_VERSION = os.getenv('OSS_TEST_HTTP_VERSION') - self.bucket = oss2.Bucket(oss2.make_auth(OSS_ID, OSS_SECRET, OSS_AUTH_VERSION), OSS_ENDPOINT, OSS_BUCKET) + self.bucket = oss2.Bucket(oss2.make_auth(OSS_ID, OSS_SECRET, OSS_AUTH_VERSION), OSS_ENDPOINT, OSS_BUCKET, http_version=OSS_HTTP_VERSION) try: self.bucket.create_bucket() @@ -91,10 +93,11 @@ def setUp(self): pass self.rsa_crypto_bucket = oss2.CryptoBucket(oss2.make_auth(OSS_ID, OSS_SECRET, OSS_AUTH_VERSION), OSS_ENDPOINT, OSS_BUCKET, - crypto_provider=oss2.LocalRsaProvider()) + crypto_provider=oss2.LocalRsaProvider(), http_version=OSS_HTTP_VERSION) + # Special handle for http20, Because now KMS don't support http20, kms_crypto_bucket create with http11 self.kms_crypto_bucket = oss2.CryptoBucket(oss2.make_auth(OSS_ID, OSS_SECRET, OSS_AUTH_VERSION), OSS_ENDPOINT, OSS_BUCKET, - crypto_provider=oss2.AliKMSProvider(OSS_ID, OSS_SECRET, OSS_REGION, OSS_CMK)) + crypto_provider=oss2.AliKMSProvider(OSS_ID, OSS_SECRET, OSS_REGION, OSS_CMK), http_version=oss2.HTTP_VERSION_11) self.key_list = [] self.temp_files = [] diff --git a/tests/test_bucket.py b/tests/test_bucket.py index 8f75849b..8bfdb519 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -541,6 +541,26 @@ def test_xml_input_output(self): self.assertEqual(result.allow_empty_referer, True) self.assertEqual(result.referers[0], to_string(u'阿里云')) +class TestHttp20OverBucket(TestBucket): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestBucket，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverBucket, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverBucket, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverBucket, self).tearDown() if __name__ == '__main__': unittest.main() diff --git a/tests/test_chinese.py b/tests/test_chinese.py index f73fb994..cb19f40a 100644 --- a/tests/test_chinese.py +++ b/tests/test_chinese.py @@ -63,7 +63,27 @@ def test_get_symlink(self): result = self.bucket.get_symlink(symlink) self.assertEqual(result.target_key, key) - + +class TestHttp20OverChinese(TestChinese): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestChinese，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverChinese, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverChinese, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverChinese, self).tearDown() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_download.py b/tests/test_download.py index d171a4c6..142c3ed0 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -609,6 +609,27 @@ def test_resumable_incomplete_download(self): except: self.assertTrue(False) +class TestHttp20OverDownload(TestDownload): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestDownload，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverDownload, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverDownload, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverDownload, self).tearDown() + if __name__ == '__main__': unittest.main() diff --git a/tests/test_image.py b/tests/test_image.py index ff8c6253..93d0f144 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -19,7 +19,13 @@ def __prepare(self): def __test(self, original_image, new_image, image_style): original_image_content = self.bucket.get_object(original_image, process=image_style) - self.bucket.put_object(new_image, original_image_content) + + # Special handle for http20, Need add Content-Length to request header in this test case + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + test_header = {"Content-Length" :original_image_content.headers['Content-Length']} + self.bucket.put_object(new_image, original_image_content, headers=test_header) + else: + self.bucket.put_object(new_image, original_image_content) def __test_to_file(self, original_image, new_image, image_style): self.bucket.get_object_to_file(original_image, new_image, process=image_style) @@ -86,5 +92,26 @@ def test_resize_to_file(self): self.__check(new_image, 100, 100, 3267, 'jpg') +class TestHttp20OverImage(TestImage): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestImage，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverImage, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverImage, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverImage, self).tearDown() + if __name__ == '__main__': unittest.main() diff --git a/tests/test_iterator.py b/tests/test_iterator.py index bce09f01..910beeec 100644 --- a/tests/test_iterator.py +++ b/tests/test_iterator.py @@ -182,6 +182,26 @@ def test_live_channel_iterator(self): for live_channel in channel_name_list: self.bucket.delete_live_channel(live_channel) +class TestHttp20OverIterator(TestIterator): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestIterator，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverIterator, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverIterator, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverIterator, self).tearDown() if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_live_channel.py b/tests/test_live_channel.py index b099734f..89a458a7 100644 --- a/tests/test_live_channel.py +++ b/tests/test_live_channel.py @@ -251,8 +251,29 @@ def test_anonymous_auth(self): self.assertEqual(signed_url, self._get_publish_url(self.bucket.bucket_name, channel_name) + "?playlistName=" + playlist_name) - self.assertRaises(oss2.exceptions.AccessDenied, bucket.delete_live_channel, 'test-live-chan') - + self.assertRaises(oss2.exceptions.AccessDenied, bucket.delete_live_channel, 'test-live-chan') + +class TestHttp20OverLiveChannel(TestLiveChannel): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestLiveChannel，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverLiveChannel, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverLiveChannel, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverLiveChannel, self).tearDown() + if __name__ == '__main__': unittest.main() diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 3bf840d0..5a071804 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -97,6 +97,27 @@ def test_upload_part_copy(self): self.assertEqual(len(content_got), len(content)) self.assertEqual(content_got, content) +class TestHttp20OverMultipart(TestMultipart): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestMultipart，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverMultipart, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverMultipart, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverMultipart, self).tearDown() + if __name__ == '__main__': unittest.main() diff --git a/tests/test_object.py b/tests/test_object.py index 31d22417..ff9a8c49 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -188,7 +188,13 @@ def test_object_empty(self): key = self.random_key() content = b'' - self.bucket.put_object(key, content) + # Special handle for http20, Need add Content-Length to request header in this test case + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + test_header = {"Content-Length":"0"} + self.bucket.put_object(key, content, headers=test_header) + else: + self.bucket.put_object(key, content) + res = self.bucket.get_object(key) self.assertEqual(res.read(), b'') @@ -203,7 +209,13 @@ def test_file_empty(self): with open(input_filename, 'wb') as f: f.write(content) - self.bucket.put_object_from_file(key, input_filename) + # Special handle for http20, Need add Content-Length to request header in this test case + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + test_header = {"Content-Length":"0"} + self.bucket.put_object_from_file(key, input_filename, headers=test_header) + else: + self.bucket.put_object_from_file(key, input_filename) + self.bucket.get_object_to_file(key, output_filename) self.assertTrue(filecmp.cmp(input_filename, output_filename)) @@ -218,7 +230,13 @@ def test_streaming(self): # 获取OSS上的文件，一边读取一边写入到另外一个OSS文件 src = self.bucket.get_object(src_key) - result = self.bucket.put_object(dst_key, src) + + # Special handle for http20, Need add Content-Length to request header in this test case + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + test_header = {"Content-Length" : "1048576"} + result = self.bucket.put_object(dst_key, src, headers=test_header) + else: + result = self.bucket.put_object(dst_key, src) # verify self.assertTrue(src.client_crc is not None) @@ -239,6 +257,10 @@ def generator(): return generator() def test_data_generator(self): + # Special handle for http20, Ignore this test if http20 + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + return + key = self.random_key() key2 = self.random_key() content = random_bytes(1024 * 1024 + 1) @@ -406,7 +428,7 @@ def test_private_download_url(self): resp = requests.get(url) self.assertEqual(content, resp.content) - + def test_sign_url_with_callback(self): key = self.random_key() @@ -587,8 +609,12 @@ def test_object_acl(self): self.bucket.delete_object(key) def test_object_exists(self): + # Special handle for http20, Ignore this test if http20 + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + return + key = self.random_key() - + auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower()) self.assertRaises(NoSuchBucket, bucket.object_exists, key) @@ -609,20 +635,23 @@ def test_user_meta(self): self.assertEqual(headers['x-oss-meta-key2'], 'value2') def test_get_object_meta(self): + # Special handle for http20, Ignore this test if http20 + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + return + key = self.random_key() content = 'hello' - + # bucket no exist auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket = oss2.Bucket(auth, OSS_ENDPOINT, random_string(63).lower()) - + self.assertRaises(NoSuchBucket, bucket.get_object_meta, key) - + # object no exist self.assertRaises(NoSuchKey, self.bucket.get_object_meta, key) self.bucket.put_object(key, content) - # get meta normal result = self.bucket.get_object_meta(key) @@ -861,6 +890,10 @@ def test_get_symlink(self): self.assertEqual(result.target_key, key) def test_process_object(self): + # Special handle for http20, Ignore this test if http20 + if self.bucket.session.http_version is oss2.HTTP_VERSION_20: + return + key = self.random_key(".jpg") result = self.bucket.put_object_from_file(key, "tests/example.jpg") self.assertEqual(result.status, 200) @@ -910,6 +943,47 @@ def tearDown(self): os.environ['OSS_TEST_AUTH_VERSION'] = oss2.AUTH_VERSION_2 super(TestSign, self).tearDown() +class TestHttp20OverObject(TestObject): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestObject，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverObject, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverObject, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverObject, self).tearDown() + +class TestHttp20OverSign(TestSign): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestSign，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverSign, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSign, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSign, self).tearDown() if __name__ == '__main__': unittest.main() diff --git a/tests/test_sts.py b/tests/test_sts.py index 03243aa5..89282d2f 100644 --- a/tests/test_sts.py +++ b/tests/test_sts.py @@ -72,7 +72,8 @@ def init_bucket(self): self.token = fetch_sts_token(OSS_STS_ID, OSS_STS_KEY, OSS_STS_ARN) auth = oss2.StsAuth(self.token.access_key_id, self.token.access_key_secret, self.token.security_token) - self.bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET) + OSS_HTTP_VERSION = os.getenv('OSS_TEST_HTTP_VERSION') + self.bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET, http_version=OSS_HTTP_VERSION) def test_object(self): self.init_bucket() @@ -132,4 +133,49 @@ def tearDown(self): os.environ['OSS_TEST_AUTH_VERSION'] = oss2.AUTH_VERSION_1 else: os.environ['OSS_TEST_AUTH_VERSION'] = oss2.AUTH_VERSION_2 - super(TestSign, self).tearDown() \ No newline at end of file + super(TestSign, self).tearDown() + + class TestHttp20OverSts(TestSts): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestSts，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverSts, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSts, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSts, self).tearDown() + + class TestHttp20OverSign(TestSign): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestSign，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverSign, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSign, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverSign, self).tearDown() + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_upload.py b/tests/test_upload.py index 6a851234..4bfe4ad0 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -258,6 +258,27 @@ def check_not_sane(key, value): check_not_sane('key', None) check_not_sane('parts', None) +class TestHttp20OverUpload(TestUpload): + """ + 当环境变量使用oss2.HTTP11时，则重新设置为HTTP20, 再运行TestUpload，反之亦然 + """ + def __init__(self, *args, **kwargs): + super(TestHttp20OverUpload, self).__init__(*args, **kwargs) + + def setUp(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverUpload, self).setUp() + + def tearDown(self): + if os.getenv('OSS_TEST_HTTP_VERSION') == oss2.HTTP_VERSION_11: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_20 + else: + os.environ['OSS_TEST_HTTP_VERSION'] = oss2.HTTP_VERSION_11 + super(TestHttp20OverUpload, self).tearDown() + if __name__ == '__main__': unittest.main() From b7f952ad7ecd523c06699a469cf5df1643b043fe Mon Sep 17 00:00:00 2001 From: wanyuanyang Date: Sat, 29 Sep 2018 21:06:19 +0800 Subject: [PATCH 6/6] add set_content_length to headers --- oss2/headers.py | 3 +++ tests/test_headers.py | 10 +++++++++- tests/test_image.py | 4 +++- tests/test_object.py | 10 +++++++--- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/oss2/headers.py b/oss2/headers.py index 5ddf76ca..eed2da5b 100644 --- a/oss2/headers.py +++ b/oss2/headers.py @@ -47,3 +47,6 @@ def set_server_side_encryption(self, algorithm=None, cmk_id=None): self[OSS_SERVER_SIDE_ENCRYPTION] = "KMS" if cmk_id is not None: self[OSS_SERVER_SIDE_ENCRYPTION_KEY_ID] = cmk_id + + def set_content_length(self, content_length): + self["Content-Length"] = content_length diff --git a/tests/test_headers.py b/tests/test_headers.py index 62bc3797..757871a6 100644 --- a/tests/test_headers.py +++ b/tests/test_headers.py @@ -23,6 +23,14 @@ def test_check_requestHeader(self): self.assertTrue("x-oss-server-side-encryption" not in myHeader) self.assertTrue("x-oss-server-side-encryption-key-id" not in myHeader) + def test_set_content_length(self): + myHeader = RequestHeader() + + myHeader.set_content_length("100") + self.assertTrue(myHeader["Content-Length"] is "100") + + myHeader.set_content_length("200") + self.assertTrue(myHeader["Content-Length"] is "200") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_image.py b/tests/test_image.py index 93d0f144..b991b10f 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -6,6 +6,7 @@ import json from common import * +from oss2.headers import * class TestImage(OssTestCase): @@ -22,7 +23,8 @@ def __test(self, original_image, new_image, image_style): # Special handle for http20, Need add Content-Length to request header in this test case if self.bucket.session.http_version is oss2.HTTP_VERSION_20: - test_header = {"Content-Length" :original_image_content.headers['Content-Length']} + test_header = RequestHeader() + test_header.set_content_length(original_image_content.headers['Content-Length']) self.bucket.put_object(new_image, original_image_content, headers=test_header) else: self.bucket.put_object(new_image, original_image_content) diff --git a/tests/test_object.py b/tests/test_object.py index ff9a8c49..19c86e5e 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -10,6 +10,7 @@ NotFound, NoSuchKey, Conflict, PositionNotEqualToLength, ObjectNotAppendable) from oss2.compat import is_py2, is_py33 +from oss2.headers import * from common import * @@ -190,7 +191,8 @@ def test_object_empty(self): # Special handle for http20, Need add Content-Length to request header in this test case if self.bucket.session.http_version is oss2.HTTP_VERSION_20: - test_header = {"Content-Length":"0"} + test_header = RequestHeader() + test_header.set_content_length("0") self.bucket.put_object(key, content, headers=test_header) else: self.bucket.put_object(key, content) @@ -211,7 +213,8 @@ def test_file_empty(self): # Special handle for http20, Need add Content-Length to request header in this test case if self.bucket.session.http_version is oss2.HTTP_VERSION_20: - test_header = {"Content-Length":"0"} + test_header = RequestHeader() + test_header.set_content_length("0") self.bucket.put_object_from_file(key, input_filename, headers=test_header) else: self.bucket.put_object_from_file(key, input_filename) @@ -233,7 +236,8 @@ def test_streaming(self): # Special handle for http20, Need add Content-Length to request header in this test case if self.bucket.session.http_version is oss2.HTTP_VERSION_20: - test_header = {"Content-Length" : "1048576"} + test_header = RequestHeader() + test_header.set_content_length("1048576") result = self.bucket.put_object(dst_key, src, headers=test_header) else: result = self.bucket.put_object(dst_key, src)