From 827a0cb414ac38f1315f23d7491177fb95508242 Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 7 Mar 2022 21:54:32 -0500 Subject: [PATCH 1/3] add bucket name delimiter --- README.rst | 1 + guillotina_s3storage/storage.py | 16 +++++++++++++++- guillotina_s3storage/tests/fixtures.py | 1 + guillotina_s3storage/tests/test_storage.py | 17 +++++++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9ce0656..11c66f9 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,7 @@ Example config.json: "aws_client_id": "", "aws_client_secret": "", "bucket": "", + "bucket_name_format": "{container}{delimiter}{base}", "endpoint_url": null, "ssl": true, "verify_ssl": null, diff --git a/guillotina_s3storage/storage.py b/guillotina_s3storage/storage.py index b8fa708..ce2ca14 100644 --- a/guillotina_s3storage/storage.py +++ b/guillotina_s3storage/storage.py @@ -315,9 +315,23 @@ def __init__(self, settings, loop=None): self._bucket_name = settings["bucket"] + self._bucket_name_format = settings.get( + "bucket_name_format", "{container}{delimiter}{base}" + ) + async def get_bucket_name(self): container = task_vars.container.get() - bucket_name = container.id.lower() + "." + self._bucket_name + + if "." in self._bucket_name: + char_delimiter = "." + else: + char_delimiter = "-" + + bucket_name = self._bucket_name_format.format( + container=container.id.lower(), + delimiter=char_delimiter, + base=self._bucket_name, + ) bucket_name = bucket_name.replace("_", "-") diff --git a/guillotina_s3storage/tests/fixtures.py b/guillotina_s3storage/tests/fixtures.py index 20ed4c1..40aa117 100644 --- a/guillotina_s3storage/tests/fixtures.py +++ b/guillotina_s3storage/tests/fixtures.py @@ -17,6 +17,7 @@ def settings_configurator(settings): "factory": "guillotina_s3storage.storage.S3BlobStore", "settings": { "bucket": os.environ.get("S3CLOUD_BUCKET", "testbucket"), + "bucket_name_format": "{container}{delimiter}{base}", "aws_client_id": os.environ.get("S3CLOUD_ID", "x" * 10), "aws_client_secret": os.environ.get("S3CLOUD_SECRET", "x" * 10), # noqa }, diff --git a/guillotina_s3storage/tests/test_storage.py b/guillotina_s3storage/tests/test_storage.py index 7bcdd74..2ead2dc 100644 --- a/guillotina_s3storage/tests/test_storage.py +++ b/guillotina_s3storage/tests/test_storage.py @@ -775,3 +775,20 @@ async def test_read_range(own_dummy_request, mock_txn): async for chunk in s3mng.read_range(100, 200): assert len(chunk) == 100 assert chunk == _test_gif[100:200] + + +async def test_custom_bucket_name(dummy_request): + util = get_utility(IS3BlobStore) + request = dummy_request # noqa + login() + container = create_content(Container, id="test-container") + task_vars.container.set(container) + + # No dots in the name, delimiter is - + bucket_name = await util.get_bucket_name() + assert bucket_name.startswith("test-container-testbucket") + + # Dots in the name, delimiter is . + util._bucket_name = "testbucket.com" + bucket_name = await util.get_bucket_name() + assert bucket_name.startswith("test-container.testbucket.com") From ea9b75861de0396fec7380deff84815abb34568a Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 7 Mar 2022 21:54:39 -0500 Subject: [PATCH 2/3] changelog --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index daab6fd..2925c3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ -5.0.9 (unreleased) +5.0.9 ------------------ -- Nothing changed yet. +- Allow bucket name delimiter config 5.0.8 (2021-05-06) From 9fa1c95755444226c6ea51d068f6275f17bfb45d Mon Sep 17 00:00:00 2001 From: Ismael Date: Mon, 7 Mar 2022 22:02:33 -0500 Subject: [PATCH 3/3] formatting --- guillotina_s3storage/interfaces.py | 9 +++------ guillotina_s3storage/storage.py | 4 ++-- guillotina_s3storage/tests/test_storage.py | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/guillotina_s3storage/interfaces.py b/guillotina_s3storage/interfaces.py index 0c66f5c..d5470b9 100644 --- a/guillotina_s3storage/interfaces.py +++ b/guillotina_s3storage/interfaces.py @@ -5,18 +5,15 @@ class IS3FileField(IFileField): - """Field marked as S3FileField - """ + """Field marked as S3FileField""" # Configuration Utility class IS3BlobStore(Interface): - """Configuration utility. - """ + """Configuration utility.""" class IS3File(IFile): - """Marker for a S3File - """ + """Marker for a S3File""" diff --git a/guillotina_s3storage/storage.py b/guillotina_s3storage/storage.py index ce2ca14..7d1868d 100644 --- a/guillotina_s3storage/storage.py +++ b/guillotina_s3storage/storage.py @@ -3,8 +3,10 @@ import logging from typing import AsyncIterator +import aiobotocore import aiohttp import backoff +import botocore from guillotina import configure from guillotina import task_vars from guillotina.component import get_utility @@ -19,8 +21,6 @@ from guillotina.schema import Object from zope.interface import implementer -import aiobotocore -import botocore from guillotina_s3storage.interfaces import IS3BlobStore from guillotina_s3storage.interfaces import IS3File from guillotina_s3storage.interfaces import IS3FileField diff --git a/guillotina_s3storage/tests/test_storage.py b/guillotina_s3storage/tests/test_storage.py index 2ead2dc..6ee9650 100644 --- a/guillotina_s3storage/tests/test_storage.py +++ b/guillotina_s3storage/tests/test_storage.py @@ -3,6 +3,7 @@ from hashlib import md5 import backoff +import botocore.exceptions import pytest from guillotina import task_vars from guillotina.component import get_utility @@ -16,7 +17,6 @@ from guillotina.tests.utils import login from zope.interface import Interface -import botocore.exceptions from guillotina_s3storage.interfaces import IS3BlobStore from guillotina_s3storage.storage import CHUNK_SIZE from guillotina_s3storage.storage import RETRIABLE_EXCEPTIONS