diff --git a/requirements-dev.txt b/requirements-dev.txt index 62f7e9e836..762d6146bb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,7 +12,7 @@ types-setuptools types-boto types-pytz flake8>=3.8.4,<7 -flake8-bugbear>=20.11.1,<21 +flake8-bugbear>=20.11.1,<24 black isort pydocstyle diff --git a/src/toil/common.py b/src/toil/common.py index 728a4a8b9c..2f1c0723c6 100644 --- a/src/toil/common.py +++ b/src/toil/common.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os import pickle @@ -36,6 +37,7 @@ ContextManager, Dict, List, + MutableMapping, Optional, Set, Tuple, @@ -67,7 +69,7 @@ QueueSizeMessage, gen_message_bus_path) from toil.fileStores import FileID -from toil.lib.aws import zone_to_region +from toil.lib.aws import zone_to_region, build_tag_dict_from_env from toil.lib.compatibility import deprecated from toil.lib.conversions import bytes2human, human2bytes from toil.lib.io import try_path diff --git a/src/toil/jobStores/aws/jobStore.py b/src/toil/jobStores/aws/jobStore.py index 864437f336..b164120c69 100644 --- a/src/toil/jobStores/aws/jobStore.py +++ b/src/toil/jobStores/aws/jobStore.py @@ -35,6 +35,7 @@ from botocore.exceptions import ClientError import toil.lib.encryption as encryption +from toil.lib.aws import build_tag_dict_from_env from toil.fileStores import FileID from toil.jobStores.abstractJobStore import (AbstractJobStore, ConcurrentFileModificationException, @@ -56,8 +57,7 @@ ReadableTransformingPipe, WritablePipe) from toil.lib.aws.session import establish_boto3_session -from toil.lib.aws.utils import (build_tag_dict_from_env, - create_s3_bucket, +from toil.lib.aws.utils import (create_s3_bucket, flatten_tags, get_bucket_region, get_object_for_url, diff --git a/src/toil/lib/aws/__init__.py b/src/toil/lib/aws/__init__.py index e3a9e9549a..03c4584896 100644 --- a/src/toil/lib/aws/__init__.py +++ b/src/toil/lib/aws/__init__.py @@ -25,6 +25,7 @@ Dict, Iterable, List, + MutableMapping, Optional, TypeVar, Union) @@ -172,3 +173,23 @@ def running_on_ecs() -> bool: """ # We only care about relatively current ECS return 'ECS_CONTAINER_METADATA_URI_V4' in os.environ + +def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]: + tags = dict() + owner_tag = environment.get('TOIL_OWNER_TAG') + if owner_tag: + tags.update({'Owner': owner_tag}) + + user_tags = environment.get('TOIL_AWS_TAGS') + if user_tags: + try: + json_user_tags = json.loads(user_tags) + if isinstance(json_user_tags, dict): + tags.update(json.loads(user_tags)) + else: + logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') + exit(1) + except json.decoder.JSONDecodeError: + logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') + exit(1) + return tags diff --git a/src/toil/lib/aws/utils.py b/src/toil/lib/aws/utils.py index 2b0d89466e..38f44fabb7 100644 --- a/src/toil/lib/aws/utils.py +++ b/src/toil/lib/aws/utils.py @@ -28,7 +28,8 @@ Optional, Set, Union, - cast) + cast, + MutableMapping) from urllib.parse import ParseResult from toil.lib.aws import session @@ -41,10 +42,8 @@ retry) if sys.version_info >= (3, 8): - from typing import Literal, MutableMapping + from typing import Literal else: - from typing import MutableMapping - from typing_extensions import Literal try: @@ -403,28 +402,6 @@ def list_objects_for_url(url: ParseResult) -> List[str]: logger.debug('Found in %s items: %s', url, listing) return listing - -def build_tag_dict_from_env(environment: MutableMapping[str, str] = os.environ) -> Dict[str, str]: - tags = dict() - owner_tag = environment.get('TOIL_OWNER_TAG') - if owner_tag: - tags.update({'Owner': owner_tag}) - - user_tags = environment.get('TOIL_AWS_TAGS') - if user_tags: - try: - json_user_tags = json.loads(user_tags) - if isinstance(json_user_tags, dict): - tags.update(json.loads(user_tags)) - else: - logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') - exit(1) - except json.decoder.JSONDecodeError: - logger.error('TOIL_AWS_TAGS must be in JSON format: {"key" : "value", ...}') - exit(1) - return tags - - def flatten_tags(tags: Dict[str, str]) -> List[Dict[str, str]]: """ Convert tags from a key to value dict into a list of 'Key': xxx, 'Value': xxx dicts. diff --git a/src/toil/test/lib/aws/test_utils.py b/src/toil/test/lib/aws/test_utils.py index 2f34d8a1d3..3de3862a94 100644 --- a/src/toil/test/lib/aws/test_utils.py +++ b/src/toil/test/lib/aws/test_utils.py @@ -18,7 +18,7 @@ import pytest -from toil.lib.aws.utils import build_tag_dict_from_env +from toil.lib.aws import build_tag_dict_from_env from toil.test import ToilTest logger = logging.getLogger(__name__) diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py index 9e9a676e04..27bea8ad77 100644 --- a/src/toil/utils/toilLaunchCluster.py +++ b/src/toil/utils/toilLaunchCluster.py @@ -19,7 +19,7 @@ from toil import applianceSelf from toil.common import parser_with_common_options -from toil.lib.aws.utils import build_tag_dict_from_env +from toil.lib.aws import build_tag_dict_from_env from toil.provisioners import (check_valid_node_types, cluster_factory, parse_node_types) diff --git a/version_template.py b/version_template.py index 6116aae7ae..678c18a4f1 100644 --- a/version_template.py +++ b/version_template.py @@ -28,7 +28,7 @@ # - don't import even standard modules at global scope without renaming them # to have leading/trailing underscores -baseVersion = '5.9.1' +baseVersion = '5.9.2' cgcloudVersion = '1.6.0a1.dev393'