From e3410d57110f1d6643314706a03f3cc65a15d19e Mon Sep 17 00:00:00 2001 From: Gang Li Date: Thu, 14 Mar 2024 15:31:27 +0800 Subject: [PATCH] Add support for CloudFront invalidating --- charon.spec | 8 ++ charon/cache.py | 134 ++++++++++++++++++++++++++ charon/cmd/cmd_delete.py | 2 + charon/cmd/cmd_index.py | 45 ++++----- charon/cmd/cmd_upload.py | 2 + charon/cmd/internal.py | 3 +- charon/config.py | 4 + charon/pkgs/indexing.py | 21 ++-- charon/pkgs/maven.py | 43 ++++++++- charon/pkgs/npm.py | 24 ++++- charon/pkgs/pkg_utils.py | 61 +++++++++--- setup.py | 2 +- tests/base.py | 36 ++++--- tests/constants.py | 29 ++++++ tests/requirements.txt | 4 +- tests/test_cf_in_maven_upload.py | 57 +++++++++++ tests/test_cfclient.py | 65 +++++++++++++ tests/test_manifest_del.py | 4 +- tests/test_manifest_upload.py | 4 +- tests/test_maven_del.py | 4 +- tests/test_maven_del_multi_tgts.py | 4 +- tests/test_maven_index.py | 10 +- tests/test_maven_index_multi_tgts.py | 4 +- tests/test_maven_sign.py | 4 +- tests/test_maven_upload.py | 4 +- tests/test_maven_upload_multi_tgts.py | 4 +- tests/test_npm_del.py | 4 +- tests/test_npm_del_multi_tgts.py | 4 +- tests/test_npm_dist_gen.py | 4 +- tests/test_npm_index.py | 22 ++++- tests/test_npm_index_multi_tgts.py | 4 +- tests/test_npm_meta.py | 4 +- tests/test_npm_upload.py | 4 +- tests/test_npm_upload_multi_tgts.py | 4 +- tests/test_pkgs_dryrun.py | 4 +- tests/test_s3client.py | 4 +- 36 files changed, 532 insertions(+), 108 deletions(-) create mode 100644 charon/cache.py create mode 100644 tests/test_cf_in_maven_upload.py create mode 100644 tests/test_cfclient.py diff --git a/charon.spec b/charon.spec index ffb77655..195f0806 100644 --- a/charon.spec +++ b/charon.spec @@ -80,6 +80,14 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog +* Fri Mar 29 2024 Gang Li +- 1.3.0 release +- Add validate command: validate the checksum for maven artifacts +- Add index command: support to re-index of the speicified folder +- Add CF invalidating features: + - Invalidate generated metadata files (maven-metadata*/package.json/index.html) after product uploading/deleting in CloudFront + - Add command to do CF invalidating and checking + * Mon Sep 18 2023 Harsh Modi - 1.2.2 release - hot fix for "dist_tags" derived issue diff --git a/charon/cache.py b/charon/cache.py new file mode 100644 index 00000000..b289f8ca --- /dev/null +++ b/charon/cache.py @@ -0,0 +1,134 @@ +from boto3 import session +from botocore.exceptions import ClientError +from typing import Dict, List +import os +import logging +import uuid + +logger = logging.getLogger(__name__) + +ENDPOINT_ENV = "aws_endpoint_url" + +DEFAULT_BUCKET_TO_DOMAIN = { + "prod-maven-ga": "maven.repository.redhat.com", + "prod-maven-ea": "maven.repository.redhat.com", + "stage-maven-ga": "maven.strage.repository.redhat.com", + "stage-maven-ea": "maven.strage.repository.redhat.com", + "prod-npm": "npm.repository.redhat.com", + "stage-npm": "npm.stage.repository.redhat.com" +} + + +class CFClient(object): + """The CFClient is a wrapper of the original boto3 clouldfrong client, + which will provide CloudFront functions to be used in the charon. + """ + + def __init__( + self, + aws_profile=None, + extra_conf=None + ) -> None: + self.__client = self.__init_aws_client(aws_profile, extra_conf) + + def __init_aws_client( + self, aws_profile=None, extra_conf=None + ): + if aws_profile: + logger.debug("Using aws profile: %s", aws_profile) + cf_session = session.Session(profile_name=aws_profile) + else: + cf_session = session.Session() + endpoint_url = self.__get_endpoint(extra_conf) + return cf_session.client( + 'cloudfront', + endpoint_url=endpoint_url + ) + + def __get_endpoint(self, extra_conf) -> str: + endpoint_url = os.getenv(ENDPOINT_ENV) + if not endpoint_url or not endpoint_url.strip(): + if isinstance(extra_conf, Dict): + endpoint_url = extra_conf.get(ENDPOINT_ENV, None) + if endpoint_url: + logger.info("Using endpoint url for aws client: %s", endpoint_url) + else: + logger.debug("No user-specified endpoint url is used.") + return endpoint_url + + def invalidate_paths(self, distr_id: str, paths: List[str]) -> Dict[str, str]: + """Send a invalidating requests for the paths in distribution to CloudFront. + This will invalidate the paths in the distribution to enforce the refreshment + from backend S3 bucket for these paths. For details see: + https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html + * The distr_id is the id for the distribution. This id can be get through + get_dist_id_by_domain(domain) function + * Can specify the invalidating paths through paths param. + """ + caller_ref = str(uuid.uuid4()) + logger.debug("[CloudFront] Creating invalidation for paths: %s", paths) + try: + response = self.__client.create_invalidation( + DistributionId=distr_id, + InvalidationBatch={ + 'CallerReference': caller_ref, + 'Paths': { + 'Quantity': len(paths), + 'Items': paths + } + } + ) + if response: + invalidation = response.get('Invalidation', {}) + return { + 'Id': invalidation.get('Id', None), + 'Status': invalidation.get('Status', None) + } + except Exception as err: + logger.error( + "[CloudFront] Error occurred while creating invalidation, error: %s", err + ) + + def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict: + try: + response = self.__client.get_invalidation( + DistributionId=distr_id, + Id=invalidation_id + ) + if response: + invalidation = response.get('Invalidation', {}) + return { + 'Id': invalidation.get('Id', None), + 'CreateTime': invalidation.get('CreateTime', None), + 'Status': invalidation.get('Status', None) + } + except Exception as err: + logger.error( + "[CloudFront] Error occurred while check invalidation of id %s, " + "error: %s", invalidation_id, err + ) + + def get_dist_id_by_domain(self, domain: str) -> str: + """Get distribution id by a domain name. The id can be used to send invalidating + request through #invalidate_paths function + * Domain are Ronda domains, like "maven.repository.redhat.com" + or "npm.repository.redhat.com" + """ + try: + response = self.__client.list_distributions() + if response: + dist_list_items = response.get("DistributionList", {}).get("Items", []) + for distr in dist_list_items: + aliases_items = distr.get('Aliases', {}).get('Items', []) + if aliases_items and domain in aliases_items: + return distr['Id'] + logger.error("[CloudFront]: Distribution not found for domain %s", domain) + except ClientError as err: + logger.error( + "[CloudFront]: Error occurred while get distribution for domain %s: %s", + domain, err + ) + return None + + def get_domain_by_bucket(self, bucket: str) -> str: + return DEFAULT_BUCKET_TO_DOMAIN.get(bucket, None) diff --git a/charon/cmd/cmd_delete.py b/charon/cmd/cmd_delete.py index fd2bba5f..dda57d2e 100644 --- a/charon/cmd/cmd_delete.py +++ b/charon/cmd/cmd_delete.py @@ -158,6 +158,7 @@ def delete( buckets=buckets, aws_profile=aws_profile, dir_=work_dir, + cf_enable=conf.is_aws_cf_enable(), dry_run=dryrun, manifest_bucket_name=manifest_bucket_name ) @@ -178,6 +179,7 @@ def delete( buckets=buckets, aws_profile=aws_profile, dir_=work_dir, + cf_enable=conf.is_aws_cf_enable(), dry_run=dryrun, manifest_bucket_name=manifest_bucket_name ) diff --git a/charon/cmd/cmd_index.py b/charon/cmd/cmd_index.py index 281ed876..e9a3e18c 100644 --- a/charon/cmd/cmd_index.py +++ b/charon/cmd/cmd_index.py @@ -87,34 +87,31 @@ def index( # log is recorded get_target sys.exit(1) - aws_bucket = None - prefix = None - for b in conf.get_target(target): + for b in tgt: aws_bucket = b.get('bucket') - prefix = b.get('prefix', '') - package_type = None - if "maven" in aws_bucket: - logger.info( - "The target is a maven repository. Will refresh the index as maven package type" - ) - package_type = PACKAGE_TYPE_MAVEN - elif "npm" in aws_bucket: - package_type = PACKAGE_TYPE_NPM - logger.info( - "The target is a npm repository. Will refresh the index as npm package type" - ) - else: - logger.error( - "The target is not supported. Only maven or npm target is supported." - ) - sys.exit(1) + package_type = None + if "maven" in aws_bucket: + logger.info( + "The target is a maven repository. Will refresh the index as maven package type" + ) + package_type = PACKAGE_TYPE_MAVEN + elif "npm" in aws_bucket: + package_type = PACKAGE_TYPE_NPM + logger.info( + "The target is a npm repository. Will refresh the index as npm package type" + ) + else: + logger.error( + "The target %s is not supported. Only maven or npm target is supported.", + target + ) - if not aws_bucket: - logger.error("No bucket specified!") - sys.exit(1) + if not aws_bucket: + logger.error("No bucket specified for target %s!", target) + else: + re_index(b, path, package_type, aws_profile, dryrun) - re_index(aws_bucket, prefix, path, package_type, aws_profile, dryrun) except Exception: print(traceback.format_exc()) sys.exit(2) # distinguish between exception and bad config or bad state diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index 71c30295..2fe19901 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -178,6 +178,7 @@ def upload( aws_profile=aws_profile, dir_=work_dir, gen_sign=contain_signature, + cf_enable=conf.is_aws_cf_enable(), key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name @@ -200,6 +201,7 @@ def upload( aws_profile=aws_profile, dir_=work_dir, gen_sign=contain_signature, + cf_enable=conf.is_aws_cf_enable(), key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name diff --git a/charon/cmd/internal.py b/charon/cmd/internal.py index cb76559a..edc87c05 100644 --- a/charon/cmd/internal.py +++ b/charon/cmd/internal.py @@ -35,7 +35,8 @@ def _get_buckets(targets: List[str], conf: CharonConfig) -> List[Tuple[str, str, aws_bucket = bucket.get('bucket') prefix = bucket.get('prefix', '') registry = bucket.get('registry', DEFAULT_REGISTRY) - buckets.append((target, aws_bucket, prefix, registry)) + cf_domain = bucket.get('domain', None) + buckets.append((target, aws_bucket, prefix, registry, cf_domain)) return buckets diff --git a/charon/config.py b/charon/config.py index 8f128617..f9b6403c 100644 --- a/charon/config.py +++ b/charon/config.py @@ -38,6 +38,7 @@ def __init__(self, data: Dict): self.__manifest_bucket: str = data.get("manifest_bucket", None) self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) + self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -63,6 +64,9 @@ def get_ignore_signature_suffix(self, package_type: str) -> List[str]: def get_detach_signature_command(self) -> str: return self.__signature_command + def is_aws_cf_enable(self) -> bool: + return self.__aws_cf_enable + def get_config() -> Optional[CharonConfig]: config_file_path = os.path.join(os.getenv("HOME"), ".charon", CONFIG_FILE) diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 0ab43057..bd9192ab 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -15,13 +15,15 @@ """ from charon.config import get_template from charon.storage import S3Client +from charon.cache import CFClient +from charon.pkgs.pkg_utils import invalidate_cf_paths from charon.constants import (INDEX_HTML_TEMPLATE, NPM_INDEX_HTML_TEMPLATE, PACKAGE_TYPE_MAVEN, PACKAGE_TYPE_NPM, PROD_INFO_SUFFIX) from charon.utils.files import digest_content from jinja2 import Template import os import logging -from typing import List, Set +from typing import List, Set, Tuple from charon.utils.strings import remove_prefix @@ -259,21 +261,23 @@ def __compare(self, other) -> int: def re_index( - bucket: str, - prefix: str, + bucket: Tuple[str, str, str, str, str], path: str, package_type: str, aws_profile: str = None, + cf_enable: bool = False, dry_run: bool = False ): """Refresh the index.html for the specified folder in the bucket. """ + bucket_name = bucket[1] + prefix = bucket[2] s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run) real_prefix = prefix if prefix.strip() != "/" else "" s3_folder = os.path.join(real_prefix, path) if path.strip() == "" or path.strip() == "/": s3_folder = prefix - items: List[str] = s3_client.list_folder_content(bucket, s3_folder) + items: List[str] = s3_client.list_folder_content(bucket_name, s3_folder) contents = [i for i in items if not i.endswith(PROD_INFO_SUFFIX)] if PACKAGE_TYPE_NPM == package_type: if any([True if "package.json" in c else False for c in contents]): @@ -303,14 +307,17 @@ def re_index( index_path = os.path.join(path, "index.html") if path == "/": index_path = "index.html" - s3_client.simple_delete_file(index_path, (bucket, real_prefix)) + s3_client.simple_delete_file(index_path, (bucket_name, real_prefix)) s3_client.simple_upload_file( - index_path, index_content, (bucket, real_prefix), + index_path, index_content, (bucket_name, real_prefix), "text/html", digest_content(index_content) ) + if cf_enable: + cf_client = CFClient(aws_profile=aws_profile) + invalidate_cf_paths(cf_client, bucket, [index_path]) else: logger.warning( "The path %s does not contain any contents in bucket %s. " "Will not do any re-indexing", - path, bucket + path, bucket_name ) diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9fd57422..72bf6bf8 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -20,7 +20,12 @@ from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix from charon.storage import S3Client -from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process +from charon.cache import CFClient +from charon.pkgs.pkg_utils import ( + upload_post_process, + rollback_post_process, + invalidate_cf_paths +) from charon.config import CharonConfig, get_template, get_config from charon.constants import (META_FILE_GEN_KEY, META_FILE_DEL_KEY, META_FILE_FAILED, MAVEN_METADATA_TEMPLATE, @@ -257,11 +262,12 @@ def handle_maven_uploading( prod_key: str, ignore_patterns=None, root="maven-repository", - buckets: List[Tuple[str, str, str, str]] = None, + buckets: List[Tuple[str, str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, gen_sign=False, + cf_enable=False, key=None, dry_run=False, manifest_bucket_name=None @@ -322,6 +328,9 @@ def handle_maven_uploading( succeeded = True generated_signs = [] for bucket in buckets: + # prepare cf invalidate files + cf_invalidate_paths = [] + # 5. Do manifest uploading if not manifest_bucket_name: logger.warning( @@ -360,6 +369,9 @@ def handle_maven_uploading( ) failed_metas.extend(_failed_metas) logger.info("maven-metadata.xml updating done in bucket %s\n", bucket_name) + # Add maven-metadata.xml to CF invalidate paths + if cf_enable: + cf_invalidate_paths.extend(meta_files.get(META_FILE_GEN_KEY, [])) # 8. Determine refreshment of archetype-catalog.xml if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")): @@ -386,6 +398,9 @@ def handle_maven_uploading( ) failed_metas.extend(_failed_metas) logger.info("archetype-catalog.xml updating done in bucket %s\n", bucket_name) + # Add archtype-catalog to invalidate paths + if cf_enable: + cf_invalidate_paths.extend(archetype_files) # 10. Generate signature file if contain_signature is set to True if gen_sign: @@ -436,9 +451,17 @@ def handle_maven_uploading( ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") + # Add index files to Cf invalidate paths + if cf_enable: + cf_invalidate_paths.extend(created_indexes) else: logger.info("Bypass indexing") + # Finally do the CF invalidating for metadata files + if cf_enable and len(cf_invalidate_paths) > 0: + cf_client = CFClient(aws_profile=aws_profile) + invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths, top_level) + upload_post_process(failed_files, failed_metas, prod_key, bucket_name) succeeded = succeeded and len(failed_files) <= 0 and len(failed_metas) <= 0 @@ -450,10 +473,11 @@ def handle_maven_del( prod_key: str, ignore_patterns=None, root="maven-repository", - buckets: List[Tuple[str, str, str, str]] = None, + buckets: List[Tuple[str, str, str, str, str]] = None, aws_profile=None, dir_=None, do_index=True, + cf_enable=False, dry_run=False, manifest_bucket_name=None ) -> Tuple[str, bool]: @@ -487,6 +511,9 @@ def handle_maven_del( logger.debug("Valid poms: %s", valid_poms) succeeded = True for bucket in buckets: + # prepare cf invalidation paths + cf_invalidate_paths = [] + prefix = remove_prefix(bucket[2], "/") s3_client = S3Client(aws_profile=aws_profile, dry_run=dry_run) bucket_name = bucket[1] @@ -544,6 +571,8 @@ def handle_maven_del( if len(_failed_metas) > 0: failed_metas.extend(_failed_metas) logger.info("maven-metadata.xml updating done\n") + if cf_enable: + cf_invalidate_paths.extend(meta_files.get(META_FILE_GEN_KEY, [])) # 7. Determine refreshment of archetype-catalog.xml if os.path.exists(os.path.join(top_level, "archetype-catalog.xml")): @@ -578,6 +607,8 @@ def handle_maven_del( if len(_failed_metas) > 0: failed_metas.extend(_failed_metas) logger.info("archetype-catalog.xml updating done\n") + if cf_enable: + cf_invalidate_paths.extend(archetype_files) if do_index: logger.info("Start generating index files for all changed entries") @@ -596,9 +627,15 @@ def handle_maven_del( if len(_failed_index_files) > 0: failed_metas.extend(_failed_index_files) logger.info("Index files updating done.\n") + if cf_enable: + cf_invalidate_paths.extend(created_indexes) else: logger.info("Bypassing indexing") + if cf_enable and len(cf_invalidate_paths): + cf_client = CFClient(aws_profile=aws_profile) + invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths, top_level) + rollback_post_process(failed_files, failed_metas, prod_key, bucket_name) succeeded = succeeded and len(failed_files) == 0 and len(failed_metas) == 0 diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 684e8457..376116c3 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -28,8 +28,13 @@ from charon.config import CharonConfig, get_config from charon.constants import META_FILE_GEN_KEY, META_FILE_DEL_KEY, PACKAGE_TYPE_NPM from charon.storage import S3Client +from charon.cache import CFClient from charon.utils.archive import extract_npm_tarball -from charon.pkgs.pkg_utils import upload_post_process, rollback_post_process +from charon.pkgs.pkg_utils import ( + upload_post_process, + rollback_post_process, + invalidate_cf_paths +) from charon.utils.strings import remove_prefix from charon.utils.files import write_manifest from charon.utils.map import del_none, replace_field @@ -78,6 +83,7 @@ def handle_npm_uploading( dir_=None, do_index=True, gen_sign=False, + cf_enable=False, key=None, dry_run=False, manifest_bucket_name=None @@ -96,9 +102,13 @@ def handle_npm_uploading( Returns the directory used for archive processing and if uploading is successful """ + client = S3Client(aws_profile=aws_profile, dry_run=dry_run) generated_signs = [] for bucket in buckets: + # prepare cf invalidate files + cf_invalidate_paths = [] + bucket_name = bucket[1] prefix = remove_prefix(bucket[2], "/") registry = bucket[3] @@ -159,6 +169,8 @@ def handle_npm_uploading( client, bucket_name, target_dir, package_metadata, prefix ) logger.info("package.json generation done\n") + if cf_enable: + cf_invalidate_paths.extend(meta_files.get(META_FILE_GEN_KEY, [])) if META_FILE_GEN_KEY in meta_files: _failed_metas = client.upload_metadatas( @@ -218,9 +230,16 @@ def handle_npm_uploading( ) failed_metas.extend(_failed_metas) logger.info("Index files updating done\n") + if cf_enable: + cf_invalidate_paths.extend(created_indexes) else: logger.info("Bypass indexing\n") + # Do CloudFront invalidating for generated metadata + if cf_enable and len(cf_invalidate_paths): + cf_client = CFClient(aws_profile=aws_profile) + invalidate_cf_paths(cf_client, bucket, cf_invalidate_paths) + upload_post_process(failed_files, failed_metas, product, bucket_name) succeeded = succeeded and len(failed_files) == 0 and len(failed_metas) == 0 @@ -234,6 +253,7 @@ def handle_npm_del( aws_profile=None, dir_=None, do_index=True, + cf_enable=False, dry_run=False, manifest_bucket_name=None ) -> Tuple[str, str]: @@ -476,7 +496,7 @@ def _scan_for_version(path: str): logger.error('Error: Failed to parse json!') -def _is_latest_version(source_version: str, versions: list()): +def _is_latest_version(source_version: str, versions: List[str]): for v in versions: if compare(source_version, v) <= 0: return False diff --git a/charon/pkgs/pkg_utils.py b/charon/pkgs/pkg_utils.py index 20ffc71b..a206f697 100644 --- a/charon/pkgs/pkg_utils.py +++ b/charon/pkgs/pkg_utils.py @@ -1,5 +1,7 @@ -from typing import List +from typing import List, Tuple +from charon.cache import CFClient import logging +import os logger = logging.getLogger(__name__) @@ -44,15 +46,52 @@ def __post_process( product_key, operation, bucket) else: total = len(failed_files) + len(failed_metas) - logger.error("%d file(s) occur errors/warnings in bucket %s, " - "please see errors.log for details.\n", - bucket, total) - logger.error("Product release %s is %s Ronda service in bucket %s, " - "but has some failures as below:", - product_key, operation, bucket) + logger.error( + "%d file(s) occur errors/warnings in bucket %s, " + "please see errors.log for details.\n", + bucket, total + ) + logger.error( + "Product release %s is %s Ronda service in bucket %s, " + "but has some failures as below:", + product_key, operation, bucket + ) if len(failed_files) > 0: - logger.error("Failed files: \n%s\n", - failed_files) + logger.error("Failed files: \n%s\n", failed_files) if len(failed_metas) > 0: - logger.error("Failed metadata files: \n%s\n", - failed_metas) + logger.error("Failed metadata files: \n%s\n", failed_metas) + + +def invalidate_cf_paths( + cf_client: CFClient, + bucket: Tuple[str, str, str, str, str], + invalidate_paths: List[str], + root="/" +): + logger.info("Invalidating CF cache for %s", bucket[1]) + bucket_name = bucket[1] + prefix = bucket[2] + prefix = "/" + prefix if not prefix.startswith("/") else prefix + domain = bucket[4] + slash_root = root + if not root.endswith("/"): + slash_root = slash_root + "/" + final_paths = [] + for full_path in invalidate_paths: + path = full_path + if path.startswith(slash_root): + path = path[len(slash_root):] + if prefix: + path = os.path.join(prefix, path) + final_paths.append(path) + logger.debug("Invalidating paths: %s", final_paths) + if not domain: + domain = cf_client.get_domain_by_bucket(bucket_name) + distr_id = cf_client.get_dist_id_by_domain(domain) + if distr_id: + result = cf_client.invalidate_paths(distr_id, final_paths) + if result: + logger.info( + "The CF invalidating request for metadata/indexing is sent, " + "request id %s, status is %s", result['Id'], result['Status'] + ) diff --git a/setup.py b/setup.py index ae737355..98d6be4d 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ """ from setuptools import setup, find_packages -version = "1.2.2" +version = "1.3.0" # f = open('README.md') # long_description = f.read().strip() diff --git a/tests/base.py b/tests/base.py index 49cd2f1e..00f47cbf 100644 --- a/tests/base.py +++ b/tests/base.py @@ -25,9 +25,12 @@ from charon.pkgs.pkg_utils import is_metadata from charon.storage import PRODUCT_META_KEY, CHECKSUM_META_KEY from tests.commons import TEST_BUCKET, TEST_MANIFEST_BUCKET -from moto import mock_s3 - from tests.constants import HERE +from moto import mock_aws +import logging + +logging.basicConfig(level=logging.INFO) +logging.getLogger("charon").setLevel(logging.DEBUG) SHORT_TEST_PREFIX = "ga" LONG_TEST_PREFIX = "earlyaccess/all" @@ -38,7 +41,21 @@ def setUp(self): self.change_home() config_base = self.get_config_base() self.__prepare_template(config_base) - default_config_content = """ + config_content = self.get_config_content() + self.prepare_config(config_base, config_content) + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + os.environ = self.old_environ + + def change_home(self): + self.old_environ = os.environ.copy() + self.tempdir = tempfile.mkdtemp(prefix='charon-test-') + # Configure environment and copy templates + os.environ['HOME'] = self.tempdir + + def get_config_content(self): + return """ ignore_patterns: - ".*^(redhat).*" - ".*snapshot.*" @@ -69,17 +86,6 @@ def setUp(self): aws_profile: "test" manifest_bucket: "manifest" """ - self.prepare_config(config_base, default_config_content) - - def tearDown(self): - shutil.rmtree(self.tempdir, ignore_errors=True) - os.environ = self.old_environ - - def change_home(self): - self.old_environ = os.environ.copy() - self.tempdir = tempfile.mkdtemp(prefix='charon-test-') - # Configure environment and copy templates - os.environ['HOME'] = self.tempdir def __prepare_template(self, config_base): template_path = os.path.join(config_base, 'template') @@ -101,7 +107,7 @@ def get_config_base(self) -> str: return os.path.join(self.get_temp_dir(), '.charon') -@mock_s3 +@mock_aws class PackageBaseTest(BaseTest): def setUp(self): super().setUp() diff --git a/tests/constants.py b/tests/constants.py index 2e6d111f..7ca6df89 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -17,3 +17,32 @@ HERE = os.path.dirname(__file__) INPUTS = os.path.join(HERE, 'input') + +TEST_DS_CONFIG = { + 'CallerReference': 'test', + "Aliases": { + "Quantity": 1, + "Items": [ + "maven.repository.redhat.com" + ] + }, + "Origins": { + "Quantity": 1, + "Items": [ + { + "Id": "prod-maven-ga", + "DomainName": "prod-maven-ga.s3.us-east-1.amazonaws.com", + "OriginPath": "", + "CustomHeaders": { + "Quantity": 0 + }, + } + ] + }, + "DefaultCacheBehavior": { + "TargetOriginId": "prod-maven-ga", + "ViewerProtocolPolicy": "allow-all", + }, + "Comment": "", + "Enabled": True + } diff --git a/tests/requirements.txt b/tests/requirements.txt index ff6f91ae..4acad34d 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,9 @@ flexmock>=0.10.6 -responses>=0.9.0,<0.10.8 +responses>=0.9.0 pytest<=7.1.3 pytest-cov pytest-html flake8 requests-mock -moto==3.0.7 +moto==5.0.3 python-gnupg==0.5.0 diff --git a/tests/test_cf_in_maven_upload.py b/tests/test_cf_in_maven_upload.py new file mode 100644 index 00000000..0651e606 --- /dev/null +++ b/tests/test_cf_in_maven_upload.py @@ -0,0 +1,57 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from charon.pkgs.maven import handle_maven_uploading +from tests.base import PackageBaseTest +from tests.constants import TEST_DS_CONFIG +from tests.commons import TEST_BUCKET +from tests.constants import INPUTS +from moto import mock_aws +import boto3 +import os + + +@mock_aws +class CFInMavenUploadTest(PackageBaseTest): + def setUp(self): + super().setUp() + # mock_cf is used to generate expected content + self.mock_cf = self.__prepare_cf() + response = self.mock_cf.create_distribution(DistributionConfig=TEST_DS_CONFIG) + self.test_dist_id = response.get('Distribution').get('Id') + + def __prepare_cf(self): + return boto3.client('cloudfront') + + def test_cf_after_upload(self): + response = self.mock_cf.list_invalidations(DistributionId=self.test_dist_id) + self.assertIsNotNone(response) + self.assertEqual(0, response.get('InvalidationList').get('Quantity')) + + test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") + product = "commons-client-4.5.6" + handle_maven_uploading( + test_zip, product, + buckets=[('', TEST_BUCKET, 'ga', '', 'maven.repository.redhat.com')], + dir_=self.tempdir, + do_index=False, + cf_enable=True + ) + + response = self.mock_cf.list_invalidations(DistributionId=self.test_dist_id) + self.assertEqual(1, response.get('InvalidationList').get('Quantity')) + items = response.get('InvalidationList').get('Items') + self.assertEqual(1, len(items)) + self.assertEqual('completed', str.lower(items[0].get('Status'))) diff --git a/tests/test_cfclient.py b/tests/test_cfclient.py new file mode 100644 index 00000000..610c454b --- /dev/null +++ b/tests/test_cfclient.py @@ -0,0 +1,65 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from tests.base import BaseTest +from tests.constants import TEST_DS_CONFIG +from charon.cache import CFClient +from moto import mock_aws +import boto3 +import pytest + + +@mock_aws +class CFClientTest(BaseTest): + def setUp(self): + super().setUp() + # mock_cf is used to generate expected content + self.mock_cf = self.__prepare_cf() + response = self.mock_cf.create_distribution(DistributionConfig=TEST_DS_CONFIG) + self.test_dist_id = response.get('Distribution').get('Id') + # cf_client is the client we will test + self.cf_client = CFClient() + + def tearDown(self): + self.mock_cf.delete_distribution(Id=self.test_dist_id, IfMatch=".") + super().tearDown() + + def __prepare_cf(self): + return boto3.client('cloudfront') + + def test_get_distribution_id(self): + dist_id = self.cf_client.get_dist_id_by_domain("maven.repository.redhat.com") + self.assertIsNotNone(dist_id) + dist_id = self.cf_client.get_dist_id_by_domain("notexists.redhat.com") + self.assertIsNone(dist_id) + + def test_invalidate_paths(self): + dist_id = self.cf_client.get_dist_id_by_domain("maven.repository.redhat.com") + result = self.cf_client.invalidate_paths(dist_id, ["/*"]) + self.assertIsNotNone(result['Id']) + self.assertEqual('completed', str.lower(result['Status'])) + status = self.cf_client.invalidate_paths("noexists_id", ["/*"]) + self.assertIsNone(status) + + @pytest.mark.skip(reason=""" + Because current moto 5.0.3 has not implemented the get_invalidation(), + this test will fail. Will enable it when the it is implemented in future moto + """) + def test_check_invalidation(self): + dist_id = self.cf_client.get_dist_id_by_domain("maven.repository.redhat.com") + result = self.cf_client.invalidate_paths(dist_id, ["/*"]) + invalidation = self.cf_client.check_invalidation(dist_id, result['Id']) + self.assertIsNotNone(invalidation['Id']) + self.assertEqual('completed', str.lower(result['Status'])) diff --git a/tests/test_manifest_del.py b/tests/test_manifest_del.py index fc5ff35c..b5d42255 100644 --- a/tests/test_manifest_del.py +++ b/tests/test_manifest_del.py @@ -15,7 +15,7 @@ """ import os -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.maven import handle_maven_uploading, handle_maven_del from charon.pkgs.npm import handle_npm_uploading, handle_npm_del @@ -28,7 +28,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class ManifestDeleteTest(PackageBaseTest): def test_maven_manifest_delete(self): diff --git a/tests/test_manifest_upload.py b/tests/test_manifest_upload.py index e6aa43e9..8a76de8d 100644 --- a/tests/test_manifest_upload.py +++ b/tests/test_manifest_upload.py @@ -15,7 +15,7 @@ """ import os -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.maven import handle_maven_uploading from charon.pkgs.npm import handle_npm_uploading @@ -29,7 +29,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class ManifestUploadTest(PackageBaseTest): def test_maven_manifest_upload(self): diff --git a/tests/test_maven_del.py b/tests/test_maven_del.py index c26e6d4a..9ce85eaa 100644 --- a/tests/test_maven_del.py +++ b/tests/test_maven_del.py @@ -24,13 +24,13 @@ ARCHETYPE_CATALOG, ARCHETYPE_CATALOG_FILES, COMMONS_CLIENT_459_MVN_NUM, COMMONS_CLIENT_META_NUM ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenDeleteTest(PackageBaseTest): def test_maven_deletion(self): self.__test_prefix_deletion("") diff --git a/tests/test_maven_del_multi_tgts.py b/tests/test_maven_del_multi_tgts.py index ffc60954..c3c93713 100644 --- a/tests/test_maven_del_multi_tgts.py +++ b/tests/test_maven_del_multi_tgts.py @@ -24,13 +24,13 @@ ARCHETYPE_CATALOG, ARCHETYPE_CATALOG_FILES, COMMONS_CLIENT_459_MVN_NUM, COMMONS_CLIENT_META_NUM, TEST_BUCKET_2 ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenDeleteMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_maven_index.py b/tests/test_maven_index.py index 7468310d..445b8e92 100644 --- a/tests/test_maven_index.py +++ b/tests/test_maven_index.py @@ -24,13 +24,13 @@ COMMONS_LOGGING_INDEXES, COMMONS_CLIENT_INDEX, COMMONS_CLIENT_456_INDEX, COMMONS_LOGGING_INDEX, COMMONS_ROOT_INDEX ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenFileIndexTest(PackageBaseTest): def test_uploading_index(self): @@ -177,7 +177,11 @@ def test_re_index(self): Key=commons_client_457_test, Body="Just a test content" ) - re_index(TEST_BUCKET, "", commons_client_root, "maven") + re_index( + (TEST_BUCKET, TEST_BUCKET, "", "", None), + commons_client_root, "maven", + cf_enable=True + ) indedx_obj = test_bucket.Object(COMMONS_CLIENT_INDEX) index_content = str(indedx_obj.get()["Body"].read(), "utf-8") self.assertIn('../', index_content) diff --git a/tests/test_maven_index_multi_tgts.py b/tests/test_maven_index_multi_tgts.py index a02707f2..ddd7bb12 100644 --- a/tests/test_maven_index_multi_tgts.py +++ b/tests/test_maven_index_multi_tgts.py @@ -23,13 +23,13 @@ COMMONS_LOGGING_INDEXES, COMMONS_CLIENT_INDEX, COMMONS_CLIENT_456_INDEX, COMMONS_LOGGING_INDEX, COMMONS_ROOT_INDEX, TEST_BUCKET_2 ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenFileIndexMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_maven_sign.py b/tests/test_maven_sign.py index 41cab15e..52df5690 100644 --- a/tests/test_maven_sign.py +++ b/tests/test_maven_sign.py @@ -19,13 +19,13 @@ TEST_BUCKET, COMMONS_CLIENT_456_SIGNS, COMMONS_LOGGING_SIGNS, COMMONS_CLIENT_456_INDEX, COMMONS_CLIENT_459_SIGNS ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenFileSignTest(PackageBaseTest): def test_uploading_sign(self): diff --git a/tests/test_maven_upload.py b/tests/test_maven_upload.py index 431475a8..c47d1695 100644 --- a/tests/test_maven_upload.py +++ b/tests/test_maven_upload.py @@ -23,13 +23,13 @@ COMMONS_CLIENT_456_MVN_NUM, COMMONS_CLIENT_MVN_NUM, COMMONS_CLIENT_META_NUM ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenUploadTest(PackageBaseTest): def test_fresh_upload(self): self.__test_prefix_upload("") diff --git a/tests/test_maven_upload_multi_tgts.py b/tests/test_maven_upload_multi_tgts.py index ffb41d20..921e8a9d 100644 --- a/tests/test_maven_upload_multi_tgts.py +++ b/tests/test_maven_upload_multi_tgts.py @@ -24,13 +24,13 @@ COMMONS_CLIENT_456_MVN_NUM, COMMONS_CLIENT_MVN_NUM, COMMONS_CLIENT_META_NUM, TEST_BUCKET_2 ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class MavenUploadMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_npm_del.py b/tests/test_npm_del.py index 8d6a6df0..5f734b26 100644 --- a/tests/test_npm_del.py +++ b/tests/test_npm_del.py @@ -14,7 +14,7 @@ limitations under the License. """ import os -from moto import mock_s3 +from moto import mock_aws from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY @@ -23,7 +23,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class NPMDeleteTest(PackageBaseTest): def test_npm_deletion(self): self.__test_prefix() diff --git a/tests/test_npm_del_multi_tgts.py b/tests/test_npm_del_multi_tgts.py index ac1e2c32..a6401db6 100644 --- a/tests/test_npm_del_multi_tgts.py +++ b/tests/test_npm_del_multi_tgts.py @@ -14,7 +14,7 @@ limitations under the License. """ import os -from moto import mock_s3 +from moto import mock_aws from charon.constants import PROD_INFO_SUFFIX, DEFAULT_REGISTRY from charon.pkgs.npm import handle_npm_uploading, handle_npm_del from charon.storage import CHECKSUM_META_KEY @@ -23,7 +23,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class NPMDeleteMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_npm_dist_gen.py b/tests/test_npm_dist_gen.py index 438cc094..7fbf58c0 100644 --- a/tests/test_npm_dist_gen.py +++ b/tests/test_npm_dist_gen.py @@ -15,7 +15,7 @@ """ import os import subresource_integrity -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.npm import handle_npm_uploading from charon.utils.files import digest, HashType from tests.base import PackageBaseTest @@ -26,7 +26,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class NPMUploadTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_npm_index.py b/tests/test_npm_index.py index 02dc64e0..f6745c3c 100644 --- a/tests/test_npm_index.py +++ b/tests/test_npm_index.py @@ -22,7 +22,7 @@ TEST_BUCKET, CODE_FRAME_7_14_5_INDEXES, CODE_FRAME_7_15_8_INDEXES, COMMONS_ROOT_INDEX ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS @@ -30,7 +30,7 @@ NAMESPACE_BABEL_INDEX = "@babel/index.html" -@mock_s3 +@mock_aws class NpmFileIndexTest(PackageBaseTest): def test_uploading_index(self): self.__test_upload_prefix() @@ -222,7 +222,11 @@ def test_re_index(self): test_bucket.put_object( Key=test_file_path, Body="test content" ) - re_index(TEST_BUCKET, prefix, "@babel/", "npm") + re_index( + (TEST_BUCKET, TEST_BUCKET, prefix, "", None), + "@babel/", "npm", + cf_enable=True + ) index_obj = test_bucket.Object(prefixed_namespace_babel_index) index_content = str(index_obj.get()["Body"].read(), "utf-8") self.assertIn( @@ -249,7 +253,11 @@ def test_re_index(self): test_bucket.put_object( Key=test_file_path, Body="test content" ) - re_index(TEST_BUCKET, prefix, "/", "npm") + re_index( + (TEST_BUCKET, TEST_BUCKET, prefix, "", None), + "/", "npm", + cf_enable=True + ) index_obj = test_bucket.Object(prefixed_root_index) index_content = str(index_obj.get()["Body"].read(), "utf-8") self.assertIn('@babel/', index_content) @@ -277,7 +285,11 @@ def test_re_index(self): test_bucket.put_object( Key=test_file_path, Body="test content" ) - re_index(TEST_BUCKET, prefix, metadata_path, "npm") + re_index( + (TEST_BUCKET, TEST_BUCKET, prefix, "", None), + metadata_path, "npm", + cf_enable=True + ) objs = list(test_bucket.objects.all()) actual_files = [obj.key for obj in objs] self.assertIn( diff --git a/tests/test_npm_index_multi_tgts.py b/tests/test_npm_index_multi_tgts.py index ef653303..acb882a4 100644 --- a/tests/test_npm_index_multi_tgts.py +++ b/tests/test_npm_index_multi_tgts.py @@ -22,7 +22,7 @@ CODE_FRAME_7_15_8_INDEXES, COMMONS_ROOT_INDEX, TEST_BUCKET_2 ) -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS @@ -30,7 +30,7 @@ NAMESPACE_BABEL_INDEX = "@babel/index.html" -@mock_s3 +@mock_aws class NpmFileIndexMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_npm_meta.py b/tests/test_npm_meta.py index a0627e2b..6d112efd 100644 --- a/tests/test_npm_meta.py +++ b/tests/test_npm_meta.py @@ -16,7 +16,7 @@ import os import boto3 -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.npm import handle_npm_uploading, read_package_metadata_from_content from charon.storage import S3Client @@ -27,7 +27,7 @@ MY_BUCKET = "npm_bucket" -@mock_s3 +@mock_aws class NPMMetadataOnS3Test(BaseTest): def setUp(self): super().setUp() diff --git a/tests/test_npm_upload.py b/tests/test_npm_upload.py index 3438ad61..53767301 100644 --- a/tests/test_npm_upload.py +++ b/tests/test_npm_upload.py @@ -15,7 +15,7 @@ """ import os -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.npm import handle_npm_uploading from charon.pkgs.pkg_utils import is_metadata @@ -29,7 +29,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class NPMUploadTest(PackageBaseTest): def test_npm_upload(self): diff --git a/tests/test_npm_upload_multi_tgts.py b/tests/test_npm_upload_multi_tgts.py index 82a265f7..242937a7 100644 --- a/tests/test_npm_upload_multi_tgts.py +++ b/tests/test_npm_upload_multi_tgts.py @@ -15,7 +15,7 @@ """ import os -from moto import mock_s3 +from moto import mock_aws from charon.pkgs.npm import handle_npm_uploading from charon.pkgs.pkg_utils import is_metadata @@ -29,7 +29,7 @@ from tests.constants import INPUTS -@mock_s3 +@mock_aws class NPMUploadMultiTgtsTest(PackageBaseTest): def setUp(self): super().setUp() diff --git a/tests/test_pkgs_dryrun.py b/tests/test_pkgs_dryrun.py index 7f2b004e..3b82d1b4 100644 --- a/tests/test_pkgs_dryrun.py +++ b/tests/test_pkgs_dryrun.py @@ -18,13 +18,13 @@ from charon.constants import DEFAULT_REGISTRY from tests.base import PackageBaseTest from tests.commons import TEST_BUCKET -from moto import mock_s3 +from moto import mock_aws import os from tests.constants import INPUTS -@mock_s3 +@mock_aws class PkgsDryRunTest(PackageBaseTest): def test_maven_upload_dry_run(self): test_zip = os.path.join(INPUTS, "commons-client-4.5.6.zip") diff --git a/tests/test_s3client.py b/tests/test_s3client.py index b33e68d7..48063daa 100644 --- a/tests/test_s3client.py +++ b/tests/test_s3client.py @@ -19,7 +19,7 @@ from charon.utils.files import overwrite_file, read_sha1 from charon.constants import PROD_INFO_SUFFIX from tests.base import BaseTest, SHORT_TEST_PREFIX -from moto import mock_s3 +from moto import mock_aws import boto3 import os import sys @@ -35,7 +35,7 @@ COMMONS_LANG3_ZIP_MVN_ENTRY = 26 -@mock_s3 +@mock_aws class S3ClientTest(BaseTest): def setUp(self): super().setUp()