Add support for CloudFront invalidating

.gitignore

@@ -11,6 +11,7 @@ coverage
 .vscode
 package/
 .local
+local
 .DS_Store
 
 # Unit test

charon.spec

@@ -80,6 +80,13 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
 
 %changelog
+* Mon Mar 25 2024 Gang Li <[email protected]>
+- 1.3.0 release
+- Add validate command: validate the checksum for maven artifacts
+- Add index command: support to re-index of the speicified folder
+- Add CF invalidating feature: invalidate generated metadata files (maven-metadata*/package.json/index.html) after product uploading/deleting in CloudFront
+- Add CF invalidating feature: add command to do CF invalidating and checking
+
 * Mon Sep 18 2023 Harsh Modi <[email protected]>
 - 1.2.2 release
 - hot fix for "dist_tags" derived issue

charon/cache.py

@@ -0,0 +1,137 @@
+from boto3 import session
+from botocore.exceptions import ClientError
+from typing import Dict, List
+import os
+import logging
+import uuid
+
+logger = logging.getLogger(__name__)
+
+ENDPOINT_ENV = "aws_endpoint_url"
+
+DEFAULT_BUCKET_TO_DOMAIN = {
+    "prod-maven-ga": "maven.repository.redhat.com",
+    "prod-maven-ea": "maven.repository.redhat.com",
+    "stage-maven-ga": "maven.strage.repository.redhat.com",
+    "stage-maven-ea": "maven.strage.repository.redhat.com",
+    "prod-npm": "npm.repository.redhat.com",
+    "stage-npm": "npm.stage.repository.redhat.com"
+}
+
+
+class CFClient(object):
+    """The CFClient is a wrapper of the original boto3 clouldfrong client,
+    which will provide CloudFront functions to be used in the charon.
+    """
+
+    def __init__(
+        self,
+        aws_profile=None,
+        extra_conf=None
+    ) -> None:
+        self.__client = self.__init_aws_client(aws_profile, extra_conf)
+
+    def __init_aws_client(
+        self, aws_profile=None, extra_conf=None
+    ):
+        if aws_profile:
+            logger.debug("[CloudFront] Using aws profile: %s", aws_profile)
+            cf_session = session.Session(profile_name=aws_profile)
+        else:
+            cf_session = session.Session()
+        endpoint_url = self.__get_endpoint(extra_conf)
+        return cf_session.client(
+            'cloudfront',
+            endpoint_url=endpoint_url
+        )
+
+    def __get_endpoint(self, extra_conf) -> str:
+        endpoint_url = os.getenv(ENDPOINT_ENV)
+        if not endpoint_url or not endpoint_url.strip():
+            if isinstance(extra_conf, Dict):
+                endpoint_url = extra_conf.get(ENDPOINT_ENV, None)
+        if endpoint_url:
+            logger.info(
+                "[CloudFront] Using endpoint url for aws CF client: %s",
+                endpoint_url
+            )
+        else:
+            logger.debug("[CloudFront] No user-specified endpoint url is used.")
+        return endpoint_url
+
+    def invalidate_paths(self, distr_id: str, paths: List[str]) -> Dict[str, str]:
+        """Send a invalidating requests for the paths in distribution to CloudFront.
+        This will invalidate the paths in the distribution to enforce the refreshment
+        from backend S3 bucket for these paths. For details see:
+        https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html
+            * The distr_id is the id for the distribution. This id can be get through
+            get_dist_id_by_domain(domain) function
+            * Can specify the invalidating paths through paths param.
+        """
+        caller_ref = str(uuid.uuid4())
+        logger.debug("[CloudFront] Creating invalidation for paths: %s", paths)
+        try:
+            response = self.__client.create_invalidation(
+                DistributionId=distr_id,
+                InvalidationBatch={
+                    'CallerReference': caller_ref,
+                    'Paths': {
+                        'Quantity': len(paths),
+                        'Items': paths
+                    }
+                }
+            )
+            if response:
+                invalidation = response.get('Invalidation', {})
+                return {
+                    'Id': invalidation.get('Id', None),
+                    'Status': invalidation.get('Status', None)
+                }
+        except Exception as err:
+            logger.error(
+                "[CloudFront] Error occurred while creating invalidation, error: %s", err
+            )
+
+    def check_invalidation(self, distr_id: str, invalidation_id: str) -> dict:
+        try:
+            response = self.__client.get_invalidation(
+                DistributionId=distr_id,
+                Id=invalidation_id
+            )
+            if response:
+                invalidation = response.get('Invalidation', {})
+                return {
+                    'Id': invalidation.get('Id', None),
+                    'CreateTime': invalidation.get('CreateTime', None),
+                    'Status': invalidation.get('Status', None)
+                }
+        except Exception as err:
+            logger.error(
+                "[CloudFront] Error occurred while check invalidation of id %s, "
+                "error: %s", invalidation_id, err
+            )
+
+    def get_dist_id_by_domain(self, domain: str) -> str:
+        """Get distribution id by a domain name. The id can be used to send invalidating
+        request through #invalidate_paths function
+           * Domain are Ronda domains, like "maven.repository.redhat.com"
+           or "npm.repository.redhat.com"
+        """
+        try:
+            response = self.__client.list_distributions()
+            if response:
+                dist_list_items = response.get("DistributionList", {}).get("Items", [])
+                for distr in dist_list_items:
+                    aliases_items = distr.get('Aliases', {}).get('Items', [])
+                    if aliases_items and domain in aliases_items:
+                        return distr['Id']
+            logger.error("[CloudFront]: Distribution not found for domain %s", domain)
+        except ClientError as err:
+            logger.error(
+                "[CloudFront]: Error occurred while get distribution for domain %s: %s",
+                domain, err
+            )
+        return None
+
+    def get_domain_by_bucket(self, bucket: str) -> str:
+        return DEFAULT_BUCKET_TO_DOMAIN.get(bucket, None)

charon/cmd/__init__.py

@@ -18,6 +18,7 @@
 from charon.cmd.cmd_delete import delete
 from charon.cmd.cmd_index import index
 from charon.cmd.cmd_checksum import validate
+from charon.cmd.cmd_cache import clear_cf
 
 
 @group()
@@ -33,3 +34,4 @@ def cli():
 cli.add_command(delete)
 cli.add_command(index)
 cli.add_command(validate)
+cli.add_command(clear_cf)

charon/cmd/cmd_cache.py

@@ -0,0 +1,126 @@
+"""
+Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon)
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from charon.config import get_config
+from charon.cmd.internal import _decide_mode, _get_buckets
+from charon.cache import CFClient
+from charon.pkgs.pkg_utils import invalidate_cf_paths
+from click import command, option
+from typing import List
+
+import traceback
+import logging
+import sys
+import os
+
+logger = logging.getLogger(__name__)
+
+
+@option(
+    "--target",
+    "-t",
+    "target",
+    help="""
+    The target to do the uploading, which will decide which s3 bucket
+    and what root path where all files will be uploaded to.
+    Can accept more than one target.
+    """,
+    required=True
+)
+@option(
+    "--path",
+    "-p",
+    "paths",
+    help="""
+    The paths which will be invalidated in CF. The path can use the format as CF defining
+    in https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html
+    """,
+    multiple=True
+)
+@option(
+    "--path-file",
+    "-f",
+    "path_file",
+    help="""
+    The file which contain the paths to be invalidated in CF. Pahts in this file follow the
+    format of CF defining too, and each path should be in a single line.
+    """
+)
+@option(
+    "--debug",
+    "-D",
+    "debug",
+    help="Debug mode, will print all debug logs for problem tracking.",
+    is_flag=True,
+    default=False
+)
+@option(
+    "--quiet",
+    "-q",
+    "quiet",
+    help="Quiet mode, will shrink most of the logs except warning and errors.",
+    is_flag=True,
+    default=False
+)
+@command()
+def clear_cf(
+    target: str,
+    paths: List[str],
+    path_file: str,
+    quiet: bool = False,
+    debug: bool = False
+):
+    """This command will do invalidating on AWS CloudFront for the specified paths.
+    """
+    _decide_mode(
+        f"cfclear-{target}", "",
+        is_quiet=quiet, is_debug=debug
+    )
+    if not paths and not path_file:
+        logger.error(
+            "No path specified, please specify at least one path "
+            "through --path or --path-file.")
+        sys.exit(1)
+
+    work_paths = []
+    if paths:
+        work_paths.extend(paths)
+
+    if path_file:
+        with open(path_file, "r", encoding="utf-8") as f:
+            for line in f.readlines():
+                work_paths.append(str(line).strip())
+
+    try:
+        conf = get_config()
+        if not conf:
+            sys.exit(1)
+
+        aws_profile = os.getenv("AWS_PROFILE") or conf.get_aws_profile()
+        if not aws_profile:
+            logger.error("No AWS profile specified!")
+            sys.exit(1)
+
+        buckets = _get_buckets([target], conf)
+
+        for b in buckets:
+            cf_client = CFClient(aws_profile=aws_profile)
+            invalidate_cf_paths(
+                cf_client, b, work_paths
+            )
+    except Exception:
+        print(traceback.format_exc())
+        sys.exit(2)

charon/cmd/cmd_delete.py

@@ -158,6 +158,7 @@ def delete(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
+                cf_enable=conf.is_aws_cf_enable(),
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )
@@ -178,6 +179,7 @@ def delete(
                 buckets=buckets,
                 aws_profile=aws_profile,
                 dir_=work_dir,
+                cf_enable=conf.is_aws_cf_enable(),
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
             )

charon/cmd/cmd_index.py

@@ -87,34 +87,31 @@ def index(
             # log is recorded get_target
             sys.exit(1)
 
-        aws_bucket = None
-        prefix = None
-        for b in conf.get_target(target):
+        for b in tgt:
             aws_bucket = b.get('bucket')
-            prefix = b.get('prefix', '')
 
-        package_type = None
-        if "maven" in aws_bucket:
-            logger.info(
-                "The target is a maven repository. Will refresh the index as maven package type"
-            )
-            package_type = PACKAGE_TYPE_MAVEN
-        elif "npm" in aws_bucket:
-            package_type = PACKAGE_TYPE_NPM
-            logger.info(
-                "The target is a npm repository. Will refresh the index as npm package type"
-            )
-        else:
-            logger.error(
-                "The target is not supported. Only maven or npm target is supported."
-            )
-            sys.exit(1)
+            package_type = None
+            if "maven" in aws_bucket:
+                logger.info(
+                    "The target is a maven repository. Will refresh the index as maven package type"
+                )
+                package_type = PACKAGE_TYPE_MAVEN
+            elif "npm" in aws_bucket:
+                package_type = PACKAGE_TYPE_NPM
+                logger.info(
+                    "The target is a npm repository. Will refresh the index as npm package type"
+                )
+            else:
+                logger.error(
+                    "The target %s is not supported. Only maven or npm target is supported.",
+                    target
+                )
 
-        if not aws_bucket:
-            logger.error("No bucket specified!")
-            sys.exit(1)
+            if not aws_bucket:
+                logger.error("No bucket specified for target %s!", target)
+            else:
+                re_index(b, path, package_type, aws_profile, dryrun)
 
-        re_index(aws_bucket, prefix, path, package_type, aws_profile, dryrun)
     except Exception:
         print(traceback.format_exc())
         sys.exit(2)  # distinguish between exception and bad config or bad state

charon/cmd/cmd_upload.py

@@ -178,6 +178,7 @@ def upload(
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 gen_sign=contain_signature,
+                cf_enable=conf.is_aws_cf_enable(),
                 key=sign_key,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name
@@ -200,6 +201,7 @@ def upload(
                 aws_profile=aws_profile,
                 dir_=work_dir,
                 gen_sign=contain_signature,
+                cf_enable=conf.is_aws_cf_enable(),
                 key=sign_key,
                 dry_run=dryrun,
                 manifest_bucket_name=manifest_bucket_name

charon/cmd/internal.py

@@ -35,7 +35,8 @@ def _get_buckets(targets: List[str], conf: CharonConfig) -> List[Tuple[str, str,
             aws_bucket = bucket.get('bucket')
             prefix = bucket.get('prefix', '')
             registry = bucket.get('registry', DEFAULT_REGISTRY)
-            buckets.append((target, aws_bucket, prefix, registry))
+            cf_domain = bucket.get('domain', None)
+            buckets.append((target, aws_bucket, prefix, registry, cf_domain))
     return buckets