vpc-* commands version checked; fixed circular import dependency

Signed-off-by: Chris Helma <[email protected]>

manage_arkime/commands/cluster_create.py

@@ -20,6 +20,7 @@
 from core.local_file import LocalFile, S3File
 from core.usage_report import UsageReport
 from core.price_report import PriceReport
+import core.compatibility as compat
 from core.capacity_planning import (get_capture_node_capacity_plan, get_viewer_node_capacity_plan, get_ecs_sys_resource_plan, get_os_domain_plan,
                                     ClusterPlan, VpcPlan, MINIMUM_TRAFFIC, DEFAULT_SPI_DAYS, DEFAULT_REPLICAS, get_capture_vpc_plan,
                                     S3Plan, DEFAULT_S3_STORAGE_CLASS, DEFAULT_S3_STORAGE_DAYS, DEFAULT_HISTORY_DAYS,
@@ -42,8 +43,8 @@ def cmd_cluster_create(profile: str, region: str, name: str, expected_traffic: f
     is_initial_invocation = _is_initial_invocation(name, aws_provider)
     if not is_initial_invocation:
         try:
-            ver.confirm_aws_aio_version_compatibility(name, aws_provider)
-        except (ver.CliClusterVersionMismatch, ver.CaptureViewerVersionMismatch, ver.UnableToRetrieveClusterVersion) as e:
+            compat.confirm_aws_aio_version_compatibility(name, aws_provider)
+        except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
             logger.error(e)
             logger.warning("Aborting...")
             return

manage_arkime/commands/cluster_deregister_vpc.py

@@ -4,9 +4,9 @@
 from aws_interactions.aws_client_provider import AwsClientProvider
 import aws_interactions.iam_interactions as iami
 import aws_interactions.ssm_operations as ssm_ops
+import core.compatibility as compat
 import core.constants as constants
 from core.cross_account_wrangling import CrossAccountAssociation, remove_vpce_permissions
-import core.versioning as ver
 
 logger = logging.getLogger(__name__)
 
@@ -17,8 +17,8 @@ def cmd_cluster_deregister_vpc(profile: str, region: str, cluster_name: str, vpc
     aws_provider = AwsClientProvider(aws_profile=profile, aws_region=region)
 
     try:
-        ver.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
-    except (ver.CliClusterVersionMismatch, ver.CaptureViewerVersionMismatch, ver.UnableToRetrieveClusterVersion) as e:
+        compat.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
         logger.error(e)
         logger.warning("Aborting...")
         return

manage_arkime/commands/cluster_destroy.py

@@ -8,8 +8,8 @@
 from aws_interactions.ssm_operations import get_ssm_param_json_value, get_ssm_param_value, get_ssm_names_by_path, delete_ssm_param, ParamDoesNotExist
 from cdk_interactions.cdk_client import CdkClient
 from core.capacity_planning import ClusterPlan
+import core.compatibility as compat
 import core.constants as constants
-import core.versioning as ver
 import cdk_interactions.cdk_context as context
 
 logger = logging.getLogger(__name__)
@@ -26,8 +26,8 @@ def cmd_cluster_destroy(profile: str, region: str, name: str, destroy_everything
     cdk_client = CdkClient(aws_provider.get_aws_env())
 
     try:
-        ver.confirm_aws_aio_version_compatibility(name, aws_provider)
-    except (ver.CliClusterVersionMismatch, ver.CaptureViewerVersionMismatch, ver.UnableToRetrieveClusterVersion) as e:
+        compat.confirm_aws_aio_version_compatibility(name, aws_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
         logger.error(e)
         logger.warning("Aborting...")
         return

manage_arkime/commands/cluster_register_vpc.py

@@ -3,9 +3,9 @@
 
 from aws_interactions.aws_client_provider import AwsClientProvider
 import aws_interactions.ssm_operations as ssm_ops
+import core.compatibility as compat
 import core.constants as constants
 from core.cross_account_wrangling import CrossAccountAssociation, ensure_cross_account_role_exists, add_vpce_permissions
-import core.versioning as ver
 
 logger = logging.getLogger(__name__)
 
@@ -17,8 +17,8 @@ def cmd_cluster_register_vpc(profile: str, region: str, cluster_name: str, vpc_a
     aws_env = aws_provider.get_aws_env()
 
     try:
-        ver.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
-    except (ver.CliClusterVersionMismatch, ver.CaptureViewerVersionMismatch, ver.UnableToRetrieveClusterVersion) as e:
+        compat.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
         logger.error(e)
         logger.warning("Aborting...")
         return

manage_arkime/commands/config_update.py

@@ -9,6 +9,7 @@
 import aws_interactions.ecs_interactions as ecs
 import aws_interactions.s3_interactions as s3
 import aws_interactions.ssm_operations as ssm_ops
+import core.compatibility as compat
 import core.constants as constants
 from core.local_file import LocalFile, S3File
 import core.versioning as ver
@@ -31,8 +32,8 @@ def cmd_config_update(profile: str, region: str, cluster_name: str, capture: boo
     aws_env = aws_provider.get_aws_env()
 
     try:
-        ver.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
-    except (ver.CliClusterVersionMismatch, ver.CaptureViewerVersionMismatch, ver.UnableToRetrieveClusterVersion) as e:
+        compat.confirm_aws_aio_version_compatibility(cluster_name, aws_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
         logger.error(e)
         logger.warning("Aborting...")
         return

manage_arkime/commands/vpc_add.py

@@ -9,6 +9,7 @@
 import cdk_interactions.cdk_context as context
 from cdk_interactions.cdk_client import CdkClient
 import cdk_interactions.cfn_wrangling as cfn
+import core.compatibility as compat
 import core.constants as constants
 from core.cross_account_wrangling import CrossAccountAssociation
 from core.vni_provider import SsmVniProvider, VniAlreadyUsed, VniOutsideRange, VniPoolExhausted
@@ -68,11 +69,11 @@ def cmd_vpc_add(profile: str, region: str, cluster_name: str, vpc_id: str, user_
             logger.warning("Aborting...")
             return
 
-    # Confirm the Cluster exists before proceeding
+    # Confirm the Cluster exists and is compatible before proceeding
     try:
-        ssm_ops.get_ssm_param_value(constants.get_cluster_ssm_param_name(cluster_name), cluster_acct_provider)
-    except ssm_ops.ParamDoesNotExist:
-        logger.error(f"The cluster {cluster_name} does not exist; try using the clusters-list command to see the clusters you have created.")
+        compat.confirm_aws_aio_version_compatibility(cluster_name, cluster_acct_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
+        logger.error(e)
         logger.warning("Aborting...")
         return
 

manage_arkime/commands/vpc_remove.py

@@ -6,6 +6,7 @@
 import aws_interactions.ssm_operations as ssm_ops
 from cdk_interactions.cdk_client import CdkClient
 import cdk_interactions.cdk_context as context
+import core.compatibility as compat
 import core.constants as constants
 from core.cross_account_wrangling import CrossAccountAssociation
 from core.vni_provider import SsmVniProvider
@@ -44,15 +45,20 @@ def cmd_vpc_remove(profile: str, region: str, cluster_name: str, vpc_id: str):
                      + " Aborting...")
         return
 
-    # Confirm the Cluster exists before proceeding
+    # Confirm the Cluster exists and is compatible before proceeding
     try:
-        vpce_service_id = ssm_ops.get_ssm_param_json_value(constants.get_cluster_ssm_param_name(cluster_name), "vpceServiceId", cluster_acct_provider)
-    except ssm_ops.ParamDoesNotExist:
-        logger.error(f"The cluster {cluster_name} does not exist; try using the clusters-list command to see the clusters you have created.")
+        compat.confirm_aws_aio_version_compatibility(cluster_name, cluster_acct_provider)
+    except (compat.CliClusterVersionMismatch, compat.CaptureViewerVersionMismatch, compat.UnableToRetrieveClusterVersion) as e:
+        logger.error(e)
         logger.warning("Aborting...")
         return
 
     # Pull all our deployed configuration from SSM and tear down the ENI-specific resources
+    vpce_service_id = ssm_ops.get_ssm_param_json_value(
+        constants.get_cluster_ssm_param_name(cluster_name),
+        "vpceServiceId",
+        cluster_acct_provider
+    )
     vpc_ssm_param = constants.get_vpc_ssm_param_name(cluster_name, vpc_id)
     event_bus_arn = ssm_ops.get_ssm_param_json_value(vpc_ssm_param, "busArn", vpc_acct_provider)
     subnet_search_path = f"{vpc_ssm_param}/subnets"

manage_arkime/core/compatibility.py

@@ -0,0 +1,60 @@
+import json
+
+import arkime_interactions.config_wrangling as config_wrangling
+from aws_interactions.aws_client_provider import AwsClientProvider
+import aws_interactions.ssm_operations as ssm_ops
+import core.constants as constants
+from core.versioning import AWS_AIO_VERSION
+
+class UnableToRetrieveClusterVersion(Exception):
+    def __init__(self, cluster_name: str, cli_version: int):
+        super().__init__(f"It appears the cluster {cluster_name} does not exist.  There's also a chance the AWS AIO version"
+                        + f" of the CLI ({cli_version}) is incompatible with your Cluster.  If you're confident the Cluster"
+                        + " exists, you can try checking the AWS AIO version of your cluster using the clusters-list"
+                        + " command.  The CLI and Cluster versions must match.")
+
+class CaptureViewerVersionMismatch(Exception):
+    def __init__(self, capture_version: int, viewer_version: int):
+        super().__init__(f"The AWS AIO versions of your Capture ({capture_version}) and Viewer ({viewer_version})"
+                         + " components do not match.  This is unexpected and should not happen.  Please cut us a"
+                         + " ticket at: https://github.com/arkime/aws-aio/issues/new")
+
+class CliClusterVersionMismatch(Exception):
+    def __init__(self, cli_version: int, cluster_version: int):
+        super().__init__(f"The AWS AIO versions of your CLI ({cli_version}) and Cluster ({cluster_version}) do not"
+                         + " match.  This is likely to result in unexpected behavior.  Please change your CLI to the"
+                         + f" latest minor version under the major version ({cluster_version}).  Check out the"
+                         + " following README section for more details:"
+                         + " https://github.com/arkime/aws-aio#aws-aio-version-mismatch")
+
+def confirm_aws_aio_version_compatibility(cluster_name: str, aws_provider: AwsClientProvider,
+                                          cli_version: int = AWS_AIO_VERSION):
+    # Unfortunately, it currently appears impossible to distinguish between the scenarios where the cluster doesn't
+    # exist and the cluster exists but is a different version.  In either case, we could get the ParamDoesNotExist
+    # exception.
+    try:
+        raw_capture_details_val = ssm_ops.get_ssm_param_value(
+            constants.get_capture_config_details_ssm_param_name(cluster_name),
+            aws_provider
+        )
+        capture_config_details = config_wrangling.ConfigDetails.from_dict(json.loads(raw_capture_details_val))
+
+        raw_viewer_details_val = ssm_ops.get_ssm_param_value(
+            constants.get_viewer_config_details_ssm_param_name(cluster_name),
+            aws_provider
+        )
+        viewer_config_details = config_wrangling.ConfigDetails.from_dict(json.loads(raw_viewer_details_val))
+    except ssm_ops.ParamDoesNotExist:
+        raise UnableToRetrieveClusterVersion(cluster_name, cli_version)
+
+    capture_version = int(capture_config_details.version.aws_aio_version)
+    viewer_version = int(viewer_config_details.version.aws_aio_version)
+
+    if capture_version != viewer_version:
+        raise CaptureViewerVersionMismatch(capture_version, viewer_version)
+
+    if capture_version != cli_version:
+        raise CliClusterVersionMismatch(cli_version, capture_version)
+
+    # Everything matches, we're good to go
+    return

manage_arkime/core/versioning.py

@@ -1,14 +1,9 @@
 from dataclasses import dataclass
 from datetime import datetime, timezone
 import hashlib
-import json
 import logging
 from typing import Dict
 
-import arkime_interactions.config_wrangling as config_wrangling
-from aws_interactions.aws_client_provider import AwsClientProvider
-import aws_interactions.ssm_operations as ssm_ops
-import core.constants as constants
 from core.local_file import LocalFile
 from core.shell_interactions import call_shell_command
 
@@ -81,56 +76,3 @@ def get_version_info(config_file: LocalFile, config_version: str = None) -> Vers
         get_source_version(),
         datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
     )
-
-class UnableToRetrieveClusterVersion(Exception):
-    def __init__(self, cluster_name: str, cli_version: int):
-        super().__init__(f"It appears the cluster {cluster_name} does not exist.  There's also a chance the AWS AIO version"
-                        + f" of the CLI ({cli_version}) is incompatible with your Cluster.  If you're confident the Cluster"
-                        + " exists, you can try checking the AWS AIO version of your cluster using the clusters-list"
-                        + " command.  The CLI and Cluster versions must match.")
-
-class CaptureViewerVersionMismatch(Exception):
-    def __init__(self, capture_version: int, viewer_version: int):
-        super().__init__(f"The AWS AIO versions of your Capture ({capture_version}) and Viewer ({viewer_version})"
-                         + " components do not match.  This is unexpected and should not happen.  Please cut us a"
-                         + " ticket at: https://github.com/arkime/aws-aio/issues/new")
-
-class CliClusterVersionMismatch(Exception):
-    def __init__(self, cli_version: int, cluster_version: int):
-        super().__init__(f"The AWS AIO versions of your CLI ({cli_version}) and Cluster ({cluster_version}) do not"
-                         + " match.  This is likely to result in unexpected behavior.  Please change your CLI to the"
-                         + f" latest minor version under the major version ({cluster_version}).  Check out the"
-                         + " following README section for more details:"
-                         + " https://github.com/arkime/aws-aio#aws-aio-version-mismatch")
-
-def confirm_aws_aio_version_compatibility(cluster_name: str, aws_provider: AwsClientProvider,
-                                          cli_version: int = AWS_AIO_VERSION):
-    # Unfortunately, it currently appears impossible to distinguish between the scenarios where the cluster doesn't
-    # exist and the cluster exists but is a different version.  In either case, we could get the ParamDoesNotExist
-    # exception.
-    try:
-        raw_capture_details_val = ssm_ops.get_ssm_param_value(
-            constants.get_capture_config_details_ssm_param_name(cluster_name),
-            aws_provider
-        )
-        capture_config_details = config_wrangling.ConfigDetails.from_dict(json.loads(raw_capture_details_val))
-
-        raw_viewer_details_val = ssm_ops.get_ssm_param_value(
-            constants.get_viewer_config_details_ssm_param_name(cluster_name),
-            aws_provider
-        )
-        viewer_config_details = config_wrangling.ConfigDetails.from_dict(json.loads(raw_viewer_details_val))
-    except ssm_ops.ParamDoesNotExist:
-        raise UnableToRetrieveClusterVersion(cluster_name, cli_version)
-
-    capture_version = int(capture_config_details.version.aws_aio_version)
-    viewer_version = int(viewer_config_details.version.aws_aio_version)
-
-    if capture_version != viewer_version:
-        raise CaptureViewerVersionMismatch(capture_version, viewer_version)
-
-    if capture_version != cli_version:
-        raise CliClusterVersionMismatch(cli_version, capture_version)
-
-    # Everything matches, we're good to go
-    return

test_manage_arkime/commands/test_cluster_create.py

@@ -13,14 +13,15 @@
 from commands.cluster_create import (cmd_cluster_create, _set_up_viewer_cert, _get_next_capacity_plan, _get_next_user_config, _confirm_usage,
                                      _get_previous_capacity_plan, _get_previous_user_config, _configure_ism, _set_up_arkime_config,
                                      _tag_domain, _should_proceed_with_operation, _is_initial_invocation, _get_stacks_to_deploy, _get_cdk_context)
+from core.compatibility import CliClusterVersionMismatch
 import core.constants as constants
 from core.capacity_planning import (CaptureNodesPlan, ViewerNodesPlan, EcsSysResourcePlan, MINIMUM_TRAFFIC, OSDomainPlan, DataNodesPlan, MasterNodesPlan,
                                     VpcPlan, ClusterPlan, DEFAULT_SPI_DAYS, DEFAULT_REPLICAS, DEFAULT_NUM_AZS, S3Plan,
                                     DEFAULT_S3_STORAGE_CLASS, DEFAULT_S3_STORAGE_DAYS, DEFAULT_HISTORY_DAYS, Cidr, DEFAULT_VPC_CIDR, DEFAULT_CAPTURE_PUBLIC_MASK,
                                     DEFAULT_VIEWER_PUBLIC_MASK)
 import core.local_file as local_file
 from core.user_config import UserConfig
-from core.versioning import VersionInfo, CliClusterVersionMismatch
+from core.versioning import VersionInfo
 
 
 @mock.patch("commands.cluster_create.AwsClientProvider")
@@ -110,7 +111,7 @@ def test_WHEN_cmd_cluster_create_called_THEN_cdk_command_correct(mock_cdk_client
     assert expected_tag_calls == mock_tag.call_args_list
 
 @mock.patch("commands.cluster_create.AwsClientProvider", mock.Mock())
-@mock.patch("commands.cluster_create.ver.confirm_aws_aio_version_compatibility")
+@mock.patch("commands.cluster_create.compat.confirm_aws_aio_version_compatibility")
 @mock.patch("commands.cluster_create._is_initial_invocation")
 @mock.patch("commands.cluster_create._tag_domain")
 @mock.patch("commands.cluster_create._set_up_arkime_config")