feat: extended validation to compare with target

README.md

@@ -99,12 +99,13 @@ You can run this tool locally or using Docker.
     Please find sample inputs in the `sample/inputs` folder
     * [sample opdk input](sample/inputs/opdk.input.properties)
     * [sample saas input](sample/inputs/saas.input.properties)
+    * [sample x/hybrid input](sample/inputs/x.input.properties)
 
     Refer the below table to set the required inputs in the `input` section of `input.properties` file.
 
     | Section  | Input   | Description |
     | -------- | ------- | ------- |
-    | `input`  | `SOURCE_URL`    | Apigee OPDK/Edge Management URL |
+    | `input`  | `SOURCE_URL`    | Apigee OPDK/Edge/X/Hybrid Management URL |
     | `input`  | `SOURCE_ORG`     | Apigee OPDK/Edge Organization|
     | `input`  | `SOURCE_AUTH_TYPE`    | Apigee OPDK/Edge auth type , `basic` OR `oauth`|
     | `input`  | `SOURCE_UI_URL`    | Apigee OPDK/Edge UI URL, use default|

core_wrappers.py

@@ -55,7 +55,7 @@
 from topology import ApigeeTopology
 from utils import (
     create_dir, get_source_auth_token,
-    get_access_token, write_csv_report)
+    get_access_token)
 import sharding
 from base_logger import logger
 
@@ -203,7 +203,7 @@ def export_artifacts(cfg, resources_list):
     return export_data
 
 
-def validate_artifacts(cfg, export_data):  # noqa pylint: disable=R0914
+def validate_artifacts(cfg, resources_list, export_data):  # noqa pylint: disable=R0914
     """Validates exported artifacts against the target environment.
 
     Validates the exported Apigee artifacts against the constraints of
@@ -225,45 +225,48 @@ def validate_artifacts(cfg, export_data):  # noqa pylint: disable=R0914
     report = {}
     target_dir = cfg.get('inputs', 'TARGET_DIR')
     export_dir = f"{target_dir}/{cfg.get('export', 'EXPORT_DIR')}"
+    target_export_dir = f"{target_dir}/target"
+    api_export_dir = f"{target_export_dir}/apis"
+    sf_export_dir = f"{target_export_dir}/sharedflows"
+    create_dir(api_export_dir)
+    create_dir(sf_export_dir)
     gcp_project_id = cfg.get('inputs', 'GCP_PROJECT_ID')
     gcp_env_type = cfg.get('inputs', 'GCP_ENV_TYPE',
                            fallback=DEFAULT_GCP_ENV_TYPE)
     gcp_token = get_access_token()
+    apigee_export = ApigeeExporter(
+        'https://apigee.googleapis.com/v1',
+        gcp_project_id,
+        gcp_token,
+        'oauth',
+        True
+    )
+    target_resources = ['targetservers', 'flowhooks', 'resourcefiles', 'apis', 'sharedflows']  # noqa pylint: disable=C0301
+    target_resource_list = []
+    if 'all' in resources_list:
+        target_resource_list = target_resources
+    else:
+        target_resource_list = [ r for r in resources_list if r in target_resources]  # noqa pylint: disable=C0301
 
-    apigee_validator = ApigeeValidator(gcp_project_id, gcp_token, gcp_env_type)
+    target_export_data = apigee_export.get_export_data(target_resource_list, target_export_dir)  # noqa pylint: disable=C0301
+    apigee_validator = ApigeeValidator(gcp_project_id, gcp_token, gcp_env_type, target_export_data)  # noqa pylint: disable=C0301
 
     for env, _ in export_data['envConfig'].items():
         logger.info(f'Environment -- {env}')  # pylint: disable=W1203
         target_servers = export_data['envConfig'][env]['targetServers']
         resourcefiles = export_data['envConfig'][env]['resourcefiles']
         flowhooks = export_data['envConfig'][env]['flowhooks']
         report[env + SEPERATOR +
-               'targetServers'] = apigee_validator.validate_env_targetservers(target_servers)  # noqa pylint: disable=C0301
+               'targetServers'] = apigee_validator.validate_env_targetservers(env, target_servers)  # noqa pylint: disable=C0301
         report[env + SEPERATOR +
-               'resourcefiles'] = apigee_validator.validate_env_resourcefiles(resourcefiles)  # noqa pylint: disable=C0301
+               'resourcefiles'] = apigee_validator.validate_env_resourcefiles(env, resourcefiles)  # noqa pylint: disable=C0301
         report[env + SEPERATOR +
                'flowhooks'] = apigee_validator.validate_env_flowhooks(env, flowhooks)  # noqa
 
     validation = apigee_validator.validate_proxy_bundles(export_dir)
     # Todo  # pylint: disable=W0511
     # validate proxy unifier output bundles
     report.update(validation)
-    report_header = ['Type', 'Name', 'Importable', 'Reason']
-    report_rows = []
-    for each_type, type_data in report.items():
-        for each_item in type_data:
-            report_rows.append(
-                [
-                    each_type,
-                    each_item.get('name', None),
-                    each_item.get('importable', None),
-                    "" if len(each_item.get('reason', [])) == 0 else json.dumps(   # noqa
-                        each_item.get('reason', []), indent=2)
-                ]
-            )
-
-    csv_report = cfg.get('validate', 'CSV_REPORT')
-    write_csv_report(f'{target_dir}/{csv_report}', report_header, report_rows)
     return report
 
 

main.py

@@ -122,7 +122,7 @@ def main():
 
     if (not report.get('report', False) or
             not export_data.get('validation_report', False)):
-        report = validate_artifacts(cfg, export_data)
+        report = validate_artifacts(cfg, resources_list, export_data)
         report['report'] = True
         export_data['validation_report'] = report
         write_json(export_data_file, export_data)

sample/inputs/x.input.properties

@@ -0,0 +1,29 @@
+[inputs]
+SOURCE_URL=https://apigee.googleapis.com/v1
+SOURCE_ORG=apigee-org-1234
+SOURCE_AUTH_TYPE=oauth
+SOURCE_UI_URL=https://console.cloud.google.conm/apigee?project=apigee-org-1234
+SOURCE_APIGEE_VERSION=SAAS
+GCP_PROJECT_ID=apigee-org-4321
+GCP_ENV_TYPE=ENVIRONMENT_TYPE_UNSPECIFIED
+API_URL=https://apidocs.apigee.com/docs
+TARGET_DIR=target
+SSL_VERIFICATION=true
+
+[export]
+EXPORT_DIR=export
+EXPORT_FILE=export_data.json
+
+[topology]
+TOPOLOGY_DIR=topology
+NW_TOPOLOGY_MAPPING=pod_component_mapping.json
+DATA_CENTER_MAPPING=data_center_mapping.json
+
+[report]
+QUALIFICATION_REPORT=qualification_report.xlsx
+
+[validate]
+CSV_REPORT=report.csv
+
+[visualize]
+VISUALIZATION_GRAPH_FILE=visualization.html

validator.py

@@ -40,7 +40,7 @@ class ApigeeValidator():
     rules and compatibility checks.
     """
 
-    def __init__(self, project_id, token, env_type):
+    def __init__(self, project_id, token, env_type, target_export_data):
         """Initializes ApigeeValidator.
 
         Args:
@@ -51,8 +51,9 @@ def __init__(self, project_id, token, env_type):
         """
         self.project_id = project_id
         self.xorhybrid = ApigeeNewGen(project_id, token, env_type)
+        self.target_export_data = target_export_data
 
-    def validate_env_targetservers(self, target_servers):
+    def validate_env_targetservers(self, env, target_servers):
         """Validates environment target servers.
 
         Args:
@@ -68,6 +69,11 @@ def validate_env_targetservers(self, target_servers):
         for _, target_server_data in target_servers.items():
             obj = copy.copy(target_server_data)
             obj['importable'], obj['reason'] = self.validate_env_targetserver_resource(target_server_data)   # noqa pylint: disable=C0301
+            ts = self.target_export_data.get('envConfig', {}).get(env, {}).get('targetServers', {}).keys()    # noqa pylint: disable=C0301
+            if target_server_data['name'] in ts:
+                obj['imported'] = True
+            else:
+                obj['imported'] = False
             validation_targetservers.append(obj)
         return validation_targetservers
 
@@ -95,7 +101,7 @@ def validate_env_targetserver_resource(self, targetservers):
             return True, []
         return False, errors
 
-    def validate_env_resourcefiles(self, resourcefiles):
+    def validate_env_resourcefiles(self, env, resourcefiles):
         """Validates environment resource files.
 
         Args:
@@ -111,6 +117,11 @@ def validate_env_resourcefiles(self, resourcefiles):
         for resourcefile in resourcefiles.keys():
             obj = copy.copy(resourcefiles[resourcefile])
             obj['importable'], obj['reason'] = self.validate_env_resourcefile_resource(resourcefiles[resourcefile])    # noqa pylint: disable=C0301
+            rf = self.target_export_data.get('envConfig', {}).get(env, {}).get('resourcefiles', {}).keys()    # noqa pylint: disable=C0301
+            if resourcefile in rf:
+                obj['imported'] = True
+            else:
+                obj['imported'] = False
             validation_rfiles.append(obj)
         return validation_rfiles
 
@@ -147,8 +158,8 @@ def validate_proxy_bundles(self, export_dir):
             dict: Validation results for APIs and
                 sharedflows.
         """
-        apis = self.xorhybrid.list_org_objects('apis')
-        sharedflows = self.xorhybrid.list_org_objects('sharedflows')
+        apis = self.target_export_data.get('orgConfig', {}).get('apis', {}).keys()    # noqa pylint: disable=C0301
+        sharedflows = self.target_export_data.get('orgConfig', {}).get('sharedflows', {}).keys()    # noqa pylint: disable=C0301
         apis_sf_list = {'apis': apis, 'sharedflows': sharedflows}
         validation = {'apis': [], 'sharedflows': []}
         for each_api_type in ['apis', 'sharedflows']:
@@ -208,6 +219,11 @@ def validate_env_flowhooks(self, env, flowhooks):
             obj = copy.copy(flowhooks[flowhook])
             obj['name'] = flowhook
             obj['importable'], obj['reason'] = self.validate_env_flowhooks_resource(env, flowhooks[flowhook])   # noqa pylint: disable=C0301
+            fh = self.target_export_data.get('envConfig', {}).get(env, {}).get('flowhooks', {}).keys()    # noqa pylint: disable=C0301
+            if flowhook in fh:
+                obj['imported'] = True
+            else:
+                obj['imported'] = False
             validation_flowhooks.append(obj)
         return validation_flowhooks