Qradar flows (#236)

main.py

@@ -50,7 +50,8 @@ def __main__():
 
     # positional arguments
     translate_parser.add_argument(
-        'module', choices=stix_translation.TRANSLATION_MODULES, help='The translation module to use')
+        'module', 
+        help='The translation module to use')
     translate_parser.add_argument('translate_type', choices=[
         stix_translation.RESULTS, stix_translation.QUERY, stix_translation.PARSE, stix_translation.SUPPORTED_ATTRIBUTES], help='The translation action to perform')
     translate_parser.add_argument(
@@ -73,7 +74,7 @@ def __main__():
 
     # positional arguments
     transmit_parser.add_argument(
-        'module', choices=stix_transmission.TRANSMISSION_MODULES,
+        'module', 
         help='Choose which connection module to use'
     )
     transmit_parser.add_argument(
@@ -105,11 +106,11 @@ def __main__():
     execute_parser = parent_subparsers.add_parser(EXECUTE, help='Translate and fully execute a query')
     # positional arguments
     execute_parser.add_argument(
-        'transmission_module', choices=stix_transmission.TRANSMISSION_MODULES,
+        'transmission_module', 
         help='Which connection module to use'
     )
     execute_parser.add_argument(
-        'translation_module', choices=stix_translation.TRANSLATION_MODULES,
+        'translation_module', 
         help='Which translation module to use'
     )
     execute_parser.add_argument(
@@ -205,12 +206,14 @@ def is_async():
         # Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
         translation = stix_translation.StixTranslation()
         dsl = translation.translate(args.translation_module, 'query', args.data_source, args.query, {'validate_pattern': True})
+        print("\n\n***** TRANSLATED QUERIES *****\n\n{}".format(dsl))
         connection_dict = json.loads(args.connection)
         configuration_dict = json.loads(args.configuration)
 
         transmission = stix_transmission.StixTransmission(args.transmission_module, connection_dict, configuration_dict)
 
         results = []
+        print("\n\n***** SEARCH RESULTS *****\n\n")
         for query in dsl['queries']:
             search_result = transmission.query(query)
 
@@ -240,8 +243,8 @@ def is_async():
 
         # Translate results to STIX
         result = translation.translate(args.translation_module, 'results', args.data_source, json.dumps(results), {"stix_validator": True})
-        print(result)
-
+
+        print("\n\n***** STIX BUNDLE *****\n\n{}".format(result))
         exit(0)
 
     elif args.command == TRANSLATE:

stix_shifter/stix_translation/src/json_to_stix/json_to_stix.py

@@ -2,6 +2,7 @@
 from . import json_to_stix_translator
 from ..modules.base.base_result_translator import BaseResultTranslator
 from stix_shifter.stix_translation.src.utils import transformers
+from stix_shifter.stix_translation.src.utils.exceptions import LoadJsonResultsException, TranslationResultException
 
 # Concrete BaseResultTranslator
 
@@ -20,16 +21,21 @@ def translate_results(self, data_source, data, options, mapping=None):
         :rtype: str
         """
         self.mapping = options['mapping'] if 'mapping' in options else {}
-        json_data = json.loads(data)
-        data_source = json.loads(data_source)
+        try:
+            json_data = json.loads(data)
+            data_source = json.loads(data_source)
+        except Exception:
+            raise LoadJsonResultsException()
 
         if(not self.mapping):
             map_file = open(self.default_mapping_file_path).read()
             map_data = json.loads(map_file)
         else:
             map_data = self.mapping
 
-        results = json_to_stix_translator.convert_to_stix(data_source, map_data,
-                                                          json_data, transformers.get_all_transformers(), options, self.callback)
+        try:
+            results = json_to_stix_translator.convert_to_stix(data_source, map_data, json_data, transformers.get_all_transformers(), options, self.callback)
+        except Exception as ex:
+            raise TranslationResultException("Error when converting results to STIX: {}".format(ex))
 
         return json.dumps(results, indent=4, sort_keys=False)

stix_shifter/stix_translation/src/json_to_stix/json_to_stix_translator.py

@@ -58,7 +58,8 @@ def _get_value(obj, ds_key, transformer):
             print('{} not found in object'.format(ds_key))
             return None
         ret_val = obj[ds_key]
-        if transformer is not None:
+        # Is this getting his with a none-type value?
+        if ret_val and transformer is not None:
             return transformer.transform(ret_val)
         return ret_val
 
@@ -121,7 +122,10 @@ def _valid_stix_value(props_map, key, stix_value):
         :return: whether STIX value is valid for this STIX property
         :rtype: bool
         """
-        if stix_value is None:
+
+        #  Causing a couple of failing tests in MSATP
+        if stix_value is None or stix_value == '':
+            print("Removing invalid value '{}' for {}".format(stix_value, key))
             return False
         elif key in props_map and 'valid_regex' in props_map[key]:
             pattern = re.compile(props_map[key]['valid_regex'])
@@ -134,7 +138,7 @@ def _transform(self, object_map, observation, ds_map, ds_key, obj):
         to_map = obj[ds_key]
 
         if ds_key not in ds_map:
-            print('{} is not found in map, skipping'.format(ds_key))
+            # print('{} is not found in map, skipping'.format(ds_key))
             return
 
         if isinstance(to_map, dict):
@@ -175,15 +179,21 @@ def _transform(self, object_map, observation, ds_map, ds_key, obj):
             group = False
             if ds_key_def.get('cybox', self.cybox_default):
                 object_name = ds_key_def.get('object')
-                print("ds_key_def inside {}".format(ds_key_def))
                 if 'references' in ds_key_def:
                     references = ds_key_def['references']
                     if isinstance(references, list):
                         stix_value = []
                         for ref in references:
-                            stix_value.append(object_map[ref])
+                            val = object_map.get(ref)
+                            if not DataSourceObjToStixObj._valid_stix_value(self.properties, key_to_add, val):
+                                continue
+                            stix_value.append(val)
+                        if not stix_value:
+                            continue
                     else:
-                        stix_value = object_map[references]
+                        stix_value = object_map.get(references)
+                        if not DataSourceObjToStixObj._valid_stix_value(self.properties, key_to_add, stix_value):
+                            continue
                 else:
                     stix_value = DataSourceObjToStixObj._get_value(obj, ds_key, transformer)
                     if not DataSourceObjToStixObj._valid_stix_value(self.properties, key_to_add, stix_value):
@@ -192,7 +202,6 @@ def _transform(self, object_map, observation, ds_map, ds_key, obj):
                 # Group Values
                 if 'group' in ds_key_def:
                     group = True
-
                 DataSourceObjToStixObj._handle_cybox_key_def(key_to_add, observation, stix_value, object_map, object_name, group)
             else:
                 # get the object name defined for custom attributes
@@ -222,6 +231,7 @@ def transform(self, obj):
         :param obj: the datasource object that is being converted to stix
         :return: the input object converted to stix valid json
         """
+        NUMBER_OBSERVED_KEY = 'number_observed'
         object_map = {}
         stix_type = 'observed-data'
         ds_map = self.ds_to_stix_map
@@ -241,6 +251,10 @@ def transform(self, obj):
                 self._transform(object_map, observation, ds_map, ds_key, obj)
         else:
             print("Not a dict: {}".format(obj))
+
+        # Add required property to the observation if it wasn't added via the mapping
+        if NUMBER_OBSERVED_KEY not in observation:
+            observation[NUMBER_OBSERVED_KEY] = 1
 
         # Validate each STIX object
         if self.stix_validator:

stix_shifter/stix_translation/src/json_to_stix/observable.py

@@ -1,11 +1,11 @@
 REGEX = {
     'date': '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d+)?Z',
     'ipv4': ('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'),  # noqa: E501
-    'ipv6': ('(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))'),
+    'ipv6': ('^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$'),
     'mac': ('^(([0-9a-fA-F]{2}[:-]){5}([0-9a-fA-F]{2})|([0-9a-fA-F]{3}[\.]){3}([0-9a-fA-F]{3}))$'),
-    'ipv4_cidr': ('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/[0-2][0-5]?$'),  # noqa: E501
-    'domain_name': ('^([a-z0-9]+(-[a-z0-9]+)*)*[\.a-z]*$|^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*){1}(\.[a-zA-Z0-9]+(-['
-                    'a-zA-Z0-9]+)*)*(\.[a-zA-Z]{2,})+$')
+    'ipv4_cidr': ('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\/(([1-2][0-9])|(3[0-2])|[0-9])$'),  # noqa: E501
+    'domain_name': ('^([a-z0-9]+(-[a-z0-9]+)*)*[\.a-z]*$|^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*){1}(\.[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)*(\.[a-zA-Z]{2,})+$'),
+    'ipv6_cidr': ('^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:' '[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|' '([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|' '[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%' '[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}' '[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/((1[0-2][0-8])|([1-9][0-9])|[0-9])$')
 }
 
 properties = {

stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/aws_cloud_watch_logs_translator.py

@@ -6,7 +6,7 @@
 
 class Translator(BaseTranslator):
 
-    def __init__(self):
+    def __init__(self, dialect=None):
         basepath = path.dirname(__file__)
         filepath = path.abspath(
             path.join(basepath, "json", "to_stix_map.json"))

stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/data_mapping.py

@@ -1,61 +1,12 @@
 from os import path
-import glob
-import json
 from stix_shifter.stix_translation.src.modules.base.base_data_mapper import BaseDataMapper
 
 
 class DataMapper(BaseDataMapper):
 
     def __init__(self, options):
-        mapping_json = options['mapping'] if 'mapping' in options else {}
         basepath = path.dirname(__file__)
-        self.from_stix_files_cnt = self.json_files_to_fetch(path.abspath(
-            path.join(basepath, "json", "from_*.json")))
-        self.map_data = mapping_json or self.fetch_mapping()
+        self.dialect = options.get('dialect') or 'guardduty'
+        self.map_data = self.fetch_mapping(basepath)
 
-    @staticmethod
-    def json_files_to_fetch(file_path):
-        return glob.glob(file_path)
 
-    def fetch_mapping(self):
-        """
-        Fetches STIX-to-datasource mapping JSON from the module's from_stix_map.json file
-        :param basepath: path of data source translation module
-        :type basepath: str
-        """
-        map_data = {}
-        try:
-            for each_file in self.from_stix_files_cnt:
-                map_file = open(each_file).read()
-                map_data[path.basename(each_file)] = json.loads(map_file)
-            return map_data
-        except Exception as ex:
-            print('exception in main():', ex)
-            return {}
-
-    def map_field(self, stix_object_name, stix_property_name):
-        """
-        :param stix_object_name: str, stix object
-        :param stix_property_name: str, stix field
-        :return: list
-        """
-        mapped_field_lst = []
-        for each_model_mapper in self.map_data.values():
-            if stix_object_name in each_model_mapper and stix_property_name in \
-                    each_model_mapper[stix_object_name]["fields"]:
-                mapped_field_lst.append(each_model_mapper[stix_object_name]["fields"][stix_property_name])
-        return mapped_field_lst
-
-    def map_field_json(self, stix_object_name, stix_property_name, json_file):
-        """
-        Return mapped fields from json file
-        :param stix_object_name:str, stix object
-        :param stix_property_name:str, stix field
-        :param json_file:str, json file name
-        :return: list
-        """
-        if stix_object_name in self.map_data.get(json_file) and stix_property_name in \
-                self.map_data.get(json_file)[stix_object_name]["fields"]:
-            return self.map_data.get(json_file)[stix_object_name]["fields"][stix_property_name]
-        else:
-            return []

stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/json/from_stix_guardduty_map.json → stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/json/guardduty_from_stix_map.json

@@ -1,5 +1,4 @@
 {
-  "log-type": "guardduty",
   "ipv4-addr": {
     "fields": {
       "value": ["eth0_private_ip", "eth1_private_ip", "public_ip", "remote_ip"],

stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/json/from_stix_vpcflow_map.json → stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/json/vpcflow_from_stix_map.json

@@ -1,5 +1,4 @@
 {
-  "log-type": "vpcflow",
   "ipv4-addr": {
     "fields": {
       "value": ["srcAddr", "dstAddr"],

stix_shifter/stix_translation/src/modules/aws_cloud_watch_logs/query_constructor.py

@@ -32,12 +32,11 @@ class QueryStringPatternTranslator:
     }
     aws_query = "fields {fields}{stix_filter} {parse_filter}{logtype_filter} | filter ({filter_query})"
 
-    def __init__(self, pattern: Pattern, data_model_mapper, time_range, from_stix_json_filename):
+    def __init__(self, pattern: Pattern, data_model_mapper, time_range):
         self.dmm = data_model_mapper
         self.pattern = pattern
         self._time_range = time_range
-        self.json_file = from_stix_json_filename
-        self.log_type = self.load_json('json/' + path.basename(from_stix_json_filename)).get('log-type')
+        self.log_type = self.dmm.dialect
         self._log_config_data = self.load_json(MASTER_CONFIG_FILE)
         self._protocol_lookup_needed = True if self.log_type in ['vpcflow'] else False
         self._parse_statement = []
@@ -316,7 +315,7 @@ def __eval_comparison_exp(self, expression):
                     raise NotImplementedError("Un-supported protocol '{}' for operation '{}' for aws '{}' logs".format(
                         expression.value, expression.comparator, path.basename(self.log_type)))
                 expression.value = value
-        mapped_fields_array = self.dmm.map_field_json(stix_object, stix_field, path.basename(self.json_file))
+        mapped_fields_array = self.dmm.map_field(stix_object, stix_field)
         comparator = self._lookup_comparison_operator(self, expression.comparator)
         comparison_string = self.__eval_comparison_value(comparator, expression, mapped_fields_array, stix_field)
         if stix_field.lower() == 'protocols[*]':
@@ -374,17 +373,16 @@ def translate_pattern(pattern: Pattern, data_model_mapping, options):
     timerange = options['timerange']
     limit = options['result_limit']
     final_queries = []
-    for each_json_file in data_model_mapping.from_stix_files_cnt:
-        queries_obj = QueryStringPatternTranslator(pattern, data_model_mapping, timerange, each_json_file)
-        qualifier_list = list(zip(*queries_obj.time_range_lst))
-        queries_string = queries_obj.qualified_queries
-        for index, each_query in enumerate(queries_string, start=0):
-            translate_query_dict = dict()
-            translate_query_dict['logType'] = queries_obj.log_type
-            translate_query_dict['limit'] = limit
-            translate_query_dict['queryString'] = each_query
-            translate_query_dict['startTime'] = qualifier_list[0][index]
-            translate_query_dict['endTime'] = qualifier_list[1][index]
-            translate_query_dict = json.dumps(translate_query_dict)
-            final_queries.append(translate_query_dict)
+    queries_obj = QueryStringPatternTranslator(pattern, data_model_mapping, timerange)
+    qualifier_list = list(zip(*queries_obj.time_range_lst))
+    queries_string = queries_obj.qualified_queries
+    for index, each_query in enumerate(queries_string, start=0):
+        translate_query_dict = dict()
+        translate_query_dict['logType'] = queries_obj.log_type
+        translate_query_dict['limit'] = limit
+        translate_query_dict['queryString'] = each_query
+        translate_query_dict['startTime'] = qualifier_list[0][index]
+        translate_query_dict['endTime'] = qualifier_list[1][index]
+        translate_query_dict = json.dumps(translate_query_dict)
+        final_queries.append(translate_query_dict)
     return final_queries

stix_shifter/stix_translation/src/modules/base/base_data_mapper.py

@@ -1,7 +1,7 @@
 from abc import ABCMeta, abstractmethod
 from os import path
 import json
-from stix_shifter.stix_translation.src.utils.exceptions import DataMappingException
+import glob
 
 
 class BaseDataMapper(object, metaclass=ABCMeta):
@@ -13,9 +13,10 @@ def fetch_mapping(self, basepath):
         :type basepath: str
         """
         try:
-            filepath = path.abspath(
-                path.join(basepath, "json", "from_stix_map.json"))
-
+            if hasattr(self, 'dialect') and not(self.dialect == 'default'):
+                filepath = self._fetch_from_stix_mapping_file(basepath)
+            else:
+                filepath = path.abspath(path.join(basepath, "json", 'from_stix_map.json'))
             map_file = open(filepath).read()
             map_data = json.loads(map_file)
             return map_data
@@ -38,3 +39,7 @@ def map_field(self, stix_object_name, stix_property_name):
             return self.map_data[stix_object_name]["fields"][stix_property_name]
         else:
             return []
+
+    def _fetch_from_stix_mapping_file(self, basepath):
+        mapping_paths = glob.glob(path.abspath(path.join(basepath, "json", "{}_from_stix*.json".format(self.dialect))))
+        return mapping_paths[0]

stix_shifter/stix_translation/src/modules/carbonblack/data_mapping.py

@@ -10,8 +10,8 @@ def __init__(self, options):
         assert "process" in self.map_data and "binary" in self.map_data
 
     def fetch_mapping(self):
-        process_mapping = self._fetch_mapping_file("process_api_from_stix_map.json")
-        binary_mapping = self._fetch_mapping_file("binary_api_from_stix_map.json")
+        process_mapping = self._fetch_mapping_file("process_from_stix_map.json")
+        binary_mapping = self._fetch_mapping_file("binary_from_stix_map.json")
         return {"binary": binary_mapping, "process": process_mapping}
 
     def _fetch_mapping_file(self, filename):