Update bundle adapter for the pipeline (#141)

requirements.txt

@@ -4,4 +4,4 @@ python-dateutil==2.7.3
 antlr4-python3-runtime==4.7
 boto3==1.9.76
 xmltodict==0.11.0
-git+git://github.com/oasis-open/cti-pattern-matcher.git@master#egg=stix2-matcher 
+git+git://github.com/oasis-open/cti-pattern-matcher.git@v0.1.0#egg=stix2-matcher

setup.py

@@ -135,7 +135,8 @@
     # For an analysis of "install_requires" vs pip's requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
     install_requires=['stix2-patterns==1.1.0', 'stix2-validator==0.5.0',
-                      'antlr4-python3-runtime==4.7', 'python-dateutil==2.7.3'],  # Optional
+                      'antlr4-python3-runtime==4.7', 'python-dateutil==2.7.3',
+                      'stix2-matcher@https://github.com/oasis-open/cti-pattern-matcher/archive/v0.1.0.zip#egg=stix2-matcher'],  # Optional
 
     # List additional groups of dependencies here (e.g. development
     # dependencies). Users will be able to install these using the "extras"

stix_shifter/stix_translation/src/modules/stix_bundle/stix_bundle_translator.py

@@ -0,0 +1,39 @@
+from ..base.base_translator import BaseTranslator
+import json
+import requests
+import re
+import uuid
+
+START_STOP_PATTERN = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?"
+
+
+class Translator(BaseTranslator):
+    def transform_query(self, data, antlr_parsing_object={}, data_model_mapper={}, options={}, mapping=None):
+        # Data is a STIX pattern.
+        # stix2-matcher will break on START STOP qualifiers so remove before returning pattern.
+        # Remove this when ever stix2-matcher supports proper qualifier timestamps
+        data = re.sub(START_STOP_PATTERN, " ", data)
+        return data
+
+    def translate_results(self, data_source, data, options, mapping=None):
+        # Wrap data in a STIX bundle and insert the data_source identity object as the first object
+        bundle = {
+            "type": "bundle",
+            "id": "bundle--" + str(uuid.uuid4()),
+            "objects": []
+        }
+
+        data_source = json.loads(data_source)
+        bundle['objects'] += [data_source]
+        # Data is already STIX and we don't want to touch it
+        bundle_data = json.loads(data)
+
+        for obs in bundle_data:
+            obs["created_by_ref"] = data_source['id']
+
+        bundle['objects'] += bundle_data
+        return json.dumps(bundle, indent=4, sort_keys=False)
+
+    def __init__(self):
+        self.result_translator = self
+        self.query_translator = self

stix_shifter/stix_translation/stix_translation.py

@@ -10,13 +10,14 @@
 from stix_shifter.stix_translation.src.utils.unmapped_attribute_stripper import strip_unmapped_attributes
 import sys
 
-TRANSLATION_MODULES = ['qradar', 'dummy', 'car', 'cim', 'splunk', 'elastic', 'bigfix', 'csa', 'csa:at', 'csa:nf', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy', 'bundle']
+TRANSLATION_MODULES = ['qradar', 'dummy', 'car', 'cim', 'splunk', 'elastic', 'bigfix', 'csa', 'csa:at', 'csa:nf', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy', 'stix_bundle']
 
 RESULTS = 'results'
 QUERY = 'query'
 PARSE = 'parse'
 DEFAULT_LIMIT = 10000
 DEFAULT_TIMERANGE = 5
+START_STOP_PATTERN = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?"
 SHARED_DATA_MAPPERS = {'elastic': car_data_mapping, 'splunk': cim_data_mapping, 'cim': cim_data_mapping, 'car': car_data_mapping}
 
 
@@ -30,8 +31,7 @@ def __init__(self):
 
     def _validate_pattern(self, pattern):
         # Validator doesn't support START STOP qualifier so strip out before validating pattern
-        start_stop_pattern = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?"
-        pattern_without_start_stop = re.sub(start_stop_pattern, " ", pattern)
+        pattern_without_start_stop = re.sub(START_STOP_PATTERN, " ", pattern)
         errors = run_validator(pattern_without_start_stop)
         if (errors != []):
             raise StixValidationException("The STIX pattern has the following errors: {}".format(errors))

stix_shifter/stix_transmission/src/modules/stix_bundle/stix_bundle_connector.py

@@ -0,0 +1,100 @@
+from ..base.base_connector import BaseConnector
+
+from stix2matcher.matcher import Pattern
+from stix2matcher.matcher import MatchListener
+from stix2validator import validate_instance
+import json
+import requests
+from .....utils.error_response import ErrorResponder
+
+
+class UnexpectedResponseException(Exception):
+    pass
+
+
+class Connector(BaseConnector):
+    def __init__(self, connection, configuration):
+
+        self.is_async = False
+        self.connection = connection
+        self.configuration = configuration
+        self.results_connector = self
+        self.query_connector = self
+        self.ping_connector = self
+        self.status_connector = self
+
+    # We re-implement this method so we can fetch all the "bindings", as their method only
+    # returns the first for some reason
+    def match(self, pattern, observed_data_sdos, verbose=False):
+        compiled_pattern = Pattern(pattern)
+        matcher = MatchListener(observed_data_sdos, verbose)
+        compiled_pattern.walk(matcher)
+
+        found_bindings = matcher.matched()
+
+        if found_bindings:
+            matching_sdos = []
+            for binding in found_bindings:
+                matching_sdos = matching_sdos + matcher.get_sdos_from_binding(binding)
+        else:
+            matching_sdos = []
+
+        return matching_sdos
+
+    def ping(self):
+        return {"success": True}
+
+    def create_query_connection(self, query):
+        return {"success": True, "search_id": query}
+
+    def create_status_connection(self, search_id):
+        return {"success": True, "status": "COMPLETED", "progress": 100}
+
+    def create_results_connection(self, search_id, offset, length):
+        # search_id is the pattern
+        observations = []
+        return_obj = dict()
+
+        bundle_url = self.connection.get('host')
+        auth = self.configuration.get('auth')
+
+        if auth is not None:
+            response = requests.get(bundle_url, auth=(auth.get('username'), auth.get('password')))
+        else:
+            response = requests.get(bundle_url)
+
+        response_code = response.status_code
+
+        if response_code != 200:
+            response_txt = response.raise_for_status()
+            if ErrorResponder.is_plain_string(response_txt):
+                ErrorResponder.fill_error(return_obj, message=response_txt)
+            elif ErrorResponder.is_json_string(response_txt):
+                response_json = json.loads(response_txt)
+                ErrorResponder.fill_error(return_obj, response_json, ['reason'])
+            else:
+                raise UnexpectedResponseException
+        else:
+            bundle = response.json()
+
+            if "validate" in self.configuration and self.configuration["validate"] is True:
+                results = validate_instance(bundle)
+
+                if results.is_valid is not True:
+                    return {"success": False, "message": "Invalid STIX received: " + json.dumps(results)}
+
+            for obj in bundle["objects"]:
+                if obj["type"] == "observed-data":
+                    observations.append(obj)
+
+            # Pattern match
+            results = self.match(search_id, observations, False)
+
+            if len(results) != 0:
+                return_obj['success'] = True
+                return_obj['data'] = results[int(offset):int(offset + length)]
+            else:
+                return_obj['success'] = True
+                return_obj['data'] = []
+
+        return return_obj

stix_shifter/stix_transmission/stix_transmission.py

@@ -1,7 +1,7 @@
 import importlib
 from ..utils.error_response import ErrorResponder
 
-TRANSMISSION_MODULES = ['async_dummy', 'synchronous_dummy', 'qradar', 'splunk', 'bigfix', 'csa', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy','bundle']
+TRANSMISSION_MODULES = ['async_dummy', 'synchronous_dummy', 'qradar', 'splunk', 'bigfix', 'csa', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy','stix_bundle']
 
 RESULTS = 'results'
 QUERY = 'query'