diff --git a/requirements.txt b/requirements.txt index 8bed89ed0..461711035 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ python-dateutil==2.7.3 antlr4-python3-runtime==4.7 boto3==1.9.76 xmltodict==0.11.0 -git+git://github.com/oasis-open/cti-pattern-matcher.git@master#egg=stix2-matcher +git+git://github.com/oasis-open/cti-pattern-matcher.git@v0.1.0#egg=stix2-matcher \ No newline at end of file diff --git a/setup.py b/setup.py index 2a847fdee..ff2f819b5 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,8 @@ # For an analysis of "install_requires" vs pip's requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['stix2-patterns==1.1.0', 'stix2-validator==0.5.0', - 'antlr4-python3-runtime==4.7', 'python-dateutil==2.7.3'], # Optional + 'antlr4-python3-runtime==4.7', 'python-dateutil==2.7.3', + 'stix2-matcher@https://github.com/oasis-open/cti-pattern-matcher/archive/v0.1.0.zip#egg=stix2-matcher'], # Optional # List additional groups of dependencies here (e.g. development # dependencies). Users will be able to install these using the "extras" diff --git a/stix_shifter/stix_translation/src/modules/bundle/bundle_translator.py b/stix_shifter/stix_translation/src/modules/bundle/bundle_translator.py deleted file mode 100644 index 65d63a460..000000000 --- a/stix_shifter/stix_translation/src/modules/bundle/bundle_translator.py +++ /dev/null @@ -1,21 +0,0 @@ -from ..base.base_translator import BaseTranslator -import json -import requests - - -class Translator(BaseTranslator): - def transform_query(self, data, antlr_parsing_object={}, data_model_mapper={}, options={}, mapping=None): - # Data is a STIX pattern and we don't want to touch it - return data - - def translate_results(self, data_source, data, options, mapping=None): - # Data is already STIX and we don't want to touch it - bundle_data = json.loads(data) - data_source = json.loads(data_source) - for obs in bundle_data: - obs["created_by_ref"] = data_source['id'] - return json.dumps(bundle_data, indent=4, sort_keys=False) - - def __init__(self): - self.result_translator = self - self.query_translator = self diff --git a/stix_shifter/stix_translation/src/modules/bundle/__init__.py b/stix_shifter/stix_translation/src/modules/stix_bundle/__init__.py similarity index 100% rename from stix_shifter/stix_translation/src/modules/bundle/__init__.py rename to stix_shifter/stix_translation/src/modules/stix_bundle/__init__.py diff --git a/stix_shifter/stix_translation/src/modules/stix_bundle/stix_bundle_translator.py b/stix_shifter/stix_translation/src/modules/stix_bundle/stix_bundle_translator.py new file mode 100644 index 000000000..102eb737c --- /dev/null +++ b/stix_shifter/stix_translation/src/modules/stix_bundle/stix_bundle_translator.py @@ -0,0 +1,39 @@ +from ..base.base_translator import BaseTranslator +import json +import requests +import re +import uuid + +START_STOP_PATTERN = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?" + + +class Translator(BaseTranslator): + def transform_query(self, data, antlr_parsing_object={}, data_model_mapper={}, options={}, mapping=None): + # Data is a STIX pattern. + # stix2-matcher will break on START STOP qualifiers so remove before returning pattern. + # Remove this when ever stix2-matcher supports proper qualifier timestamps + data = re.sub(START_STOP_PATTERN, " ", data) + return data + + def translate_results(self, data_source, data, options, mapping=None): + # Wrap data in a STIX bundle and insert the data_source identity object as the first object + bundle = { + "type": "bundle", + "id": "bundle--" + str(uuid.uuid4()), + "objects": [] + } + + data_source = json.loads(data_source) + bundle['objects'] += [data_source] + # Data is already STIX and we don't want to touch it + bundle_data = json.loads(data) + + for obs in bundle_data: + obs["created_by_ref"] = data_source['id'] + + bundle['objects'] += bundle_data + return json.dumps(bundle, indent=4, sort_keys=False) + + def __init__(self): + self.result_translator = self + self.query_translator = self diff --git a/stix_shifter/stix_translation/stix_translation.py b/stix_shifter/stix_translation/stix_translation.py index eab89a066..d0f5d0289 100644 --- a/stix_shifter/stix_translation/stix_translation.py +++ b/stix_shifter/stix_translation/stix_translation.py @@ -10,13 +10,14 @@ from stix_shifter.stix_translation.src.utils.unmapped_attribute_stripper import strip_unmapped_attributes import sys -TRANSLATION_MODULES = ['qradar', 'dummy', 'car', 'cim', 'splunk', 'elastic', 'bigfix', 'csa', 'csa:at', 'csa:nf', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy', 'bundle'] +TRANSLATION_MODULES = ['qradar', 'dummy', 'car', 'cim', 'splunk', 'elastic', 'bigfix', 'csa', 'csa:at', 'csa:nf', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy', 'stix_bundle'] RESULTS = 'results' QUERY = 'query' PARSE = 'parse' DEFAULT_LIMIT = 10000 DEFAULT_TIMERANGE = 5 +START_STOP_PATTERN = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?" SHARED_DATA_MAPPERS = {'elastic': car_data_mapping, 'splunk': cim_data_mapping, 'cim': cim_data_mapping, 'car': car_data_mapping} @@ -30,8 +31,7 @@ def __init__(self): def _validate_pattern(self, pattern): # Validator doesn't support START STOP qualifier so strip out before validating pattern - start_stop_pattern = "\s?START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP\s?t'\d{4}(-\d{2}){2}T(\d{2}:){2}\d{2}.\d{1,3}Z'\s?" - pattern_without_start_stop = re.sub(start_stop_pattern, " ", pattern) + pattern_without_start_stop = re.sub(START_STOP_PATTERN, " ", pattern) errors = run_validator(pattern_without_start_stop) if (errors != []): raise StixValidationException("The STIX pattern has the following errors: {}".format(errors)) diff --git a/stix_shifter/stix_transmission/src/modules/bundle/bundle_connector.py b/stix_shifter/stix_transmission/src/modules/bundle/bundle_connector.py deleted file mode 100644 index 96038bb4a..000000000 --- a/stix_shifter/stix_transmission/src/modules/bundle/bundle_connector.py +++ /dev/null @@ -1,73 +0,0 @@ -from ..base.base_connector import BaseConnector - -from stix2matcher.matcher import Pattern -from stix2matcher.matcher import MatchListener -from stix2validator import validate_instance -import json, requests - -class Connector(BaseConnector): - def __init__(self, connection, configuration): - - self.is_async = False - - self.connection = connection - self.configuration = configuration - - self.results_connector = self - self.query_connector = self - self.ping_connector = self - - #We re-implement this method so we can fetch all the "bindings", as their method only - #returns the first for some reason - def match(self, pattern, observed_data_sdos, verbose=False): - compiled_pattern = Pattern(pattern) - matcher = MatchListener(observed_data_sdos, verbose) - compiled_pattern.walk(matcher) - - found_bindings = matcher.matched() - - if found_bindings: - matching_sdos = [] - for binding in found_bindings: - matching_sdos = matching_sdos + matcher.get_sdos_from_binding(binding) - else: - matching_sdos = [] - - return matching_sdos - - def ping(self): - return {"success":True} - - def create_query_connection(self, query): - return { "success": True, "search_id": query } - - def create_results_connection(self, search_id, offset, length): - #search_id is the pattern - observations = [] - - if "http_user" in self.configuration: - response = requests.get(self.configuration["bundle_url"],auth=(self.configuration["http_user"], self.configuration["http_password"])) - else: - response = requests.get(self.configuration["bundle_url"]) - - if response.status_code != 200: - response.raise_for_status() - - bundle = response.json() - - if "validate" in self.configuration and self.configuration["validate"] is True: - results = validate_instance(bundle) - - if results.is_valid is not True: - return { "success":False, "message":"Invalid STIX recieved: " + json.dumps(results) } - - for obj in bundle["objects"]: - if obj["type"] == "observed-data": - observations.append( obj ) - - #Pattern match - results = self.match(search_id, observations, False) - - return results[ int(offset):int(offset + length) ] - - diff --git a/stix_shifter/stix_transmission/src/modules/bundle/__init__.py b/stix_shifter/stix_transmission/src/modules/stix_bundle/__init__.py similarity index 100% rename from stix_shifter/stix_transmission/src/modules/bundle/__init__.py rename to stix_shifter/stix_transmission/src/modules/stix_bundle/__init__.py diff --git a/stix_shifter/stix_transmission/src/modules/stix_bundle/stix_bundle_connector.py b/stix_shifter/stix_transmission/src/modules/stix_bundle/stix_bundle_connector.py new file mode 100644 index 000000000..9c4421efa --- /dev/null +++ b/stix_shifter/stix_transmission/src/modules/stix_bundle/stix_bundle_connector.py @@ -0,0 +1,100 @@ +from ..base.base_connector import BaseConnector + +from stix2matcher.matcher import Pattern +from stix2matcher.matcher import MatchListener +from stix2validator import validate_instance +import json +import requests +from .....utils.error_response import ErrorResponder + + +class UnexpectedResponseException(Exception): + pass + + +class Connector(BaseConnector): + def __init__(self, connection, configuration): + + self.is_async = False + self.connection = connection + self.configuration = configuration + self.results_connector = self + self.query_connector = self + self.ping_connector = self + self.status_connector = self + + # We re-implement this method so we can fetch all the "bindings", as their method only + # returns the first for some reason + def match(self, pattern, observed_data_sdos, verbose=False): + compiled_pattern = Pattern(pattern) + matcher = MatchListener(observed_data_sdos, verbose) + compiled_pattern.walk(matcher) + + found_bindings = matcher.matched() + + if found_bindings: + matching_sdos = [] + for binding in found_bindings: + matching_sdos = matching_sdos + matcher.get_sdos_from_binding(binding) + else: + matching_sdos = [] + + return matching_sdos + + def ping(self): + return {"success": True} + + def create_query_connection(self, query): + return {"success": True, "search_id": query} + + def create_status_connection(self, search_id): + return {"success": True, "status": "COMPLETED", "progress": 100} + + def create_results_connection(self, search_id, offset, length): + # search_id is the pattern + observations = [] + return_obj = dict() + + bundle_url = self.connection.get('host') + auth = self.configuration.get('auth') + + if auth is not None: + response = requests.get(bundle_url, auth=(auth.get('username'), auth.get('password'))) + else: + response = requests.get(bundle_url) + + response_code = response.status_code + + if response_code != 200: + response_txt = response.raise_for_status() + if ErrorResponder.is_plain_string(response_txt): + ErrorResponder.fill_error(return_obj, message=response_txt) + elif ErrorResponder.is_json_string(response_txt): + response_json = json.loads(response_txt) + ErrorResponder.fill_error(return_obj, response_json, ['reason']) + else: + raise UnexpectedResponseException + else: + bundle = response.json() + + if "validate" in self.configuration and self.configuration["validate"] is True: + results = validate_instance(bundle) + + if results.is_valid is not True: + return {"success": False, "message": "Invalid STIX received: " + json.dumps(results)} + + for obj in bundle["objects"]: + if obj["type"] == "observed-data": + observations.append(obj) + + # Pattern match + results = self.match(search_id, observations, False) + + if len(results) != 0: + return_obj['success'] = True + return_obj['data'] = results[int(offset):int(offset + length)] + else: + return_obj['success'] = True + return_obj['data'] = [] + + return return_obj diff --git a/stix_shifter/stix_transmission/stix_transmission.py b/stix_shifter/stix_transmission/stix_transmission.py index 97fd474c6..69f1a996a 100644 --- a/stix_shifter/stix_transmission/stix_transmission.py +++ b/stix_shifter/stix_transmission/stix_transmission.py @@ -1,7 +1,7 @@ import importlib from ..utils.error_response import ErrorResponder -TRANSMISSION_MODULES = ['async_dummy', 'synchronous_dummy', 'qradar', 'splunk', 'bigfix', 'csa', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy','bundle'] +TRANSMISSION_MODULES = ['async_dummy', 'synchronous_dummy', 'qradar', 'splunk', 'bigfix', 'csa', 'aws_security_hub', 'carbonblack', 'elastic_ecs', 'proxy','stix_bundle'] RESULTS = 'results' QUERY = 'query'