From 54f2f0d4cb973c1f7f1a4cbd8d0502e523b0a745 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Aug 2024 14:02:21 +0200 Subject: [PATCH 01/39] fix: [stix2 import] Fixed generic info field to use the title set by users - Title set by users is defining the info field if set - Producer information is included as well if set - If title is not set, then we have the fallback option with a generic title based on the Bundle id and producer --- misp_stix_converter/stix2misp/stix2_to_misp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 2d7f497..de25c3e 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -317,9 +317,11 @@ def event_tags(self) -> list: @property def generic_info_field(self) -> str: - message = f'STIX {self.stix_version} Bundle ({self._identifier})' if self.event_title is not None: - message = f'{self.event_title} {message}' + if self.producer is not None: + return f'{self.event_title} produced by {self.producer}' + return self.event_title + message = f'STIX {self.stix_version} Bundle ({self._identifier})' if self.producer is not None: message += f' produced by {self.producer}' return f'{message} and converted with the MISP-STIX import feature.' From 1031533a7114bf1996e9d22dc8680cb863eabab9 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Aug 2024 14:38:32 +0200 Subject: [PATCH 02/39] chg: [stix2 import] Added separation in the generic Event info field, between the title and information on the producer --- misp_stix_converter/stix2misp/stix2_to_misp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index de25c3e..80bf1c3 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -319,11 +319,11 @@ def event_tags(self) -> list: def generic_info_field(self) -> str: if self.event_title is not None: if self.producer is not None: - return f'{self.event_title} produced by {self.producer}' + return f'{self.event_title} - produced by {self.producer}' return self.event_title message = f'STIX {self.stix_version} Bundle ({self._identifier})' if self.producer is not None: - message += f' produced by {self.producer}' + message += f' - produced by {self.producer}' return f'{message} and converted with the MISP-STIX import feature.' @property From ee243cf0285c20c1aa9790435e0a5aea8a8fe4eb Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 19 Aug 2024 22:58:20 +0200 Subject: [PATCH 03/39] chg: [stix2 import] Better handling of the STIX2 Parser class arguments - Made the different arguments available with the command line feature part of the parsing method rather than setting them with the Parser init --- misp_stix_converter/misp_stix_converter.py | 40 ++++++++--- .../stix2misp/external_stix2_to_misp.py | 51 +++++++------ misp_stix_converter/stix2misp/importparser.py | 71 ++++++++++++++----- .../stix2misp/internal_stix2_to_misp.py | 22 +++--- .../stix2misp/stix2_to_misp.py | 39 ++++------ 5 files changed, 143 insertions(+), 80 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 08918c7..3f0fb60 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -687,12 +687,12 @@ def stix_2_to_misp(filename: _files_type, return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( _from_misp(bundle.objects), distribution, sharing_group_id, - title, producer, galaxies_as_tags, organisation_uuid, - cluster_distribution, cluster_sharing_group_id + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser = parser(*args) + stix_parser = parser() stix_parser.load_stix_bundle(bundle) - stix_parser.parse_stix_bundle(single_event) + stix_parser.parse_stix_bundle(**args) if output_dir is None: output_dir = filename.parent if stix_parser.single_event: @@ -731,12 +731,12 @@ def stix2_to_misp_instance( return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( _from_misp(bundle.objects), distribution, sharing_group_id, - title, producer, galaxies_as_tags, organisation_uuid, - cluster_distribution, cluster_sharing_group_id + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser = parser(*args) + stix_parser = parser() stix_parser.load_stix_bundle(bundle) - stix_parser.parse_stix_bundle(single_event) + stix_parser.parse_stix_bundle(**args) if stix_parser.single_event: misp_event = misp.add_event(stix_parser.misp_event, pythonify=True) if not isinstance(misp_event, MISPEvent): @@ -773,9 +773,29 @@ def _from_misp(stix_objects): return False -def _get_stix2_parser(from_misp: bool, *args: tuple) -> tuple: +def _get_stix2_parser(from_misp: bool, distribution: int, + sharing_group_id: Union[int, None], + title: Union[str, None], producer: Union[str, None], + galaxies_as_tags: bool, single_event: bool, + organisation_uuid: str, cluster_distribution: int, + cluster_sharing_group_id: Union[int, None]) -> tuple: + args = { + 'distribution': distribution, + 'galaxies_as_tags': galaxies_as_tags, + 'producer': producer, + 'sharing_group_id': sharing_group_id, + 'single_event': single_event, + 'title': title + } if from_misp: - return InternalSTIX2toMISPParser, args[:-3] + return InternalSTIX2toMISPParser, args + args.update( + { + 'cluster_distribution': cluster_distribution, + 'cluster_sharing_group_id': cluster_sharing_group_id, + 'organisation_uuid': organisation_uuid + } + ) return ExternalSTIX2toMISPParser, args diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 7d238d8..997a96d 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -20,22 +20,8 @@ class ExternalSTIX2toMISPParser(STIX2toMISPParser): - def __init__(self, distribution: Optional[int] = 0, - sharing_group_id: Optional[int] = None, - title: Optional[str] = None, - producer: Optional[str] = None, - galaxies_as_tags: Optional[bool] = False, - organisation_uuid: Optional[str] = MISP_org_uuid, - cluster_distribution: Optional[int] = 0, - cluster_sharing_group_id: Optional[int] = None): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) - self._set_cluster_distribution( - self._sanitise_distribution(cluster_distribution), - self._sanitise_sharing_group_id(cluster_sharing_group_id) - ) - self.__organisation_uuid = organisation_uuid + def __init__(self): + super().__init__() self._mapping = ExternalSTIX2toMISPMapping # parsers self._attack_pattern_parser: ExternalSTIX2AttackPatternConverter @@ -53,6 +39,21 @@ def __init__(self, distribution: Optional[int] = 0, self._tool_parser: ExternalSTIX2ToolConverter self._vulnerability_parser: ExternalSTIX2VulnerabilityConverter + def parse_stix_bundle( + self, cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + organisation_uuid: Optional[str] = MISP_org_uuid, **kwargs): + self._set_parameters(**kwargs) + self._set_cluster_distribution( + cluster_distribution, cluster_sharing_group_id + ) + self.__organisation_uuid = organisation_uuid + self._parse_stix_bundle() + + ############################################################################ + # PROPERTIES # + ############################################################################ + @property def cluster_distribution(self) -> dict: return self.__cluster_distribution @@ -67,6 +68,10 @@ def observable_object_parser(self) -> STIX2ObservableObjectConverter: def organisation_uuid(self) -> str: return self.__organisation_uuid + ############################################################################ + # PARSER SETTERS # + ############################################################################ + def _set_attack_pattern_parser(self): self._attack_pattern_parser = ExternalSTIX2AttackPatternConverter(self) @@ -75,10 +80,16 @@ def _set_campaign_parser(self): def _set_cluster_distribution( self, distribution: int, sharing_group_id: Union[int, None]): - cluster_distribution = {'distribution': distribution} - if distribution == 4 and sharing_group_id is not None: - cluster_distribution['sharing_group_id'] = sharing_group_id - self.__cluster_distribution = cluster_distribution + cl_dis = {'distribution': self._sanitise_distribution(distribution)} + if distribution == 4: + if sharing_group_id is not None: + cl_dis['sharing_group_id'] = self._sanitise_sharing_group_id( + sharing_group_id + ) + else: + cl_dis['distribution'] = 0 + self._cluster_distribution_and_sharing_group_id_error() + self.__cluster_distribution = cl_dis def _set_course_of_action_parser(self): self._course_of_action_parser = ExternalSTIX2CourseOfActionConverter(self) diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 31ddaf6..27e8199 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -23,6 +23,8 @@ ] _DATA_PATH = Path(__file__).parents[1].resolve() / 'data' +_DEFAULT_DISTRIBUTION = 0 + _VALID_DISTRIBUTIONS = (0, 1, 2, 3, 4) _RFC_VERSIONS = (1, 3, 4, 5) _UUIDv4 = UUID('76beed5f-7251-457e-8c2a-b45f7b589d3d') @@ -71,29 +73,20 @@ def _load_json_file(path): class STIXtoMISPParser(metaclass=ABCMeta): - def __init__(self, distribution: int, sharing_group_id: Union[int, None], - title: Union[str, None], producer: Union[str, None], - galaxies_as_tags: bool): + def __init__(self): self._identifier: str + self.__distribution: int + self.__galaxies_as_tags: bool + self.__galaxy_feature: str + self.__producer: Union[str, None] self.__relationship_types: dict + self.__sharing_group_id: Union[int, None] + self.__title: Union[str, None] self._clusters: dict = {} + self._galaxies: dict = {} self.__errors: defaultdict = defaultdict(set) self.__warnings: defaultdict = defaultdict(set) - self.__distribution = self._sanitise_distribution(distribution) - self.__sharing_group_id = self._sanitise_sharing_group_id( - sharing_group_id - ) - self.__title = title - self.__producer = producer - self.__galaxies_as_tags = self._sanitise_galaxies_as_tags( - galaxies_as_tags - ) - if self.galaxies_as_tags: - self.__galaxy_feature = 'as_tag_names' - else: - self._galaxies: dict = {} - self.__galaxy_feature = 'as_container' self.__replacement_uuids: dict = {} def _sanitise_distribution(self, distribution: int) -> int: @@ -107,7 +100,8 @@ def _sanitise_distribution(self, distribution: int) -> int: self._distribution_value_error(sanitised) return 0 - def _sanitise_galaxies_as_tags(self, galaxies_as_tags: bool): + def _sanitise_galaxies_as_tags( + self, galaxies_as_tags: Union[bool, str, int]) -> bool: if isinstance(galaxies_as_tags, bool): return galaxies_as_tags if galaxies_as_tags in ('true', 'True', '1', 1): @@ -127,6 +121,32 @@ def _sanitise_sharing_group_id( self._sharing_group_id_error(error) return None + def _set_parameters(self, distribution: int = _DEFAULT_DISTRIBUTION, + sharing_group_id: Optional[int] = None, + galaxies_as_tags: Optional[bool] = False, + single_event: Optional[bool] = False, + producer: Optional[str] = None, + title: Optional[str] = None): + self.__distribution = self._sanitise_distribution(distribution) + self.__sharing_group_id = self._sanitise_sharing_group_id( + sharing_group_id + ) + if self.sharing_group_id is None and self.distribution == 4: + self.__distribution = 0 + self._distribution_and_sharing_group_id_error() + self.__galaxies_as_tags = self._sanitise_galaxies_as_tags( + galaxies_as_tags + ) + self.__galaxy_feature = ( + 'as_tag_names' if self.galaxies_as_tags else 'as_container' + ) + self.__single_event = single_event + self.__producer = producer + self.__title = title + + def _set_single_event(self, single_event: bool): + self.__single_event = single_event + ############################################################################ # PROPERTIES # ############################################################################ @@ -179,6 +199,10 @@ def replacement_uuids(self) -> dict: def sharing_group_id(self) -> Union[int, None]: return self.__sharing_group_id + @property + def single_event(self) -> bool: + return self.__single_event + @property def synonyms_mapping(self) -> dict: try: @@ -208,6 +232,12 @@ def _attribute_from_pattern_parsing_error(self, indicator_id: str): f'Error while parsing pattern from indicator with id {indicator_id}' ) + def _cluster_distribution_and_sharing_group_id_error(self): + self.__errors['init'].add( + 'Invalid Cluster Sharing Group ID - ' + 'cannot be None when distribution is 4' + ) + def _course_of_action_error( self, course_of_action_id: str, exception: Exception): self.__errors[self._identifier].add( @@ -226,6 +256,11 @@ def _custom_object_error(self, custom_object_id: str, exception: Exception): f'{custom_object_id}: {self._parse_traceback(exception)}' ) + def _distribution_and_sharing_group_id_error(self): + self.__errors['init'].add( + 'Invalid Sharing Group ID - cannot be None when distribution is 4' + ) + def _distribution_error(self, exception: Exception): self.__errors['init'].add( f'Wrong distribution format: {exception}' diff --git a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py index d4247c2..821fc24 100644 --- a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py @@ -26,14 +26,8 @@ class InternalSTIX2toMISPParser(STIX2toMISPParser): - def __init__(self, distribution: Optional[int] = 0, - sharing_group_id: Optional[int] = None, - title: Optional[str] = None, - producer: Optional[str] = None, - galaxies_as_tags: Optional[bool] = False): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) + def __init__(self): + super().__init__() self._mapping = InternalSTIX2toMISPMapping # parsers self._attack_pattern_parser: InternalSTIX2AttackPatternConverter @@ -52,6 +46,14 @@ def __init__(self, distribution: Optional[int] = 0, self._tool_parser: InternalSTIX2ToolConverter self._vulnerability_parser: InternalSTIX2VulnerabilityConverter + def parse_stix_bundle(self, **kwargs): + self._set_parameters(**kwargs) + self._parse_stix_bundle() + + ############################################################################ + # PROPERTIES # + ############################################################################ + @property def custom_object_parser(self) -> STIX2CustomObjectConverter: if not hasattr(self, '_custom_object_parser'): @@ -70,6 +72,10 @@ def observed_data_parser(self) -> InternalSTIX2ObservedDataConverter: self, '_observed_data_parser', self._set_observed_data_parser() ) + ############################################################################ + # PARSER SETTERS # + ############################################################################ + def _set_attack_pattern_parser(self) -> InternalSTIX2AttackPatternConverter: self._attack_pattern_parser = InternalSTIX2AttackPatternConverter(self) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 80bf1c3..4117db0 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -202,12 +202,8 @@ class STIX2toMISPParser(STIXtoMISPParser, metaclass=ABCMeta): - def __init__(self, distribution: int, sharing_group_id: Union[int, None], - title: Union[str, None], producer: Union[str, None], - galaxies_as_tags: bool): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) + def __init__(self): + super().__init__() self._creators: set = set() self._mapping: Union[ ExternalSTIX2toMISPMapping, InternalSTIX2toMISPMapping @@ -259,8 +255,17 @@ def load_stix_bundle(self, bundle: Union[Bundle_v20, Bundle_v21]): self._critical_error(exception) self.__n_report = 2 if n_report >= 2 else n_report - def parse_stix_bundle(self, single_event: Optional[bool] = False): - self.__single_event = single_event + def parse_stix_content( + self, filename: str, single_event: Optional[bool] = False): + try: + bundle = _load_stix2_content(filename) + except Exception as exception: + sys.exit(exception) + self.load_stix_bundle(bundle) + del bundle + self.parse_stix_bundle(single_event) + + def _parse_stix_bundle(self): try: feature = self._mapping.bundle_to_misp_mapping(str(self.__n_report)) except AttributeError: @@ -279,16 +284,6 @@ def parse_stix_bundle(self, single_event: Optional[bool] = False): if hasattr(self, feature): setattr(self, feature, {}) - def parse_stix_content( - self, filename: str, single_event: Optional[bool] = False): - try: - bundle = _load_stix2_content(filename) - except Exception as exception: - sys.exit(exception) - self.load_stix_bundle(bundle) - del bundle - self.parse_stix_bundle(single_event) - ############################################################################ # PROPERTIES # ############################################################################ @@ -378,10 +373,6 @@ def observed_data_parser(self) -> _OBSERVED_DATA_PARSER_TYPING: self._set_observed_data_parser() return self._observed_data_parser - @property - def single_event(self) -> bool: - return self.__single_event - @property def stix_version(self) -> str: return self.__stix_version @@ -685,13 +676,13 @@ def _parse_bundle_with_multiple_reports(self): self.__misp_events.append(self.misp_event) def _parse_bundle_with_no_report(self): - self.__single_event = True + self._set_single_event(True) self.__misp_event = self._create_generic_event() self._parse_loaded_features() self._handle_unparsed_content() def _parse_bundle_with_single_report(self): - self.__single_event = True + self._set_single_event(True) if hasattr(self, '_report') and self._report is not None: for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) From 24c6edd22b6546aaf6dadd85ddbb301f7b3aed11 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Aug 2024 08:59:48 +0200 Subject: [PATCH 04/39] add: [tests] Tests for STIX 2.x Bundle import with specific producer or title set by user --- tests/test_external_stix20_bundles.py | 12 +++++++ tests/test_external_stix20_import.py | 44 +++++++++++++++++++++++-- tests/test_external_stix21_bundles.py | 12 +++++++ tests/test_external_stix21_import.py | 47 ++++++++++++++++++++++++--- 4 files changed, 108 insertions(+), 7 deletions(-) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 9e0489f..9087efb 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -1536,6 +1536,18 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ] return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ + # EVENTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_with_event_title(cls): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES + ] + return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ # GALAXIES SAMPLES # ############################################################################ diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 3b40295..71f6fdc 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -2,12 +2,50 @@ # -*- coding: utf-8 -*- from .test_external_stix20_bundles import TestExternalSTIX20Bundles -from ._test_stix import TestSTIX21 -from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import +from ._test_stix import TestSTIX20 +from ._test_stix_import import TestExternalSTIX2Import, TestSTIX20Import from uuid import uuid5 -class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): +class TestExternalSTIX20Import(TestExternalSTIX2Import, TestSTIX20, TestSTIX20Import): + + ############################################################################ + # MISP EVENT IMPORT TESTS. # + ############################################################################ + + def test_stix20_bundle_with_event_title(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(title='Malicious IP addresses report') + event = self.parser.misp_event + self.assertEqual(event.info, self.parser.event_title) + + def test_stix20_bundle_with_event_title_and_producer(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle( + title='Malicious IP addresses report', + producer='MISP Project' + ) + event = self.parser.misp_event + self.assertEqual( + event.info, + f'{self.parser.event_title} - produced by {self.parser.producer}' + ) + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) + + def test_stix20_bundle_with_producer(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(producer='MISP Project') + event = self.parser.misp_event + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) ############################################################################ # MISP GALAXIES IMPORT TESTS # diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 71c7063..d96b264 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -1831,6 +1831,18 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ] return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ + # EVENTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_with_event_title(cls): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES + ] + return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ # GALAXIES SAMPLES # ############################################################################ diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 5f3d6fe..833a7fd 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -10,11 +10,50 @@ class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): - ################################################################################ - # MISP GALAXIES IMPORT TESTS # - ################################################################################ + ############################################################################ + # MISP EVENT IMPORT TESTS. # + ############################################################################ + + def test_stix21_bundle_with_event_title(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(title='Malicious IP addresses report') + event = self.parser.misp_event + self.assertEqual(event.info, self.parser.event_title) + + def test_stix21_bundle_with_event_title_and_producer(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle( + title='Malicious IP addresses report', + producer='MISP Project' + ) + event = self.parser.misp_event + self.assertEqual( + event.info, + f'{self.parser.event_title} - produced by {self.parser.producer}' + ) + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) + + def test_stix21_bundle_with_producer(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(producer='MISP Project') + event = self.parser.misp_event + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) + + ############################################################################ + # MISP GALAXIES IMPORT TESTS # + ############################################################################ - def _check_location_galaxy_features(self, galaxies, stix_object, galaxy_type, cluster_value=None): + def _check_location_galaxy_features( + self, galaxies, stix_object, galaxy_type, cluster_value=None): self.assertEqual(len(galaxies), 1) galaxy = galaxies[0] self.assertEqual(len(galaxy.clusters), 1) From 3773bea8a2b10c540e60f16426f445bb7b865169 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Aug 2024 12:12:21 +0200 Subject: [PATCH 05/39] chg: [stix2 import] Excluding the producer from the event info title --- misp_stix_converter/stix2misp/stix2_to_misp.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 4117db0..26c58aa 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -313,12 +313,8 @@ def event_tags(self) -> list: @property def generic_info_field(self) -> str: if self.event_title is not None: - if self.producer is not None: - return f'{self.event_title} - produced by {self.producer}' return self.event_title message = f'STIX {self.stix_version} Bundle ({self._identifier})' - if self.producer is not None: - message += f' - produced by {self.producer}' return f'{message} and converted with the MISP-STIX import feature.' @property From 1a47495b0c3309c03c850d22a50e6d7916cecdd7 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Aug 2024 13:36:51 +0200 Subject: [PATCH 06/39] fix: [tests] Fixed tests for STIX 2.x Bundles imported as MISP Events where producer and info values are set by user --- tests/test_external_stix20_bundles.py | 2 +- tests/test_external_stix20_import.py | 24 ++---------------------- tests/test_external_stix21_bundles.py | 2 +- tests/test_external_stix21_import.py | 24 ++---------------------- 4 files changed, 6 insertions(+), 46 deletions(-) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 9087efb..557731e 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -1541,7 +1541,7 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ############################################################################ @classmethod - def get_bundle_with_event_title(cls): + def get_bundle_without_report(cls): bundle = deepcopy(cls.__bundle) bundle['objects'] = [ deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 71f6fdc..06d9d94 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -13,35 +13,15 @@ class TestExternalSTIX20Import(TestExternalSTIX2Import, TestSTIX20, TestSTIX20Im # MISP EVENT IMPORT TESTS. # ############################################################################ - def test_stix20_bundle_with_event_title(self): - bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() - self.parser.load_stix_bundle(bundle) - self.parser.parse_stix_bundle(title='Malicious IP addresses report') - event = self.parser.misp_event - self.assertEqual(event.info, self.parser.event_title) - def test_stix20_bundle_with_event_title_and_producer(self): - bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() + bundle = TestExternalSTIX20Bundles.get_bundle_without_report() self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle( title='Malicious IP addresses report', producer='MISP Project' ) event = self.parser.misp_event - self.assertEqual( - event.info, - f'{self.parser.event_title} - produced by {self.parser.producer}' - ) - self.assertEqual( - event.tags[0]['name'], - f'misp-galaxy:producer="{self.parser.producer}"' - ) - - def test_stix20_bundle_with_producer(self): - bundle = TestExternalSTIX20Bundles.get_bundle_with_event_title() - self.parser.load_stix_bundle(bundle) - self.parser.parse_stix_bundle(producer='MISP Project') - event = self.parser.misp_event + self.assertEqual(event.info, self.parser.event_title) self.assertEqual( event.tags[0]['name'], f'misp-galaxy:producer="{self.parser.producer}"' diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index d96b264..352c5e6 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -1836,7 +1836,7 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ############################################################################ @classmethod - def get_bundle_with_event_title(cls): + def get_bundle_without_grouping(cls): bundle = deepcopy(cls.__bundle) bundle['objects'] = [ deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 833a7fd..d2ada9b 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -14,35 +14,15 @@ class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Im # MISP EVENT IMPORT TESTS. # ############################################################################ - def test_stix21_bundle_with_event_title(self): - bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() - self.parser.load_stix_bundle(bundle) - self.parser.parse_stix_bundle(title='Malicious IP addresses report') - event = self.parser.misp_event - self.assertEqual(event.info, self.parser.event_title) - def test_stix21_bundle_with_event_title_and_producer(self): - bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() + bundle = TestExternalSTIX21Bundles.get_bundle_without_grouping() self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle( title='Malicious IP addresses report', producer='MISP Project' ) event = self.parser.misp_event - self.assertEqual( - event.info, - f'{self.parser.event_title} - produced by {self.parser.producer}' - ) - self.assertEqual( - event.tags[0]['name'], - f'misp-galaxy:producer="{self.parser.producer}"' - ) - - def test_stix21_bundle_with_producer(self): - bundle = TestExternalSTIX21Bundles.get_bundle_with_event_title() - self.parser.load_stix_bundle(bundle) - self.parser.parse_stix_bundle(producer='MISP Project') - event = self.parser.misp_event + self.assertEqual(event.info, f'{self.parser.event_title}') self.assertEqual( event.tags[0]['name'], f'misp-galaxy:producer="{self.parser.producer}"' From dd4f9c14db80e442c9bba1afea9f677ccb9b3cd6 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Aug 2024 15:31:59 +0200 Subject: [PATCH 07/39] fix: [misp_stix_converter] Fixed some argparse help values --- misp_stix_converter/__init__.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 08838e8..4282700 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -110,8 +110,7 @@ def main(): # IMPORT SUBPARSER import_parser = subparsers.add_parser( - 'import', help='Import STIX to MISP - try ' - '`misp_stix_converter import -h` for more help.' + 'import', help='Import STIX to MISP - try `misp_stix_converter import -h` for more help.' ) import_parser.add_argument( '-f', '--file', nargs='+', type=Path, required=True, @@ -138,7 +137,14 @@ def main(): ) import_parser.add_argument( '-d', '--distribution', type=int, default=0, choices=[0, 1, 2, 3, 4], - help='Distribution level for the imported MISP content - default is 0' + help=''' + Distribution level for the imported MISP content (default is 0) + - 0: Your organisation only + - 1: This community only + - 2: Connected communities + - 3: All communities + - 4: Sharing Group + ''' ) import_parser.add_argument( '-sg', '--sharing_group', type=int, default=None, @@ -153,7 +159,15 @@ def main(): ) import_parser.add_argument( '-cd', '--cluster_distribution', type=int, default=0, choices=[0, 1, 2, 3, 4], - help='Galaxy Clusters distribution level in case of External STIX 2 content - default id 0' + help=''' + Galaxy Clusters distribution level + in case of External STIX 2 content (default id 0) + - 0: Your organisation only + - 1: This community only + - 2: Connected communities + - 3: All communities + - 4: Sharing Group + ''' ) import_parser.add_argument( '-cg', '--cluster_sharing_group', type=int, default=None, @@ -161,7 +175,7 @@ def main(): ) import_parser.add_argument( '-t', '--title', type=str, default=None, - help='Title prefix to add to the MISP Event `info` field.' + help='Title used to set the MISP Event `info` field.' ) import_parser.add_argument( '-p', '--producer', From f11d7e76bb29b41c3e1cdd72c35044e65955d461 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Aug 2024 16:58:51 +0200 Subject: [PATCH 08/39] chg: [stix2 import] Defining a separate abstract class for methods related to external STIX only --- .../stix2misp/external_stix2_to_misp.py | 40 +++---------------- misp_stix_converter/stix2misp/importparser.py | 28 +++++++++++++ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 997a96d..06b598b 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -11,15 +11,13 @@ ExternalSTIX2MalwareConverter, ExternalSTIX2ObservedDataConverter, ExternalSTIX2ThreatActorConverter, ExternalSTIX2ToolConverter, ExternalSTIX2VulnerabilityConverter, STIX2ObservableObjectConverter) +from .importparser import ExternalSTIXtoMISPParser from .stix2_to_misp import STIX2toMISPParser, _OBSERVABLE_TYPING from collections import defaultdict from typing import Optional, Union -MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' - - -class ExternalSTIX2toMISPParser(STIX2toMISPParser): +class ExternalSTIX2toMISPParser(STIX2toMISPParser, ExternalSTIXtoMISPParser): def __init__(self): super().__init__() self._mapping = ExternalSTIX2toMISPMapping @@ -39,35 +37,22 @@ def __init__(self): self._tool_parser: ExternalSTIX2ToolConverter self._vulnerability_parser: ExternalSTIX2VulnerabilityConverter - def parse_stix_bundle( - self, cluster_distribution: Optional[int] = 0, - cluster_sharing_group_id: Optional[int] = None, - organisation_uuid: Optional[str] = MISP_org_uuid, **kwargs): + def parse_stix_bundle(self, cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + organisation_uuid: Optional[str] = None, **kwargs): self._set_parameters(**kwargs) self._set_cluster_distribution( cluster_distribution, cluster_sharing_group_id ) - self.__organisation_uuid = organisation_uuid + self._set_organisation_uuid(organisation_uuid) self._parse_stix_bundle() - ############################################################################ - # PROPERTIES # - ############################################################################ - - @property - def cluster_distribution(self) -> dict: - return self.__cluster_distribution - @property def observable_object_parser(self) -> STIX2ObservableObjectConverter: if not hasattr(self, '_observable_object_parser'): self._set_observable_object_parser() return self._observable_object_parser - @property - def organisation_uuid(self) -> str: - return self.__organisation_uuid - ############################################################################ # PARSER SETTERS # ############################################################################ @@ -78,19 +63,6 @@ def _set_attack_pattern_parser(self): def _set_campaign_parser(self): self._campaign_parser = ExternalSTIX2CampaignConverter(self) - def _set_cluster_distribution( - self, distribution: int, sharing_group_id: Union[int, None]): - cl_dis = {'distribution': self._sanitise_distribution(distribution)} - if distribution == 4: - if sharing_group_id is not None: - cl_dis['sharing_group_id'] = self._sanitise_sharing_group_id( - sharing_group_id - ) - else: - cl_dis['distribution'] = 0 - self._cluster_distribution_and_sharing_group_id_error() - self.__cluster_distribution = cl_dis - def _set_course_of_action_parser(self): self._course_of_action_parser = ExternalSTIX2CourseOfActionConverter(self) diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 27e8199..dd64e62 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -72,6 +72,34 @@ def _load_json_file(path): return json.load(f) +class ExternalSTIXtoMISPParser(metaclass=ABCMeta): + _MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' + + def _set_cluster_distribution( + self, distribution: int, sharing_group_id: Union[int, None]): + cl_dis = {'distribution': self._sanitise_distribution(distribution)} + if distribution == 4: + if sharing_group_id is not None: + cl_dis['sharing_group_id'] = self._sanitise_sharing_group_id( + sharing_group_id + ) + else: + cl_dis['distribution'] = 0 + self._cluster_distribution_and_sharing_group_id_error() + self.__cluster_distribution = cl_dis + + def _set_organisation_uuid(self, organisation_uuid: Union[str, None]): + self.__organisation_uuid = organisation_uuid or self._MISP_org_uuid + + @property + def cluster_distribution(self) -> dict: + return self.__cluster_distribution + + @property + def organisation_uuid(self) -> str: + return self.__organisation_uuid + + class STIXtoMISPParser(metaclass=ABCMeta): def __init__(self): self._identifier: str From 0c9e543cc9fb06babca57825648d1fca5c03e41b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Aug 2024 17:03:03 +0200 Subject: [PATCH 09/39] fix: [stix2 import] Fixed the method to directly load and parse STIX Bundle giving a filename - Giving the whole set of required arguments needed by the Bundles parsing methods --- misp_stix_converter/stix2misp/stix2_to_misp.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 26c58aa..72b704b 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -255,15 +255,14 @@ def load_stix_bundle(self, bundle: Union[Bundle_v20, Bundle_v21]): self._critical_error(exception) self.__n_report = 2 if n_report >= 2 else n_report - def parse_stix_content( - self, filename: str, single_event: Optional[bool] = False): + def parse_stix_content(self, filename: str, **kwargs): try: bundle = _load_stix2_content(filename) except Exception as exception: sys.exit(exception) self.load_stix_bundle(bundle) del bundle - self.parse_stix_bundle(single_event) + self.parse_stix_bundle(**kwargs) def _parse_stix_bundle(self): try: From 9acfcd507ad28427822ccea68ff257e53999a883 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 23 Aug 2024 17:26:23 +0200 Subject: [PATCH 10/39] fix: [stix2 import] Making the MISP_org_uuid available while putting its declaration at the right place --- misp_stix_converter/misp_stix_converter.py | 5 ++--- misp_stix_converter/stix2misp/__init__.py | 4 ++-- misp_stix_converter/stix2misp/importparser.py | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 3f0fb60..f641fb1 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -14,9 +14,8 @@ from .misp2stix.misp_to_stix21 import MISPtoSTIX21Parser from .misp2stix.stix1_mapping import NS_DICT, SCHEMALOC_DICT from .stix2misp.external_stix1_to_misp import ExternalSTIX1toMISPParser -from .stix2misp.external_stix2_to_misp import ( - ExternalSTIX2toMISPParser, MISP_org_uuid) -from .stix2misp.importparser import _load_stix2_content +from .stix2misp.external_stix2_to_misp import ExternalSTIX2toMISPParser +from .stix2misp.importparser import _load_stix2_content, MISP_org_uuid from .stix2misp.internal_stix1_to_misp import InternalSTIX1toMISPParser from .stix2misp.internal_stix2_to_misp import InternalSTIX2toMISPParser from collections import defaultdict diff --git a/misp_stix_converter/stix2misp/__init__.py b/misp_stix_converter/stix2misp/__init__.py index b4f7633..f9f09f3 100644 --- a/misp_stix_converter/stix2misp/__init__.py +++ b/misp_stix_converter/stix2misp/__init__.py @@ -1,7 +1,7 @@ from .external_stix1_to_misp import ExternalSTIX1toMISPParser # noqa from .external_stix2_mapping import ExternalSTIX2toMISPMapping # noqa -from .external_stix2_to_misp import ExternalSTIX2toMISPParser, MISP_org_uuid # noqa -from .importparser import _load_stix2_content # noqa +from .external_stix2_to_misp import ExternalSTIX2toMISPParser # noqa +from .importparser import _load_stix2_content, MISP_org_uuid # noqa from .internal_stix1_to_misp import InternalSTIX1toMISPParser # noqa from .internal_stix2_mapping import InternalSTIX2toMISPMapping # noqa from .internal_stix2_to_misp import InternalSTIX2toMISPParser # noqa diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index dd64e62..9c98492 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -23,6 +23,8 @@ ] _DATA_PATH = Path(__file__).parents[1].resolve() / 'data' +MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' + _DEFAULT_DISTRIBUTION = 0 _VALID_DISTRIBUTIONS = (0, 1, 2, 3, 4) @@ -73,8 +75,6 @@ def _load_json_file(path): class ExternalSTIXtoMISPParser(metaclass=ABCMeta): - _MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' - def _set_cluster_distribution( self, distribution: int, sharing_group_id: Union[int, None]): cl_dis = {'distribution': self._sanitise_distribution(distribution)} @@ -89,7 +89,7 @@ def _set_cluster_distribution( self.__cluster_distribution = cl_dis def _set_organisation_uuid(self, organisation_uuid: Union[str, None]): - self.__organisation_uuid = organisation_uuid or self._MISP_org_uuid + self.__organisation_uuid = organisation_uuid or MISP_org_uuid @property def cluster_distribution(self) -> dict: From f74114ede708cdfb863d6a87db4b628fc4883d7d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 27 Aug 2024 23:30:52 +0200 Subject: [PATCH 11/39] chg: [stix2 import] Better error and warning messages handling - It is easier to find out where an error message is generated when it is next to the exception than in a parent class - As a result we move the error message generation to where it is more easily identifiable --- .../stix2_attack_pattern_converter.py | 6 +- .../stix2_course_of_action_converter.py | 8 +- .../stix2_custom_object_converter.py | 6 +- .../converters/stix2_identity_converter.py | 6 +- .../converters/stix2_indicator_converter.py | 195 ++++++++-------- .../converters/stix2_location_converter.py | 6 +- .../converters/stix2_malware_converter.py | 12 +- .../converters/stix2_observable_converter.py | 4 +- .../stix2_observed_data_converter.py | 29 ++- .../stix2_threat_actor_converter.py | 6 +- .../converters/stix2_tool_converter.py | 6 +- .../stix2_vulnerability_converter.py | 6 +- .../stix2misp/external_stix2_to_misp.py | 12 +- misp_stix_converter/stix2misp/importparser.py | 210 +----------------- .../stix2misp/internal_stix2_to_misp.py | 2 +- .../stix2misp/stix2_to_misp.py | 66 +++++- 16 files changed, 224 insertions(+), 356 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py b/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py index bb0b79c..45cbaa5 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py @@ -116,7 +116,11 @@ def parse(self, attack_pattern_ref: str): try: parser(attack_pattern) except Exception as exception: - self.main_parser._attack_pattern_error(attack_pattern.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Attack Pattern object with id ' + f'{attack_pattern.id}: {_traceback}' + ) def _create_cluster(self, attack_pattern: _ATTACK_PATTERN_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py b/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py index 458e145..32072c3 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py @@ -87,8 +87,10 @@ def parse(self, course_of_action_ref: str): try: parser(course_of_action) except Exception as exception: - self.main_parser._course_of_action_error( - course_of_action.id, exception + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Course of Action object with id ' + f'{course_of_action.id}: {_traceback}' ) def _create_cluster(self, course_of_action: _COURSE_OF_ACTION_TYPING, @@ -121,4 +123,4 @@ def _parse_course_of_action_object( ) for attribute in self._generic_parser(course_of_action): misp_object.add_attribute(**attribute) - self.main_parser._add_misp_object(misp_object, course_of_action) \ No newline at end of file + self.main_parser._add_misp_object(misp_object, course_of_action) diff --git a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py index 24c964e..55f1efd 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py @@ -53,7 +53,11 @@ def parse(self, custom_ref: str): try: parser(custom_object) except Exception as exception: - self.main_parser._custom_object_error(custom_object.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Custom object with id ' + f'{custom_object.id}: {_traceback} + ) def _parse_custom_attribute(self, custom_attribute: _CUSTOM_OBJECT_TYPING): attribute = { diff --git a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py index 1ee282f..d8066d4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py @@ -322,7 +322,11 @@ def parse(self, identity_ref: str): try: parser(identity) except Exception as exception: - self.main_parser._identity_error(identity.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Identity object with id ' + f'{identity.id}: {_traceback} + ) def _parse_employee_object(self, identity: _IDENTITY_TYPING): misp_object = self._create_misp_object('employee', identity) diff --git a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py index da10f64..2153543 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py @@ -19,14 +19,18 @@ from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject -from stix2.v21.sdo import Indicator +from stix2.v20.sdo import Indicator as Indicator_v20 +from stix2.v21.sdo import Indicator as Indicator_v21 from stix2patterns.inspector import _PatternData as PatternData +from types import GeneratorType from typing import TYPE_CHECKING, Tuple, Union if TYPE_CHECKING: from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser +_INDICATOR_TYPING = Union[Indicator_v20, Indicator_v21] + class STIX2IndicatorMapping(STIX2Mapping, metaclass=ABCMeta): # SINGLE ATTRIBUTES MAPPING @@ -308,8 +312,8 @@ def parse(self, indicator_ref: str): try: parser(indicator) except UnknownPatternMappingError as error: - self.main_parser._unknown_pattern_mapping_warning( - indicator.id, error.__str__() + self._unknown_pattern_mapping_warning( + indicator.id, error.__str__().split('_') ) self._create_stix_pattern_object(indicator) except InvalidSTIXPatternError as error: @@ -352,7 +356,7 @@ def _compile_stix_pattern( return self._pattern_parser.pattern def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: - if isinstance(indicator, Indicator): + if isinstance(indicator, (Indicator_v20, Indicator_v21)): pattern_type = indicator.pattern_type if pattern_type != 'stix': try: @@ -367,6 +371,14 @@ def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: return '_create_stix_pattern_object' return '_parse_stix_pattern' + # Errors handlin + def _no_converted_content_from_pattern_warning( + self, indicator: _INDICATOR_TYPING): + self.main_parser._add_warning( + "No content extracted from the following Indicator's (id: " + f'{indicator.id}) pattern: {indicator.pattern}' + ) + ############################################################################ # INDICATORS PARSING METHODS # ############################################################################ @@ -380,7 +392,7 @@ def _parse_asn_pattern( field = keys[0] mapping = self._mapping.asn_pattern_mapping(field) if mapping is None: - self.main_parser._unmapped_pattern_warning(indicator.id, field) + self._unmapped_pattern_warning(indicator.id, field) continue if not isinstance(values, tuple): attributes.append( @@ -405,9 +417,7 @@ def _parse_asn_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -417,9 +427,7 @@ def _parse_asn_pattern( if 'asn' in (attr['object_relation'] for attr in attributes): self._handle_import_case(indicator, attributes, 'asn') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_directory_pattern( @@ -430,9 +438,7 @@ def _parse_directory_pattern( continue mapping = self._mapping.directory_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -442,9 +448,7 @@ def _parse_directory_pattern( if misp_object.attributes: self.main_parser._add_misp_object(misp_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_domain_ip_port_pattern( @@ -459,9 +463,7 @@ def _parse_domain_ip_port_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -469,7 +471,7 @@ def _parse_domain_ip_port_pattern( else: attributes.append({'value': values, **mapping}) if any(key not in features for key in pattern.comparisons.keys()): - self.main_parser._unknown_pattern_mapping_warning( + self._unknown_pattern_mapping_warning( indicator.id, ( key for key in pattern.comparisons.keys() @@ -482,9 +484,7 @@ def _parse_domain_ip_port_pattern( 'first-seen', 'last-seen' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_email_address_pattern( @@ -495,9 +495,7 @@ def _parse_email_address_pattern( continue mapping = self._mapping.email_address_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -507,9 +505,7 @@ def _parse_email_address_pattern( if attributes: self._handle_import_case(indicator, attributes, 'email') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_email_message_pattern( @@ -521,7 +517,7 @@ def _parse_email_message_pattern( field = '.'.join(keys) if len(keys) > 1 else keys[0] mapping = self._mapping.email_message_mapping(field) if mapping is None: - self.main_parser._unmapped_pattern_warning(indicator.id, field) + self._unmapped_pattern_warning(indicator.id, field) continue if isinstance(values, tuple): for value in values: @@ -534,9 +530,7 @@ def _parse_email_message_pattern( 'bcc', 'cc', 'to' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_file_and_pe_pattern( @@ -554,7 +548,7 @@ def _parse_file_and_pe_pattern( _, _, _, index, _, hash_type = keys mapping = self._mapping.file_hashes_mapping(hash_type) if mapping is None: - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -571,7 +565,7 @@ def _parse_file_and_pe_pattern( _, _, _, index, feature = keys mapping = self._mapping.pe_section_pattern_mapping(feature) if mapping is None: - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -605,9 +599,7 @@ def _parse_file_and_pe_pattern( **{'value': values, **attribute} ) continue - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue file_attributes = self._parse_file_attribute( keys, values, indicator.id @@ -635,9 +627,7 @@ def _parse_file_and_pe_pattern( if file_object.attributes: self.main_parser._add_misp_object(file_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_file_attribute(self, keys: list, values: Union[str, tuple], @@ -653,9 +643,7 @@ def _parse_file_attribute(self, keys: list, values: Union[str, tuple], else: yield {'value': values, **mapping} else: - self.main_parser._unmapped_pattern_warning( - indicator_id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator_id, '.'.join(keys)) def _parse_file_pattern( self, pattern: PatternData, indicator: _INDICATOR_TYPING): @@ -678,9 +666,7 @@ def _parse_file_pattern( 'file-encoding', 'fullpath', 'modification-time', 'path' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_ip_address_pattern( @@ -692,7 +678,7 @@ def _parse_ip_address_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -708,9 +694,7 @@ def _parse_ip_address_pattern( if attributes: self._handle_import_case(indicator, attributes, 'ip-port') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_mutex_pattern( @@ -733,9 +717,7 @@ def _parse_mutex_pattern( if attributes: self._handle_import_case(indicator, attributes, 'mutex', 'name') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_network_connection_pattern( @@ -765,9 +747,7 @@ def _parse_network_connection_pattern( if misp_object.attributes: self.main_parser._add_misp_object(misp_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_network_socket_pattern( @@ -781,9 +761,7 @@ def _parse_network_socket_pattern( keys[-1] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -824,9 +802,7 @@ def _parse_network_traffic_attribute( return mapping = getattr(self._mapping, f'{name}_pattern_mapping')(field) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator_id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator_id, '.'.join(keys)) return if isinstance(values, tuple): for value in values: @@ -861,9 +837,7 @@ def _parse_process_pattern( continue mapping = self._mapping.process_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -876,9 +850,7 @@ def _parse_process_pattern( 'args', 'command-line', 'current-directory', 'name', 'pid' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_registry_key_pattern( @@ -891,9 +863,7 @@ def _parse_registry_key_pattern( keys[-1 if 'values' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -906,12 +876,10 @@ def _parse_registry_key_pattern( 'data', 'data-type', 'name' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) - def _parse_sigma_pattern(self, indicator: Indicator): + def _parse_sigma_pattern(self, indicator: _INDICATOR_TYPING): if hasattr(indicator, 'name') or hasattr(indicator, 'external_references'): attributes = [] for field, mapping in self._mapping.sigma_object_mapping().items(): @@ -963,7 +931,7 @@ def _parse_sigma_pattern(self, indicator: Indicator): indicator ) - def _parse_snort_pattern(self, indicator: Indicator): + def _parse_snort_pattern(self, indicator: _INDICATOR_TYPING): self.main_parser._add_misp_attribute( { 'value': indicator.pattern, @@ -981,9 +949,7 @@ def _parse_software_pattern( continue mapping = self._mapping.software_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -993,9 +959,7 @@ def _parse_software_pattern( if attributes: self._handle_object_case(indicator, attributes, 'software') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_stix_pattern(self, indicator: _INDICATOR_TYPING): @@ -1011,7 +975,7 @@ def _parse_stix_pattern(self, indicator: _INDICATOR_TYPING): raise UnknownParsingFunctionError(feature) parser(compiled_pattern, indicator) - def _parse_suricata_pattern(self, indicator: Indicator): + def _parse_suricata_pattern(self, indicator: _INDICATOR_TYPING): misp_object = self._create_misp_object('suricata', indicator) for feature, mapping in self._mapping.suricata_object_mapping().items(): if hasattr(indicator, feature): @@ -1032,9 +996,7 @@ def _parse_url_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1046,16 +1008,14 @@ def _parse_url_pattern( {'value': values, **self._mapping.url_attribute()} ) if any(key != 'url' for key in pattern.comparisons.keys()): - self.main_parser._unknown_pattern_mapping_warning( + self._unknown_pattern_mapping_warning( indicator.id, (key for key in pattern.comparisons.keys() if key != 'url') ) if attributes: self._handle_import_case(indicator, attributes, 'url') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_user_account_pattern( @@ -1068,9 +1028,7 @@ def _parse_user_account_pattern( keys[-1 if 'unix-account-ext' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1080,9 +1038,7 @@ def _parse_user_account_pattern( if attributes: self._handle_object_case(indicator, attributes, 'user-account') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_x509_pattern( @@ -1095,9 +1051,7 @@ def _parse_x509_pattern( keys[1 if 'hashes' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1113,10 +1067,10 @@ def _parse_x509_pattern( 'validity-not-before', 'version' ) else: - self.main_parser._no_converted_content_from_pattern_warning(indicator) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) - def _parse_yara_pattern(self, indicator: Indicator): + def _parse_yara_pattern(self, indicator: _INDICATOR_TYPING): if hasattr(indicator, 'pattern_version'): misp_object = self._create_misp_object('yara', indicator) for feature, mapping in self._mapping.yara_object_mapping().items(): @@ -1146,6 +1100,23 @@ def _parse_yara_pattern(self, indicator: Indicator): indicator ) + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _unknown_pattern_mapping_warning( + self, indicator_id: str, pattern_types: GeneratorType): + self._add_warning( + f'Unable to map pattern from the Indicator with id {indicator_id}, ' + f"containing the following types: {', '.join(pattern_types)}" + ) + + def _unmapped_pattern_warning(self, indicator_id: str, feature: str): + self._add_warning( + 'Unmapped pattern part in indicator with id ' + f'{indicator_id}: {feature}' + ) + class InternalSTIX2IndicatorMapping( STIX2IndicatorMapping, InternalSTIX2Mapping): @@ -1476,10 +1447,17 @@ def parse(self, indicator_ref: str): raise UnknownParsingFunctionError(f"{feature}_indicator") try: parser(indicator) - except AttributeFromPatternParsingError as error: - self.main_parser._attribute_from_pattern_parsing_error(error) + except AttributeFromPatternParsingError as indicator_id: + self.main_parser._add_error( + 'Error while parsing pattern from ' + f'indicator with id {indicator_id}' + ) except Exception as exception: - self.main_parser._indicator_error(indicator.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Indicator object with id ' + f'{indicator.id}: {_traceback}' + ) ############################################################################ # ATTRIBUTES PARSING METHODS # @@ -1562,7 +1540,7 @@ def _attribute_from_malware_sample_indicator( self.main_parser._add_misp_attribute(attribute, indicator) def _attribute_from_patterning_language_indicator( - self, indicator: Indicator): + self, indicator: _INDICATOR_TYPING): attribute = self._create_attribute_dict(indicator) attribute['value'] = indicator.pattern self.main_parser._add_misp_attribute(attribute, indicator) @@ -1981,7 +1959,8 @@ def _object_from_parler_account_indicator( indicator, 'parler-account' ) - def _object_from_patterning_language_indicator(self, indicator: Indicator): + def _object_from_patterning_language_indicator( + self, indicator: _INDICATOR_TYPING): name = ( 'suricata' if indicator.pattern_type == 'snort' else indicator.pattern_type diff --git a/misp_stix_converter/stix2misp/converters/stix2_location_converter.py b/misp_stix_converter/stix2misp/converters/stix2_location_converter.py index ce8ca17..ba4fbee 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_location_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_location_converter.py @@ -189,4 +189,8 @@ def parse(self, location_ref: str): try: parser(location) except Exception as exception: - self.main_parser._location_error(location.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Location object with id ' + f'{location.id}: {_traceback}' + ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py b/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py index 625217f..1150845 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py @@ -94,6 +94,14 @@ def _parse_malware_object(self, malware: _MALWARE_TYPING): sample = feature(sample_ref, malware) sample.add_reference(malware_object.uuid, 'sample-of') + # Error handling + def _malware_error(self, malware_id: str, exception: Exception): + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Malware object with id ' + f'{malware_id}: {_traceback}' + ) + class ExternalSTIX2MalwareMapping(STIX2MalwareMapping, ExternalSTIX2Mapping): pass @@ -119,7 +127,7 @@ def parse(self, malware_ref: str): else: self._parse_malware_object(malware) except Exception as exception: - self.main_parser._malware_error(malware.id, exception) + self._malware_error(malware.id, exception) def _convert_malware_objects( self, malware: _MALWARE_TYPING) -> Iterator[MISPObject]: @@ -207,7 +215,7 @@ def parse(self, malware_ref: str): try: parser(malware) except Exception as exception: - self.main_parser._malware_error(malware.id, exception) + self._malware_error(malware.id, exception) def _create_cluster(self, malware: _MALWARE_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 2c73d58..f961e6b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -275,7 +275,9 @@ def _parse_file_observable( for hash_type, value in observable.hashes.items(): attribute = self._mapping.file_hashes_mapping(hash_type) if attribute is None: - self.main_parser._hash_type_error(hash_type) + self.main_parser._add_error( + f'Wrong hash_type: {hash_type}' + ) continue yield from self._populate_object_attributes( attribute, value, object_id diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 31b7fb8..9f9a7af 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -65,6 +65,15 @@ def _fetch_observables(self, object_refs: Union[tuple, str]) -> Generator: for object_ref in object_refs: yield self.main_parser._observable[object_ref] + # Errors handling + def _observable_mapping_error( + self, observed_data_id: str, observable_types: Exception): + self.main_parser._add_error( + 'Unable to map observable objects related to the Observed Data ' + f'object with id {observed_data_id} containing the folowing types' + f": {observable_types.__str__().replace('_', ', ')}" + ) + class ExternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, ExternalSTIX2ObservableConverter): @@ -95,9 +104,7 @@ def parse(self, observed_data_ref: str): else: self._parse_observable_objects(observed_data) except UnknownObservableMappingError as observable_types: - self.main_parser._observable_mapping_error( - observed_data.id, observable_types - ) + self._observable_mapping_error(observed_data.id, observable_types) def parse_relationships(self): for misp_object in self.main_parser.misp_event.objects: @@ -204,9 +211,7 @@ def _parse_multiple_observable_object_refs( object_type = object_ref.split('--')[0] mapping = self._mapping.observable_mapping(object_type) if mapping is None: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue feature = f'_parse_{mapping}_observable_object_refs' try: @@ -255,17 +260,13 @@ def _parse_multiple_observable_objects( observable_objects.update(observables) continue if len(observable_types) == 1: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue mapping = self._mapping.observable_mapping( observed_data.objects[object_id]['type'] ) if mapping is None: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue feature = f'_parse_{mapping}_observable_objects' try: @@ -2256,9 +2257,7 @@ def parse(self, observed_data_ref: str): try: parser(observed_data) except UnknownObservableMappingError as observable_types: - self.main_parser._observable_mapping_error( - observed_data.id, observable_types - ) + self._observable_mapping_error(observed_data.id, observable_types) ############################################################################ # ATTRIBUTES PARSING METHODS # diff --git a/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py b/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py index c026768..fec3e40 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py @@ -104,4 +104,8 @@ def parse(self, threat_actor_ref: str): try: parser(threat_actor) except Exception as exception: - self.main_parser._threat_actor_error(threat_actor.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Threat Actor object with id ' + f'{threat_actor.id}: {_traceback}' + ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py b/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py index 756c755..f996fad 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py @@ -97,7 +97,11 @@ def parse(self, tool_ref: str): try: parser(tool) except Exception as exception: - self.main_parser._tool_error(tool.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Tool object with id ' + f'{tool.id}: {_traceback}' + ) def _create_cluster(self, tool: _TOOL_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py b/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py index 2bf01fb..8791cc2 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py @@ -108,7 +108,11 @@ def parse(self, vulnerability_ref: str): try: parser(vulnerability) except Exception as exception: - self.main_parser._vulnerability_error(vulnerability.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Vulnerability object with id ' + f'{vulnerability.id}: {_traceback}' + ) def _parse_vulnerability_attribute( self, vulnerability: _VULNERABILITY_TYPING): diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 06b598b..d39a68c 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -146,18 +146,24 @@ def _handle_unparsed_content(self): continue feature = self._mapping.observable_mapping(observable_type) if feature is None: - self._observable_object_mapping_error( - unparsed_content[observable_type][0] + observable_id = unparsed_content[observable_type][0] + self._add_error( + f'Unable to map observable object with id {observable_id}' ) continue to_call = f'_parse_{feature}_observable_object' for object_id in unparsed_content[observable_type]: if self._observable[object_id]['used'][self.misp_event.uuid]: + # if object_id.split('--')[0] not in _force_observables_list: continue try: getattr(self.observable_object_parser, to_call)(object_id) except Exception as exception: - self._observable_object_error(object_id, exception) + _traceback = self._parse_traceback(exception) + self._add_error( + 'Error parsing the Observable object with id ' + f'{object_id}: {_traceback}' + ) super()._handle_unparsed_content() def _parse_loaded_features(self): diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 9c98492..b1ce39f 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -247,18 +247,11 @@ def warnings(self) -> defaultdict: # ERRORS AND WARNINGS HANDLING METHODS # ############################################################################ - def _attack_pattern_error( - self, attack_pattern_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - 'Error parsing the Attack Pattern object with id ' - f'{attack_pattern_id}: {tb}' - ) + def _add_error(self, error: str): + self.__errors[self._identifier].add(error) - def _attribute_from_pattern_parsing_error(self, indicator_id: str): - self.__errors[self._identifier].add( - f'Error while parsing pattern from indicator with id {indicator_id}' - ) + def _add_warning(self, warning: str): + self.__warnings[self._identifier].add(warning) def _cluster_distribution_and_sharing_group_id_error(self): self.__errors['init'].add( @@ -266,24 +259,6 @@ def _cluster_distribution_and_sharing_group_id_error(self): 'cannot be None when distribution is 4' ) - def _course_of_action_error( - self, course_of_action_id: str, exception: Exception): - self.__errors[self._identifier].add( - 'Error parsing the Course of Action object with id' - f'{course_of_action_id}: {self._parse_traceback(exception)}' - ) - - def _critical_error(self, exception: Exception): - self.__errors[self._identifier].add( - f'The following exception was raised: {exception}' - ) - - def _custom_object_error(self, custom_object_id: str, exception: Exception): - self.__errors[self._identifier].add( - 'Error parsing the Custom object with id' - f'{custom_object_id}: {self._parse_traceback(exception)}' - ) - def _distribution_and_sharing_group_id_error(self): self.__errors['init'].add( 'Invalid Sharing Group ID - cannot be None when distribution is 4' @@ -304,88 +279,6 @@ def _galaxies_as_tags_error(self, galaxies_as_tags): f'Invalid galaxies_as_tags flag: {galaxies_as_tags} (bool expected)' ) - def _hash_type_error(self, hash_type: str): - self.__errors[self._identifier].add(f'Wrong hash type: {hash_type}') - - def _identity_error(self, identity_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Identity object with id {identity_id}: {tb}' - ) - - def _indicator_error(self, indicator_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Indicator object with id {indicator_id}: {tb}' - ) - - def _intrusion_set_error(self, intrusion_set_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Intrusion Set object with id {intrusion_set_id}' - f': {self._parse_traceback(exception)}' - ) - - def _location_error(self, location_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Location object with id {location_id}: {tb}' - ) - - def _malware_error(self, malware_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Malware object with id {malware_id}: {tb}' - ) - - def _marking_definition_error(self, marking_definition_id: str): - self.__errors[self._identifier].add( - f'Error parsing the Marking Definition object with id ' - f'{marking_definition_id}' - ) - - def _no_converted_content_from_pattern_warning( - self, indicator: _INDICATOR_TYPING): - self.__warnings[self._identifier].add( - "No content to extract from the following Indicator's (id: " - f'{indicator.id}) pattern: {indicator.pattern}' - ) - - def _object_ref_loading_error(self, object_ref: str): - self.__errors[self._identifier].add( - f'Error loading the STIX object with id {object_ref}' - ) - - def _object_type_loading_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Error loading the STIX object of type {object_type}' - ) - - def _observable_mapping_error( - self, observed_data_id: str, observable_types: str): - self.__errors[self._identifier].add( - 'Unable to map observable objects related to the Observed Data ' - f'object with id {observed_data_id} containing the folowing types' - f": {observable_types.__str__().replace('_', ', ')}" - ) - - def _observable_object_error( - self, observable_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Observable object with id {observable_id}' - f': {self._parse_traceback(exception)}' - ) - - def _observable_object_mapping_error(self, observable_id: str): - self.__errors[self._identifier].add( - f'Unable to map observable object with id {observable_id}.' - ) - - def _observed_data_error(self, observed_data_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Observed Data object with id {observed_data_id}' - f': {self._parse_traceback(exception)}' - ) - @staticmethod def _parse_traceback(exception: Exception) -> str: tb = ''.join(traceback.format_tb(exception.__traceback__)) @@ -396,105 +289,12 @@ def _sharing_group_id_error(self, exception: Exception): f'Wrong sharing group id format: {exception}' ) - def _threat_actor_error(self, threat_actor_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Threat Actor object with id {threat_actor_id}' - f': {self._parse_traceback(exception)}' - ) - - def _tool_error(self, tool_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Tool object with id {tool_id}: {tb}' - ) - - def _unable_to_load_stix_object_type_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Unable to load STIX object type: {object_type}' - ) - - def _undefined_object_error(self, object_id: str): - self.__errors[self._identifier].add( - f'Unable to define the object identified with the id: {object_id}' - ) - - def _unknown_attribute_type_warning(self, attribute_type: str): - self.__warnings[self._identifier].add( - f'MISP attribute type not mapped: {attribute_type}' - ) - - def _unknown_marking_object_warning(self, marking_ref: str): - self.__warnings[self._identifier].add( - f'Unknown marking definition object referenced by id {marking_ref}' - ) - - def _unknown_marking_ref_warning(self, marking_ref: str): - self.__warnings[self._identifier].add( - f'Unknown marking ref: {marking_ref}' - ) - - def _unknown_network_protocol_warning( - self, protocol: str, object_id: str, - object_type: Optional[str] = 'indicator'): - message = ( - 'in patterning expression within the indicator with id' - if object_type == 'indicator' else - f'within the {object_type} object with id' - ) - self.__warnings[self._identifier].add( - f'Unknown network protocol: {protocol}, {message} {object_id}' - ) - - def _unknown_object_name_warning(self, name: str): - self.__warnings[self._identifier].add( - f'MISP object name not mapped: {name}' - ) - - def _unknown_parsing_function_error(self, feature: str): - self.__errors[self._identifier].add( - f'Unknown STIX parsing function name: {feature}' - ) - - def _unknown_pattern_mapping_warning( - self, indicator_id: str, pattern_types: Union[GeneratorType, str]): - if not isinstance(pattern_types, GeneratorType): - pattern_types = pattern_types.split('_') - self.__warnings[self._identifier].add( - f'Unable to map pattern from the Indicator with id {indicator_id}, ' - f"containing the following types: {', '.join(pattern_types)}" - ) - - def _unknown_pattern_type_error(self, indicator_id: str, pattern_type: str): - self.__errors[self._identifier].add( - f'Unknown pattern type in indicator with id {indicator_id}' - f': {pattern_type}' - ) - - def _unknown_stix_object_type_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Unknown STIX object type: {object_type}' - ) - - def _unmapped_pattern_warning(self, indicator_id: str, feature: str): - self.__warnings[self._identifier].add( - f'Unmapped pattern part in indicator with id {indicator_id}' - f': {feature}' - ) - - def _vulnerability_error(self, vulnerability_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Vulnerability object with id {vulnerability_id}' - f': {self._parse_traceback(exception)}' - ) - ############################################################################ # MISP OBJECT RELATIONSHIPS MAPPING CREATION METHODS # ############################################################################ def __get_relationship_types(self): - relationships_path = Path( - AbstractMISP().resources_path / 'misp-objects' / 'relationships' - ) + relationships_path = resources_path / 'misp-objects' / 'relationships' relationships = _load_json_file(relationships_path / 'definition.json') self.__relationship_types = { relationship['name']: relationship['opposite'] for relationship diff --git a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py index 821fc24..9d5bba8 100644 --- a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py @@ -17,7 +17,7 @@ from pymisp import MISPSighting from stix2.v20.sdo import CustomObject as CustomObject_v20 from stix2.v21.sdo import CustomObject as CustomObject_v21 -from typing import Optional, Union +from typing import Union _CUSTOM_TYPING = Union[ CustomObject_v20, diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 72b704b..5f9f140 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -243,14 +243,19 @@ def load_stix_bundle(self, bundle: Union[Bundle_v20, Bundle_v21]): n_report += 1 feature = self._mapping.stix_object_loading_mapping(object_type) if feature is None: - self._unable_to_load_stix_object_type_error(object_type) + self._add_error( + f'Unable to load STIX object type: {object_type}' + ) continue if hasattr(stix_object, 'created_by_ref'): self._creators.add(stix_object.created_by_ref) try: getattr(self, feature)(stix_object) - except MarkingDefinitionLoadingError as error: - self._marking_definition_error(error) + except MarkingDefinitionLoadingError as marking_definition_id: + self._add_error( + 'Error whil parsing the Marking Definition ' + f'object with id {marking_definition_id}' + ) except AttributeError as exception: self._critical_error(exception) self.__n_report = 2 if n_report >= 2 else n_report @@ -589,18 +594,26 @@ def _handle_object(self, object_type: str, object_ref: str): self._object_type_loading_error(error) except UndefinedIndicatorError as error: self._undefined_indicator_error(error) - except UndefinedSTIXObjectError as error: - self._undefined_object_error(error) + except UndefinedSTIXObjectError as object_id: + self._add_error( + 'Unable to define the object identified ' + f'with the id {object_id}' + ) except UndefinedObservableError as error: self._undefined_observable_error(error) - except UnknownAttributeTypeError as error: - self._unknown_attribute_type_warning(error) - except UnknownObjectNameError as error: - self._unknown_object_name_warning(error) + except UnknownAttributeTypeError as attribute_type: + self._add_warning( + f'MISP attribute type not mapped: {attribute_type}' + ) + except UnknownObjectNameError as name: + self._add_warning(f'MISP object name not mapped: {name}') except UnknownParsingFunctionError as error: self._unknown_parsing_function_error(error) - except UnknownPatternTypeError as error: - self._unknown_pattern_type_error(object_ref, error) + except UnknownPatternTypeError as pattern_type: + self._add_error( + 'Unknown pattern type in Indicator object with id ' + f'{object_ref}: {pattern_type}' + ) def _handle_misp_event_tags( self, misp_event: MISPEvent, stix_object: _GROUPING_REPORT_TYPING): @@ -1233,3 +1246,34 @@ def _timestamp_from_date(date: datetime) -> int: time.strptime(date.split('+')[0], "%Y-%m-%dT%H:%M:%S.%fZ") ) ) + + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _critical_error(self, exception: Exception): + self._add_error(f'The following exception was raised: {exception}') + + def _object_ref_loading_error(self, object_ref: str): + self._add_error(f'Error loading the STIX object with id {object_ref}') + + def _object_type_loading_error(self, object_type: str): + self._add_error(f'Error loading the STIX object of type {object_type}') + + def _unknown_network_protocol_warning( + self, protocol: str, object_id: str, + object_type: Optional[str] = 'indicator'): + message = ( + 'in patterning expression within the indicator with id' + if object_type == 'indicator' else + f'within the {object_type} object with id' + ) + self._add_warning( + f'Unknown network protocol: {protocol}, {message} {object_id}' + ) + + def _unknown_parsing_function_error(self, feature: Exception): + self._add_error(f'Unknown STIX parsing function name: {feature}') + + def _unknown_stix_object_type_error(self, object_type: Exception): + self._add_error(f'Unknown STIX object type: {object_type}') From 5da587241e9f5aa574d0c63d0b5e8040839b4ddf Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 28 Aug 2024 20:27:05 +0200 Subject: [PATCH 12/39] fix: [stix2 import] Code monkey typo fixed --- .../stix2misp/converters/stix2_custom_object_converter.py | 2 +- .../stix2misp/converters/stix2_identity_converter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py index 55f1efd..47abe9a 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py @@ -56,7 +56,7 @@ def parse(self, custom_ref: str): _traceback = self.main_parser._parse_traceback(exception) self.main_parser._add_error( 'Error parsing the Custom object with id ' - f'{custom_object.id}: {_traceback} + f'{custom_object.id}: {_traceback}' ) def _parse_custom_attribute(self, custom_attribute: _CUSTOM_OBJECT_TYPING): diff --git a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py index d8066d4..48439bc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py @@ -325,7 +325,7 @@ def parse(self, identity_ref: str): _traceback = self.main_parser._parse_traceback(exception) self.main_parser._add_error( 'Error while parsing the Identity object with id ' - f'{identity.id}: {_traceback} + f'{identity.id}: {_traceback}' ) def _parse_employee_object(self, identity: _IDENTITY_TYPING): From de7bd394e89bd84be704d8881d36b70c9929f8b4 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 28 Aug 2024 20:33:55 +0200 Subject: [PATCH 13/39] fix: [stix2 import] Fixed test on indicator version - `pattern_type` is a 2.1 field --- .../stix2misp/converters/stix2_indicator_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py index 2153543..94a597f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py @@ -356,7 +356,7 @@ def _compile_stix_pattern( return self._pattern_parser.pattern def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: - if isinstance(indicator, (Indicator_v20, Indicator_v21)): + if isinstance(indicator, Indicator_v21): pattern_type = indicator.pattern_type if pattern_type != 'stix': try: From 48da1e2cfdabd2ac9f3a304d72eb20dc2289f50a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 29 Aug 2024 22:13:40 +0200 Subject: [PATCH 14/39] fix: [stix2 import] Removed unused part of the datetime to timestamp conversion method --- misp_stix_converter/stix2misp/importparser.py | 12 +++++++++++- misp_stix_converter/stix2misp/stix2_to_misp.py | 14 -------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index b1ce39f..ab5df72 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -5,8 +5,10 @@ from .exceptions import UnavailableGalaxyResourcesError from abc import ABCMeta from collections import defaultdict +from datetime import datetime from pathlib import Path -from pymisp import AbstractMISP, MISPEvent, MISPObject +from pymisp import MISPEvent, MISPObject +from pymisp.abstract import resources_path from stix2.exceptions import InvalidValueError from stix2.parsing import dict_to_stix2, parse as stix2_parser, ParseError from stix2.v20.bundle import Bundle as Bundle_v20 @@ -449,3 +451,11 @@ def _sanitise_uuid(self, object_id: str) -> str: self.replacement_uuids[object_uuid] = sanitised_uuid return sanitised_uuid return object_uuid + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + @staticmethod + def _timestamp_from_date(date: datetime) -> int: + return int(date.timestamp()) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 5f9f140..47d71d3 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import sys -import time from .converters import ( ExternalSTIX2AttackPatternConverter, ExternalSTIX2MalwareAnalysisConverter, ExternalSTIX2CampaignConverter, InternalSTIX2CampaignConverter, @@ -31,7 +30,6 @@ from .internal_stix2_mapping import InternalSTIX2toMISPMapping from abc import ABCMeta from collections import defaultdict -from datetime import datetime from pymisp import ( MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, MISPObject, MISPSighting) @@ -1235,18 +1233,6 @@ def _parse_confidence_level(confidence_level: int) -> str: return 'misp:confidence-level="rarely-confident"' return 'misp:confidence-level="unconfident"' - @staticmethod - def _timestamp_from_date(date: datetime) -> int: - return int(date.timestamp()) - try: - return int(date.timestamp()) - except AttributeError: - return int( - time.mktime( - time.strptime(date.split('+')[0], "%Y-%m-%dT%H:%M:%S.%fZ") - ) - ) - ############################################################################ # ERRORS AND WARNINGS HANDLING METHODS # ############################################################################ From 8e8c6e635ede8a1477d7a5dd9e35b45ec397ef2a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 16 Sep 2024 23:09:09 +0200 Subject: [PATCH 15/39] chg: [stix2 import] More specific name for the method to check is a STIX 2.x file was generated from MISP --- misp_stix_converter/__init__.py | 4 ++-- misp_stix_converter/misp_stix_converter.py | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 4282700..6242b6e 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -10,13 +10,13 @@ from .misp2stix import stix20_framing, stix21_framing # noqa # Helpers from .misp_stix_converter import ( # noqa - _from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2, + _is_stix2_from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2, misp_event_collection_to_stix1, misp_to_stix1, misp_to_stix2, stix_1_to_misp, stix_2_to_misp, stix2_to_misp_instance) # STIX 1 special helpers from .misp_stix_converter import ( # noqa _get_campaigns, _get_courses_of_action, _get_events, _get_indicators, - _get_observables, _get_threat_actors, _get_ttps, _from_misp) + _get_observables, _get_threat_actors, _get_ttps) # STIX 1 footers from .misp_stix_converter import ( # noqa _get_campaigns_footer, _get_courses_of_action_footer, _get_indicators_footer, diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index f641fb1..789d6cf 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -685,7 +685,7 @@ def stix_2_to_misp(filename: _files_type, except Exception as error: return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( - _from_misp(bundle.objects), distribution, sharing_group_id, + _is_stix2_from_misp(bundle.objects), distribution, sharing_group_id, title, producer, galaxies_as_tags, single_event, organisation_uuid, cluster_distribution, cluster_sharing_group_id ) @@ -729,7 +729,7 @@ def stix2_to_misp_instance( except Exception as error: return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( - _from_misp(bundle.objects), distribution, sharing_group_id, + _is_stix2_from_misp(bundle.objects), distribution, sharing_group_id, title, producer, galaxies_as_tags, single_event, organisation_uuid, cluster_distribution, cluster_sharing_group_id ) @@ -762,14 +762,6 @@ def stix2_to_misp_instance( # STIX CONTENT LOADING FUNCTIONS # ################################################################################ -def _from_misp(stix_objects): - for stix_object in stix_objects: - labels = stix_object.get('labels', []) - if stix_object['type'] not in _STIX2_event_types or not labels: - continue - if any(tag in labels for tag in _MISP_STIX_tags): - return True - return False def _get_stix2_parser(from_misp: bool, distribution: int, @@ -798,6 +790,16 @@ def _get_stix2_parser(from_misp: bool, distribution: int, return ExternalSTIX2toMISPParser, args +def _is_stix2_from_misp(stix_objects: list): + for stix_object in stix_objects: + labels = stix_object.get('labels', []) + if stix_object['type'] not in _STIX2_event_types or not labels: + continue + if any(tag in labels for tag in _MISP_STIX_tags): + return True + return False + + def _load_stix_event(filename, tries=0): try: return STIXPackage.from_xml(filename) From 6e0fa214eb7e9d25d153239bdc5958defb14c661 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 23 Sep 2024 17:14:20 +0200 Subject: [PATCH 16/39] fix: [stix2 import] Fixed `synonyms_mapping` call - The `synonyms_mapping` attribute is reached when we want to convert some SDOs into tags while they're usually converted as custom Galaxy Cluster --- misp_stix_converter/stix2misp/converters/stix2converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index f288e0b..c830ab8 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -212,9 +212,9 @@ def parse(self, stix_object_ref: str): ############################################################################ def _check_existing_galaxy_name(self, stix_object_name: str) -> Union[list, None]: - if stix_object_name in self.synonyms_mapping: - return self.synonyms_mapping[stix_object_name] - for name, tag_names in self.synonyms_mapping.items(): + if stix_object_name in self.main_parser.synonyms_mapping: + return self.main_parser.synonyms_mapping[stix_object_name] + for name, tag_names in self.main_parser.synonyms_mapping.items(): if stix_object_name in name: return tag_names From 1e47a32576d57cc22b9ce5dc08cbfb9b0812ac6e Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 23 Sep 2024 18:06:14 +0200 Subject: [PATCH 17/39] wip: [stix1 import] First version of a STIX 1 import feature porting from the MISP core code base --- .../stix2misp/external_stix1_to_misp.py | 430 ++++++++++- misp_stix_converter/stix2misp/importparser.py | 56 ++ .../stix2misp/internal_stix1_to_misp.py | 430 ++++++++++- .../stix2misp/stix1_mapping.py | 307 +++++++- .../stix2misp/stix1_to_misp.py | 694 +++++++++++++++++- 5 files changed, 1897 insertions(+), 20 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix1_to_misp.py b/misp_stix_converter/stix2misp/external_stix1_to_misp.py index 129e4ad..e848ab7 100644 --- a/misp_stix_converter/stix2misp/external_stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix1_to_misp.py @@ -1,9 +1,431 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .stix1_to_misp import STIX1toMISPParser +from .importparser import ExternalSTIXtoMISPParser +from .stix1_mapping import ExternalSTIX1toMISPMapping +from .stix1_to_misp import StixObjectTypeError, STIX1toMISPParser +from collections import defaultdict +from cybox.core import Observable, Observables +from pymisp.abstract import misp_objects_path +from pymisp import MISPAttribute, MISPEvent, MISPObject +from stix.data_marking import MarkingSpecification +from stix.extensions.marking.ais import AISMarkingStructure +from stix.extensions.marking.tlp import TLPMarkingStructure +from stix.indicator import Indicator +from stix.threat_actor import ThreatActor +from stix.ttp import TTP +from typing import Optional - -class ExternalSTIX1toMISPParser(STIX1toMISPParser): +class ExternalSTIX1toMISPParser(STIX1toMISPParser, ExternalSTIXtoMISPParser): def __init__(self): - super().__init__() \ No newline at end of file + super().__init__() + self._mapping = ExternalSTIX1toMISPMapping + self.__dns_objects = defaultdict(dict) + self.__dns_ips = [] + + def parse_stix_package(self, cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + organisation_uuid: Optional[str] = None, **kwargs): + self._set_parameters(**kwargs) + self._set_single_event(True) + self._set_cluster_distribution( + cluster_distribution, cluster_sharing_group_id + ) + self._set_organisation_uuid(organisation_uuid) + self._set_misp_event(MISPEvent()) + if self.stix_package.timestamp: + stix_date = self.stix_package.timestamp + try: + self.misp_event.date = stix_date.date() + except AttributeError: + self.misp_event.date = stix_date + self.misp_event.timestamp = self._timestamp_from_date(stix_date) + self.misp_event.info = self._get_event_info() + header = self.stix_package.stix_header + if getattr(getattr(header, 'description', None), 'value', None): + self.misp_event.add_attribute( + **{ + 'type': 'text', 'value': header.description.value, + 'comment': 'STIX Header Description' + } + ) + if getattr(header, 'handling', None): + for handling in header.handling: + for tag in self._parse_marking(handling): + self.misp_event.add_tag(tag) + if self.stix_package.indicators: + for indicator in self.stix_package.indicators: + if indicator.related_indicators: + for related_indicator in indicator.related_indicators: + self._parse_indicator(related_indicator) + else: + self._parse_indicator(indicator) + if self.stix_package.observables: + self._parse_observables() + if self.stix_package.ttps: + for ttp in self.stix_package.ttps.ttp: + self._parse_ttp(ttp) + if self.stix_package.courses_of_action: + for course_of_action in self.stix_package.courses_of_action: + self._parse_course_of_action(course_of_action) + if self.stix_package.threat_actors: + for threat_actor in self.stix_package.threat_actors: + self._parse_threat_actor(threat_actor) + if self.dns_objects: + for domain in self.dns_objects['domain'].values(): + domain_attribute = domain['data'] + ip_reference = domain['related'] + if ip_reference in self.dns_objects['ip']: + misp_object = MISPObject( + 'passive-dns', misp_objects_path_custom=misp_objects_path + ) + domain_attribute['object_relation'] = "rrname" + misp_object.add_attribute(**domain_attribute) + ip_address = self.dns_objects['ip'][ip_reference]['value'] + misp_object.add_attribute( + **{ + "type": "text", "object_relation": "rdata", + "value": ip_address + } + ) + misp_object.add_attribute( + **{ + 'type': 'text', 'object_relation': 'rrtype', + 'value': "AAAA" if ":" in ip_address else "A" + } + ) + self.misp_event.add_object(misp_object) + else: + self.misp_event.add_attribute(**domain_attribute) + for ip, ip_attribute in self.dns_objects['ip'].items(): + if ip not in self.dns_ips: + self.misp_event.add_attribute(**ip_attribute) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def dns_ips(self) -> list: + return self.__dns_ips + + @property + def dns_objects(self) -> dict: + return self.__dns_objects + + ############################################################################ + # STIX OBJECTS PARSING METHODS # + ############################################################################ + + def _parse_attributes_from_ttp(self, ttp: TTP, galaxies: set): + attributes = [] + if ttp.resources and getattr(ttp.resources, 'infrastructure', None).observable_characterization: + observables = ttp.resources.infrastructure.observable_characterization + if observables.observables: + for observable in observables.observables: + if not self._has_properties(observable): + continue + properties = observable.object_.properties + try: + attribute_type, attribute_value, _ = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, ttp.id_) + continue + if isinstance(attribute_value, list): + attributes.extend( + {'type': attribute_type, 'value': value, 'to_ids': False} + for value in attribute_value + ) + else: + attributes.append( + { + 'type': attribute_type, + 'value': attribute_value, + 'to_ids': False + } + ) + if ttp.exploit_targets and ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + if vulnerability.cve_id: + attributes.append( + { + 'type': 'vulnerability', + 'value': vulnerability.cve_id + } + ) + elif vulnerability.title: + title = vulnerability.title + if title in self.synonyms_mapping: + galaxies.update(self.synonyms_mapping[title]) + else: + galaxies.add(f'misp-galaxy:branded-vulnerability="{title}"') + if len(attributes) == 1: + attributes[0].update(self._sanitise_attribute_uuid(ttp.id_)) + return attributes + + def _parse_description(self, stix_object: Indicator | Observable): + if stix_object.description: + misp_attribute = { + 'type': 'text', 'value': stix_object.description.value + } + if stix_object.timestamp: + misp_attribute['timestamp'] = self._timestamp_from_date( + stix_object.timestamp + ) + self.misp_event.add_attribute(**misp_attribute) + + def _parse_galaxies_from_ttp(self, ttp: TTP): + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + yield from self._parse_galaxy(attack_pattern, 'title', 'misp-attack-pattern') + if ttp.behavior.malware_instances: + for malware_instance in ttp.behavior.malware_instances: + yield from self._parse_galaxy(malware_instance, 'title', 'ransomware') + if ttp.resources and ttp.resources.tools: + for tool in ttp.resources.tools: + yield from self._parse_galaxy(tool, 'name', 'tool') + + def _parse_indicator(self, indicator: Indicator): + if hasattr(indicator, 'observable') and indicator.observable: + observable = indicator.observable + if self._has_properties(observable): + properties = observable.object_.properties + uuid = self._sanitise_uuid(observable.object_.id_) + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, indicator.id_) + return + if isinstance(attribute_value, (str, int)): + if observable.object_.related_objects: + related_objects = observable.object_.related_objects + resolving = ( + attribute_type == "url" and len(related_objects) == 1 and + related_objects[0].relationship.value == "Resolved_To" + ) + if resolving: + related_ip = self._sanitise_uuid(related_objects[0].idref) + self.dns_objects['domain'][uuid] = { + "related": related_ip, "data": { + "type": "text", "value": attribute_value + } + } + if related_ip not in self.dns_ips: + self.dns_ips.append(related_ip) + return + # if the returned value is a simple value, we build an attribute + attribute = {'to_ids': True, 'uuid': uuid} + if indicator.timestamp: + attribute['timestamp'] = self._timestamp_from_date(indicator.timestamp) + if hasattr(observable, 'handling') and observable.handling: + attribute['Tag'] = [] + for handling in observable.handling: + attribute['Tag'].extend(self._parse_marking(handling)) + if attribute_type in ('ip-src', 'ip-dst'): + attribute.update( + {'type': attribute_type, 'value': attribute_value} + ) + self.dns_objects['ip'][uuid] = attribute + return + self._handle_attribute_case(attribute_type, attribute_value, compl_data, attribute) + elif attribute_value: + if all(isinstance(value, dict) for value in attribute_value): + # it is a list of attributes, so we build an object + test_mechanisms = [] + if hasattr(indicator, 'test_mechanisms') and indicator.test_mechanisms: + for test_mechanism in indicator.test_mechanisms: + attribute_type = self._mapping.test_mechanisms_mapping(test_mechanism._XSI_TYPE) + if attribute_type is None: + self._add_error( + 'Unknown Test Mechanism type' + f': {test_mechanism._XSI_TYPE}' + ) + continue + if test_mechanism.rule.value is None: + continue + self.misp_event.add_attribute( + **{ + 'type': attribute_type, + 'value': test_mechanism.rule.value + } + ) + test_mechanisms.append(attribute.uuid) + self._handle_object_case( + attribute_type, attribute_value, compl_data, + to_ids=True, object_uuid=uuid, + test_mechanisms=test_mechanisms + ) + else: + # it is a list of attribute values, so we add single attributes + for value in attribute_value: + self.misp_event.add_attribute(**{'type': attribute_type, 'value': value, 'to_ids': True}) + elif hasattr(observable, 'observable_composition') and observable.observable_composition: + self._parse_observables(observable.observable_composition.observables, to_ids=True) + else: + self._parse_description(indicator) + + def _parse_marking(self, handling: MarkingSpecification): + if getattr(handling, 'marking_structures', None): + for marking in handling.marking_structures: + parser = self._mapping.marking_mapping(marking._XSI_TYPE) + if parser is not None: + yield from getattr(self, parser)(marking) + + def _parse_observables(self, observables: Optional[Observables] = None, to_ids: bool = False): + for observable in observables or self.stix_package.observables: + if self._has_properties(observable): + observable_object = observable.object_ + properties = observable_object.properties + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties, title=observable.title) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + uuid = self._sanitise_uuid(observable_object.id_) + if isinstance(attribute_value, (str, int)): + if observable.object_.related_objects: + related_objects = observable.object_.related_objects + resolving = ( + attribute_type == "url" and len(related_objects) == 1 and + related_objects[0].relationship.value == "Resolved_To" + ) + if resolving: + related_ip = self._sanitise_uuid(related_objects[0].idref) + self.dns_objects['domain'][uuid] = { + "related": related_ip, "data": { + "type": "text", "value": attribute_value + } + } + if related_ip not in self.dns_ips: + self.dns_ips.append(related_ip) + continue + # if the returned value is a simple value, we build an attribute + attribute = {'to_ids': to_ids, 'uuid': uuid} + if hasattr(observable, 'handling') and observable.handling: + attribute['Tag'] = [] + for handling in observable.handling: + attribute['Tag'].extend(self._parse_marking(handling)) + if attribute_type in ('ip-src', 'ip-dst'): + attribute.update( + {'type': attribute_type, 'value': attribute_value} + ) + self.dns_objects['ip'][uuid] = attribute + continue + elif attribute_value: + if all(isinstance(value, dict) for value in attribute_value): + # it is a list of attributes, so we build an object + self._handle_object_case( + attribute_type, attribute_value, compl_data, + to_ids=to_ids, object_uuid=uuid + ) + else: + # it is a list of attribute values, so we add single attributes + for value in attribute_value: + self.misp_event.add_attribute( + **{'type': attribute_type, 'value': value, 'to_ids': to_ids} + ) + elif observable_object.related_objects: + for related_object in observable_object.related_objects: + relationship = related_object.relationship.value.lower().replace('_', '-') + self.references[uuid].append( + { + "idref": self.fetch_uuid(related_object.idref), + "relationship": relationship + } + ) + else: + self._parse_description(observable) + + def _parse_threat_actor(self, threat_actor: ThreatActor): + if getattr(threat_actor, 'title', None) is not None: + self.galaxies.update(self._parse_galaxy(threat_actor, 'title', 'threat-actor')) + elif getattr(threat_actor, 'identity', None) is not None: + identity = threat_actor.identity + if getattr(identity, 'name', None) is not None: + self.galaxies.update(self._resolve_galaxy(identity.name, 'threat-actor')) + elif hasattr(identity, 'specification') and getattr(identity.specification, 'party_name', None) is not None: + party_name = identity.specification.party_name + if getattr(party_name, 'person_names', None) is not None: + for person_name in party_name.person_names: + self.galaxies.update( + self._resolve_galaxy(person_name.name_elements[0].value, 'threat-actor') + ) + elif getattr(party_name, 'organisation_names', None) is not None: + for organisation_name in party_name.organisation_names: + self.galaxies.update( + self._resolve_galaxy(organisation_name.name_elements[0].value, 'threat-actor') + ) + + def _parse_ttp(self, ttp: TTP): + galaxies = set(self._parse_galaxies_from_ttp(ttp)) + if self._has_ttp_content(ttp): + attributes = self._parse_attributes_from_ttp(ttp, galaxies) + if attributes: + for attribute in attributes: + misp_attribute = MISPAttribute() + misp_attribute.from_dict(**attribute) + for galaxy in galaxies: + misp_attribute.add_tag(galaxy) + self.misp_event.add_attribute(**misp_attribute) + return + self.galaxies.update(galaxies) + + ############################################################################ + # MARKING DEFINITIONS PARSING METHODS. # + ############################################################################ + + @staticmethod + def _parse_AIS_marking(marking: AISMarkingStructure): + for feature in ('is_proprietary', 'not_proprietary'): + proprietary = getattr(marking, feature) + if proprietary is None: + continue + yield f'ais-marking:AISMarking="{feature.title()}"' + if hasattr(proprietary, 'cisa_proprietary'): + cisa_proprietary = ( + 'true' if proprietary.cisa_proprietary.numerator == 1 + else 'false' + ) + yield f'ais-marking:CISA_Proprietary="{cisa_proprietary}"' + if hasattr(proprietary, 'ais_consent'): + consent = proprietary.ais_consent.consent + yield f'ais-marking:AISConsent="{consent}"' + if hasattr(proprietary, 'tlp_marking'): + color = proprietary.tlp_marking.color + yield f'ais-marking:TLPMarking="{color}"' + + @staticmethod + def _parse_TLP_marking(marking: TLPMarkingStructure): + yield f'tlp:{marking.color.lower()}' + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + def _get_event_info(self): + if hasattr(self.stix_package, 'title'): + return self.stix_package.title + if hasattr(getattr(self.stix_package, 'stix_header', None), 'title'): + return self.stix_package.stix_header.title + return f"Imported from external STIX {self.stix_version} Package" + + @staticmethod + def _has_properties(observable): + if not hasattr(observable, 'object_') or not observable.object_: + return False + if hasattr(observable.object_, 'properties') and observable.object_.properties: + return True + return False + + def _has_ttp_content(self, ttp: TTP) -> bool: + if ttp.resources is not None and ttp.resources.infrastructure is not None: + return True + if ttp.exploit_targets is None or ttp.exploit_targets.exploit_target is None: + return False + return any( + exploit_target.item.vulnerability is not None + for exploit_target in ttp.exploit_targets.exploit_target + ) diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index ab5df72..cfd49aa 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 import json +import sys import traceback from .exceptions import UnavailableGalaxyResourcesError from abc import ABCMeta from collections import defaultdict from datetime import datetime +from mixbox.namespaces import NamespaceNotFoundError from pathlib import Path from pymisp import MISPEvent, MISPObject from pymisp.abstract import resources_path +from stix.core import STIXPackage from stix2.exceptions import InvalidValueError from stix2.parsing import dict_to_stix2, parse as stix2_parser, ParseError from stix2.v20.bundle import Bundle as Bundle_v20 @@ -60,6 +63,26 @@ def _handle_stix2_loading_error(stix2_content: dict): return bundle(*stix2_content, allow_custom=True, interoperability=True) +def _load_stix1_package(filename, tries=0): + try: + return STIXPackage.from_xml(filename) + except NamespaceNotFoundError: + if tries > 0: + sys.exit('Cannot handle STIX namespace') + _update_namespaces() + return _load_stix1_package(filename, tries + 1) + except NotImplementedError: + sys.exit('Missing python library: stix_edh') + except Exception: + try: + import maec + return STIXPackage.from_xml(filename) + except ImportError: + sys.exit('Missing python library: maec') + except Exception as error: + sys.exit(f'Error while loading STIX1 package: {error.__str__()}') + + def _load_stix2_content(filename): with open(filename, 'rt', encoding='utf-8') as f: stix2_content = f.read() @@ -76,6 +99,20 @@ def _load_json_file(path): return json.load(f) +def _update_namespaces(): + from mixbox.namespaces import Namespace, register_namespace + # LIST OF ADDITIONAL NAMESPACES + # can add additional ones whenever it is needed + ADDITIONAL_NAMESPACES = [ + Namespace('http://us-cert.gov/ciscp', 'CISCP', + 'http://www.us-cert.gov/sites/default/files/STIX_Namespace/ciscp_vocab_v1.1.1.xsd'), + Namespace('http://taxii.mitre.org/messages/taxii_xml_binding-1.1', 'TAXII', + 'http://docs.oasis-open.org/cti/taxii/v1.1.1/cs01/schemas/TAXII-XMLMessageBinding-Schema.xsd') + ] + for namespace in ADDITIONAL_NAMESPACES: + register_namespace(namespace) + + class ExternalSTIXtoMISPParser(metaclass=ABCMeta): def _set_cluster_distribution( self, distribution: int, sharing_group_id: Union[int, None]): @@ -119,6 +156,9 @@ def __init__(self): self.__warnings: defaultdict = defaultdict(set) self.__replacement_uuids: dict = {} + def _populate_misp_event(self): + self.misp_events.append(self.misp_event) + def _sanitise_distribution(self, distribution: int) -> int: try: sanitised = int(distribution) @@ -151,6 +191,12 @@ def _sanitise_sharing_group_id( self._sharing_group_id_error(error) return None + def _set_misp_event(self, misp_event: MISPEvent): + self.__misp_event = misp_event + + def _set_misp_events(self): + self.__misp_events = [] + def _set_parameters(self, distribution: int = _DEFAULT_DISTRIBUTION, sharing_group_id: Optional[int] = None, galaxies_as_tags: Optional[bool] = False, @@ -209,6 +255,16 @@ def galaxy_definitions(self) -> Path: def galaxy_feature(self) -> str: return self.__galaxy_feature + @property + def misp_event(self) -> MISPEvent: + return self.__misp_event + + @property + def misp_events(self) -> Union[list, MISPEvent]: + return getattr( + self, '_STIXtoMISPParser__misp_events', self.__misp_event + ) + @property def producer(self) -> Union[str, None]: return self.__producer diff --git a/misp_stix_converter/stix2misp/internal_stix1_to_misp.py b/misp_stix_converter/stix2misp/internal_stix1_to_misp.py index ef2c411..9fa58cd 100644 --- a/misp_stix_converter/stix2misp/internal_stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix1_to_misp.py @@ -1,9 +1,435 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .stix1_to_misp import STIX1toMISPParser +from .stix1_mapping import InternalSTIX1toMISPMapping +from .stix1_to_misp import StixObjectTypeError, STIX1toMISPParser +from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp.abstract import resources_path +from pymisp.api import describe_types +from stix.exploit_target import Vulnerability, Weakness +from stix.indicator import Indicator, Observable +from stix.ttp import TTP +from stix.ttp.attack_pattern import AttackPattern +from typing import Optional + +_MISP_categories = describe_types.get('categories') +_MISP_objects_path = resources_path / 'objects' class InternalSTIX1toMISPParser(STIX1toMISPParser): def __init__(self): - super().__init__() \ No newline at end of file + super().__init__() + self._mapping = InternalSTIX1toMISPMapping + self.__dates = set() + self.__timestamps = set() + self.__titles = set() + + def parse_stix_package(self, **kwargs): + self._set_parameters(**kwargs) + self._set_misp_event(MISPEvent()) + for item in self.stix_package.related_packages.related_package: + package = item.item + self._event = package.incidents[0] + object_references = [] + for coa_taken in self._event.coa_taken: + self._parse_course_of_action(coa_taken.course_of_action) + if self._event.attributed_threat_actors: + object_references.extend( + threat_actor.item.idref for threat_actor + in self._event.attributed_threat_actors.threat_actor + ) + if self._event.leveraged_ttps and self._event.leveraged_ttps.ttp: + object_references.extend( + ttp.item.idref for ttp in self._event.leveraged_ttps.ttp + ) + object_references = tuple( + '-'.join(part for part in reference.split('-')[-5:]) + for reference in object_references if reference is not None + ) + if self._event.timestamp: + stix_date = self._event.timestamp + try: + self.dates.add(stix_date.date()) + except AttributeError: + self.dates.add(stix_date) + self.timestamps.add(self._timestamp_from_date(stix_date)) + self.titles.add(self._get_event_info()) + if self._event.related_indicators: + for indicator in self._event.related_indicators.indicator: + self._parse_indicator(indicator) + if self._event.related_observables: + for observable in self._event.related_observables.observable: + self._parse_observable(observable) + if self._event.history: + for entry in self.event.history.history_items: + journal_entry = entry.journal_entry.value + try: + entry_type, entry_value = journal_entry.split(': ') + if entry_type == "MISP Tag": + self.misp_event.add_tag(entry_value) + elif entry_type.startswith('attribute['): + _, category, attribute_type = entry_type.split('[') + self.misp_event.add_attribute( + **{ + 'type': attribute_type[:-1], + 'category': category[:-1], + 'value': entry_value + } + ) + elif entry_type == "Event Threat Level": + threat_level = self._mapping.threat_level_mapping( + entry_value + ) + if threat_level is not None: + self.misp_event.threat_level_id = threat_level + except ValueError: + continue + if self._event.information_source and self._event.information_source.references: + for reference in self._event.information_source.references: + self.misp_event.add_attribute(**{'type': 'link', 'value': reference}) + if package.courses_of_action: + for course_of_action in package.courses_of_action: + self.galaxies.update( + self._parse_galaxy(course_of_action, 'title', 'course-of-action') + ) + if package.threat_actors: + for threat_actor in package.threat_actors: + self.galaxies.update( + self._parse_galaxy(threat_actor, 'title', 'threat-actor') + ) + if package.ttps: + for ttp in package.ttps.ttp: + ttp_id = '-'.join((part for part in ttp.id_.split('-')[-5:])) + if ttp_id not in object_references: + self._parse_ttp(ttp) + continue + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + self._parse_attack_pattern_object(attack_pattern, ttp_id) + continue + if ttp.exploit_targets and ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + self._parse_vulnerability_object(vulnerability, ttp_id) + if exploit_target.item.weaknesses: + for weakness in exploit_target.item.weaknesses: + self._parse_weakness_object(weakness, ttp_id) + # if ttp.handling: + # self.parse_tlp_marking(ttp.handling) + self._set_distribution() + self.misp_event.info = ' - '.join(self.titles) + self.misp_event.date = max(self.dates) + self.misp_event.timestamp = max(self.timestamps) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def dates(self) -> set: + return self.__dates + + @property + def timestamps(self) -> set: + return self.__timestamps + + @property + def titles(self) -> set: + return self.__titles + + ############################################################################ + # STIX OBJECTS PARSING METHODS # + ############################################################################ + + def _parse_attack_pattern_object(self, attack_pattern: AttackPattern, ttp_id: str): + attributes = [] + for key, relation in self._mapping.attack_pattern_object_mapping().items(): + value = getattr(attack_pattern, key) + if value: + attributes.append( + (relation, value if isinstance(value, str) else value.value) + ) + if attributes: + attack_pattern_object = MISPObject('attack-pattern') + attack_pattern_object.uuid = ttp_id + for attribute in attributes: + attack_pattern_object.add_attribute(*attribute) + self.misp_event.add_object(attack_pattern_object) + + # Parse indicators of a STIX document coming from our exporter + def _parse_indicator(self, indicator: Indicator): + # define is an indicator will be imported as attribute or object + if indicator.relationship in _MISP_categories: + self._parse_misp_attribute_indicator(indicator) + else: + self._parse_misp_object_indicator(indicator) + + def _parse_observable(self, observable: Observable): + if observable.relationship in _MISP_categories: + self.parse_misp_attribute_observable(observable) + else: + self.parse_misp_object_observable(observable) + + def _parse_ttp(self, ttp: TTP): + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + self.galaxies.update(self._parse_galaxy(attack_pattern, 'title', 'misp-attack-pattern')) + if ttp.behavior.malware_instances: + for malware_instance in ttp.behavior.malware_instances: + if not malware_instance._XSI_TYPE or 'stix-maec' not in malware_instance._XSI_TYPE: + self.galaxies.update(self._parse_galaxy(malware_instance, 'title', 'ransomware')) + elif ttp.exploit_targets: + if ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + self.galaxies.update( + self._parse_galaxy(vulnerability, 'title', 'branded-vulnerability') + ) + elif ttp.resources: + if ttp.resources.tools: + for tool in ttp.resources.tools: + self.galaxies.update(self._parse_galaxy(tool, 'name', 'tool')) + + def _parse_vulnerability_object(self, vulnerability: Vulnerability, ttp_id: str): + attributes = [] + for key, mapping in self._mapping.vulnerability_object_mapping().items(): + value = getattr(vulnerability, key) + if value: + attribute_type, relation = mapping + attributes.append( + { + 'type': attribute_type, 'object_relation': relation, + 'value': value if isinstance(value, str) else value.value + } + ) + if attributes: + if len(attributes) == 1 and attributes[0]['object_relation'] == 'id': + attributes = attributes[0] + attributes['uuid'] = ttp_id + self.misp_event.add_attribute(**attributes) + else: + vulnerability_object = MISPObject('vulnerability') + vulnerability_object.uuid = ttp_id + for attribute in attributes: + vulnerability_object.add_attribute(*attribute) + self.misp_event.add_object(vulnerability_object) + + def _parse_weakness_object(self, weakness: Weakness, ttp_id: str): + attributes = [] + for key, relation in self._mapping.weakness_object_mapping().items(): + value = getattr(weakness, key) + if value: + attributes.append( + (relation, value if isinstance(value, str) else value.value) + ) + if attributes: + weakness_object = MISPObject('weakness') + weakness_object.uuid = ttp_id + for attribute in attributes: + weakness_object.add_attribute(*attribute) + self.misp_event.add_object(weakness_object) + + ############################################################################ + # MISP PARSING METHODS # + ############################################################################ + + # Parse STIX objects that we know will give MISP attributes + def _parse_misp_attribute_indicator(self, indicator: Indicator): + item = indicator.item + if item.observable: + misp_attribute = { + 'to_ids': True, 'category': str(indicator.relationship), + 'timestamp': self._timestamp_from_date(item.timestamp) + } + misp_attribute.update(self._sanitise_attribute_uuid(indicator.id_)) + observable = item.observable + self._parse_misp_attribute(observable, misp_attribute, indicator.id_, to_ids=True) + + def _parse_misp_attribute_observable(self, observable): + if observable.item: + misp_attribute = { + 'to_ids': False, 'category': str(observable.relationship) + } + misp_attribute.update( + self._sanitise_attribute_uuid(observable.item.id_) + ) + self._parse_misp_attribute(observable.item, misp_attribute, observable.id_) + + def _parse_misp_attribute( + self, observable: Observable, misp_attribute: dict, + stix_object_id: str, to_ids: Optional[bool] = False): + if getattr(observable.object_, 'properties', None) is not None: + properties = observable.object_.properties + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type( + properties, title=observable.title + ) + if isinstance(attribute_value, (str, int)): + self._handle_attribute_case(attribute_type, attribute_value, compl_data, misp_attribute) + else: + self._handle_object_case(attribute_type, attribute_value, compl_data, to_ids=to_ids) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, stix_object_id) + elif getattr(observable.observable_composition, 'observables', None) is not None: + attribute_dict = {} + for observables in observable.observable_composition.observables: + properties = observables.object_.properties + try: + attribute_type, attribute_value, _ = self._handle_attribute_type( + properties, observable_id=observable.id_ + ) + attribute_dict[attribute_type] = attribute_value + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, stix_object_id) + if attribute_dict: + attribute_type, attribute_value = self._composite_type(attribute_dict) + self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attribute) + + # Parse STIX object that we know will give MISP objects + def _parse_misp_object_indicator(self, indicator: Indicator): + name = self._define_name(indicator.item.observable, indicator.relationship) + if name == 'passive-dns' and str(indicator.relationship) != "misc": + self._add_error( + f'Unable to parse the Indicator object with id {indicator.id_}' + ) + else: + self._fill_misp_object(indicator.item, name, to_ids=True) + + def _parse_misp_object_observable(self, observable: Observable): + name = self._define_name(observable.item, observable.relationship) + try: + self._fill_misp_object(observable, name) + except Exception: + self._add_error( + 'Unable to parse the Observable ' + f'object with id {observable.id_}' + ) + + ############################################################################ + # MISP OBJECTS PARSING METHODS # + ############################################################################ + + # Create a MISP object, its attributes, and add it in the MISP event + def _fill_misp_object(self, item, name, to_ids=False): + composition = any( + ( + ( + hasattr(item, 'observable') and + hasattr(item.observable, 'observable_composition') and + item.observable.observable_composition + ), + ( + hasattr(item, 'observable_composition') and + item.observable_composition + ) + ) + ) + if composition: + misp_object = MISPObject(name, misp_objects_path_custom=_MISP_objects_path) + self._sanitise_object_uuid(misp_object, item.id_) + if to_ids: + observables = item.observable.observable_composition.observables + misp_object.timestamp = self._get_imestamp_from_date(item.timestamp) + else: + observables = item.observable_composition.observables + args = (misp_object, observables, to_ids) + self._handle_file_composition(*args) if name == 'file' else self._handle_composition(*args) + self.misp_event.add_object(**misp_object) + else: + properties = item.observable.object_.properties if to_ids else item.object_.properties + self._parse_observable_object(properties, to_ids, self._sanitise_uuid(item.id_)) + + def _handle_composition(self, misp_object, observables, to_ids): + for observable in observables: + properties = observable.object_.properties + try: + attribute = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + misp_attribute = MISPAttribute() + misp_attribute.type, misp_attribute.value, misp_attribute.object_relation = attribute + if 'Port' in observable.id_: + misp_attribute.object_relation = '-'.join( + ( + observable.id_.split('-')[0].split(':')[1][:3], + misp_attribute.object_relation + ) + ) + misp_attribute.to_ids = to_ids + misp_object.add_attribute(**misp_attribute) + return misp_object + + def _handle_file_composition(self, misp_object, observables, to_ids): + for observable in observables: + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type( + observable.object_.properties, title=observable.title + ) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + if isinstance(attribute_value, str): + misp_object.add_attribute( + **{ + 'type': attribute_type, 'value': attribute_value, + 'object_relation': attribute_type, 'to_ids': to_ids, + 'data': compl_data + } + ) + else: + for attribute in attribute_value: + attribute['to_ids'] = to_ids + misp_object.add_attribute(**attribute) + return misp_object + + # Create a MISP attribute and add it in its MISP object + def _parse_observable_object(self, properties, to_ids, uuid): + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties) + if isinstance(attribute_value, (str, int)): + attribute = {'to_ids': to_ids, 'uuid': uuid} + self._handle_attribute_case(attribute_type, attribute_value, compl_data, attribute) + else: + self._handle_object_case(attribute_type, attribute_value, compl_data, to_ids=to_ids, object_uuid=uuid) + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + # Return type & value of a composite attribute in MISP + @staticmethod + def _composite_type(attributes: dict): + if "port" in attributes: + if "ip-src" in attributes: + return "ip-src|port", f"{attributes['ip-src']}|{attributes['port']}" + elif "ip-dst" in attributes: + return "ip-dst|port", f"{attributes['ip-dst']}|{attributes['port']}" + elif "hostname" in attributes: + return "hostname|port", f"{attributes['hostname']}|{attributes['port']}" + elif "domain" in attributes: + if "ip-src" in attributes: + ip_value = attributes["ip-src"] + elif "ip-dst" in attributes: + ip_value = attributes["ip-dst"] + return "domain|ip", f"{attributes['domain']}|{ip_value}" + + def _define_name(self, observable: Observable, relationship): + observable_id = observable.id_ + if relationship == "file": + return "registry-key" if "WinRegistryKey" in observable_id else "file" + if "Custom" in observable_id: + return observable_id.split("Custom")[0].split(":")[1] + if relationship == "network" and "ObservableComposition" in observable_id: + return observable_id.split("_")[0].split(":")[1] + return self._mapping.cybox_to_misp_object()[observable_id.split('-')[0].split(':')[1]] + + def _get_event_info(self): + if hasattr(self._event, 'title'): + return self._event.title + if hasattr(getattr(self._event, 'stix_header', None), 'title'): + return self.event.stix_header.title + return f"Imported from STIX {self.stix_version} Package generated with MISP" diff --git a/misp_stix_converter/stix2misp/stix1_mapping.py b/misp_stix_converter/stix2misp/stix1_mapping.py index 8d9a75b..b3c824b 100644 --- a/misp_stix_converter/stix2misp/stix1_mapping.py +++ b/misp_stix_converter/stix2misp/stix1_mapping.py @@ -2,17 +2,302 @@ # -*- coding: utf-8 -*- from .. import Mapping +from typing import Union -class STIX1Mapping: - def __init__(self): - self.__threat_level_mapping = Mapping( - High = '1', - Medium = '2', - Low = '3', - Undefined = '4' - ) +class STIX1toMISPMapping: + __attribute_types_mapping = Mapping( + AccountObjectType = '_handle_credential', + AddressObjectType = '_handle_address', + ArtifactObjectType = '_handle_attachment', + ASObjectType = '_handle_as', + CustomObjectType = '_handle_custom', + DNSRecordObjectType = '_handle_dns', + DomainNameObjectType = '_handle_domain_or_url', + EmailMessageObjectType = '_handle_email_attribute', + FileObjectType = '_handle_file', + HostnameObjectType = '_handle_hostname', + HTTPSessionObjectType = '_handle_http', + LinkObjectType = '_handle_link', + MutexObjectType = '_handle_mutex', + NetworkConnectionObjectType = '_handle_network_connection', + NetworkSocketObjectType = '_handle_network_socket', + PDFFileObjectType = '_handle_file', + PipeObjectType = '_handle_pipe', + PortObjectType = '_handle_port', + ProcessObjectType = '_handle_process', + SocketAddressObjectType = '_handle_socket_address', + SystemObjectType = '_handle_system', + UnixUserAccountObjectType = '_handle_unix_user', + URIObjectType = '_handle_domain_or_url', + UserAccountObjectType = '_handle_user', + WhoisObjectType = '_handle_whois', + WindowsExecutableFileObjectType = '_handle_pe', + WindowsFileObjectType = '_handle_file', + WindowsRegistryKeyObjectType = '_handle_regkey', + WindowsServiceObjectType = '_handle_windows_service', + WindowsUserAccountObjectType = '_handle_windows_user', + X509CertificateObjectType = '_handle_x509' + ) + _file_attribute_type = ('filename', 'filename') + __event_types = Mapping( + ArtifactObjectType = {"type": "attachment", "relation": "attachment"}, + DomainNameObjectType = {"type": "domain", "relation": "domain"}, + FileObjectType = _file_attribute_type, + HostnameObjectType = {"type": "hostname", "relation": "host"}, + MutexObjectType = {"type": "mutex", "relation": "mutex"}, + PDFFileObjectType = _file_attribute_type, + PortObjectType = {"type": "port", "relation": "port"}, + URIObjectType = {"type": "url", "relation": "url"}, + WindowsFileObjectType = _file_attribute_type, + WindowsExecutableFileObjectType = _file_attribute_type, + WindowsRegistryKeyObjectType = {"type": "regkey", "relation": ""} + ) - @property - def threat_level_mapping(self) -> dict: - return self.__threat_level_mapping \ No newline at end of file + # Objects mappings + _AS_attribute = ('AS', 'asn') + __as_mapping = Mapping( + number = _AS_attribute, + handle = _AS_attribute, + name = ('text', 'description') + ) + __credential_authentication_mapping = Mapping( + authentication_type = ('text', 'value', 'type'), + authentication_data = ('text', 'value', 'password'), + structured_authentication_mechanism = ('text', 'description.value', 'format') + ) + __credential_custom_types = ("username", "origin", "notification") + __email_mapping = Mapping( + boundary = ("email-mime-boundary", 'value', "mime-boundary"), + from_ = ("email-src", "address_value.value", "from"), + message_id = ("email-message-id", "value", "message-id"), + reply_to = ("email-reply-to", 'address_value.value', "reply-to"), + subject = ("email-subject", 'value', "subject"), + user_agent = ("text", 'value', "user-agent"), + x_mailer = ("email-x-mailer", 'value', "x-mailer") + ) + _file_mapping = Mapping( + file_path = ('text', 'file_path.value', 'path'), + full_path = ('text', 'full_path.value', 'fullpath'), + file_format = ('mime-type', 'file_format.value', 'mimetype'), + byte_runs = ('pattern-in-file', 'byte_runs[0].byte_run_data', 'pattern-in-file'), + size_in_bytes = ('size-in-bytes', 'size_in_bytes.value', 'size-in-bytes'), + peak_entropy = ('float', 'peak_entropy.value', 'entropy') + ) + __network_connection_fields = ('source_socket_address', 'destination_socket_address') + __network_fields = ('src', 'dst') + __network_reference_mapping = Mapping( + ip_address = ('ip-{}', 'address_value', 'ip-{}'), + port = ('port', 'port_value', '{}-port'), + hostname = ('hostname', 'hostname_value', 'hostname-{}') + ) + __network_socket_fields = ('local_address', 'remote_address') + __network_socket_mapping = Mapping( + protocol = ('text', 'protocol.value', 'protocol'), + address_family = ('text', 'address_family.value', 'address-family'), + domain = ('text', 'domain.value', 'domain-family') + ) + __pe_header_mapping = Mapping( + characteristics = ('hex', 'characteristics-hex'), + machine = ('hex', 'machine-hex'), + number_of_sections = ('counter', 'number-of-sections'), + pointer_to_symbol_table = ('hex', 'pointer-to-symbol-table'), + size_of_optional_header = ('counter', 'size-of-optional-header') + ) + __pe_mapping = Mapping( + **{ + 'file_name': ('filename', 'original-filename'), + 'type': ('text', 'type') + } + ) + __process_mapping = Mapping( + creation_time = ('datetime', 'creation-time'), + start_time = ('datetime', 'start-time'), + name = ('text', 'name'), + pid = ('text', 'pid'), + parent_pid = ('text', 'parent-pid') + ) + __regkey_mapping = Mapping( + **{'hive': ('text', 'hive'), 'key': ('regkey', 'key')} + ) + __regkey_value_mapping = Mapping( + data = ('text', 'data'), + datatype = ('text', 'data-type'), + name = ('text', 'name') + ) + __user_account_object_mapping = Mapping( + username = ('text', 'username'), + full_name = ('text', 'display-name'), + disabled = ('boolean', 'disabled'), + creation_date = ('datetime', 'created'), + last_login = ('datetime', 'last_login'), + home_directory = ('text', 'home_dir'), + script_path = ('text', 'shell') + ) + __whois_mapping = Mapping( + registrar_info = ('whois-registrar', 'value', 'whois-registrar'), + ip_address = ('ip-src', 'address_value.value', 'ip-address'), + domain_name = ('domain', 'value.value', 'domain') + ) + __whois_registrant_mapping = Mapping( + email_address = ('whois-registrant-email', 'address_value.value', 'registrant-email'), + name = ('whois-registrant-name', 'value', 'registrant-name'), + phone_number = ('whois-registrant-phone', 'value', 'registrant-phone'), + organization = ('whois-registrant-org', 'value', 'registrant-org') + ) + __x509_certificate_types = ('version', 'serial_number', 'issuer', 'subject') + __x509_datetime_types = ('not_before', 'not_after') + __x509_pubkey_types = ('exponent', 'modulus') + + @classmethod + def as_mapping(cls) -> dict: + return cls.__as_mapping + + @classmethod + def attribute_types_mapping(cls, object_type: str) -> Union[str, None]: + return cls.__attribute_types_mapping.get(object_type) + + @classmethod + def credential_authentication_mapping(cls) -> dict: + return cls.__credential_authentication_mapping + + @classmethod + def credential_custom_types(cls) -> tuple: + return cls.__credential_custom_types + + @classmethod + def email_mapping(cls) -> dict: + return cls.__email_mapping + + @classmethod + def event_types(cls, object_type: str) -> Union[dict, None]: + return cls.__event_types.get(object_type) + + @classmethod + def file_mapping(cls) -> dict: + return cls._file_mapping + + @classmethod + def network_fields(cls) -> tuple: + return cls.__network_fields + + @classmethod + def network_connection_fields(cls) -> tuple: + return cls.__network_connection_fields + + @classmethod + def network_reference_mapping(cls) -> dict: + return cls.__network_reference_mapping + + @classmethod + def network_socket_fields(cls) -> tuple: + return cls.__network_socket_fields + + @classmethod + def network_socket_mapping(cls) -> dict: + return cls.__network_socket_mapping + + @classmethod + def pe_header_mapping(cls) -> dict: + return cls.__pe_header_mapping + + @classmethod + def pe_mapping(cls) -> dict: + return cls.__pe_mapping + + @classmethod + def process_mapping(cls) -> dict: + return cls.__process_mapping + + @classmethod + def regkey_mapping(cls) -> dict: + return cls.__regkey_mapping + + @classmethod + def regkey_value_mapping(cls) -> dict: + return cls.__regkey_value_mapping + + @classmethod + def user_account_object_mapping(cls) -> dict: + return cls.__user_account_object_mapping + + @classmethod + def whois_mapping(cls) -> dict: + return cls.__whois_mapping + + @classmethod + def whois_registrant_mapping(cls) -> dict: + return cls.__whois_registrant_mapping + + @classmethod + def x509_certificate_types(cls) -> tuple: + return cls.__x509_certificate_types + + @classmethod + def x509_datetime_types(cls) -> tuple: + return cls.__x509_datetime_types + + @classmethod + def x509_pubkey_types(cls) -> tuple: + return cls.__x509_pubkey_types + + +class ExternalSTIX1toMISPMapping(STIX1toMISPMapping): + __marking_mapping = Mapping( + **{ + 'AIS:AISMarkingStructure': '_parse_AIS_marking', + 'tlpMarking:TLPMarkingStructureType': '_parse_TLP_marking' + } + ) + __test_mechanism_mapping = Mapping( + **{ + 'yaraTM:YaraTestMechanismType': 'yara' + } + ) + + @classmethod + def marking_mapping(cls, marking_type: str) -> Union[str, None]: + return cls.__marking_mapping.get(marking_type) + + @classmethod + def test_mechanism_mapping(cls, test_mechanism_type: str) -> Union[str, None]: + return cls.__test_mechanism_mapping.get(test_mechanism_type) + + +class InternalSTIX1toMISPMapping(STIX1toMISPMapping): + __attack_pattern_object_mapping = Mapping( + capec_id = 'id', + title = 'name', + description = 'summary' + ) + __threat_level_mapping = Mapping( + High = '1', + Medium = '2', + Low = '3', + Undefined = '4' + ) + __vulnerability_object_mapping = Mapping( + cve_id = ('vulnerability', 'id'), + description = ('text', 'summary'), + published_datetime = ('datetime', 'published') + ) + __weakness_object_mapping = Mapping( + cwe_id = 'id', + description = 'description' + ) + + @classmethod + def attack_pattern_object_mappin(cls) -> dict: + return cls.__attack_pattern_object_mapping + + @classmethod + def threat_level_mapping(cls, threat_level: str) -> Union[str, None]: + return cls.__threat_level_mapping.get(threat_level) + + @classmethod + def vulnerability_object_mapping(cls) -> dict: + return cls.__vulnerability_object_mapping + + @classmethod + def weakness_object_mapping(cls) -> dict: + return cls.__weakness_object_mapping diff --git a/misp_stix_converter/stix2misp/stix1_to_misp.py b/misp_stix_converter/stix2misp/stix1_to_misp.py index 4e02a58..0a2847a 100644 --- a/misp_stix_converter/stix2misp/stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/stix1_to_misp.py @@ -1,9 +1,697 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python3 -from .importparser import STIXtoMISPParser +from .importparser import STIXtoMISPParser, _load_stix1_package +from .stix1_mapping import STIX1toMISPMapping +from abc import ABCMeta +from base64 import b64decode, b64encode +from collections import defaultdict +from cybox.common import Hash +from cybox.objects import ( + account_object, address_object, artifact_object, as_object, + email_message_object, dns_record_object, domain_name_object, file_object, + hostname_object, http_session_object, link_object, mutex_object, + network_connection_object, network_socket_object, pipe_object, + process_object, socket_address_object, system_object, uri_object, + unix_user_account_object, user_account_object, whois_object, + win_executable_file_object, win_registry_key_object, win_service_object, + win_user_account_object, x509_certificate_object) +from operator import attrgetter +from pathlib import Path +from pymisp.abstract import misp_objects_path +from pymisp import MISPAttribute, MISPEvent, MISPObject +from stix.coa import CourseOfAction +from stix.core import STIXPackage +from stix.threat_actor import ThreatActor +from typing import Union +from uuid import uuid4 +_ADDRESS_TYPING = Union[address_object.Address, address_object.EmailAddress] +_NETWORK_PROPERTIES_TYPING = Union[ + network_connection_object.NetworkConnection, + network_socket_object.NetworkSocket +] +_PROPERTIES_TYPING = Union[ + account_object.Authentication, email_message_object.EmailHeader, + whois_object.WhoisEntry, whois_object.WhoisRegistrant +] +_PARTIAL_PROPERTIES_TYPING = Union[ + as_object.AS, process_object.Process, user_account_object.UserAccount, + win_executable_file_object.WinExecutableFile, win_registry_key_object.WinRegistryKey, + win_registry_key_object.RegistryValue +] +_SIMPLE_PROPERTIES_TYPING = Union[ + file_object.File, network_socket_object.NetworkSocket +] +_STIX_OBJECT_TYPING = Union[CourseOfAction, ThreatActor] -class STIX1toMISPParser(STIXtoMISPParser): + +class StixObjectTypeError(Exception): + pass + + +class STIX1toMISPParser(STIXtoMISPParser, metaclass=ABCMeta): def __init__(self): - super.__init__() \ No newline at end of file + super().__init__() + self._mapping = STIX1toMISPMapping + self.__galaxies = set() + self.__references = defaultdict(list) + + def load_stix_package(self, stix_package: STIXPackage): + self.__stix_package = stix_package + + def parse_stix_content(self, filename: Union[Path, str], **kwargs): + self.__stix_package = _load_stix1_package(filename) + self.parse_stix_package(**kwargs) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def galaxies(self) -> set: + return self.__galaxies + + @property + def references(self) -> dict: + return self.__references + + @property + def stix_package(self) -> STIXPackage: + return self.__stix_package + + @property + def stix_version(self) -> str: + return getattr(self.__stix_package, 'stix_version', '1.1.1') + + ############################################################################ + # PARSING METHODS USED BY BOTH CHILD CLASSES # + ############################################################################ + + # Define type & value of an attribute or object in MISP + def _handle_attribute_type(self, properties, is_object=False, title=None): + xsi_type = properties._XSI_TYPE + args = [properties] + if xsi_type in ("FileObjectType", "PDFFileObjectType", "WindowsFileObjectType"): + args.append(is_object) + elif xsi_type == "ArtifactObjectType": + args.append(title) + parser = self._mapping.attribute_types_mapping(xsi_type) + if parser is None: + raise StixObjectTypeError(xsi_type) + return getattr(self, parser)(*args) + + def _handle_attribute_case(self, attribute_type, attribute_value, data, attribute): + if attribute_type in ('attachment', 'malware-sample'): + attribute['data'] = data + elif attribute_type == 'text': + attribute['comment'] = data + self.misp_event.add_attribute(attribute_type, attribute_value, **attribute) + + # The value returned by the indicators or observables parser is a list of dictionaries + # These dictionaries are the attributes we add in an object, itself added in the MISP event + def _handle_object_case(self, name, attribute_value, compl_data, to_ids=False, object_uuid=None, test_mechanisms=[]): + misp_object = MISPObject(name, misp_objects_path_custom=misp_objects_path) + if object_uuid: + misp_object.uuid = object_uuid + for attribute in attribute_value: + print(attribute) + attribute['to_ids'] = to_ids + misp_object.add_attribute(**attribute) + print() + if isinstance(compl_data, dict): + # if some complementary data is a dictionary containing an uuid, + # it means we are using it to add an object reference + if "pe_uuid" in compl_data: + misp_object.add_reference(compl_data['pe_uuid'], 'includes') + if "process_uuid" in compl_data: + for uuid in compl_data["process_uuid"]: + misp_object.add_reference(uuid, 'connected-to') + if test_mechanisms: + for test_mechanism in test_mechanisms: + misp_object.add_reference(test_mechanism, 'detected-with') + self.misp_event.add_object(misp_object) + + # Parse a course of action and add a MISP object to the event + def parse_course_of_action(self, course_of_action): + misp_object = MISPObject('course-of-action', misp_objects_path_custom=misp_objects_path) + misp_object.uuid = self.fetch_uuid(course_of_action.id_) + if course_of_action.title: + attribute = {'type': 'text', 'object_relation': 'name', + 'value': course_of_action.title} + misp_object.add_attribute(**attribute) + for prop, properties_key in self._mapping._coa_mapping().items(): + if getattr(course_of_action, prop): + attribute = { + 'type': 'text', 'object_relation': prop.replace('_', ''), + 'value': attrgetter('{}.{}'.format(prop, properties_key))(course_of_action) + } + misp_object.add_attribute(**attribute) + if course_of_action.parameter_observables: + for observable in course_of_action.parameter_observables.observables: + properties = observable.object_.properties + attribute = MISPAttribute() + attribute.type, attribute.value, _ = self.handle_attribute_type(properties) + referenced_uuid = str(uuid4()) + attribute.uuid = referenced_uuid + self.misp_event.add_attribute(**attribute) + misp_object.add_reference(referenced_uuid, 'observable', None, **attribute) + self.misp_event.add_object(misp_object) + + ############################################################################ + # OBSERVABLE OBJECTS PARSING METHODS # + ############################################################################ + + @staticmethod + def _handle_address(properties: _ADDRESS_TYPING) -> tuple: + if properties.category == 'e-mail': + return 'email-src', properties.address_value.value, 'from' + return "ip-src" if properties.is_source else "ip-dst", properties.address_value.value, 'ip' + + def _handle_as(self, properties: as_object.AS) -> tuple: + attributes = tuple( + self._fetch_attributes_with_partial_key_parsing(properties, 'as_mapping') + ) + return attributes[0] if len(attributes) == 1 else ('asn', self._return_object_attributes(attributes), '') + + # Return type & value of an attachment attribute + def _handle_attachment(self, properties: artifact_object.Artifact, title: str) -> tuple: + if properties.hashes: + return "malware-sample", f"{title}|{properties.hashes[0]}", properties.raw_artifact.value + return self._mapping.event_types(properties._XSI_TYPE)['type'], title, properties.raw_artifact.value + + # Return type & attributes of a credential object + def _handle_credential(self, properties: account_object.Account) -> tuple: + attributes = [] + if properties.description: + attributes.append(["text", properties.description.value, "text"]) + if properties.authentication: + for authentication in properties.authentication: + attributes.extend( + self._fetch_attributes_with_key_parsing(authentication, 'credential_authentication_mapping') + ) + if properties.custom_properties: + for prop in properties.custom_properties: + if prop.name in self._mapping.credential_custom_types: + attributes.append(['text', prop.value, prop.name]) + return attributes[0] if len(attributes) == 1 else ("credential", self._return_object_attributes(attributes), "") + + # Return type & attributes of a dns object + def _handle_dns(self, properties: dns_record_object.DNSRecord) -> tuple: + relation = [] + if properties.domain_name: + relation.append(["domain", str(properties.domain_name.value), ""]) + if properties.ip_address: + relation.append(["ip-dst", str(properties.ip_address.value), ""]) + if relation: + if len(relation) == '2': + domain = relation[0][1] + ip = relation[1][1] + attributes = [["text", domain, "rrname"], ["text", ip, "rdata"]] + rrtype = "AAAA" if ":" in ip else "A" + attributes.append(["text", rrtype, "rrtype"]) + return "passive-dns", self._return_object_attributes(attributes), "" + return relation[0] + + # Return type & value of a domain or url attribute + def _handle_domain_or_url(self, properties: Union[domain_name_object.DomainName, uri_object.URI]) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.value.value, event_types['relation'] + + # Return type & value of an email attribute + def _handle_email_attribute(self, properties: email_message_object.EmailMessage) -> tuple: + if properties.header: + header = properties.header + attributes = self._fetch_attributes_with_key_parsing(header, '_email_mapping') + if header.to: + for to in header.to: + attributes.append(["email-dst", to.address_value.value, "to"]) + if header.cc: + for cc in header.cc: + attributes.append(["email-dst", cc.address_value.value, "cc"]) + else: + attributes = [] + if properties.attachments: + attributes.extend(self._handle_email_attachment(properties)) + return attributes[0] if len(attributes) == 1 else ("email", self._return_object_attributes(attributes), "") + + # Return type & value of an email attachment + def _handle_email_attachment(self, properties: email_message_object.EmailMessage): + related_objects = ( + {related.id_: related.properties for related in properties.parent.related_objects} + if properties.parent.related_objects else {} + ) + for attachment in (attachment.object_reference for attachment in properties.attachments): + if attachment in related_objects: + yield ("email-attachment", related_objects[attachment].file_name.value, "attachment") + else: + parent_id = self._sanitise_uuid(properties.parent.id_) + referenced_id = self._sanitise_uuid(attachment) + self.references[parent_id].append( + {'idref': referenced_id, 'relationship': 'attachment'} + ) + + # Return type & attributes of a file object + def _handle_file(self, properties: file_object.File, is_object: bool) -> tuple: + b_hash, b_file = False, False + attributes = [] + if properties.hashes: + b_hash = True + for hash_property in properties.hashes: + attributes.append(self._handle_hashes_attribute(hash_property)) + if properties.file_name: + value = properties.file_name.value + if value: + b_file = True + attribute_type, relation = self._mapping.event_types(properties._XSI_TYPE) + attributes.append([attribute_type, value, relation]) + attributes.extend(self._fetch_attributes_with_keys(properties, 'file_mapping')) + if len(attributes) == 1: + attribute = attributes[0] + return attribute[0] if attribute[2] != "fullpath" else "filename", attribute[1], "" + if len(attributes) == 2: + if b_hash and b_file: + return self._handle_filename_object(attributes, is_object) + path, filename = self._handle_filename_path_case(attributes) + if path and filename: + attribute_value = f"{path}\\{filename}" + if '\\' in filename and path == filename: + attribute_value = filename + return "filename", attribute_value, "" + return "file", self._return_object_attributes(attributes), "" + + # Determine path & filename from a complete path or filename attribute + @staticmethod + def _handle_filename_path_case(attributes: list) -> tuple: + path, filename = [""] * 2 + if attributes[0][2] == 'filename' and attributes[1][2] == 'path': + path = attributes[1][1] + filename = attributes[0][1] + elif attributes[0][2] == 'path' and attributes[1][2] == 'filename': + path = attributes[0][1] + filename = attributes[1][1] + return path, filename + + # Return the appropriate type & value when we have 1 filename & 1 hash value + @staticmethod + def _handle_filename_object(attributes: list, is_object: bool) -> tuple: + for attribute in attributes: + attribute_type, attribute_value, _ = attribute + if attribute_type == "filename": + filename_value = attribute_value + else: + hash_type, hash_value = attribute_type, attribute_value + value = f"{filename_value}|{hash_value}" + if is_object: + # file object attributes cannot be filename|hash, so it is malware-sample + attr_type = "malware-sample" + return attr_type, value, attr_type + # it could be malware-sample as well, but STIX is losing this information + return f"filename|{hash_type}", value, "" + + # Return type & value of a hash attribute + @staticmethod + def _handle_hashes_attribute(hash_property: Hash) -> tuple: + hash_type = hash_property.type_.value.lower() + try: + hash_value = hash_property.simple_hash_value.value + except AttributeError: + hash_value = hash_property.fuzzy_hash_value.value + return hash_type, hash_value, hash_type + + # Return type & value of a hostname attribute + def _handle_hostname(self, properties: hostname_object.Hostname) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.hostname_value.value, event_types['relation'] + + # Return type & value of a http request attribute + @staticmethod + def _handle_http(properties: http_session_object.HTTPSession) -> tuple: + client_request = properties.http_request_response[0].http_client_request + if client_request.http_request_header: + request_header = client_request.http_request_header + if request_header.parsed_header: + value = request_header.parsed_header.user_agent.value + return "user-agent", value, "user-agent" + elif request_header.raw_header: + value = request_header.raw_header.value + return "http-method", value, "method" + elif client_request.http_request_line: + value = client_request.http_request_line.http_method.value + return "http-method", value, "method" + + # Return type & value of a link attribute + @staticmethod + def _handle_link(properties: link_object.Link) -> tuple: + return "link", properties.value.value, "link" + + # Return type & value of a mutex attribute + def _handle_mutex(self, properties: mutex_object.Mutex) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.name.value, event_types['relation'] + + def _handle_network(self, properties: _NETWORK_PROPERTIES_TYPING, mapping: str): + for feature, field in zip(self._mapping.network_fields(), getattr(self._mapping, mapping)()): + address_property = getattr(properties, field) + if address_property is None: + continue + for prop, attribute in self._mapping.network_reference_mapping().items(): + if getattr(address_property, prop): + attribute_type, key, relation = attribute + yield ( + attribute_type.format(feature), + attrgetter(f'{prop}.{key}.value')(address_property), + relation.format(feature) + ) + + # Return type & attributes of a network connection object + def _handle_network_connection(self, properties: network_connection_object.NetworkConnection) -> tuple: + attributes = list(self._handle_network(properties, 'network_connection_addresses')) + for feature in ('layer3_protocol', 'layer4_protocol', 'layer7_protocol'): + if getattr(properties, feature): + attributes.append( + ('text', attrgetter(f"{feature}.value")(properties), feature.replace('_', '-')) + ) + if attributes: + return "network-connection", self._return_object_attributes(attributes), "" + + # Return type & attributes of a network socket objet + def _handle_network_socket(self, properties: network_socket_object.NetworkSocket) -> tuple: + attributes = list(self._handle_network(properties, 'network_socket_addresses')) + attributes.extend(self._fetch_attributes_with_keys(properties, 'network_socket_mapping')) + for prop in ('is_listening', 'is_blocking'): + if getattr(properties, prop): + attributes.append(("text", prop.split('_')[1], "state")) + if attributes: + return "network-socket", self._return_object_attributes(attributes), "" + + # Return type & attributes of the file defining a portable executable object + def _handle_pe(self, properties: win_executable_file_object.WinExecutableFile) -> tuple: + pe_object = MISPObject('pe', misp_objects_path_custom=misp_objects_path) + for attribute in self._fetch_attributes_with_partial_key_parsing(properties, 'pe_mapping'): + attribute_type, value, relation = attribute + pe_object.add_attribute(relation, value, type=attribute_type) + if getattr(properties.headers, 'file_header', None) is not None: + header = properties.headers.file_header + for attribute in self._fetch_attributes_with_partial_key_parsing(header, 'pe_header_mapping'): + attribute_type, value, relation = attribute + pe_object.add_attribute(relation, value, type=attribute_type) + misp_object = self.misp_event.add_object(pe_object) + if properties.sections: + for section in properties.sections: + section_uuid = self._handle_pe_section(section) + misp_object.add_reference(section_uuid, 'includes') + file_type, file_value, _ = self._handle_file(properties, False) + return file_type, file_value, {'pe_uuid': misp_object.uuid} + + def _handle_pe_section(self, section: win_executable_file_object.PESection) -> str: + section_object = MISPObject('pe-section', misp_objects_path_custom=misp_objects_path) + header_hashes = section.header_hashes + if header_hashes is None: + header_hashes = section.data_hashes + for _hash in header_hashes: + hash_type, hash_value, _ = self._handle_hashes_attribute(_hash) + section_object.add_attribute(hash_type, hash_value) + if section.entropy: + section_object.add_attribute("entropy", section.entropy.value.value) + if section.section_header: + section_header = section.section_header + section_object.add_attribute("name", section_header.name.value) + section_object.add_attribute("size-in-bytes", section_header.size_of_raw_data.value) + return self.misp_event.add_object(section_object).uuid + + # Return type & value of a names pipe attribute + @staticmethod + def _handle_pipe(properties: pipe_object.Pipe) -> tuple: + return "named pipe", properties.name.value, "" + + # Return type & value of a port attribute + def _handle_port(self, *args): + properties = args[0] + event_types = self._mapping.event_types(properties._XSI_TYPE) + relation = event_types['relation'] + if len(args) > 1: + observable_id = args[1] + if "srcPort" in observable_id: + return event_types['type'], properties.port_value.value, f"src-{relation}" + if "dstPort" in observable_id: + return event_types['type'], properties.port_value.value, f"dst-{relation}" + return event_types['type'], properties.port_value.value, relation + + # Return type & attributes of a process object + def _handle_process(self, properties: process_object.Process): + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, '_process_mapping' + ) + ) + if properties.child_pid_list: + for child in properties.child_pid_list: + attributes.append(["text", child.value, "child-pid"]) + if properties.port_list: + for port in properties.port_list: + attributes.append(["port", port.port_value.value, "port"]) + if properties.image_info: + if properties.image_info.file_name: + attributes.append(["filename", properties.image_info.file_name.value, "image"]) + if properties.image_info.command_line: + attributes.append(["text", properties.image_info.command_line.value, "command-line"]) + if properties.network_connection_list: + references = [] + for connection in properties.network_connection_list: + object_name, object_attributes, _ = self._handle_network_connection(connection) + misp_object = MISPObject(object_name, misp_objects_path_custom=misp_objects_path) + for attribute in object_attributes: + misp_object.add_attribute(**attribute) + self.misp_event.add_object(**misp_object) + references.append(misp_object.uuid) + return "process", self._return_object_attributes(attributes), {"process_uuid": references} + return "process", self._return_object_attributes(attributes), "" + + # Return type & value of a regkey attribute + def _handle_regkey(self, properties: win_registry_key_object.WinRegistryKey): + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, '_regkey_mapping' + ) + ) + if properties.values: + value = properties.values[0] + attributes.extend( + self._fetch_attributes_with_partial_key_parsing( + value, '_regkey_value_mapping' + ) + ) + if len(attributes) in (2,3): + d_regkey = {key: value for (_, value, key) in attributes} + if 'hive' in d_regkey and 'key' in d_regkey: + regkey = f"{d_regkey['hive']}\\{d_regkey['key']}" + if 'data' in d_regkey: + return "regkey|value", f"{regkey} | {d_regkey['data']}", "" + return "regkey", regkey, "" + return "registry-key", self._return_object_attributes(attributes), "" + + # Parse a socket address object in order to return type & value + # of a composite attribute ip|port or hostname|port + def _handle_socket_address(self, properties: socket_address_object.SocketAddress) -> tuple: + if properties.ip_address: + type1, value1, _ = self._handle_address(properties.ip_address) + elif properties.hostname: + type1 = "hostname" + value1 = properties.hostname.hostname_value.value + if properties.port: + return f"{type1}|port", f"{value1}|{properties.port.port_value.value}", "" + return type1, value1, '' + + # Parse a system object to extract a mac-address attribute + @staticmethod + def _handle_system(properties: system_object.System) -> tuple: + if properties.network_interface_list: + return "mac-address", str(properties.network_interface_list[0].mac), "" + + # Parse a UNIX user account object + def _handle_unix_user(self, properties: unix_user_account_object.UnixUserAccount) -> tuple: + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + if properties.user_id: + attributes.append(['text', properties.user_id.value, 'user-id']) + if properties.group_id: + attributes.append(['text', properties.group_id.value, 'group-id']) + return 'user-account', self._return_object_attributes(attributes), '' + + # Parse a user account object + def _handle_user(self, properties: user_account_object.UserAccount) -> tuple: + attributes = tuple( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + return 'user-account', self.return_attributes(attributes), '' + + # Parse a whois object: + # Return type & attributes of a whois object if we have the required fields + # Otherwise create attributes and return type & value of the last attribute to avoid crashing the parent function + def _handle_whois(self, properties: whois_object.WhoisEntry): + attributes = list(self._fetch_attributes_with_key_parsing(properties, '_whois_mapping')) + required_one_of = True if attributes else False + if properties.registrants: + registrant = properties.registrants[0] + attributes.append(self._fetch_attributes_with_key_parsing(registrant, '_whois_registrant_mapping')) + if properties.creation_date: + attributes.append(("datetime", properties.creation_date.value.strftime('%Y-%m-%d'), "creation-date")) + required_one_of = True + if properties.updated_date: + attributes.append(("datetime", properties.updated_date.value.strftime('%Y-%m-%d'), "modification-date")) + if properties.expiration_date: + attributes.append(("datetime", properties.expiration_date.value.strftime('%Y-%m-%d'), "expiration-date")) + if properties.nameservers: + for nameserver in properties.nameservers: + attributes.append(("hostname", nameserver.value.value, "nameserver")) + if properties.remarks: + attribute_type = "text" + relation = "comment" if attributes else attribute_type + attributes.append([attribute_type, properties.remarks.value, relation]) + required_one_of = True + # Testing if we have the required attribute types for Object whois + if required_one_of: + # if yes, we return the object type and the attributes + return "whois", self._return_object_attributes(attributes), "" + # otherwise, attributes are added in the event, and one attribute is returned to not make the function crash + if len(attributes) == 1: + return attributes[0] + last_attribute = attributes.pop(-1) + for attribute in attributes: + attribute_type, attribute_value, attribute_relation = attribute + misp_attributes = {"comment": f"Whois {attribute_relation}"} + self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes) + return last_attribute + + # Return type & value of a windows service object + @staticmethod + def _handle_windows_service(properties: win_service_object.WinService) -> tuple: + if properties.name: + return "windows-service-name", properties.name.value, "" + + # Parse a windows user account object + def _handle_windows_user(self, properties: win_user_account_object.WinUser) -> tuple: + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + if properties.security_id: + attributes.append(['text', properties.security_id.value, 'user-id']) + return 'user-account', self._return_object_attributes(attributes), '' + + def _handle_x509(self, properties: x509_certificate_object.X509Certificate) -> tuple: + attributes = list(self.handle_x509_certificate(properties)) + if properties.raw_certificate: + raw = properties.raw_certificate.value + try: + relation = "raw-base64" if raw == b64encode(b64decode(raw)).strip() else "pem" + except Exception: + relation = "pem" + attributes.append(["text", raw, relation]) + if properties.certificate_signature: + signature = properties.certificate_signature + attribute_type = f"x509-fingerprint-{signature.signature_algorithm.value.lower()}" + attributes.append([attribute_type, signature.signature.value, attribute_type]) + return "x509", self._return_object_attributes(attributes), "" + + def _handle_x509_certificate(self, properties: x509_certificate_object.X509Certificate): + if properties.certificate is None: + return [] + certificate = properties.certificate + if certificate.validity: + validity = certificate.validity + for prop in self._mapping._x509_datetime_types(): + if getattr(validity, prop): + yield ['datetime', getattr(validity, prop).value, f"validity-{prop.replace('_', '-')}"] + if certificate.subject_public_key: + subject_pubkey = certificate.subject_public_key + if subject_pubkey.rsa_public_key: + rsa_pubkey = subject_pubkey.rsa_public_key + for prop in self._mapping._x509_pubkey_types(): + if getattr(rsa_pubkey, prop): + yield ['text', getattr(rsa_pubkey, prop).value, f'pubkey-info-{prop}'] + if subject_pubkey.public_key_algorithm: + yield ["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"] + for prop in self._mapping._x509_certificate_types(): + if getattr(certificate, prop): + yield ['text', getattr(certificate, prop).value, prop.replace('_', '-')] + + ############################################################################ + # GALAXIES PARSING SPECIFIC METHODS USED BY BOTH SUBCLASSES. # + ############################################################################ + + @staticmethod + def _get_galaxy_name(stix_object: _STIX_OBJECT_TYPING, + feature: str) -> Union[str, list, None]: + if getattr(stix_object, feature, None) is not None: + return getattr(stix_object, feature) + for feature in ('name', 'names'): + if getattr(stix_object, feature, None) is not None: + return [value.value for value in getattr(stix_object, feature)] + + def _parse_galaxy(self, stix_object: _STIX_OBJECT_TYPING, + feature: str, default_value: str): + names = self._get_galaxy_name(stix_object, feature) + if names: + if isinstance(names, list): + for name in names: + yield from self._resolve_galaxy(name, default_value) + else: + yield from self._resolve_galaxy(names, default_value) + + def _resolve_galaxy(self, galaxy_name: str, default_value: str) -> list: + if galaxy_name in self.synonyms_mapping: + return self.synonyms_mapping[galaxy_name] + for identifier in galaxy_name.split(' - '): + if identifier[0].isalpha() and any(character.isdecimal() for character in identifier[1:]): + for name, tag_names in self.synonyms_mapping.items(): + if identifier in name: + return tag_names + return [f'misp-galaxy:{default_value}="{galaxy_name}"'] + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + @staticmethod + def _extract_uuid(object_id: str) -> str: + return '-'.join(object_id.split('-')[1:]) + + def _fetch_attributes_with_keys(self, properties: _SIMPLE_PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, feature, relation = attribute + yield (attribute_type, attrgetter(feature)(properties), relation) + + def _fetch_attributes_with_key_parsing(self, properties: _PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, feature, relation = attribute + yield (attribute_type, attrgetter(f'{field}.{feature}')(properties), relation) + + def _fetch_attributes_with_partial_key_parsing(self, properties: _PARTIAL_PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, relation = attribute + yield (attribute_type, getattr(properties, field).value, relation) + + @staticmethod + def _return_object_attributes(attributes: Union[list, tuple]) -> tuple: + return tuple( + dict(zip(('type', 'value', 'object_relation'), attribute)) + for attribute in attributes + ) + + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _stix_object_type_error(self, xsi_type: str, object_id: str): + self._add_error(f"Unknown Observable type within STIX object with id {object_id}: {xsi_type}") \ No newline at end of file From cc9ca75d3c20a1c7b194debef2430f9767fa55c5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 25 Sep 2024 14:38:39 +0200 Subject: [PATCH 18/39] fix: [stix1 import] Fixing the email object handling and a few other clean-up changes --- misp_stix_converter/stix2misp/stix1_mapping.py | 2 +- misp_stix_converter/stix2misp/stix1_to_misp.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix1_mapping.py b/misp_stix_converter/stix2misp/stix1_mapping.py index b3c824b..08ed1a2 100644 --- a/misp_stix_converter/stix2misp/stix1_mapping.py +++ b/misp_stix_converter/stix2misp/stix1_mapping.py @@ -14,7 +14,7 @@ class STIX1toMISPMapping: CustomObjectType = '_handle_custom', DNSRecordObjectType = '_handle_dns', DomainNameObjectType = '_handle_domain_or_url', - EmailMessageObjectType = '_handle_email_attribute', + EmailMessageObjectType = '_handle_email', FileObjectType = '_handle_file', HostnameObjectType = '_handle_hostname', HTTPSessionObjectType = '_handle_http', diff --git a/misp_stix_converter/stix2misp/stix1_to_misp.py b/misp_stix_converter/stix2misp/stix1_to_misp.py index 0a2847a..21bd213 100644 --- a/misp_stix_converter/stix2misp/stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/stix1_to_misp.py @@ -53,7 +53,6 @@ class StixObjectTypeError(Exception): class STIX1toMISPParser(STIXtoMISPParser, metaclass=ABCMeta): def __init__(self): super().__init__() - self._mapping = STIX1toMISPMapping self.__galaxies = set() self.__references = defaultdict(list) @@ -115,10 +114,8 @@ def _handle_object_case(self, name, attribute_value, compl_data, to_ids=False, o if object_uuid: misp_object.uuid = object_uuid for attribute in attribute_value: - print(attribute) attribute['to_ids'] = to_ids misp_object.add_attribute(**attribute) - print() if isinstance(compl_data, dict): # if some complementary data is a dictionary containing an uuid, # it means we are using it to add an object reference @@ -219,10 +216,10 @@ def _handle_domain_or_url(self, properties: Union[domain_name_object.DomainName, return event_types['type'], properties.value.value, event_types['relation'] # Return type & value of an email attribute - def _handle_email_attribute(self, properties: email_message_object.EmailMessage) -> tuple: + def _handle_email(self, properties: email_message_object.EmailMessage) -> tuple: if properties.header: header = properties.header - attributes = self._fetch_attributes_with_key_parsing(header, '_email_mapping') + attributes = list(self._fetch_attributes_with_key_parsing(header, 'email_mapping')) if header.to: for to in header.to: attributes.append(["email-dst", to.address_value.value, "to"]) @@ -254,7 +251,7 @@ def _handle_email_attachment(self, properties: email_message_object.EmailMessage # Return type & attributes of a file object def _handle_file(self, properties: file_object.File, is_object: bool) -> tuple: b_hash, b_file = False, False - attributes = [] + attributes = list(self._fetch_attributes_with_keys(properties, 'file_mapping')) if properties.hashes: b_hash = True for hash_property in properties.hashes: @@ -265,7 +262,6 @@ def _handle_file(self, properties: file_object.File, is_object: bool) -> tuple: b_file = True attribute_type, relation = self._mapping.event_types(properties._XSI_TYPE) attributes.append([attribute_type, value, relation]) - attributes.extend(self._fetch_attributes_with_keys(properties, 'file_mapping')) if len(attributes) == 1: attribute = attributes[0] return attribute[0] if attribute[2] != "fullpath" else "filename", attribute[1], "" From eecdf993a476643465cb76687197eedd100e428e Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 25 Sep 2024 15:28:57 +0200 Subject: [PATCH 19/39] add: [stix1 import] STIX 1 to MISP automation methods added - Those are the methods used by the command line feature to process all the input files, convert the STIX content into MISP standard format and either save it in output files or push it directly to MISP --- misp_stix_converter/misp_stix_converter.py | 167 ++++++++++++++++----- 1 file changed, 132 insertions(+), 35 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 789d6cf..bca92e1 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -15,7 +15,8 @@ from .misp2stix.stix1_mapping import NS_DICT, SCHEMALOC_DICT from .stix2misp.external_stix1_to_misp import ExternalSTIX1toMISPParser from .stix2misp.external_stix2_to_misp import ExternalSTIX2toMISPParser -from .stix2misp.importparser import _load_stix2_content, MISP_org_uuid +from .stix2misp.importparser import ( + _load_stix1_package, _load_stix2_content, MISP_org_uuid) from .stix2misp.internal_stix1_to_misp import InternalSTIX1toMISPParser from .stix2misp.internal_stix2_to_misp import InternalSTIX2toMISPParser from collections import defaultdict @@ -638,31 +639,96 @@ def misp_to_stix2(filename: _files_type, debug: Optional[bool] = False, # STIX to MISP MAIN FUNCTIONS. # ################################################################################ -def stix_1_to_misp( - filename: _files_type, output_filename: Optional[_files_type]=None): - event = _load_stix_event(filename) - if isinstance(event, str): - return event - title = event.stix_header.title - from_misp = ( - title is not None and - all(feature in title for feature in ('Export from ', 'MISP')) - ) - stix_parser = ( - InternalSTIX1toMISPParser() if from_misp - else ExternalSTIX1toMISPParser() +def stix_1_to_misp(filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + output_dir: Optional[_files_type]=None, + output_name: Optional[_files_type]=None, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: + if isinstance(filename, str): + filename = Path(filename).resolve() + try: + stix_package = _load_stix1_package(filename) + except Exception as error: + return {'errors': [f'{filename} - {error.__str__()}']} + parser, args = _get_stix1_parser( + _is_stix1_from_misp(stix_package), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser.load_event() - stix_parser.build_misp_event(event) - if output_filename is None: - output_filename = f'{filename}.out' - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(stix_parser.misp_event.to_json(indent=4)) - return 1 + stix_parser = parser() + stix_parser.load_stix_package(stix_package) + stix_parser.parse_stix_package(**args) + if output_dir is None: + output_dir = filename.parent + if stix_parser.single_event: + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + with open(name, 'wt', encoding='utf-8') as f: + f.write(stix_parser.misp_event.to_json(indent=4)) + return _generate_traceback(debug, stix_parser, name) + output_names = [] + for misp_event in stix_parser.misp_events: + output = output_dir / f'{filename.name}.{misp_event.uuid}.misp.out' + with open(output, 'wt', encoding='utf-8') as f: + f.write(misp_event.to_json(indent=4)) + output_names.append(output) + return _generate_traceback(debug, stix_parser, *output_names) -def stix1_to_misp_instance(): - return +def stix1_to_misp_instance(misp: PyMISP, filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: + if isinstance(filename, str): + filename = Path(filename).resolve() + try: + stix_package = _load_stix1_package(filename) + except Exception as error: + return {'errors': [f'{filename} - {error.__str__()}']} + parser, args = _get_stix1_parser( + _is_stix1_from_misp(stix_package), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id + ) + stix_parser = parser() + stix_parser.load_stix_package(stix_package) + stix_parser.parse_stix_package(**args) + if stix_parser.single_event: + misp_event = misp.add_event(stix_parser.misp_event, pythonify=True) + if not isinstance(misp_event, MISPEvent): + return _generate_traceback( + debug, stix_parser, errors={ + stix_parser.misp_event.uuid: misp_event['errors'][1]['message'] + } + ) + return _generate_traceback(debug, stix_parser, misp_event.id) + event_ids = [] + errors = {} + for event in stix_parser.misp_events: + misp_event = misp.add_event(event, pythonify=True) + if not isinstance(misp_event, MISPEvent): + errors[event.uuid] = misp_event['errors'][1]['message'] + continue + event_ids.append(misp_event.id) + return _generate_traceback( + debug, stix_parser, *event_ids, errors=list(errors) + ) def stix_2_to_misp(filename: _files_type, @@ -710,18 +776,17 @@ def stix_2_to_misp(filename: _files_type, return _generate_traceback(debug, stix_parser, *output_names) -def stix2_to_misp_instance( - misp: PyMISP, filename: _files_type, - cluster_distribution: Optional[int] = 0, - cluster_sharing_group_id: Optional[int] = None, - debug: Optional[bool] = False, - distribution: Optional[int] = 0, - galaxies_as_tags: Optional[bool] = False, - organisation_uuid: Optional[str] = MISP_org_uuid, - producer: Optional[str] = None, - sharing_group_id: Optional[int] = None, - single_event: Optional[bool] = False, - title: Optional[str] = None) -> dict: +def stix2_to_misp_instance(misp: PyMISP, filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: if isinstance(filename, str): filename = Path(filename).resolve() try: @@ -762,6 +827,30 @@ def stix2_to_misp_instance( # STIX CONTENT LOADING FUNCTIONS # ################################################################################ +def _get_stix1_parser(from_misp: bool, distribution: int, + sharing_group_id: Union[int, None], + title: Union[str, None], producer: Union[str, None], + galaxies_as_tags: bool, single_event: bool, + organisation_uuid: str, cluster_distribution: int, + cluster_sharing_group_id: Union[int, None]) -> tuple: + args = { + 'distribution': distribution, + 'galaxies_as_tags': galaxies_as_tags, + 'producer': producer, + 'sharing_group_id': sharing_group_id, + 'single_event': single_event, + 'title': title + } + if from_misp: + return InternalSTIX1toMISPParser, args + args.update( + { + 'cluster_distribution': cluster_distribution, + 'cluster_sharing_group_id': cluster_sharing_group_id, + 'organisation_uuid': organisation_uuid + } + ) + return ExternalSTIX1toMISPParser, args def _get_stix2_parser(from_misp: bool, distribution: int, @@ -790,6 +879,14 @@ def _get_stix2_parser(from_misp: bool, distribution: int, return ExternalSTIX2toMISPParser, args +def _is_stix1_from_misp(stix_package: STIXPackage) -> bool: + try: + title = stix_package.stix_header.title + except AttributeError: + return False + return 'Export from ' in title and 'MISP' in title + + def _is_stix2_from_misp(stix_objects: list): for stix_object in stix_objects: labels = stix_object.get('labels', []) From dbd96221b283b417365c99b783ba73c4a03c8b62 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 25 Sep 2024 15:33:46 +0200 Subject: [PATCH 20/39] add: [misp_stix_converter] Making available the method to check the origin of STIX 1 files --- misp_stix_converter/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 6242b6e..02ac123 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -10,7 +10,8 @@ from .misp2stix import stix20_framing, stix21_framing # noqa # Helpers from .misp_stix_converter import ( # noqa - _is_stix2_from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2, + _is_stix1_from_misp, _is_stix2_from_misp, + misp_attribute_collection_to_stix1, misp_collection_to_stix2, misp_event_collection_to_stix1, misp_to_stix1, misp_to_stix2, stix_1_to_misp, stix_2_to_misp, stix2_to_misp_instance) # STIX 1 special helpers From 5903cc56101d6c1f2195a2038d78ac3d3a25daeb Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 1 Oct 2024 14:16:37 +0200 Subject: [PATCH 21/39] fix: [stix2 export] Better Analyst Note & Opinion conversion - Checking `authors` fields instead of adding empty list or facing potential error - Added `abstract` to Note object converted from the Analyst Note `comment` field --- misp_stix_converter/misp2stix/misp_to_stix21.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 5c66ef5..52c8865 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -229,11 +229,15 @@ def _handle_markings(self, object_args: dict, markings: tuple): def _handle_note_data(self, note, object_id: str): note_args = { - 'authors': [note['authors']], 'content': note['note'], + 'content': note['note'], 'created': self._datetime_from_str(note['created']), 'modified': self._datetime_from_str(note['modified']), 'id': f"note--{note['uuid']}", 'object_refs': [object_id] } + if note.get('authors'): + note_args['authors'] = [note['authors']] + if note.get('comment'): + note_args['abstract'] = note['comment'] if note.get('language'): note_args['lang'] = note['language'] getattr(self, self._results_handling_function)( @@ -248,13 +252,14 @@ def _handle_object_analyst_data( def _handle_opinion_data(self, opinion, object_id: str): opinion_value = int(opinion['opinion']) opinion_args = { - 'allow_custom': True, 'authors': [opinion['authors']], 'created': self._datetime_from_str(opinion['created']), 'modified': self._datetime_from_str(opinion['modified']), 'id': f"opinion--{opinion['uuid']}", 'object_refs': [object_id], 'opinion': self._parse_opinion_level(opinion_value), - 'x_misp_opinion': opinion_value + 'allow_custom': True, 'x_misp_opinion': opinion_value } + if opinion.get('authors'): + opinion_args['authors'] = [opinion['authors']] if opinion.get('comment'): opinion_args['explanation'] = opinion['comment'] getattr(self, self._results_handling_function)(Opinion(**opinion_args)) From 09126f1ffc65e36c8435a94f339686cc8d52b1f6 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 2 Oct 2024 20:45:45 +0200 Subject: [PATCH 22/39] fix: [stix2 import] Added typing in external mapping and made different variable checks easier --- .../stix2misp/external_stix2_mapping.py | 46 +++++++++---------- .../stix2misp/stix2_to_misp.py | 12 ++--- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix2_mapping.py b/misp_stix_converter/stix2misp/external_stix2_mapping.py index 53455d8..2410eba 100644 --- a/misp_stix_converter/stix2misp/external_stix2_mapping.py +++ b/misp_stix_converter/stix2misp/external_stix2_mapping.py @@ -472,7 +472,7 @@ class ExternalSTIX2toMISPMapping(STIX2toMISPMapping): ) @classmethod - def asn_pattern_mapping(cls, field) -> Union[dict, None]: + def asn_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__asn_pattern_mapping.get(field) @classmethod @@ -480,7 +480,7 @@ def course_of_action_object_mapping(cls) -> dict: return cls.__course_of_action_object_mapping @classmethod - def directory_pattern_mapping(cls, field) -> Union[dict, None]: + def directory_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__directory_object_mapping.get(field) @classmethod @@ -488,15 +488,15 @@ def directory_object_mapping(cls) -> dict: return cls.__directory_object_mapping @classmethod - def domain_ip_pattern_mapping(cls, field) -> Union[dict, None]: + def domain_ip_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__domain_ip_pattern_mapping.get(field) @classmethod - def email_address_pattern_mapping(cls, field) -> Union[dict, None]: + def email_address_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__email_address_pattern_mapping.get(field) @classmethod - def email_message_mapping(cls, field) -> Union[dict, None]: + def email_message_mapping(cls, field: str) -> Union[dict, None]: return cls.__email_object_mapping.get(field) @classmethod @@ -512,7 +512,7 @@ def file_hashes_object_mapping(cls) -> dict: return cls.__file_hashes_mapping @classmethod - def file_hashes_pattern_mapping(cls, field) -> Union[dict, None]: + def file_hashes_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__file_hashes_mapping.get(field) @classmethod @@ -524,23 +524,23 @@ def file_object_mapping(cls) -> dict: return cls.__file_object_mapping @classmethod - def file_pattern_mapping(cls, field) -> Union[dict, None]: + def file_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__file_object_mapping.get(field) @classmethod - def galaxy_name_mapping(cls, field) -> Union[dict, None]: + def galaxy_name_mapping(cls, field: str) -> Union[dict, None]: return cls.__galaxy_name_mapping.get(field) @classmethod - def http_request_extension_mapping(cls, field) -> Union[dict, None]: + def http_request_extension_mapping(cls, field: str) -> Union[dict, None]: return cls.__http_request_extension_mapping.get(field) @classmethod - def network_connection_object_reference_mapping(cls, field) -> Union[str, None]: + def network_connection_object_reference_mapping(cls, field: str) -> Union[str, None]: return cls.__network_connection_object_reference_mapping.get(field) @classmethod - def network_socket_object_reference_mapping(cls, field) -> Union[str, None]: + def network_socket_object_reference_mapping(cls, field: str) -> Union[str, None]: return cls.__network_socket_object_reference_mapping.get(field) @classmethod @@ -548,11 +548,11 @@ def network_traffic_object_mapping(cls) -> dict: return cls.__network_traffic_object_mapping @classmethod - def network_traffic_pattern_mapping(cls, field) -> Union[dict, None]: + def network_traffic_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__network_traffic_object_mapping.get(field) @classmethod - def observable_mapping(cls, field) -> Union[str, None]: + def observable_mapping(cls, field: str) -> Union[str, None]: return cls.__observable_mapping.get(field) @classmethod @@ -560,7 +560,7 @@ def pattern_forbidden_relations(cls) -> tuple: return cls.__pattern_forbidden_relations @classmethod - def pattern_mapping(cls, field) -> Union[str, None]: + def pattern_mapping(cls, field: str) -> Union[str, None]: return cls.__pattern_mapping.get(field) @classmethod @@ -568,7 +568,7 @@ def pe_object_mapping(cls) -> dict: return cls.__pe_object_mapping @classmethod - def pe_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_object_mapping.get(field) @classmethod @@ -576,7 +576,7 @@ def pe_optional_header_object_mapping(cls) -> dict: return cls.__pe_optional_header_mapping @classmethod - def pe_optional_header_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_optional_header_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_optional_header_mapping.get(field) @classmethod @@ -584,7 +584,7 @@ def pe_section_object_mapping(cls) -> dict: return cls.__pe_section_object_mapping @classmethod - def pe_section_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_section_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_section_object_mapping.get(field) @classmethod @@ -592,7 +592,7 @@ def process_object_mapping(cls) -> dict: return cls.__process_object_mapping @classmethod - def process_pattern_mapping(cls, field) -> Union[dict, None]: + def process_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__process_pattern_mapping.get(field) @classmethod @@ -604,7 +604,7 @@ def registry_key_object_mapping(cls) -> dict: return cls.__registry_key_object_mapping @classmethod - def registry_key_pattern_mapping(cls, field) -> Union[dict, None]: + def registry_key_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__registry_key_pattern_mapping.get(field) @classmethod @@ -616,7 +616,7 @@ def software_object_mapping(cls) -> dict: return cls.__software_object_mapping @classmethod - def software_pattern_mapping(cls, field) -> Union[dict, None]: + def software_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__software_pattern_mapping.get(field) @classmethod @@ -624,7 +624,7 @@ def user_account_object_mapping(cls) -> dict: return cls.__user_account_object_mapping @classmethod - def user_account_pattern_mapping(cls, field) -> Union[dict, None]: + def user_account_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__user_account_pattern_mapping.get(field) @classmethod @@ -632,7 +632,7 @@ def x509_hashes_object_mapping(cls) -> dict: return cls.__x509_hashes_mapping @classmethod - def x509_hashes_pattern_mapping(cls, field) -> Union[dict, None]: + def x509_hashes_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__x509_hashes_mapping.get(field) @classmethod @@ -640,7 +640,7 @@ def x509_object_mapping(cls) -> dict: return cls.__x509_object_mapping @classmethod - def x509_pattern_mapping(cls, field) -> Union[dict, None]: + def x509_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__x509_object_mapping.get(field) @classmethod diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 47d71d3..c95658d 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -659,22 +659,22 @@ def _misp_event_from_report(self, report: _REPORT_TYPING) -> MISPEvent: def _parse_bundle_with_multiple_reports(self): if self.single_event: self.__misp_event = self._create_generic_event() - if hasattr(self, '_report') and self._report is not None: + if getattr(self, '_report', None): for report in self._report.values(): self._handle_object_refs(report.object_refs) - if hasattr(self, '_grouping') and self._grouping is not None: + if getattr(self, '_grouping', None): for grouping in self._grouping.values(): self._handle_object_refs(grouping.object_refs) self._handle_unparsed_content() else: self.__misp_events = [] - if hasattr(self, '_report') and self._report is not None: + if getattr(self, '_report', None): for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) self._handle_object_refs(report.object_refs) self._handle_unparsed_content() self.__misp_events.append(self.misp_event) - if hasattr(self, '_grouping') and self._grouping is not None: + if getattr(self, '_grouping', None): for grouping in self._grouping.values(): self.__misp_event = self._misp_event_from_grouping(grouping) self._handle_object_refs(grouping.object_refs) @@ -689,11 +689,11 @@ def _parse_bundle_with_no_report(self): def _parse_bundle_with_single_report(self): self._set_single_event(True) - if hasattr(self, '_report') and self._report is not None: + if getattr(self, '_report', None): for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) self._handle_object_refs(report.object_refs) - elif hasattr(self, '_grouping') and self._grouping is not None: + elif getattr(self, '_grouping', None): for grouping in self._grouping.values(): self.__misp_event = self._misp_event_from_grouping(grouping) self._handle_object_refs(grouping.object_refs) From 45d286f0de8d8ad0edb9f9401a8ae5024e20ec86 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 3 Oct 2024 17:00:32 +0200 Subject: [PATCH 23/39] fix: [stix2 export] Removed non existing `comment` field in Analyst Note --- misp_stix_converter/misp2stix/misp_to_stix21.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 52c8865..3319a7d 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -236,8 +236,6 @@ def _handle_note_data(self, note, object_id: str): } if note.get('authors'): note_args['authors'] = [note['authors']] - if note.get('comment'): - note_args['abstract'] = note['comment'] if note.get('language'): note_args['lang'] = note['language'] getattr(self, self._results_handling_function)( From 34f32ed660358cf56367e8672c8c3f3e2d4bb747 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 7 Oct 2024 23:05:12 +0200 Subject: [PATCH 24/39] chg: [stix2 export] Exporting Event Reports also to STIX 2.0 --- .../misp2stix/misp_to_stix2.py | 61 +++++++++++++- .../misp2stix/misp_to_stix20.py | 82 +++++++++++++------ .../misp2stix/misp_to_stix21.py | 59 ++++--------- 3 files changed, 130 insertions(+), 72 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 7d29570..f67364f 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -117,14 +117,56 @@ def _parse_misp_event(self, misp_event: Union[MISPEvent, dict]): self.__object_refs = [] self.__relationships = [] self._handle_identity_from_event() - self._parse_event_data() - report = self._generate_event_report() + if self._misp_event.get('EventReport'): + self._id_parsing_function = { + 'attribute': '_define_stix_object_id_from_attribute', + 'object': '_define_stix_object_id_from_object' + } + self._event_report_matching = defaultdict(list) + self._handle_attributes_and_objects() + for event_report in self._misp_event['EventReport']: + note = self._parse_event_report(event_report) + self._append_SDO(note) + self._handle_analyst_data(note, event_report) + else: + self._id_parsing_function = { + 'attribute': '_define_stix_object_id', + 'object': '_define_stix_object_id' + } + self._handle_attributes_and_objects() + report = self._generate_report_from_event() self.__objects.insert(self.__index, report) def _define_stix_object_id( self, feature: str, misp_object: Union[MISPObject, dict]) -> str: return f"{feature}--{misp_object['uuid']}" + def _handle_attributes_and_objects(self): + if self._misp_event.get('Attribute'): + for attribute in self._misp_event['Attribute']: + self._resolve_attribute(attribute) + if self._misp_event.get('Object'): + self._objects_to_parse = defaultdict(dict) + self._resolve_objects() + if self._objects_to_parse: + self._resolve_objects_to_parse() + if self._objects_to_parse.get('annotation'): + objects_to_parse = self._objects_to_parse['annotation'] + for misp_object in objects_to_parse.values(): + to_ids, annotation_object = misp_object + custom = ( + annotation_object.get('ObjectReference') is None or + not self._annotates( + annotation_object['ObjectReference'] + ) + ) + if custom: + self._parse_custom_object(annotation_object) + else: + self._parse_annotation_object( + to_ids, annotation_object + ) + def _handle_default_identity(self): misp_identity_args = self._mapping.misp_identity_args() self.__identity_id = misp_identity_args['id'] @@ -258,7 +300,7 @@ def _append_SDO(self, stix_object): def _append_SDO_without_refs(self, stix_object): self.__objects.append(stix_object) - def _generate_event_report(self): + def _generate_report_from_event(self): report_args = { 'name': self._misp_event.get( 'info', @@ -392,6 +434,19 @@ def _handle_sighting_identity(self, uuid: str, name: str) -> str: self._handle_identity(identity_id, name) return identity_id + def _parse_event_report_references( + self, event_report: Union[MISPEventReport, dict]): + references = { + reference.split('(')[1][:-1] + for feature in ('attribute', 'object') + for reference in re.findall( + _event_report_regex % feature, event_report['content'] + ) + } + for reference in references: + if reference in self._event_report_matching: + yield self._event_report_matching[reference] + ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # ############################################################################ diff --git a/misp_stix_converter/misp2stix/misp_to_stix20.py b/misp_stix_converter/misp2stix/misp_to_stix20.py index 972cb3d..75416d6 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix20.py +++ b/misp_stix_converter/misp2stix/misp_to_stix20.py @@ -56,10 +56,9 @@ class CustomAttribute: @CustomObject( - 'x-misp-object', + 'x-misp-event-report', [ - ('id', IDProperty('x-misp-object')), - ('labels', ListProperty(StringProperty, required=True)), + ('id', IDProperty('x-misp-event-report')), ('created', TimestampProperty(required=True, precision='millisecond')), ('modified', TimestampProperty(required=True, precision='millisecond')), ( @@ -67,20 +66,20 @@ class CustomAttribute: ReferenceProperty(valid_types='identity', spec_version='2.0') ), ( - 'object_marking_refs', + 'object_refs', ListProperty( ReferenceProperty( - valid_types='marking-definition', spec_version='2.0' - ) + valid_types=_ANALYST_DATA_REFERENCE_TYPES, + spec_version='2.0' + ), + required=True ) ), - ('x_misp_name', StringProperty(required=True)), - ('x_misp_attributes', ListProperty(DictionaryProperty())), - ('x_misp_comment', StringProperty()), - ('x_misp_meta_category', StringProperty()) + ('x_misp_content', StringProperty(required=True)), + ('x_misp_name', StringProperty()), ] ) -class CustomMispObject: +class CustomEventReport: pass @@ -106,6 +105,35 @@ class CustomGalaxyCluster: pass +@CustomObject( + 'x-misp-object', + [ + ('id', IDProperty('x-misp-object')), + ('labels', ListProperty(StringProperty, required=True)), + ('created', TimestampProperty(required=True, precision='millisecond')), + ('modified', TimestampProperty(required=True, precision='millisecond')), + ( + 'created_by_ref', + ReferenceProperty(valid_types='identity', spec_version='2.0') + ), + ( + 'object_marking_refs', + ListProperty( + ReferenceProperty( + valid_types='marking-definition', spec_version='2.0' + ) + ) + ), + ('x_misp_name', StringProperty(required=True)), + ('x_misp_attributes', ListProperty(DictionaryProperty())), + ('x_misp_comment', StringProperty()), + ('x_misp_meta_category', StringProperty()) + ] +) +class CustomMispObject: + pass + + @CustomObject( 'x-misp-event-note', [ @@ -119,7 +147,7 @@ class CustomGalaxyCluster: ('x_misp_event_note', StringProperty(required=True)), ( 'object_ref', - ReferenceProperty(valid_types=['report'], spec_version='2.0') + ReferenceProperty(valid_types='report', spec_version='2.0') ) ] ) @@ -161,15 +189,21 @@ def __init__(self, interoperability=False): self._version = '2.0' self._mapping = MISPtoSTIX20Mapping - def _parse_event_data(self): - if self._misp_event.get('Attribute'): - for attribute in self._misp_event['Attribute']: - self._resolve_attribute(attribute) - if self._misp_event.get('Object'): - self._objects_to_parse = defaultdict(dict) - self._resolve_objects() - if self._objects_to_parse: - self._resolve_objects_to_parse() + def _parse_event_report( + self, event_report: Union[MISPEventReport, dict]) -> CustomEventReport: + timestamp = self._datetime_from_timestamp(event_report['timestamp']) + note_args = { + 'id': f"x-misp-event-report--{event_report['uuid']}", + 'created': timestamp, 'modified': timestamp, + 'created_by_ref': self.identity_id, + 'x_misp_content': event_report['content'] + } + if event_report.get('name'): + note_args['x_misp_name'] = event_report['name'] + references = set(self._parse_event_report_references(event_report)) + if references: + note_args['object_refs'] = list(references) + return CustomEventReport(**note_args) def _handle_empty_object_refs(self, object_id: str, timestamp: datetime): object_type = 'x-misp-event-note' @@ -245,9 +279,9 @@ def _handle_unpublished_report(self, report_args: dict) -> Report: ) return Report(**report_args) - ################################################################################ - # ATTRIBUTES PARSING FUNCTIONS # - ################################################################################ + ############################################################################ + # ATTRIBUTES PARSING FUNCTIONS # + ############################################################################ def _parse_attachment_attribute_observable( self, attribute: Union[MISPAttribute, dict]): diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 3319a7d..3af1bb8 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -95,51 +95,20 @@ def __init__(self, interoperability=False): self._version = '2.1' self._mapping = MISPtoSTIX21Mapping - def _parse_event_data(self): - if self._misp_event.get('EventReport'): - self._id_parsing_function = { - 'attribute': '_define_stix_object_id_from_attribute', - 'object': '_define_stix_object_id_from_object' - } - self._event_report_matching = defaultdict(list) - self._handle_attributes_and_objects() - regex = r'@[!]?\[%s\]\([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)' - for event_report in self._misp_event['EventReport']: - timestamp = self._datetime_from_timestamp( - event_report['timestamp'] - ) - note_args = { - 'id': f"note--{event_report['uuid']}", - 'created': timestamp, 'modified': timestamp, - 'created_by_ref': self.identity_id, - 'content': event_report['content'], - 'abstract': event_report['name'] - } - references = { - reference.split('(')[1][:-1] - for feature in ('attribute', 'object') - for reference in re.findall( - regex % feature, event_report['content'] - ) - } - object_refs = set() - for reference in references: - if reference in self._event_report_matching: - object_refs.update( - self._event_report_matching[reference] - ) - note_args['object_refs'] = ( - list(object_refs) if object_refs - else self._handle_empty_note_refs() - ) - self._append_SDO(self._create_note(note_args)) - self._handle_analyst_data(note_args['id'], event_report) - else: - self._id_parsing_function = { - 'attribute': '_define_stix_object_id', - 'object': '_define_stix_object_id' - } - self._handle_attributes_and_objects() + def _parse_event_report(self, event_report: Union[MISPEventReport, dict]) -> Note: + timestamp = self._datetime_from_timestamp(event_report['timestamp']) + note_args = { + 'id': f"note--{event_report['uuid']}", + 'created': timestamp, 'modified': timestamp, + 'created_by_ref': self.identity_id, + 'content': event_report['content'], + 'abstract': event_report['name'] + } + references = set(self._parse_event_report_references(event_report)) + note_args['object_refs'] = ( + list(references) if references else self._handle_empty_note_refs() + ) + return self._create_note(note_args) def _define_stix_object_id_from_attribute( self, feature: str, attribute: Union[MISPAttribute, dict]) -> str: From 91d00361dda1f4eddce6fd0b2290ae597a29ebc4 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 8 Oct 2024 09:01:22 +0200 Subject: [PATCH 25/39] fix: [tests] Added fallback test to avoid issues with datetime values --- tests/_test_stix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/_test_stix.py b/tests/_test_stix.py index 006b8aa..bfe3476 100644 --- a/tests/_test_stix.py +++ b/tests/_test_stix.py @@ -13,6 +13,8 @@ def _assert_multiple_equal(self, reference, *elements): @staticmethod def _datetime_from_str(timestamp): + if isinstance(timestamp, datetime): + return timestamp regex = f"%Y-%m-%d{'T' if 'T' in timestamp else ' '}%H:%M:%S" if '.' in timestamp: regex = f'{regex}.%f' From 3151b6f7a65014eab6eee075e4c5baffa8a4ea31 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 8 Oct 2024 11:39:35 +0200 Subject: [PATCH 26/39] add: [tests] Added tests with Analyst Data attached to a MISP object --- tests/test_events.py | 10 ++++ tests/test_stix21_export.py | 114 ++++++++++++++++-------------------- 2 files changed, 60 insertions(+), 64 deletions(-) diff --git a/tests/test_events.py b/tests/test_events.py index 5738296..fb28e98 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -823,6 +823,16 @@ "referenced_uuid": "f7ef1b4a-964a-4a69-9e21-808f85c56238", "relationship_type": "downloaded-from" } + ], + "Opinion": [ + { + "uuid": "74258748-78f2-4b19-bedc-27ec61b1c5df", + "authors": "john.doe@foo.bar", + "created": "2024-06-12 12:52:48", + "modified": "2024-06-12 12:53:58", + "opinion": "50", + "comment": "No warning from my antivirus" + } ] } ], diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index bc5cf47..d989236 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -34,6 +34,43 @@ def _check_spec_versions(self, stix_objects): class TestSTIX21EventExport(TestSTIX21GenericExport): + def _check_analyst_note(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"note--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.content, misp_layer['note']) + self.assertEqual(stix_object.lang, misp_layer['language']) + self.assertEqual( + stix_object.authors, [misp_layer['authors']] + ) + self.assertEqual( + stix_object.created, + self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + + def _check_analyst_opinion(self, stix_object, misp_layer, opinion): + self.assertEqual( + stix_object.id, f"opinion--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.opinion, opinion) + self.assertEqual( + stix_object.x_misp_opinion, int(misp_layer['opinion']) + ) + self.assertEqual(stix_object.explanation, misp_layer['comment']) + self.assertEqual(stix_object.authors, [misp_layer['authors']]) + self.assertEqual( + stix_object.created, + self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + def _check_attribute_confidence_tags(self, stix_object, attribute): self.assertEqual( stix_object.confidence, @@ -100,7 +137,7 @@ def _test_event_with_analyst_data(self, event): attribute = event['Attribute'][0] misp_object = event['Object'][0] self.parser.parse_misp_event(event) - stix_objects = self._check_bundle_features(10) + stix_objects = self._check_bundle_features(11) self._check_spec_versions(stix_objects) identity, grouping, *stix_objects = stix_objects timestamp = event['timestamp'] @@ -110,87 +147,36 @@ def _test_event_with_analyst_data(self, event): object_refs = self._check_grouping_features(grouping, event, identity_id) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - (attr_indicator, attr_opinion, obj_indicator, obj_note, report, - report_opinion, relationship, event_note) = stix_objects + (attr_indicator, attr_opinion, obj_indicator, obj_opinion, obj_attr_note, + report, report_opinion, relationship, event_note) = stix_objects self._assert_multiple_equal( attr_indicator.id, relationship.target_ref, - attr_opinion['object_refs'][0], + attr_opinion.object_refs[0], f"indicator--{attribute['uuid']}" ) attribute_opinion = attribute['Opinion'][0] - self.assertEqual( - attr_opinion.id, f"opinion--{attribute_opinion['uuid']}" - ) - self.assertEqual(attr_opinion.opinion, 'strongly-agree') - self.assertEqual( - attr_opinion.x_misp_opinion, int(attribute_opinion['opinion']) - ) - self.assertEqual(attr_opinion.explanation, attribute_opinion['comment']) - self.assertEqual(attr_opinion.authors, [attribute_opinion['authors']]) - self.assertEqual( - attr_opinion.created, - self._datetime_from_str(attribute_opinion['created']) - ) - self.assertEqual( - attr_opinion.modified, - self._datetime_from_str(attribute_opinion['modified']) - ) + self._check_analyst_opinion(attr_opinion, attribute_opinion, 'strongly-agree') self._assert_multiple_equal( obj_indicator.id, relationship.source_ref, - obj_note.object_refs[0], + obj_opinion.object_refs[0], + obj_attr_note.object_refs[0], f"indicator--{misp_object['uuid']}" ) - object_note = misp_object['Attribute'][0]['Note'][0] - self.assertEqual(obj_note.id, f"note--{object_note['uuid']}") - self.assertEqual(obj_note.content, object_note['note']) - self.assertEqual(obj_note.lang, object_note['language']) - self.assertEqual(obj_note.authors, [object_note['authors']]) - self.assertEqual( - obj_note.created, self._datetime_from_str(object_note['created']) - ) - self.assertEqual( - obj_note.modified, self._datetime_from_str(object_note['modified']) - ) + object_opinion = misp_object['Opinion'][0] + self._check_analyst_opinion(obj_opinion, object_opinion, 'neutral') + object_attribute_note = misp_object['Attribute'][0]['Note'][0] + self._check_analyst_note(obj_attr_note, object_attribute_note) self._assert_multiple_equal( report.id, report_opinion.object_refs[0], f"note--{event_report['uuid']}" ) event_report_opinion = event_report['Opinion'][0] - self.assertEqual( - report_opinion.id, f"opinion--{event_report_opinion['uuid']}" - ) - self.assertEqual(report_opinion.opinion, 'agree') - self.assertEqual( - report_opinion.x_misp_opinion, int(event_report_opinion['opinion']) - ) - self.assertEqual( - report_opinion.explanation, event_report_opinion['comment'] - ) - self.assertEqual( - report_opinion.authors, [event_report_opinion['authors']] - ) - self.assertEqual( - report_opinion.created, - self._datetime_from_str(event_report_opinion['created']) - ) - self.assertEqual( - report_opinion.modified, - self._datetime_from_str(event_report_opinion['modified']) - ) + self._check_analyst_opinion(report_opinion, event_report_opinion, 'agree') self.assertEqual(relationship.relationship_type, 'downloaded-from') - self.assertEqual(event_note.id, f"note--{note['uuid']}") - self.assertEqual(event_note.content, note['note']) - self.assertEqual(event_note.lang, note['language']) - self.assertEqual(event_note.authors, [note['authors']]) - self.assertEqual( - event_note.created, self._datetime_from_str(note['created']) - ) - self.assertEqual( - event_note.modified, self._datetime_from_str(note['modified']) - ) + self._check_analyst_note(event_note, note) def _test_event_with_attribute_confidence_tags(self, event): tlp_tag, *confidence_tags = event['Tag'] From 97adfdb0f1c0e9fd6fc13cb0579f699d88590b89 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 8 Oct 2024 11:46:16 +0200 Subject: [PATCH 27/39] fix: [stix2 export] Making the methods related to event reports part of the parent STIX 2 export class --- misp_stix_converter/misp2stix/misp_to_stix2.py | 16 ++++++++++++++++ misp_stix_converter/misp2stix/misp_to_stix21.py | 16 ---------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index f67364f..3f3da56 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -141,6 +141,22 @@ def _define_stix_object_id( self, feature: str, misp_object: Union[MISPObject, dict]) -> str: return f"{feature}--{misp_object['uuid']}" + def _define_stix_object_id_from_attribute( + self, feature: str, attribute: Union[MISPAttribute, dict]) -> str: + attribute_uuid = attribute['uuid'] + stix_id = f'{feature}--{attribute_uuid}' + self._event_report_matching[attribute_uuid].append(stix_id) + return stix_id + + def _define_stix_object_id_from_object( + self, feature: str, misp_object: Union[MISPObject, dict]) -> str: + object_uuid = misp_object['uuid'] + stix_id = f'{feature}--{object_uuid}' + self._event_report_matching[object_uuid].append(stix_id) + for attribute in misp_object['Attribute']: + self._event_report_matching[attribute['uuid']].append(stix_id) + return stix_id + def _handle_attributes_and_objects(self): if self._misp_event.get('Attribute'): for attribute in self._misp_event['Attribute']: diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 3af1bb8..59a6737 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -110,22 +110,6 @@ def _parse_event_report(self, event_report: Union[MISPEventReport, dict]) -> Not ) return self._create_note(note_args) - def _define_stix_object_id_from_attribute( - self, feature: str, attribute: Union[MISPAttribute, dict]) -> str: - attribute_uuid = attribute['uuid'] - stix_id = f'{feature}--{attribute_uuid}' - self._event_report_matching[attribute_uuid].append(stix_id) - return stix_id - - def _define_stix_object_id_from_object( - self, feature: str, misp_object: Union[MISPObject, dict]) -> str: - object_uuid = misp_object['uuid'] - stix_id = f'{feature}--{object_uuid}' - self._event_report_matching[object_uuid].append(stix_id) - for attribute in misp_object['Attribute']: - self._event_report_matching[attribute['uuid']].append(stix_id) - return stix_id - def _handle_analyst_data( self, object_id: str, data_layer: _MISP_DATA_LAYER = None): if data_layer is None: From e4758881e853f84e7f1eac69f4a8b8acc53093b5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 10:59:47 +0200 Subject: [PATCH 28/39] add: [tests] Added tests for Event Report export to STIX 2.0 --- tests/test_stix20_export.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index 6f877e5..953f73e 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -89,6 +89,33 @@ def _test_event_with_escaped_characters(self, event): data = b64encode(data.getvalue()).decode() self.assertIn(self._sanitize_pattern_value(data), indicator.pattern) + def _test_event_with_event_report(self, event): + orgc = event['Orgc'] + event_report = event['EventReport'][0] + self.parser.parse_misp_event(event) + bundle = self._check_bundle_features(9) + identity, report, *stix_objects = bundle.objects + timestamp = event['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + identity_id = self._check_identity_features(identity, orgc, timestamp) + object_refs = self._check_report_features(report, event, identity_id, timestamp) + self.assertEqual(report.published, timestamp) + for stix_object, object_ref in zip(stix_objects, object_refs): + self.assertEqual(stix_object.id, object_ref) + attack_pattern, ip_src, observed_data, domain_ip, note, _, marking = stix_objects + self.assertEqual(note.id, f"x-misp-event-report--{event_report['uuid']}") + timestamp = event_report['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + self._assert_multiple_equal(note.created, note.modified, timestamp) + self.assertEqual(note.x_misp_content, event_report['content']) + self.assertEqual(note.x_misp_name, event_report['name']) + object_refs = note.object_refs + self.assertEqual(len(object_refs), 3) + object_ids = {ip_src.id, observed_data.id, domain_ip.id} + self.assertEqual(set(object_refs), object_ids) + def _test_event_with_sightings(self, event): orgc = event['Orgc'] attribute1, attribute2 = event['Attribute'] @@ -189,6 +216,10 @@ def test_event_with_escaped_characters(self): event = get_event_with_escaped_values_v20() self._test_event_with_escaped_characters(event['Event']) + def test_event_with_event_report(self): + event = get_event_with_event_report() + self._test_event_with_event_report(event['Event']) + def test_event_with_sightings(self): event = get_event_with_sightings() self._test_event_with_sightings(event['Event']) @@ -215,6 +246,12 @@ def test_event_with_escaped_characters(self): misp_event.from_dict(**event) self._test_event_with_escaped_characters(misp_event) + def test_event_with_event_report(self): + event = get_event_with_event_report() + misp_event = MISPEvent() + misp_event.from_dict(**event) + self._test_event_with_event_report(misp_event) + def test_event_with_sightings(self): event = get_event_with_sightings() misp_event = MISPEvent() From a28510d90f3243ff936079e68bd61227e4bcf167 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 11:02:14 +0200 Subject: [PATCH 29/39] fix: [stix2 export] Fixed Event Report references fetching --- misp_stix_converter/misp2stix/misp_to_stix2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 3f3da56..45f524f 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -461,7 +461,7 @@ def _parse_event_report_references( } for reference in references: if reference in self._event_report_matching: - yield self._event_report_matching[reference] + yield from self._event_report_matching[reference] ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # From 384696557a1f2df67699136f8d97d36e6881b790 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 11:04:05 +0200 Subject: [PATCH 30/39] chg: [stix2 export] Making Analyst Data export to STIX 2.0 available - Using Custom Object --- .../misp2stix/misp_to_stix2.py | 140 +++++++++++------- .../misp2stix/misp_to_stix20.py | 94 +++++++++++- .../misp2stix/misp_to_stix21.py | 90 ++++------- 3 files changed, 205 insertions(+), 119 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 45f524f..0fc141d 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -12,13 +12,23 @@ from datetime import datetime from pathlib import Path from pymisp import ( - MISPAttribute, MISPEvent, MISPGalaxy, MISPGalaxyCluster, MISPObject) + MISPAttribute, MISPEvent, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, + MISPNote, MISPObject, MISPOpinion) from stix2.hashes import check_hash, Hash from stix2.properties import ListProperty, StringProperty from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v21.bundle import Bundle as Bundle_v21 +from stix2.v20.sdo import ( + Campaign as Campaign_v20, CustomObject as CustomObject_v20, + Indicator as Indicator_v20, Report as Report_v20, + Vulnerability as Vulnerability_v20) +from stix2.v21.sdo import ( + Campaign as Campaign_v21, CustomObject as CustomObject_v21, Grouping, + Indicator as Indicator_v21, Report as Report_v21, + Vulnerability as Vulnerability_v21) from typing import Generator, Optional, Tuple, Union +_event_report_regex = r'@[!]?\[%s\]\([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)' _label_fields = ('type', 'category', 'to_ids') _labelled_object_types = ('malware', 'threat-actor', 'tool') _misp_time_fields = ('first_seen', 'last_seen') @@ -30,6 +40,15 @@ 'observed-data': ('first_observed', 'last_observed') } +_MISP_DATA_LAYER = Union[ + dict, MISPAttribute, MISPEventReport, MISPObject +] +_STIX_OBJECT_TYPING = Union[ + Campaign_v20, Campaign_v21, CustomObject_v20, CustomObject_v21, Grouping, + Indicator_v20, Indicator_v21, Report_v20, Report_v21, + Vulnerability_v20, Vulnerability_v21 +] + class InvalidHashValueError(Exception): pass @@ -349,9 +368,9 @@ def _generate_report_from_event(self): 'object_refs': self.__object_refs, 'allow_custom': True } ) - if self._version == '2.1': - self._handle_analyst_data(report_id) - return self._create_report(report_args) + report = self._create_report(report_args) + self._handle_analyst_data(report) + return report return self._handle_unpublished_report(report_args) def _generate_galaxies_catalog(self): @@ -401,6 +420,21 @@ def _get_object_ids( in self._galaxies_catalog[name][object_type] ) + def _handle_analyst_data(self, stix_object: _STIX_OBJECT_TYPING, + data_layer: _MISP_DATA_LAYER = None): + if data_layer is None: + data_layer = self._misp_event + for note in data_layer.get('Note', []): + self._handle_note_data(stix_object, note) + for opinion in data_layer.get('Opinion', []): + self._handle_opinion_data(stix_object, opinion) + + def _handle_object_analyst_data(self, stix_object: _STIX_OBJECT_TYPING, + misp_object: Union[MISPObject, dict]): + self._handle_analyst_data(stix_object, misp_object) + for attribute in misp_object['Attribute']: + self._handle_analyst_data(stix_object, attribute) + def _handle_relationships(self): for relationship in self.__relationships: if relationship.get('undefined_target_ref'): @@ -507,11 +541,9 @@ def _handle_attribute_indicator( ) if markings: self._handle_markings(indicator_arguments, markings) - getattr(self, self._results_handling_function)( - self._create_indicator(indicator_arguments) - ) - if self._version == '2.1': - self._handle_analyst_data(indicator_id, attribute) + indicator = self._create_indicator(indicator_arguments) + getattr(self, self._results_handling_function)(indicator) + self._handle_analyst_data(indicator, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], indicator_id) @@ -612,11 +644,9 @@ def _parse_campaign_name_attribute( ) if markings: self._handle_markings(campaign_args, markings) - getattr(self, self._results_handling_function)( - self._create_campaign(campaign_args) - ) - if self._version == '2.1': - self._handle_analyst_data(campaign_id, attribute) + campaign = self._create_campaign(campaign_args) + getattr(self, self._results_handling_function)(campaign) + self._handle_analyst_data(campaign, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], campaign_id) @@ -641,11 +671,9 @@ def _parse_custom_attribute(self, attribute: Union[MISPAttribute, dict]): ) if markings: self._handle_markings(custom_args, markings) - getattr(self, self._results_handling_function)( - self._create_custom_attribute(custom_args) - ) - if self._version == '2.1': - self._handle_analyst_data(custom_id, attribute) + custom_attribute = self._create_custom_attribute(custom_args) + getattr(self, self._results_handling_function)(custom_attribute) + self._handle_analyst_data(custom_attribute, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], custom_id) @@ -1038,11 +1066,9 @@ def _parse_vulnerability_attribute( ) if markings: self._handle_markings(vulnerability_args, markings) - getattr(self, self._results_handling_function)( - self._create_vulnerability(vulnerability_args) - ) - if self._version == '2.1': - self._handle_analyst_data(vulnerability_id, attribute) + vulnerability = self._create_vulnerability(vulnerability_args) + getattr(self, self._results_handling_function)(vulnerability) + self._handle_analyst_data(vulnerability, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], vulnerability_id) @@ -1151,13 +1177,10 @@ def _handle_non_indicator_object( misp_object['ObjectReference'], object_id, object_args['modified'] ) - self._append_SDO( - getattr(self, f"_create_{object_type.replace('-', '_')}")( - object_args - ) - ) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, object_id) + feature = f"_create_{object_type.replace('-', '_')}" + stix_object = getattr(self, feature)(object_args) + getattr(self, self._results_handling_function)(stix_object) + self._handle_object_analyst_data(stix_object, misp_object) def _handle_object_indicator( self, misp_object: Union[MISPObject, dict], pattern: list): @@ -1186,9 +1209,9 @@ def _handle_object_indicator( misp_object['ObjectReference'], indicator_id, indicator_args['modified'] ) - self._append_SDO(self._create_indicator(indicator_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, indicator_id) + indicator = self._create_indicator(indicator_args) + getattr(self, self._results_handling_function)(indicator) + self._handle_object_analyst_data(indicator, misp_object) def _handle_object_observable( self, misp_object: Union[MISPObject, dict], @@ -1534,9 +1557,9 @@ def _parse_custom_object(self, misp_object: Union[MISPObject, dict]): self._parse_object_relationships( misp_object['ObjectReference'], custom_id, timestamp ) - self._append_SDO(self._create_custom_object(custom_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, custom_id) + custom_object = self._create_custom_object(custom_args) + getattr(self, self._results_handling_function)(custom_object) + self._handle_object_analyst_data(custom_object, misp_object) @staticmethod def _parse_custom_object_attribute( @@ -1681,9 +1704,9 @@ def _parse_employee_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_file_object(self, misp_object: Union[MISPObject, dict]): to_ids = self._fetch_ids_flag(misp_object['Attribute']) @@ -1950,9 +1973,9 @@ def _parse_legal_entity_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_lnk_object(self, misp_object: Union[MISPObject, dict]): if self._fetch_ids_flag(misp_object['Attribute']): @@ -2229,9 +2252,9 @@ def _parse_news_agency_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_person_object(self, misp_object: Union[MISPObject, dict]): identity_args = self._parse_identity_args(misp_object, 'individual') @@ -2267,9 +2290,9 @@ def _parse_person_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_organization_object(self, misp_object: Union[MISPObject, dict]): identity_args = self._parse_identity_args(misp_object, 'organization') @@ -2295,9 +2318,9 @@ def _parse_organization_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_pe_extensions_observable( self, pe_object: dict, uuids: Optional[list] = None) -> dict: @@ -4098,9 +4121,9 @@ def _handle_patterning_object_indicator( misp_object['ObjectReference'], indicator_id, indicator_args['modified'] ) - self._append_SDO(self._create_indicator(indicator_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, indicator_id) + indicator = self._create_indicator(indicator_args) + getattr(self, self._results_handling_function)(indicator) + self._handle_object_analyst_data(indicator, misp_object) ############################################################################ # UTILITY FUNCTIONS. # @@ -4196,6 +4219,13 @@ def _get_matching_email_display_name( def _get_vulnerability_references(vulnerability: str) -> dict: return {'source_name': 'cve', 'external_id': vulnerability} + def _handle_analyst_time_fields(self, stix_object, misp_object: Union[MISPNote, MISPOpinion]): + for feature in ('created', 'modified'): + if misp_object.get(feature): + yield feature, self._datetime_from_str(misp_object[feature]) + continue + yield feature, stix_object[feature] + def _handle_custom_data_field( self, values: Union[list, str, tuple]) -> Union[dict, list, str]: if isinstance(values, list): diff --git a/misp_stix_converter/misp2stix/misp_to_stix20.py b/misp_stix_converter/misp2stix/misp_to_stix20.py index 75416d6..82354a7 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix20.py +++ b/misp_stix_converter/misp2stix/misp_to_stix20.py @@ -6,10 +6,11 @@ from base64 import b64encode from collections import defaultdict from datetime import datetime -from pymisp import MISPAttribute, MISPObject +from pymisp import ( + MISPAttribute, MISPEventReport, MISPNote, MISPObject, MISPOpinion) from stix2.properties import ( - DictionaryProperty, IDProperty, ListProperty, ReferenceProperty, - StringProperty, TimestampProperty) + DictionaryProperty, IDProperty, IntegerProperty, ListProperty, + ReferenceProperty, StringProperty, TimestampProperty) from stix2.v20.bundle import Bundle from stix2.v20.observables import ( Artifact, AutonomousSystem, Directory, DomainName, EmailAddress, @@ -25,6 +26,56 @@ from stix2.v20.vocab import HASHING_ALGORITHM from typing import Optional, Union +_ANALYST_DATA_REFERENCE_TYPES = [ + 'attack-pattern', 'campaign', 'course-of-action', 'identity', 'indicator', + 'intrusion-set', 'malware', 'observed-data', 'report', 'threat-actor', + 'tool', 'vulnerability', 'x-misp-analyst-note', 'x-misp-analyst-opinion', + 'x-misp-attribute', 'x-misp-event-note', 'x-misp-event-report', + 'x-misp-galaxy-cluster', 'x-misp-object' +] + + +@CustomObject( + 'x-misp-analyst-note', + [ + ('id', IDProperty('x-misp-analyst-note')), + ('created', TimestampProperty(precision='millisecond')), + ('modified', TimestampProperty(precision='millisecond')), + ('x_misp_note', StringProperty(required=True)), + ('x_misp_author', StringProperty()), + ('x_misp_language', StringProperty()), + ( + 'object_ref', + ReferenceProperty( + valid_types=_ANALYST_DATA_REFERENCE_TYPES, spec_version='2.0' + ) + ) + ] +) +class CustomAnalystNote: + pass + + +@CustomObject( + 'x-misp-analyst-opinion', + [ + ('id', IDProperty('x-misp-analyst-opinion')), + ('created', TimestampProperty(precision='millisecond')), + ('modified', TimestampProperty(precision='millisecond')), + ('x_misp_opinion', IntegerProperty(required=True)), + ('x_misp_author', StringProperty()), + ('x_misp_comment', StringProperty()), + ( + 'object_ref', + ReferenceProperty( + valid_types=_ANALYST_DATA_REFERENCE_TYPES, spec_version='2.0' + ) + ) + ] +) +class CustomAnalystOpinion: + pass + @CustomObject( 'x-misp-attribute', @@ -236,6 +287,39 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids + def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): + note_args = { + 'id': f"x-misp-analyst-note--{note['uuid']}", + 'object_ref': stix_object.id, 'x_misp_note': note['note'], + **dict(self._handle_analyst_time_fields(stix_object, note)) + } + if note.get('authors'): + note_args['x_misp_author'] = note['authors'] + if note.get('language'): + note_args['x_misp_language'] = note['language'] + if stix_object.id.startswith('x-misp-'): + note_args['allow_custom'] = True + getattr(self, self._results_handling_function)( + CustomAnalystNote(**note_args) + ) + + def _handle_opinion_data( + self, stix_object, opinion: Union[MISPOpinion, dict]): + opinion_args = { + 'id': f"x-misp-analyst-opinion--{opinion['uuid']}", + 'object_ref': stix_object.id, 'x_misp_opinion': opinion['opinion'], + **dict(self._handle_analyst_time_fields(stix_object, opinion)) + } + if opinion.get('authors'): + opinion_args['x_misp_author'] = opinion['authors'] + if opinion.get('comment'): + opinion_args['x_misp_comment'] = opinion['comment'] + if stix_object.id.startswith('x-misp-'): + opinion_args['allow_custom'] = True + getattr(self, self._results_handling_function)( + CustomAnalystOpinion(**opinion_args) + ) + def _handle_opinion_object(self, sighting: dict, reference_id: str): opinion_args = { 'id': f"x-misp-opinion--{sighting['uuid']}", @@ -277,7 +361,9 @@ def _handle_unpublished_report(self, report_args: dict) -> Report: 'object_refs': self.object_refs, 'allow_custom': True } ) - return Report(**report_args) + report = self._create_report(report_args) + self._handle_analyst_data(report) + return report ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 59a6737..17d2ffd 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -8,7 +8,8 @@ from collections import defaultdict from datetime import datetime from pymisp import ( - MISPAttribute, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, MISPObject) + MISPAttribute, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, MISPNote, + MISPObject, MISPOpinion) from stix2.properties import ( DictionaryProperty, IDProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty) @@ -27,10 +28,6 @@ from stix2.v21.vocab import HASHING_ALGORITHM from typing import Optional, Union -_MISP_DATA_LAYER = Union[ - dict, MISPAttribute, MISPEventReport -] - @CustomObject( 'x-misp-attribute', @@ -110,41 +107,6 @@ def _parse_event_report(self, event_report: Union[MISPEventReport, dict]) -> Not ) return self._create_note(note_args) - def _handle_analyst_data( - self, object_id: str, data_layer: _MISP_DATA_LAYER = None): - if data_layer is None: - data_layer = self._misp_event - for note in data_layer.get('Note', []): - self._handle_note_data(note, object_id) - for opinion in data_layer.get('Opinion', []): - self._handle_opinion_data(opinion, object_id) - - def _handle_attributes_and_objects(self): - if self._misp_event.get('Attribute'): - for attribute in self._misp_event['Attribute']: - self._resolve_attribute(attribute) - if self._misp_event.get('Object'): - self._objects_to_parse = defaultdict(dict) - self._resolve_objects() - if self._objects_to_parse: - self._resolve_objects_to_parse() - if self._objects_to_parse.get('annotation'): - objects_to_parse = self._objects_to_parse['annotation'] - for misp_object in objects_to_parse.values(): - to_ids, annotation_object = misp_object - custom = ( - annotation_object.get('ObjectReference') is None or - not self._annotates( - annotation_object['ObjectReference'] - ) - ) - if custom: - self._parse_custom_object(annotation_object) - else: - self._parse_annotation_object( - to_ids, annotation_object - ) - def _handle_empty_object_refs(self, object_id: str, timestamp: datetime): note_args = { 'id': f"note--{self._misp_event['uuid']}", @@ -180,12 +142,12 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids - def _handle_note_data(self, note, object_id: str): + def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): note_args = { 'content': note['note'], - 'created': self._datetime_from_str(note['created']), - 'modified': self._datetime_from_str(note['modified']), - 'id': f"note--{note['uuid']}", 'object_refs': [object_id] + 'id': f"note--{note['uuid']}", + 'object_refs': [stix_object.id], + **dict(self._handle_analyst_time_fields(stix_object, note)) } if note.get('authors'): note_args['authors'] = [note['authors']] @@ -195,25 +157,24 @@ def _handle_note_data(self, note, object_id: str): self._create_note(note_args) ) - def _handle_object_analyst_data( - self, misp_object: Union[MISPObject, dict], object_id: str): - for attribute in misp_object['Attribute']: - self._handle_analyst_data(object_id, attribute) - - def _handle_opinion_data(self, opinion, object_id: str): + def _handle_opinion_data( + self, stix_object, opinion: Union[MISPOpinion, dict]): opinion_value = int(opinion['opinion']) opinion_args = { 'created': self._datetime_from_str(opinion['created']), 'modified': self._datetime_from_str(opinion['modified']), - 'id': f"opinion--{opinion['uuid']}", 'object_refs': [object_id], + 'allow_custom': True, 'id': f"opinion--{opinion['uuid']}", 'opinion': self._parse_opinion_level(opinion_value), - 'allow_custom': True, 'x_misp_opinion': opinion_value + 'object_refs': [stix_object.id], 'x_misp_opinion': opinion_value, + **dict(self._handle_analyst_time_fields(stix_object, opinion)) } if opinion.get('authors'): opinion_args['authors'] = [opinion['authors']] if opinion.get('comment'): opinion_args['explanation'] = opinion['comment'] - getattr(self, self._results_handling_function)(Opinion(**opinion_args)) + getattr(self, self._results_handling_function)( + self._create_opinion(opinion_args) + ) def _handle_opinion_object(self, sighting: dict, reference_id: str): opinion_args = { @@ -240,7 +201,9 @@ def _handle_opinion_object(self, sighting: dict, reference_id: str): ) } ) - getattr(self, self._results_handling_function)(Opinion(**opinion_args)) + getattr(self, self._results_handling_function)( + self._create_opinion(opinion_args) + ) def _handle_unpublished_report(self, report_args: dict) -> Grouping: grouping_id = f"grouping--{self._misp_event['uuid']}" @@ -253,8 +216,9 @@ def _handle_unpublished_report(self, report_args: dict) -> Grouping: 'object_refs': self.object_refs, 'allow_custom': True } ) - self._handle_analyst_data(grouping_id) - return Grouping(**report_args) + grouping = Grouping(**report_args) + self._handle_analyst_data(grouping) + return grouping ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # @@ -757,8 +721,9 @@ def _parse_annotation_object( values[0] if isinstance(values, list) and len(values) == 1 else values ) - self._append_SDO(self._create_note(note_args)) - self._handle_object_analyst_data(misp_object, note_id) + note = self._create_note(note_args) + getattr(self, self._results_handling_function)(note) + self._handle_object_analyst_data(note, misp_object) def _parse_asn_object_observable( self, misp_object: Union[MISPObject, dict]): @@ -1023,8 +988,9 @@ def _parse_geolocation_object(self, misp_object: Union[MISPObject, dict]): location_args[feature] = attributes.pop(key) if attributes: location_args.update(self._handle_observable_properties(attributes)) - self._append_SDO(self._create_location(location_args)) - self._handle_object_analyst_data(misp_object, location_id) + location = self._create_location(location_args) + getattr(self, self._results_handling_function)(location) + self._handle_object_analyst_data(location, misp_object) def _parse_http_request_object_observable( self, misp_object: Union[MISPObject, dict]): @@ -1720,6 +1686,10 @@ def _create_observed_data(self, args: dict, observables: list): for observable in observables: getattr(self, self._results_handling_function)(observable) + @staticmethod + def _create_opinion(opinion_args: dict) -> Opinion: + return Opinion(**opinion_args) + @staticmethod def _create_PE_extension(extension_args: dict) -> WindowsPEBinaryExt: return WindowsPEBinaryExt(**extension_args) From 1c65f75cbb3119474a56b8aa874398f6c48f9223 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 11:09:18 +0200 Subject: [PATCH 31/39] add: [tests] Added tests for Analyst Data export to STIX 2.0 --- tests/test_stix20_export.py | 88 +++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index 953f73e..49ab57d 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -27,6 +27,36 @@ def _check_bundle_features(self, length): class TestSTIX20EventExport(TestSTIX20GenericExport): + def _check_analyst_note(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"x-misp-analyst-note--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.x_misp_note, misp_layer['note']) + self.assertEqual(stix_object.x_misp_author, misp_layer['authors']) + self.assertEqual(stix_object.x_misp_language, misp_layer['language']) + self.assertEqual( + stix_object.created, self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + + def _check_analyst_opinion(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"x-misp-analyst-opinion--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.x_misp_opinion, int(misp_layer['opinion'])) + self.assertEqual(stix_object.x_misp_author, misp_layer['authors']) + self.assertEqual(stix_object.x_misp_comment, misp_layer['comment']) + self.assertEqual( + stix_object.created, self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + def _check_opinion_features(self, opinion, sighting, object_id): self.assertEqual(opinion.type, 'x-misp-opinion') self.assertEqual(opinion.id, f"x-misp-opinion--{sighting['uuid']}") @@ -70,6 +100,54 @@ def _test_base_event(self, event): "This MISP Event is empty and contains no attribute, object, galaxy or tag." ) + def _test_event_with_analyst_data(self, event): + orgc = event['Orgc'] + event_report = event['EventReport'][0] + note = event['Note'][0] + attribute = event['Attribute'][0] + misp_object = event['Object'][0] + self.parser.parse_misp_event(event) + bundle = self._check_bundle_features(11) + identity, report, *stix_objects = bundle.objects + timestamp = event['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + identity_id = self._check_identity_features(identity, orgc, timestamp) + object_refs = self._check_report_features(report, event, identity_id, timestamp) + self.assertEqual(report.published, timestamp) + for stix_object, object_ref in zip(stix_objects, object_refs): + self.assertEqual(stix_object.id, object_ref) + (attr_indicator, attr_opinion, obj_indicator, obj_opinion, obj_attr_note, + report, report_opinion, relationship, event_note) = stix_objects + self._assert_multiple_equal( + attr_indicator.id, + relationship.target_ref, + attr_opinion.object_ref, + f"indicator--{attribute['uuid']}" + ) + attribute_opinion = attribute['Opinion'][0] + self._check_analyst_opinion(attr_opinion, attribute_opinion) + self._assert_multiple_equal( + obj_indicator.id, + relationship.source_ref, + obj_opinion.object_ref, + obj_attr_note.object_ref, + f"indicator--{misp_object['uuid']}" + ) + object_opinion = misp_object['Opinion'][0] + self._check_analyst_opinion(obj_opinion, object_opinion) + object_attribute_note = misp_object['Attribute'][0]['Note'][0] + self._check_analyst_note(obj_attr_note, object_attribute_note) + self._assert_multiple_equal( + report.id, + report_opinion.object_ref, + f"x-misp-event-report--{event_report['uuid']}" + ) + event_report_opinion = event_report['Opinion'][0] + self._check_analyst_opinion(report_opinion, event_report_opinion) + self.assertEqual(relationship.relationship_type, 'downloaded-from') + self._check_analyst_note(event_note, note) + def _test_event_with_escaped_characters(self, event): attributes = deepcopy(event['Attribute']) self.parser.parse_misp_event(event) @@ -212,6 +290,10 @@ def test_base_event(self): event = get_base_event() self._test_base_event(event['Event']) + def test_event_with_analyst_data(self): + event = get_event_with_analyst_data() + self._test_event_with_analyst_data(event['Event']) + def test_event_with_escaped_characters(self): event = get_event_with_escaped_values_v20() self._test_event_with_escaped_characters(event['Event']) @@ -240,6 +322,12 @@ def test_base_event(self): misp_event.from_dict(**event) self._test_base_event(misp_event) + def test_event_with_analyst_data(self): + event = get_event_with_analyst_data() + misp_event = MISPEvent() + misp_event.from_dict(**event) + self._test_event_with_analyst_data(misp_event) + def test_event_with_escaped_characters(self): event = get_event_with_escaped_values_v20() misp_event = MISPEvent() From fb5ef97af16e528889773a8e504f20389c92400c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 16:13:22 +0200 Subject: [PATCH 32/39] fix: [tests] Avoiding issues with test samples being altered --- tests/test_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_events.py b/tests/test_events.py index fb28e98..810b1de 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -3195,7 +3195,7 @@ def get_base_event(): def get_event_with_analyst_data(): event = deepcopy(_BASE_EVENT) - event['Event'].update(_TEST_EVENT_WITH_ANALYST_DATA) + event['Event'].update(deepcopy(_TEST_EVENT_WITH_ANALYST_DATA)) return event From 89b7c5ae53ae97db242faa890db392448499776a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 9 Oct 2024 17:48:14 +0200 Subject: [PATCH 33/39] fix: [stix2 export] Parsing analyst data related to Observed Data objects & added a few missing typings --- .../misp2stix/misp_to_stix2.py | 32 +++++++++++++------ .../misp2stix/misp_to_stix20.py | 18 ++++++++--- .../misp2stix/misp_to_stix21.py | 20 +++++++++--- 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 0fc141d..43451bf 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -19,12 +19,18 @@ from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v21.bundle import Bundle as Bundle_v21 from stix2.v20.sdo import ( - Campaign as Campaign_v20, CustomObject as CustomObject_v20, - Indicator as Indicator_v20, Report as Report_v20, + AttackPattern as AttackPattern_v20, Campaign as Campaign_v20, + CourseOfAction as CourseOfAction_v20, CustomObject as CustomObject_v20, + Identity as Identity_v20, Indicator as Indicator_v20, + IntrusionSet as IntrusionSet_v20, Malware as Malware_v20, + ObservedData as ObservedData_v20, Report as Report_v20, Tool as Tool_v20, Vulnerability as Vulnerability_v20) from stix2.v21.sdo import ( - Campaign as Campaign_v21, CustomObject as CustomObject_v21, Grouping, - Indicator as Indicator_v21, Report as Report_v21, + AttackPattern as AttackPattern_v21, Campaign as Campaign_v21, + CourseOfAction as CourseOfAction_v21, CustomObject as CustomObject_v21, + Grouping, Identity as Identity_v21, Indicator as Indicator_v21, + IntrusionSet as IntrusionSet_v21, Location, Malware as Malware_v21, Note, + ObservedData as ObservedData_v21, Report as Report_v21, Tool as Tool_v21, Vulnerability as Vulnerability_v21) from typing import Generator, Optional, Tuple, Union @@ -44,9 +50,12 @@ dict, MISPAttribute, MISPEventReport, MISPObject ] _STIX_OBJECT_TYPING = Union[ - Campaign_v20, Campaign_v21, CustomObject_v20, CustomObject_v21, Grouping, - Indicator_v20, Indicator_v21, Report_v20, Report_v21, - Vulnerability_v20, Vulnerability_v21 + AttackPattern_v20, AttackPattern_v21, Campaign_v20, Campaign_v21, + CourseOfAction_v20, CourseOfAction_v21, CustomObject_v20, CustomObject_v21, + Grouping, Identity_v20, Identity_v21, Indicator_v20, Indicator_v21, + IntrusionSet_v20, IntrusionSet_v21, Location, Malware_v20, Malware_v21, + Note, ObservedData_v20, ObservedData_v21, Report_v20, Report_v21, + Tool_v20, Tool_v21, Vulnerability_v20, Vulnerability_v21 ] @@ -565,7 +574,8 @@ def _handle_attribute_observable( ) if markings: self._handle_markings(observable_args, markings) - self._create_observed_data(observable_args, observable) + observed_data = self._create_observed_data(observable_args, observable) + self._handle_analyst_data(observed_data, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], observable_id) @@ -1236,7 +1246,8 @@ def _handle_object_observable( misp_object['ObjectReference'], observable_id, observable_args['modified'] ) - self._create_observed_data(observable_args, observable) + observed_data = self._create_observed_data(observable_args, observable) + self._handle_object_analyst_data(observed_data, misp_object) def _handle_object_tags_and_galaxies( self, misp_object: Union[MISPObject, dict], @@ -4219,7 +4230,8 @@ def _get_matching_email_display_name( def _get_vulnerability_references(vulnerability: str) -> dict: return {'source_name': 'cve', 'external_id': vulnerability} - def _handle_analyst_time_fields(self, stix_object, misp_object: Union[MISPNote, MISPOpinion]): + def _handle_analyst_time_fields(self, stix_object: _STIX_OBJECT_TYPING, + misp_object: Union[MISPNote, MISPOpinion]): for feature in ('created', 'modified'): if misp_object.get(feature): yield feature, self._datetime_from_str(misp_object[feature]) diff --git a/misp_stix_converter/misp2stix/misp_to_stix20.py b/misp_stix_converter/misp2stix/misp_to_stix20.py index 82354a7..3bd0f0f 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix20.py +++ b/misp_stix_converter/misp2stix/misp_to_stix20.py @@ -33,6 +33,10 @@ 'x-misp-attribute', 'x-misp-event-note', 'x-misp-event-report', 'x-misp-galaxy-cluster', 'x-misp-object' ] +_STIX_OBJECT_TYPING = Union[ + AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, + IntrusionSet, Malware, ObservedData, Report, Tool, Vulnerability +] @CustomObject( @@ -287,7 +291,8 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids - def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): + def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, + note: Union[MISPNote, dict]): note_args = { 'id': f"x-misp-analyst-note--{note['uuid']}", 'object_ref': stix_object.id, 'x_misp_note': note['note'], @@ -303,8 +308,8 @@ def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): CustomAnalystNote(**note_args) ) - def _handle_opinion_data( - self, stix_object, opinion: Union[MISPOpinion, dict]): + def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, + opinion: Union[MISPOpinion, dict]): opinion_args = { 'id': f"x-misp-analyst-opinion--{opinion['uuid']}", 'object_ref': stix_object.id, 'x_misp_opinion': opinion['opinion'], @@ -1408,9 +1413,12 @@ def _create_intrusion_set(intrusion_set_args: dict) -> IntrusionSet: def _create_malware(malware_args: dict) -> Malware: return Malware(**malware_args) - def _create_observed_data(self, args: dict, observable: dict): + def _create_observed_data( + self, args: dict, observable: dict) -> ObservedData: args['objects'] = observable - getattr(self, self._results_handling_function)(ObservedData(**args)) + observed_data = ObservedData(**args) + getattr(self, self._results_handling_function)(observed_data) + return observed_data @staticmethod def _create_PE_extension(extension_args: dict) -> WindowsPEBinaryExt: diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 17d2ffd..99d0af7 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -28,6 +28,12 @@ from stix2.v21.vocab import HASHING_ALGORITHM from typing import Optional, Union +_STIX_OBJECT_TYPING = Union[ + AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, + Indicator, IntrusionSet, Location, Malware, Note, ObservedData, Report, + Tool, Vulnerability +] + @CustomObject( 'x-misp-attribute', @@ -142,7 +148,8 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids - def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): + def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, + note: Union[MISPNote, dict]): note_args = { 'content': note['note'], 'id': f"note--{note['uuid']}", @@ -157,8 +164,8 @@ def _handle_note_data(self, stix_object, note: Union[MISPNote, dict]): self._create_note(note_args) ) - def _handle_opinion_data( - self, stix_object, opinion: Union[MISPOpinion, dict]): + def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, + opinion: Union[MISPOpinion, dict]): opinion_value = int(opinion['opinion']) opinion_args = { 'created': self._datetime_from_str(opinion['created']), @@ -1680,11 +1687,14 @@ def _create_note(note_args: dict) -> Note: note_args['allow_custom'] = True return Note(**note_args) - def _create_observed_data(self, args: dict, observables: list): + def _create_observed_data( + self, args: dict, observables: list) -> ObservedData: args['object_refs'] = [observable.id for observable in observables] - getattr(self, self._results_handling_function)(ObservedData(**args)) + observed_data = ObservedData(**args) + getattr(self, self._results_handling_function)(observed_data) for observable in observables: getattr(self, self._results_handling_function)(observable) + return observed_data @staticmethod def _create_opinion(opinion_args: dict) -> Opinion: From d24860cbccf3032ae1bda020128de63cf15bfe7d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 08:27:31 +0200 Subject: [PATCH 34/39] chg: [tests] Updated samples & tests for analyst data export with content exported to Observed Data --- tests/test_events.py | 18 ++++++++++++++++++ tests/test_stix20_export.py | 22 ++++++++++++++++------ tests/test_stix21_export.py | 32 +++++++++++++++++++------------- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/tests/test_events.py b/tests/test_events.py index 810b1de..30828ed 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -770,6 +770,24 @@ "comment": "Fully agree with the malicious nature of the IP" } ] + }, + { + "type": "ip-dst", + "category": "Network activity", + "to_ids": False, + "uuid": "76fd763a-45fb-49a6-a732-64aeedbfd7d4", + "timestamp": "1603642920", + "value": "8.8.8.8", + "Note": [ + { + "uuid": "31fc7048-9ede-4db9-a423-ef97670ed4c6", + "authors": "opinion@foo.bar", + "created": "2024-06-12 12:52:45", + "modified": "2024-06-12 12:52:45", + "note": "DNS Resolver used to resolve the malicious domain", + "language": "en" + } + ] } ], "Object": [ diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index 49ab57d..a321d12 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -104,10 +104,10 @@ def _test_event_with_analyst_data(self, event): orgc = event['Orgc'] event_report = event['EventReport'][0] note = event['Note'][0] - attribute = event['Attribute'][0] + src_attribute, dst_attribute = event['Attribute'] misp_object = event['Object'][0] self.parser.parse_misp_event(event) - bundle = self._check_bundle_features(11) + bundle = self._check_bundle_features(13) identity, report, *stix_objects = bundle.objects timestamp = event['timestamp'] if not isinstance(timestamp, datetime): @@ -117,16 +117,26 @@ def _test_event_with_analyst_data(self, event): self.assertEqual(report.published, timestamp) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - (attr_indicator, attr_opinion, obj_indicator, obj_opinion, obj_attr_note, - report, report_opinion, relationship, event_note) = stix_objects + for stix_object in stix_objects: + print(stix_object.type) + (attr_indicator, attr_opinion, observed_data, observed_data_note, + obj_indicator, obj_opinion, obj_attr_note, report, report_opinion, + relationship, event_note) = stix_objects self._assert_multiple_equal( attr_indicator.id, relationship.target_ref, attr_opinion.object_ref, - f"indicator--{attribute['uuid']}" + f"indicator--{src_attribute['uuid']}" ) - attribute_opinion = attribute['Opinion'][0] + attribute_opinion = src_attribute['Opinion'][0] self._check_analyst_opinion(attr_opinion, attribute_opinion) + self._assert_multiple_equal( + observed_data.id, + observed_data_note.object_ref, + f"observed-data--{dst_attribute['uuid']}" + ) + attribute_note = dst_attribute['Note'][0] + self._check_analyst_note(observed_data_note, attribute_note) self._assert_multiple_equal( obj_indicator.id, relationship.source_ref, diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index d989236..fabaf50 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -134,10 +134,10 @@ def _test_event_with_analyst_data(self, event): orgc = event['Orgc'] event_report = event['EventReport'][0] note = event['Note'][0] - attribute = event['Attribute'][0] + src_attribute, dst_attribute = event['Attribute'] misp_object = event['Object'][0] self.parser.parse_misp_event(event) - stix_objects = self._check_bundle_features(11) + stix_objects = self._check_bundle_features(15) self._check_spec_versions(stix_objects) identity, grouping, *stix_objects = stix_objects timestamp = event['timestamp'] @@ -147,16 +147,26 @@ def _test_event_with_analyst_data(self, event): object_refs = self._check_grouping_features(grouping, event, identity_id) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - (attr_indicator, attr_opinion, obj_indicator, obj_opinion, obj_attr_note, + for stix_object in stix_objects: + print(stix_object.type) + (attr_indicator, attr_indicator_opinion, observed_data, _, + _, obs_data_note, obj_indicator, obj_opinion, obj_attr_note, report, report_opinion, relationship, event_note) = stix_objects self._assert_multiple_equal( attr_indicator.id, relationship.target_ref, - attr_opinion.object_refs[0], - f"indicator--{attribute['uuid']}" + attr_indicator_opinion.object_refs[0], + f"indicator--{src_attribute['uuid']}" + ) + attribute_opinion = src_attribute['Opinion'][0] + self._check_analyst_opinion(attr_indicator_opinion, attribute_opinion, 'strongly-agree') + self._assert_multiple_equal( + observed_data.id, + obs_data_note.object_refs[0], + f"observed-data--{dst_attribute['uuid']}" ) - attribute_opinion = attribute['Opinion'][0] - self._check_analyst_opinion(attr_opinion, attribute_opinion, 'strongly-agree') + attribute_note = dst_attribute['Note'][0] + self._check_analyst_note(obs_data_note, attribute_note) self._assert_multiple_equal( obj_indicator.id, relationship.source_ref, @@ -295,12 +305,8 @@ def _test_event_with_sightings(self, event): if not isinstance(timestamp, datetime): timestamp = self._datetime_from_timestamp(timestamp) identity_id = self._check_identity_features(identity, orgc, timestamp) - args = ( - grouping, - event, - identity_id - ) - for stix_object, object_ref in zip(stix_objects, self._check_grouping_features(*args)): + object_refs = self._check_grouping_features(grouping, event, identity_id) + for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) self._check_identities_from_sighting( identities, From 6393483eec091a1911362fd3e15cc6676d1f57be Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 12:16:15 +0200 Subject: [PATCH 35/39] fix: [stix2 export] Adding Note and Opinion IDs used at Event level to the `object_refs` list of references within the Report or Grouping object - Those are the Note and Opinion objects converted from Analyst Notes and Opinions --- .../misp2stix/misp_to_stix2.py | 25 ++++++++++--------- .../misp2stix/misp_to_stix20.py | 17 +++++++------ .../misp2stix/misp_to_stix21.py | 19 +++++++------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 43451bf..eedbb38 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -23,14 +23,14 @@ CourseOfAction as CourseOfAction_v20, CustomObject as CustomObject_v20, Identity as Identity_v20, Indicator as Indicator_v20, IntrusionSet as IntrusionSet_v20, Malware as Malware_v20, - ObservedData as ObservedData_v20, Report as Report_v20, Tool as Tool_v20, + ObservedData as ObservedData_v20, Tool as Tool_v20, Vulnerability as Vulnerability_v20) from stix2.v21.sdo import ( AttackPattern as AttackPattern_v21, Campaign as Campaign_v21, CourseOfAction as CourseOfAction_v21, CustomObject as CustomObject_v21, - Grouping, Identity as Identity_v21, Indicator as Indicator_v21, + Identity as Identity_v21, Indicator as Indicator_v21, IntrusionSet as IntrusionSet_v21, Location, Malware as Malware_v21, Note, - ObservedData as ObservedData_v21, Report as Report_v21, Tool as Tool_v21, + ObservedData as ObservedData_v21, Tool as Tool_v21, Vulnerability as Vulnerability_v21) from typing import Generator, Optional, Tuple, Union @@ -52,10 +52,10 @@ _STIX_OBJECT_TYPING = Union[ AttackPattern_v20, AttackPattern_v21, Campaign_v20, Campaign_v21, CourseOfAction_v20, CourseOfAction_v21, CustomObject_v20, CustomObject_v21, - Grouping, Identity_v20, Identity_v21, Indicator_v20, Indicator_v21, + Identity_v20, Identity_v21, Indicator_v20, Indicator_v21, IntrusionSet_v20, IntrusionSet_v21, Location, Malware_v20, Malware_v21, - Note, ObservedData_v20, ObservedData_v21, Report_v20, Report_v21, - Tool_v20, Tool_v21, Vulnerability_v20, Vulnerability_v21 + Note, ObservedData_v20, ObservedData_v21, Tool_v20, Tool_v21, + Vulnerability_v20, Vulnerability_v21, dict ] @@ -339,7 +339,7 @@ def unique_ids(self) -> dict: def _append_SDO(self, stix_object): self.__objects.append(stix_object) - self.__object_refs.append(stix_object.id) + self.object_refs.append(stix_object.id) def _append_SDO_without_refs(self, stix_object): self.__objects.append(stix_object) @@ -366,19 +366,20 @@ def _generate_report_from_event(self): marking['used'] = True if self._is_published(): report_id = f"report--{self._misp_event['uuid']}" - if not self.__object_refs: + if not self.object_refs: self._handle_empty_object_refs(report_id, self.event_timestamp) published = self._datetime_from_timestamp( self._misp_event['publish_timestamp'] ) report_args.update( { - 'id': report_id, 'type': 'report', 'published': published, - 'object_refs': self.__object_refs, 'allow_custom': True + 'id': report_id, 'type': 'report', + 'published': published, 'allow_custom': True } ) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs report = self._create_report(report_args) - self._handle_analyst_data(report) return report return self._handle_unpublished_report(report_args) @@ -4203,7 +4204,7 @@ def _fetch_included_reference_uuids( return uuids def _find_target_uuid(self, reference: str) -> Union[str, None]: - for object_ref in self.__object_refs: + for object_ref in self.object_refs: if reference in object_ref: return object_ref diff --git a/misp_stix_converter/misp2stix/misp_to_stix20.py b/misp_stix_converter/misp2stix/misp_to_stix20.py index 3bd0f0f..0cc4944 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix20.py +++ b/misp_stix_converter/misp2stix/misp_to_stix20.py @@ -35,7 +35,7 @@ ] _STIX_OBJECT_TYPING = Union[ AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, - IntrusionSet, Malware, ObservedData, Report, Tool, Vulnerability + IntrusionSet, Malware, ObservedData, Tool, Vulnerability, dict ] @@ -295,14 +295,14 @@ def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, note: Union[MISPNote, dict]): note_args = { 'id': f"x-misp-analyst-note--{note['uuid']}", - 'object_ref': stix_object.id, 'x_misp_note': note['note'], + 'object_ref': stix_object['id'], 'x_misp_note': note['note'], **dict(self._handle_analyst_time_fields(stix_object, note)) } if note.get('authors'): note_args['x_misp_author'] = note['authors'] if note.get('language'): note_args['x_misp_language'] = note['language'] - if stix_object.id.startswith('x-misp-'): + if stix_object['id'].startswith('x-misp-'): note_args['allow_custom'] = True getattr(self, self._results_handling_function)( CustomAnalystNote(**note_args) @@ -312,14 +312,15 @@ def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, opinion: Union[MISPOpinion, dict]): opinion_args = { 'id': f"x-misp-analyst-opinion--{opinion['uuid']}", - 'object_ref': stix_object.id, 'x_misp_opinion': opinion['opinion'], + 'object_ref': stix_object['id'], + 'x_misp_opinion': opinion['opinion'], **dict(self._handle_analyst_time_fields(stix_object, opinion)) } if opinion.get('authors'): opinion_args['x_misp_author'] = opinion['authors'] if opinion.get('comment'): opinion_args['x_misp_comment'] = opinion['comment'] - if stix_object.id.startswith('x-misp-'): + if stix_object['id'].startswith('x-misp-'): opinion_args['allow_custom'] = True getattr(self, self._results_handling_function)( CustomAnalystOpinion(**opinion_args) @@ -362,12 +363,12 @@ def _handle_unpublished_report(self, report_args: dict) -> Report: report_args.update( { 'id': report_id, 'type': 'report', - 'published': report_args['modified'], - 'object_refs': self.object_refs, 'allow_custom': True + 'published': report_args['modified'], 'allow_custom': True } ) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs report = self._create_report(report_args) - self._handle_analyst_data(report) return report ############################################################################ diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 99d0af7..ecb37b9 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -29,9 +29,9 @@ from typing import Optional, Union _STIX_OBJECT_TYPING = Union[ - AttackPattern, Campaign, CourseOfAction, CustomObject, Grouping, Identity, - Indicator, IntrusionSet, Location, Malware, Note, ObservedData, Report, - Tool, Vulnerability + AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, + Indicator, IntrusionSet, Location, Malware, Note, ObservedData, Tool, + Vulnerability, dict ] @@ -151,9 +151,8 @@ def _handle_markings(self, object_args: dict, markings: tuple): def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, note: Union[MISPNote, dict]): note_args = { - 'content': note['note'], - 'id': f"note--{note['uuid']}", - 'object_refs': [stix_object.id], + 'content': note['note'], 'id': f"note--{note['uuid']}", + 'object_refs': [stix_object['id']], **dict(self._handle_analyst_time_fields(stix_object, note)) } if note.get('authors'): @@ -172,7 +171,7 @@ def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, 'modified': self._datetime_from_str(opinion['modified']), 'allow_custom': True, 'id': f"opinion--{opinion['uuid']}", 'opinion': self._parse_opinion_level(opinion_value), - 'object_refs': [stix_object.id], 'x_misp_opinion': opinion_value, + 'object_refs': [stix_object['id']], 'x_misp_opinion': opinion_value, **dict(self._handle_analyst_time_fields(stix_object, opinion)) } if opinion.get('authors'): @@ -219,12 +218,12 @@ def _handle_unpublished_report(self, report_args: dict) -> Grouping: report_args.update( { 'id': grouping_id, 'type': 'grouping', - 'context': 'suspicious-activity', - 'object_refs': self.object_refs, 'allow_custom': True + 'context': 'suspicious-activity', 'allow_custom': True } ) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs grouping = Grouping(**report_args) - self._handle_analyst_data(grouping) return grouping ############################################################################ From 3a79ad67354c2bf221419baf5d26fd6a69293c5d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 12:18:58 +0200 Subject: [PATCH 36/39] add: [stix2 export] Added labels to Notes and Opinions objects converted from Analyst Data or Event Report - This makes it easier to understand directly where the Note or Opinion objects are converted from while reading the STIX content results --- misp_stix_converter/misp2stix/misp_to_stix21.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index ecb37b9..c6b09f0 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -98,14 +98,16 @@ def __init__(self, interoperability=False): self._version = '2.1' self._mapping = MISPtoSTIX21Mapping - def _parse_event_report(self, event_report: Union[MISPEventReport, dict]) -> Note: + def _parse_event_report( + self, event_report: Union[MISPEventReport, dict]) -> Note: timestamp = self._datetime_from_timestamp(event_report['timestamp']) note_args = { 'id': f"note--{event_report['uuid']}", 'created': timestamp, 'modified': timestamp, 'created_by_ref': self.identity_id, 'content': event_report['content'], - 'abstract': event_report['name'] + 'abstract': event_report['name'], + 'labels': ['misp:data-layer="Event Report"'] } references = set(self._parse_event_report_references(event_report)) note_args['object_refs'] = ( @@ -118,7 +120,10 @@ def _handle_empty_object_refs(self, object_id: str, timestamp: datetime): 'id': f"note--{self._misp_event['uuid']}", 'created': timestamp, 'modified': timestamp, 'created_by_ref': self.identity_id, 'object_refs': [object_id], - 'content': 'This MISP Event is empty and contains no attribute, object, galaxy or tag.' + 'content': ( + 'This MISP Event is empty and contains ' + 'no attribute, object, galaxy or tag.' + ) } self._append_SDO(self._create_note(note_args)) @@ -152,6 +157,7 @@ def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, note: Union[MISPNote, dict]): note_args = { 'content': note['note'], 'id': f"note--{note['uuid']}", + 'labels': ['misp:context-layer="Analyst Note"'], 'object_refs': [stix_object['id']], **dict(self._handle_analyst_time_fields(stix_object, note)) } @@ -170,6 +176,7 @@ def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, 'created': self._datetime_from_str(opinion['created']), 'modified': self._datetime_from_str(opinion['modified']), 'allow_custom': True, 'id': f"opinion--{opinion['uuid']}", + 'labels': ['misp:context-layer="Analyst Opinion"'], 'opinion': self._parse_opinion_level(opinion_value), 'object_refs': [stix_object['id']], 'x_misp_opinion': opinion_value, **dict(self._handle_analyst_time_fields(stix_object, opinion)) From a91279ce829c6d432b7a0d248f704d1cad9a628f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 12:20:50 +0200 Subject: [PATCH 37/39] fix: [stix2 export] Fixed Note and Opinion objects arguments - Removed duplication of the `created` and `modified` fields - Added the missing test to check whether a reference is a Custom object in order to use the `allow_custom` flag --- misp_stix_converter/misp2stix/misp_to_stix21.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index c6b09f0..4e87f4b 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -165,6 +165,8 @@ def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, note_args['authors'] = [note['authors']] if note.get('language'): note_args['lang'] = note['language'] + if stix_object['id'].startswith('x-misp--'): + note_args['allow_custom'] = True getattr(self, self._results_handling_function)( self._create_note(note_args) ) @@ -173,8 +175,6 @@ def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, opinion: Union[MISPOpinion, dict]): opinion_value = int(opinion['opinion']) opinion_args = { - 'created': self._datetime_from_str(opinion['created']), - 'modified': self._datetime_from_str(opinion['modified']), 'allow_custom': True, 'id': f"opinion--{opinion['uuid']}", 'labels': ['misp:context-layer="Analyst Opinion"'], 'opinion': self._parse_opinion_level(opinion_value), @@ -185,6 +185,8 @@ def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, opinion_args['authors'] = [opinion['authors']] if opinion.get('comment'): opinion_args['explanation'] = opinion['comment'] + if stix_object['id'].startswith('x-misp--'): + opinion_args['allow_custom'] = True getattr(self, self._results_handling_function)( self._create_opinion(opinion_args) ) From 9ea626ce45e8fdf8ba791cbd28879648cba8c041 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 13:59:24 +0200 Subject: [PATCH 38/39] add: [tests] Testing the Note & Opinion objects type for Analyst Data exported to STIX 2.x --- tests/test_stix20_export.py | 13 +++++++++++++ tests/test_stix21_export.py | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index a321d12..757a259 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -128,6 +128,12 @@ def _test_event_with_analyst_data(self, event): attr_opinion.object_ref, f"indicator--{src_attribute['uuid']}" ) + self._assert_multiple_equal( + attr_opinion.type, + obj_opinion.type, + report_opinion.type, + 'x-misp-analyst-opinion' + ) attribute_opinion = src_attribute['Opinion'][0] self._check_analyst_opinion(attr_opinion, attribute_opinion) self._assert_multiple_equal( @@ -135,6 +141,12 @@ def _test_event_with_analyst_data(self, event): observed_data_note.object_ref, f"observed-data--{dst_attribute['uuid']}" ) + self._assert_multiple_equal( + observed_data_note.type, + obj_attr_note.type, + event_note.type, + 'x-misp-analyst-note' + ) attribute_note = dst_attribute['Note'][0] self._check_analyst_note(observed_data_note, attribute_note) self._assert_multiple_equal( @@ -153,6 +165,7 @@ def _test_event_with_analyst_data(self, event): report_opinion.object_ref, f"x-misp-event-report--{event_report['uuid']}" ) + self.assertEqual(report.type, 'x-misp-event-report') event_report_opinion = event_report['Opinion'][0] self._check_analyst_opinion(report_opinion, event_report_opinion) self.assertEqual(relationship.relationship_type, 'downloaded-from') diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index fabaf50..68dbd52 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -51,6 +51,9 @@ def _check_analyst_note(self, stix_object, misp_layer): stix_object.modified, self._datetime_from_str(misp_layer['modified']) ) + self.assertEqual( + stix_object.labels, ['misp:context-layer="Analyst Note"'] + ) def _check_analyst_opinion(self, stix_object, misp_layer, opinion): self.assertEqual( @@ -70,6 +73,9 @@ def _check_analyst_opinion(self, stix_object, misp_layer, opinion): stix_object.modified, self._datetime_from_str(misp_layer['modified']) ) + self.assertEqual( + stix_object.labels, ['misp:context-layer="Analyst Opinion"'] + ) def _check_attribute_confidence_tags(self, stix_object, attribute): self.assertEqual( @@ -183,6 +189,7 @@ def _test_event_with_analyst_data(self, event): report_opinion.object_refs[0], f"note--{event_report['uuid']}" ) + self.assertEqual(report.labels, ['misp:data-layer="Event Report"']) event_report_opinion = event_report['Opinion'][0] self._check_analyst_opinion(report_opinion, event_report_opinion, 'agree') self.assertEqual(relationship.relationship_type, 'downloaded-from') From 6743ad7687d242e4ecc11dc5749fbea5f67ea1d7 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 10 Oct 2024 14:00:22 +0200 Subject: [PATCH 39/39] fix: [tests] Cleaned up tests for analyst data export --- tests/test_stix20_export.py | 6 ++---- tests/test_stix21_export.py | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index 757a259..ac8dd02 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -117,8 +117,6 @@ def _test_event_with_analyst_data(self, event): self.assertEqual(report.published, timestamp) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - for stix_object in stix_objects: - print(stix_object.type) (attr_indicator, attr_opinion, observed_data, observed_data_note, obj_indicator, obj_opinion, obj_attr_note, report, report_opinion, relationship, event_note) = stix_objects @@ -194,7 +192,7 @@ def _test_event_with_event_report(self, event): orgc = event['Orgc'] event_report = event['EventReport'][0] self.parser.parse_misp_event(event) - bundle = self._check_bundle_features(9) + bundle = self._check_bundle_features(7) identity, report, *stix_objects = bundle.objects timestamp = event['timestamp'] if not isinstance(timestamp, datetime): @@ -204,7 +202,7 @@ def _test_event_with_event_report(self, event): self.assertEqual(report.published, timestamp) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - attack_pattern, ip_src, observed_data, domain_ip, note, _, marking = stix_objects + ip_src, observed_data, domain_ip, note, _ = stix_objects self.assertEqual(note.id, f"x-misp-event-report--{event_report['uuid']}") timestamp = event_report['timestamp'] if not isinstance(timestamp, datetime): diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index 68dbd52..9f9743f 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -153,8 +153,6 @@ def _test_event_with_analyst_data(self, event): object_refs = self._check_grouping_features(grouping, event, identity_id) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - for stix_object in stix_objects: - print(stix_object.type) (attr_indicator, attr_indicator_opinion, observed_data, _, _, obs_data_note, obj_indicator, obj_opinion, obj_attr_note, report, report_opinion, relationship, event_note) = stix_objects @@ -264,7 +262,7 @@ def _test_event_with_event_report(self, event): timestamp = event_report['timestamp'] if not isinstance(timestamp, datetime): timestamp = self._datetime_from_timestamp(timestamp) - self.assertEqual(note.created, timestamp) + self._assert_multiple_equal(note.created, note.modified, timestamp) self.assertEqual(note.content, event_report['content']) object_refs = note.object_refs self.assertEqual(len(object_refs), 3)