From 84458a87351d789dcb78515594b99afc12c08adb Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 8 Jan 2024 16:50:37 +0100 Subject: [PATCH 001/137] fix: [stix2 import] Quick syntax fix --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 43d8a997..79f01a3f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -9,7 +9,7 @@ InternalSTIX2ObservableConverter, InternalSTIX2ObservableMapping, STIX2ObservableConverter, _AUTONOMOUS_SYSTEM_TYPING, _DIRECTORY_TYPING, _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) -from .stix2converter import (_MAIN_PARSER_TYPING) +from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject @@ -295,7 +295,7 @@ def _attribute_from_hostname_port_observable_v21( self.main_parser._add_misp_attribute(attribute, observed_data) def _attribute_from_ip_port_observable( - self, network_traffic :_NETWORK_TRAFFIC_TYPING, + self, network_traffic: _NETWORK_TRAFFIC_TYPING, ip_value: str, observed_data: _OBSERVED_DATA_TYPING): attribute = self._create_attribute_dict(observed_data) for feature in ('src_port', 'dst_port'): From 79a5f1d2e5b6e3d98ce654c024eaae754616bae3 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 8 Jan 2024 16:52:12 +0100 Subject: [PATCH 002/137] fix: [stix2 import] Observable objects fetcher moved to the parent class as it will be reused for internal & external conversion --- .../stix2_observed_data_converter.py | 73 ++++++++++++++++--- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 79f01a3f..9b499cfd 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -30,6 +30,69 @@ class STIX2ObservedDataConverter(STIX2ObservableConverter, metaclass=ABCMeta): def __init__(self, main: _MAIN_PARSER_TYPING): self._set_main_parser(main) + def _fetch_observables(self, object_refs: Union[list, str]): + if isinstance(object_refs, str): + return self.main_parser._observable[object_refs] + if len(object_refs) == 1: + return self.main_parser._observable[object_refs[0]] + return tuple( + self.main_parser._observable[object_ref] + for object_ref in object_refs + ) + + +class ExternalSTIX2ObservedDataConverter( + STIX2ObservedDataConverter, ExternalSTIX2ObservableConverter): + def __init__(self, main: 'ExternalSTIX2toMISPParser'): + super().__init__(main) + self._mapping = ExternalSTIX2ObservableMapping + + def parse(self, observed_data_ref: str): + observed_data = self.main_parser._get_stix_object(observed_data_ref) + try: + if hasattr(observed_data, 'object_refs'): + self._parse_observable_refs(observed_data) + else: + self._parse_observable_objects(observed_data) + except UnknownObservableMappingError as observable_types: + self.main_parser._observable_mapping_error( + observed_data.id, observable_types + ) + + ############################################################################ + # GENERIC OBSERVED DATA HANDLING METHODS # + ############################################################################ + + def _handle_observables_mapping(self, observable_types: set) -> str: + to_call = '_'.join(sorted(observable_types)) + mapping = self._mapping.observable_mapping(to_call) + if mapping is None: + raise UnknownObservableMappingError(to_call) + return mapping + + def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): + observable_types = set( + observable['type'] for observable in observed_data.objects.values() + ) + mapping = self._handle_observables_mapping(observable_types) + feature = f'_parse_{mapping}_observable_objects' + try: + parser = getattr(self, feature) + except AttributeError: + raise UnknownParsingFunctionError(feature) + parser(observed_data) + + def _parse_observable_refs(self, observed_data: ObservedData_v21): + observable_types = set( + reference.split('--')[0] for reference in observed_data.object_refs + ) + mapping = self._handle_observables_mapping(observable_types) + feature = f'_parse_{mapping}_observable_object_refs' + try: + parser = getattr(self, feature) + except AttributeError: + raise UnknownParsingFunctionError(feature) + parser(observed_data) class InternalSTIX2ObservedDataConverter( @@ -1310,16 +1373,6 @@ def _fetch_main_process(observables: dict) -> _PROCESS_TYPING: if any(hasattr(observable, feature) for feature in ref_features): return observable - def _fetch_observables(self, object_refs: Union[list, str]): - if isinstance(object_refs, str): - return self.main_parser._observable[object_refs] - if len(object_refs) == 1: - return self.main_parser._observable[object_refs[0]] - return tuple( - self.main_parser._observable[object_ref] - for object_ref in object_refs - ) - @staticmethod def _fetch_observables_v20(observed_data: ObservedData_v20): observables = tuple(observed_data.objects.values()) From b4e3cf1a60838a23c75950fac5e4879578770455 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 9 Jan 2024 16:15:22 +0100 Subject: [PATCH 003/137] fix: [tests] A tiny clarification change --- tests/_test_stix_import.py | 12 ++++++------ tests/test_internal_stix20_import.py | 2 +- tests/test_internal_stix21_import.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index dfe27261..7e4f6d0d 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -389,12 +389,6 @@ def _check_indicator_attribute(self, attribute, indicator): self._check_attribute_labels(attribute, indicator.labels) return indicator.pattern - def _check_indicator_object(self, misp_object, indicator): - self.assertEqual(misp_object.uuid, indicator.id.split('--')[1]) - self.assertEqual(misp_object.timestamp, indicator.modified) - self._check_object_labels(misp_object, indicator.labels, True) - return indicator.pattern - def _check_vulnerability_attribute(self, attribute, vulnerability): self.assertEqual(attribute.uuid, vulnerability.id.split('--')[1]) self.assertEqual(attribute.type, vulnerability.type) @@ -1444,6 +1438,12 @@ def _check_image_observable_object(self, attributes, observables): self.assertEqual(url.object_relation, 'url') self.assertEqual(url.value, artifact.x_misp_url) + def _check_indicator_object(self, misp_object, indicator): + self.assertEqual(misp_object.uuid, indicator.id.split('--')[1]) + self.assertEqual(misp_object.timestamp, indicator.modified) + self._check_object_labels(misp_object, indicator.labels, True) + return indicator.pattern + def _check_intrusion_set_object(self, misp_object, intrusion_set): self.assertEqual(misp_object.uuid, intrusion_set.id.split('--')[1]) self.assertEqual(misp_object.name, intrusion_set.type) diff --git a/tests/test_internal_stix20_import.py b/tests/test_internal_stix20_import.py index be81c381..35e4cd0c 100644 --- a/tests/test_internal_stix20_import.py +++ b/tests/test_internal_stix20_import.py @@ -11,7 +11,7 @@ class TestInternalSTIX20Import(TestInternalSTIX2Import, TestSTIX20, TestSTIX20Import): ############################################################################ - # MISP ATTRIBUTES CHECKING FUNCTIONS # + # SPECIFIC STIX 2.0 CHECKING FUNCTIONS # ############################################################################ def _check_observed_data_attribute(self, attribute, observed_data): diff --git a/tests/test_internal_stix21_import.py b/tests/test_internal_stix21_import.py index d79d23c7..c042a7d7 100644 --- a/tests/test_internal_stix21_import.py +++ b/tests/test_internal_stix21_import.py @@ -11,7 +11,7 @@ class TestInternalSTIX21Import(TestInternalSTIX2Import, TestSTIX21, TestSTIX21Import): ############################################################################ - # MISP ATTRIBUTES CHECKING FUNCTIONS # + # STIX 2.1 SPECIFIC CHECKING FUNCTIONS # ############################################################################ def _check_observed_data_attribute(self, attribute, observed_data): From dba0abc533269b2c4da9b365c00acac64844ff11 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 10 Jan 2024 16:49:36 +0100 Subject: [PATCH 004/137] fix: [stix2 import] Fixed the converters composition - The `getattr` statements were actually making their default argument execute itself and re-initialising each converter attribute as if it was there first call and the attribute did not exist --- .../stix2misp/external_stix2_to_misp.py | 19 +----- .../stix2misp/internal_stix2_to_misp.py | 24 ++------ .../stix2misp/stix2_to_misp.py | 60 +++++++++++-------- 3 files changed, 42 insertions(+), 61 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 72d8a7a7..edcb8421 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -159,18 +159,15 @@ def cluster_distribution(self) -> dict: @property def observable_object_parser(self) -> STIX2ObservableObjectConverter: - return getattr( - self, '_observable_objects_parser', + if not hasattr(self, '_observable_object_parser'): self._set_observable_object_parser() - ) + return self._observable_object_parser def _set_attack_pattern_parser(self) -> ExternalSTIX2AttackPatternConverter: self._attack_pattern_parser = ExternalSTIX2AttackPatternConverter(self) - return self._attack_pattern_parser def _set_campaign_parser(self) -> ExternalSTIX2CampaignConverter: self._campaign_parser = ExternalSTIX2CampaignConverter(self) - return self._campaign_parser def _set_cluster_distribution( self, distribution: int, sharing_group_id: Union[int, None]): @@ -181,47 +178,36 @@ def _set_cluster_distribution( def _set_course_of_action_parser(self) -> ExternalSTIX2CourseOfActionConverter: self._course_of_action_parser = ExternalSTIX2CourseOfActionConverter(self) - return self._course_of_action_parser def _set_identity_parser(self) -> ExternalSTIX2IdentityConverter: self._identity_parser = ExternalSTIX2IdentityConverter(self) - return self._identity_parser def _set_indicator_parser(self) -> ExternalSTIX2IndicatorConverter: self._indicator_parser = ExternalSTIX2IndicatorConverter(self) - return self._indicator_parser def _set_intrusion_set_parser(self) -> ExternalSTIX2IntrusionSetConverter: self._intrusion_set_parser = ExternalSTIX2IntrusionSetConverter(self) - return self._intrusion_set_parser def _set_location_parser(self) -> ExternalSTIX2LocationConverter: self._location_parser = ExternalSTIX2LocationConverter(self) - return self._location_parser def _set_malware_analysis_parser(self) -> ExternalSTIX2MalwareAnalysisConverter: self._malware_analysis_parser = ExternalSTIX2MalwareAnalysisConverter(self) - return self._malware_analysis_parser def _set_malware_parser(self) -> ExternalSTIX2MalwareConverter: self._malware_parser = ExternalSTIX2MalwareConverter(self) - return self._malware_parser def _set_observable_object_parser(self) -> STIX2ObservableObjectConverter: self._observable_object_parser = STIX2ObservableObjectConverter(self) - return self._observable_object_parser def _set_threat_actor_parser(self) -> ExternalSTIX2ThreatActorConverter: self._threat_actor_parser = ExternalSTIX2ThreatActorConverter(self) - return self._threat_actor_parser def _set_tool_parser(self) -> ExternalSTIX2ToolConverter: self._tool_parser = ExternalSTIX2ToolConverter(self) - return self._tool_parser def _set_vulnerability_parser(self) -> ExternalSTIX2VulnerabilityConverter: self._vulnerability_parser = ExternalSTIX2VulnerabilityConverter(self) - return self._vulnerability_parser ############################################################################ # STIX OBJECTS LOADING METHODS # @@ -1733,7 +1719,6 @@ def _populate_object_attributes_from_observable( } ) - ################################################################################ # MISP DATA STRUCTURES CREATION FUNCTIONS. # ################################################################################ diff --git a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py index d7646404..22253254 100644 --- a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py @@ -49,69 +49,57 @@ def __init__(self, distribution: Optional[int] = 0, @property def custom_object_parser(self) -> STIX2CustomObjectConverter: - return getattr( - self, '_custom_object_parser', self._set_custom_object_parser() - ) + if not hasattr(self, '_custom_object_parser'): + self._set_custom_object_parser() + return self._custom_object_parser @property def note_parser(self) -> STIX2NoteConverter: - return getattr(self, '_note_parser', self._set_note_parser()) + if not hasattr(self, '_note_parser'): + self._set_note_parser() + return self._note_parser def _set_attack_pattern_parser(self) -> InternalSTIX2AttackPatternConverter: self._attack_pattern_parser = InternalSTIX2AttackPatternConverter(self) - return self._attack_pattern_parser def _set_campaign_parser(self) -> InternalSTIX2CampaignConverter: self._campaign_parser = InternalSTIX2CampaignConverter(self) - return self._campaign_parser def _set_course_of_action_parser(self) -> InternalSTIX2CourseOfActionConverter: self._course_of_action_parser = InternalSTIX2CourseOfActionConverter(self) - return self._course_of_action_parser def _set_custom_object_parser(self) -> STIX2CustomObjectConverter: self._custom_object_parser = STIX2CustomObjectConverter(self) - return self._custom_object_parser def _set_identity_parser(self) -> InternalSTIX2IdentityConverter: self._identity_parser = InternalSTIX2IdentityConverter(self) - return self._identity_parser def _set_indicator_parser(self) -> InternalSTIX2IndicatorConverter: self._indicator_parser = InternalSTIX2IndicatorConverter(self) - return self._indicator_parser def _set_intrusion_set_parser(self) -> InternalSTIX2IntrusionSetConverter: self._intrusion_set_parser = InternalSTIX2IntrusionSetConverter(self) - return self._intrusion_set_parser def _set_location_parser(self) -> InternalSTIX2LocationConverter: self._location_parser = InternalSTIX2LocationConverter(self) - return self._location_parser def _set_malware_analysis_parser(self) -> InternalSTIX2MalwareAnalysisConverter: self._malware_analysis_parser = InternalSTIX2MalwareAnalysisConverter(self) - return self._malware_analysis_parser def _set_malware_parser(self) -> InternalSTIX2MalwareConverter: self._malware_parser = InternalSTIX2MalwareConverter(self) - return self._malware_parser def _set_note_parser(self) -> STIX2NoteConverter: self._note_parser = STIX2NoteConverter(self) - return self._note_parser def _set_threat_actor_parser(self) -> InternalSTIX2ThreatActorConverter: self._threat_actor_parser = InternalSTIX2ThreatActorConverter(self) - return self._threat_actor_parser def _set_tool_parser(self) -> InternalSTIX2ToolConverter: self._tool_parser = InternalSTIX2ToolConverter(self) - return self._tool_parser def _set_vulnerability_parser(self) -> InternalSTIX2VulnerabilityConverter: self._vulnerability_parser = InternalSTIX2VulnerabilityConverter(self) - return self._vulnerability_parser ############################################################################ # STIX OBJECTS LOADING METHODS # diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 76ff5bdb..24978c57 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -292,22 +292,21 @@ def parse_stix_content( @property def attack_pattern_parser(self) -> _ATTACK_PATTERN_PARSER_TYPING: - return getattr( - self, '_attack_pattern_parser', self._set_attack_pattern_parser() - ) + if not hasattr(self, '_attack_pattern_parser'): + self._set_attack_pattern_parser() + return self._attack_pattern_parser @property def campaign_parser(self) -> _CAMPAIGN_PARSER_TYPING: - return getattr( - self, '_campaign_parser', self._set_campaign_parser() - ) + if not hasattr(self, '_campaign_parser'): + self._set_campaign_parser() + return self._campaign_parser @property def course_of_action_parser(self) -> _COURSE_OF_ACTION_PARSER_TYPING: - return getattr( - self, '_course_of_action_parser', + if not hasattr(self, '_course_of_action_parser'): self._set_course_of_action_parser() - ) + return self._course_of_action_parser @property def generic_info_field(self) -> str: @@ -315,32 +314,39 @@ def generic_info_field(self) -> str: @property def identity_parser(self) -> _IDENTITY_PARSER_TYPING: - return getattr(self, '_identity_parser', self._set_identity_parser()) + if not hasattr(self, '_identity_parser'): + self._set_identity_parser() + return self._identity_parser @property def indicator_parser(self) -> _INDICATOR_PARSER_TYPING: - return getattr(self, '_indicator_parser', self._set_indicator_parser()) + if not hasattr(self, '_indicator_parser'): + self._set_indicator_parser() + return self._indicator_parser @property def intrusion_set_parser(self) -> _INTRUSION_SET_PARSER_TYPING: - return getattr( - self, '_intrusion_set_parser', self._set_intrusion_set_parser() - ) + if not hasattr(self, '_intrusion_set_parser'): + self._set_intrusion_set_parser() + return self._intrusion_set_parser @property def location_parser(self) -> _LOCATION_PARSER_TYPING: - return getattr(self, '_location_parser', self._set_location_parser()) + if not hasattr(self, '_location_parser'): + self._set_location_parser() + return self._location_parser @property def malware_analysis_parser(self) -> _MALWARE_ANALYSIS_PARSER_TYPING: - return getattr( - self, '_malware_analysis_parser', + if not hasattr(self, '_malware_analysis_parser'): self._set_malware_analysis_parser() - ) + return self._malware_analysis_parser @property def malware_parser(self) -> _MALWARE_PARSER_TYPING: - return getattr(self, '_malware_parser', self._set_malware_parser()) + if not hasattr(self, '_malware_parser'): + self._set_malware_parser() + return self._malware_parser @property def misp_event(self) -> MISPEvent: @@ -362,19 +368,21 @@ def stix_version(self) -> str: @property def threat_actor_parser(self) -> _THREAT_ACTOR_PARSER_TYPING: - return getattr( - self, '_threat_actor_parser', self._set_threat_actor_parser() - ) + if not hasattr(self, '_threat_actor_parser'): + self._set_threat_actor_parser() + return self._threat_actor_parser @property def tool_parser(self) -> _TOOL_PARSER_TYPING: - return getattr(self, '_tool_parser', self._set_tool_parser()) + if not hasattr(self, '_tool_parser'): + self._set_tool_parser() + return self._tool_parser @property def vulnerability_parser(self) -> _VULNERABILITY_PARSER_TYPING: - return getattr( - self, '_vulnerability_parser', self._set_vulnerability_parser() - ) + if not hasattr(self, '_vulnerability_parser'): + self._set_vulnerability_parser() + return self._vulnerability_parser ############################################################################ # STIX OBJECTS LOADING METHODS # From b8ce0ae2a0f6add6d3f7836b9d672faf53f689c5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 10 Jan 2024 23:59:03 +0100 Subject: [PATCH 005/137] wip: [stix2 import] Porting Observed Data objects conversion ability to converters, starting with Directory objects - Introducing a better conversion process - Handling complex references between observable objects amongst observed data objects --- .../stix2_observed_data_converter.py | 185 +++++++++++++++++- 1 file changed, 182 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9b499cfd..20432664 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -13,14 +13,21 @@ from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject +from stix2.v20.observables import ( + Directory as Directory_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 +from stix2.v21.observables import ( + Directory as Directory_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 -from typing import Iterator, Optional, Tuple, TYPE_CHECKING, Union +from typing import Optional, TYPE_CHECKING, Union if TYPE_CHECKING: from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser +_OBSERVABLE_OBJECTS_TYPING = Union[ + Directory_v20, Directory_v21 +] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 ] @@ -46,6 +53,13 @@ class ExternalSTIX2ObservedDataConverter( def __init__(self, main: 'ExternalSTIX2toMISPParser'): super().__init__(main) self._mapping = ExternalSTIX2ObservableMapping + self._observable_relationships: dict + + @property + def observable_relationships(self): + if not hasattr(self, '_observable_relationships'): + self._set_observable_relationships() + return self._observable_relationships def parse(self, observed_data_ref: str): observed_data = self.main_parser._get_stix_object(observed_data_ref) @@ -59,6 +73,16 @@ def parse(self, observed_data_ref: str): observed_data.id, observable_types ) + def parse_relationships(self): + for misp_object in self.main_parser.misp_event.objects: + object_uuid = misp_object.uuid + if object_uuid in self.observable_relationships: + for relationship in self.observable_relationships[object_uuid]: + misp_object.add_reference(*relationship) + + def _set_observable_relationships(self): + self._observable_relationships = defaultdict(set) + ############################################################################ # GENERIC OBSERVED DATA HANDLING METHODS # ############################################################################ @@ -94,6 +118,161 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): raise UnknownParsingFunctionError(feature) parser(observed_data) + ############################################################################ + # OBSERVABLE OBJECTS PARSING METHODS # + ############################################################################ + + def _parse_directory_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, + object_id: Optional[str] = None) -> MISPObject: + directory = observed_data.objects[object_id] + if directory.get('id') is not None: + misp_object = self._create_misp_object_from_observable_object_ref( + 'directory', directory, observed_data + ) + for attribute in self._parse_directory_observable(directory): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + object_id = f'{observed_data.id} - {object_id}' + misp_object = self._create_misp_object_from_observable_object( + 'directory', observed_data, object_id + ) + attributes = self._parse_directory_observable(directory, object_id) + for attribute in attributes: + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_directory_observable_object_ref( + self, observed_data: ObservedData_v21, + directory: _DIRECTORY_TYPING) -> MISPObject: + misp_object = self._create_misp_object_from_observable_object_ref( + 'directory', directory, observed_data + ) + for attribute in self._parse_directory_observable(directory): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_directory_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + directory = observable['observable'] + if not hasattr(directory, 'contains_refs'): + if observable['used'].get(self.event_uuid, False): + continue + misp_object = self._parse_directory_observable_object_ref( + observed_data, directory + ) + observable['misp_object'] = misp_object + observable['used'][self.event_uuid] = True + continue + misp_object = ( + observable['misp_object'] if + observable['used'].get(self.event_uuid, False) else + self._parse_directory_observable_object_ref( + observed_data, directory + ) + ) + observable['misp_object'] = misp_object + observable['used'][self.event_uuid] = True + for contained_ref in directory.contains_refs: + contained = self._fetch_observables(contained_ref) + if contained_ref not in observed_data.object_refs: + if contained['used'].get(self.event_uuid, False): + misp_object.add_reference( + contained['misp_object'].uuid, 'contains' + ) + continue + self.observable_relationships[misp_object.uuid].add( + ( + self.main_parser._sanitise_uuid(contained_ref), + 'contains' + ) + ) + continue + if contained['used'].get(self.event_uuid, False): + misp_object.add_reference( + contained['misp_object'].uuid, 'contains' + ) + continue + contained_object = self._parse_directory_observable_object_ref( + observed_data, contained['observable'] + ) + contained['misp_object'] = contained_object + contained['used'][self.event_uuid] = True + misp_object.add_reference(contained_object.uuid, 'contains') + + def _parse_directory_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + directory = next(iter(observed_data.objects.values())) + misp_object = self._create_misp_object_from_observable_object( + 'directory', observed_data + ) + attributes = self._parse_directory_observable( + directory, observed_data.id + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + observable_objects = { + object_id: {'used': False, 'observable': observable} + for object_id, observable in observed_data.objects.items() + } + for object_id, observable in observable_objects.items(): + directory = observable['observable'] + misp_object = ( + observable['misp_object'] if observable['used'] else + self._parse_directory_observable_object( + observed_data, object_id + ) + ) + observable['misp_object'] = misp_object + observable['used'] = True + if hasattr(directory, 'contains_refs'): + for contained_ref in directory.contains_refs: + contained = observable_objects[contained_ref] + if contained['used']: + misp_object.add_reference( + contained['misp_object'].uuid, 'contains' + ) + continue + contained_object = self._parse_directory_observable_object( + observed_data, contained_ref + ) + contained['misp_object'] = contained_object + contained['used'] = True + misp_object.add_reference( + contained_object.uuid, 'contains' + ) + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + def _create_misp_object_from_observable_object( + self, name: str, observed_data: _OBSERVED_DATA_TYPING, + object_id: Optional[str] = None) -> MISPObject: + if object_id is None: + return self._create_misp_object(name, observed_data) + misp_object = self._create_misp_object(name) + misp_object.from_dict( + uuid=self.main_parser._create_v5_uuid(object_id), + **self._parse_timeline(observed_data) + ) + return misp_object + + def _create_misp_object_from_observable_object_ref( + self, name: str, observable: _OBSERVABLE_OBJECTS_TYPING, + observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: + misp_object = self._create_misp_object(name) + misp_object.from_dict( + comment=f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + ) + self.main_parser._sanitise_object_uuid(misp_object, observable.id) + return misp_object + class InternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, InternalSTIX2ObservableConverter): @@ -568,14 +747,14 @@ def _object_from_domain_ip_observable_v21( ip_parsed = set() for object_ref in observed_data.object_refs: if object_ref.startswith('domain-name--'): - observable = self.main_parser._observable[object_ref] + observable = self._fetch_observables(object_ref) for attribute in self._parse_domain_ip_observable(observable): misp_object.add_attribute(**attribute) if hasattr(observable, 'resolves_to_refs'): for reference in observable.resolves_to_refs: if reference in ip_parsed: continue - address = self.main_parser._observable[reference] + address = self._fetch_observables(reference) misp_object.add_attribute( **{ 'value': address.value, From 9db712a16f838858e65a9c6ecbac766c96ba85c5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 15:20:12 +0100 Subject: [PATCH 006/137] fix: [stix2 import] Quick pep8 clean-up --- .../stix2misp/converters/stix2_observable_converter.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 170850b4..6a408048 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -11,10 +11,11 @@ Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, DomainName as DomainName_v20, EmailAddress as EmailAddress_v20, EmailMessage as EmailMessage_v20, - File as File_v20, HTTPRequestExt as HTTPRequestExt_v20, Mutex as Mutex_v20, NetworkTraffic as NetworkTraffic_v20, - Process as Process_v20, Software as Software_v20, - UNIXAccountExt as UNIXAccountExt_v20, URL as URL_v20, - UserAccount as UserAccount_v20, WindowsPEBinaryExt as WindowsExtension_v20, + File as File_v20, HTTPRequestExt as HTTPRequestExt_v20, Mutex as Mutex_v20, + NetworkTraffic as NetworkTraffic_v20, Process as Process_v20, + Software as Software_v20, UNIXAccountExt as UNIXAccountExt_v20, + URL as URL_v20, UserAccount as UserAccount_v20, + WindowsPEBinaryExt as WindowsExtension_v20, WindowsPESection as WindowsPESection_v20, WindowsRegistryKey as WindowsRegistryKey_v20, WindowsRegistryValueType as WindowsRegistryValue_v20, From f78257473b399ebd658032ab1f969783e4570903 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 15:21:11 +0100 Subject: [PATCH 007/137] fix: [stix2 import] Fixed directory mapping --- misp_stix_converter/stix2misp/converters/stix2mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 936d9838..64a372df 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -854,7 +854,7 @@ class ExternalSTIX2Mapping(STIX2Mapping): modified=STIX2Mapping.modification_time_attribute(), mtime=STIX2Mapping.modification_time_attribute(), path=STIX2Mapping.path_attribute(), - path_enc={'type': 'text', 'object_relaiton': 'path-encoding'} + path_enc={'type': 'text', 'object_relation': 'path-encoding'} ) __network_traffic_object_mapping = Mapping( src_port=STIX2Mapping.src_port_attribute(), From 2c2bdd404428ef523da2b5f818bf5fcdb84b1d96 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 15:24:23 +0100 Subject: [PATCH 008/137] wip: [tests] Tests for directory observable objects import from STIX 2.x --- tests/test_external_stix20_bundles.py | 72 ++++++++++++++++++++++ tests/test_external_stix20_import.py | 79 ++++++++++++++++++++++++ tests/test_external_stix21_bundles.py | 86 +++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 76 ++++++++++++++++++++++- 4 files changed, 312 insertions(+), 1 deletion(-) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 0b4d7db6..b80db529 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -108,6 +108,59 @@ ] } ] +_DIRECTORY_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--5e384ae7-672c-4250-9cda-3b4da964451a", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "directory", + "path": "/var/www/MISP/app/files/scripts/misp-stix", + "path_enc": "ISO-8859-1", + "created": "2021-07-21T11:44:56Z", + "modified": "2023-12-12T11:24:30Z", + "accessed": "2023-12-12T11:24:30Z" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--5ac47782-e1b8-40b6-96b4-02510a00020f", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "directory", + "path": "/var/www/MISP/app/files/scripts/", + "path_enc": "ISO-8859-6-I", + "created": "2014-07-25T10:47:08Z", + "modified": "2023-12-12T11:34:05Z", + "accessed": "2023-12-12T11:34:05Z", + "contains_refs": [ + "1" + ] + }, + "1": { + "type": "directory", + "path": "/var/www/MISP/app/files/scripts/misp-stix", + "path_enc": "ISO-8859-1", + "created": "2021-07-21T11:44:56Z", + "modified": "2023-12-12T11:24:30Z", + "accessed": "2023-12-12T11:24:30Z" + } + } + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -349,6 +402,17 @@ class TestExternalSTIX20Bundles: ] } + @classmethod + def __assemble_bundle(cls, *stix_objects): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), deepcopy(cls.__report), *stix_objects + ] + bundle['objects'][1]['object_refs'] = [ + stix_object['id'] for stix_object in stix_objects + ] + return dict_to_stix2(bundle, allow_custom=True) + @classmethod def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): relationship = { @@ -405,3 +469,11 @@ def get_bundle_with_tool_galaxy(cls): @classmethod def get_bundle_with_vulnerability_galaxy(cls): return cls.__assemble_galaxy_bundle(*_VULNERABILITY_OBJECTS) + + ############################################################################ + # MISP OBJECTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_with_directory_objects(cls): + return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 4559d0fe..e5217ac6 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -4,6 +4,7 @@ from .test_external_stix20_bundles import TestExternalSTIX20Bundles from ._test_stix import TestSTIX21 from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import +from uuid import uuid5 class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): @@ -178,3 +179,81 @@ def test_stix20_bundle_with_vulnerability_galaxy(self): meta['external_id'], attribute_vuln.external_references[0].external_id ) + + ############################################################################ + # MISP OBJECTS IMPORT TESTS. # + ############################################################################ + + def _check_directory_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'directory') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + directory = observed_data.objects[identifier] + self.assertEqual(len(misp_object.attributes), 5) + accessed, created, modified, path, path_enc = misp_object.attributes + self._assert_multiple_equal( + accessed.type, created.type, modified.type, 'datetime' + ) + self.assertEqual(accessed.object_relation, 'access-time') + self.assertEqual(accessed.value, directory.accessed) + self.assertEqual( + accessed.uuid, uuid5(self._UUIDv4, f'{object_id} - access-time - {accessed.value}') + ) + self.assertEqual(created.object_relation, 'creation-time') + self.assertEqual(created.value, directory.created) + self.assertEqual( + created.uuid, uuid5(self._UUIDv4, f'{object_id} - creation-time - {created.value}') + ) + self.assertEqual(modified.object_relation, 'modification-time') + self.assertEqual(modified.value, directory.modified) + self.assertEqual( + modified.uuid, uuid5(self._UUIDv4, f'{object_id} - modification-time - {modified.value}') + ) + self.assertEqual(path.type, 'text') + self.assertEqual(path.object_relation, 'path') + self.assertEqual(path.value, directory.path) + self.assertEqual( + path.uuid, uuid5(self._UUIDv4, f'{object_id} - path - {path.value}') + ) + self.assertEqual(path_enc.type, 'text') + self.assertEqual(path_enc.object_relation, 'path-encoding') + self.assertEqual(path_enc.value, directory.path_enc) + self.assertEqual( + path_enc.uuid, uuid5(self._UUIDv4, f'{object_id} - path-encoding - {path_enc.value}') + ) + + def _check_misp_object_fields(self, misp_object, observed_data, identifier): + if identifier is None: + self.assertEqual(misp_object.uuid, observed_data.id.split('--')[1]) + else: + self.assertEqual( + misp_object.uuid, uuid5(self._UUIDv4, f'{observed_data.id} - {identifier}') + ) + if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): + self.assertEqual(misp_object.first_seen, observed_data.first_observed) + self.assertEqual(misp_object.last_seen, observed_data.last_observed) + self.assertEqual(misp_object.timestamp, observed_data.modified) + + def test_stix20_bundle_with_directory_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_directory_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + single_directory, directory, referenced_directory = misp_objects + self._check_directory_object(single_directory, observed_data1) + self._check_directory_object(directory, observed_data2, '0') + self._check_directory_object(referenced_directory, observed_data2, '1') + reference = directory.references[0] + self._assert_multiple_equal( + reference.referenced_uuid, + referenced_directory.uuid, + uuid5(self._UUIDv4, f'{observed_data2.id} - 1') + ) + self.assertEqual(reference.relationship_type, 'contains') \ No newline at end of file diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 800f4cc1..7abde011 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -114,6 +114,73 @@ ] } ] +_DIRECTORY_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--5e384ae7-672c-4250-9cda-3b4da964451b", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "directory--5e384ae7-672c-4250-9cda-3b4da964451a", + "directory--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--e812789e-e49d-47e2-b334-8ee0e8a766ce", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "directory--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "directory", + "spec_version": "2.1", + "id": "directory--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "path": "/var/www/MISP", + "path_enc": "UTF-8", + "ctime": "2011-11-26T10:45:31Z", + "mtime": "2023-12-12T11:34:05Z", + "atime": "2023-12-12T11:34:05Z", + "contains_refs": [ + "directory--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "directory", + "spec_version": "2.1", + "id": "directory--5e384ae7-672c-4250-9cda-3b4da964451a", + "path": "/var/www/MISP/app/files/scripts", + "path_enc": "ISO-8859-6-I", + "ctime": "2014-07-25T10:47:08Z", + "mtime": "2023-12-12T11:34:05Z", + "atime": "2023-12-12T11:34:05Z", + "contains_refs": [ + "directory--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "directory", + "spec_version": "2.1", + "id": "directory--f93cb275-0366-4ecc-abf0-a17928d1e177", + "path": "/var/www/MISP/app/files/scripts/misp-stix", + "path_enc": "ISO-8859-1", + "ctime": "2021-07-21T11:44:56Z", + "mtime": "2023-12-12T11:24:30Z", + "atime": "2023-12-12T11:24:30Z" + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -392,6 +459,17 @@ class TestExternalSTIX21Bundles: ] } + @classmethod + def __assemble_bundle(cls, *stix_objects): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), deepcopy(cls.__grouping), *stix_objects + ] + bundle['objects'][1]['object_refs'] = [ + stix_object['id'] for stix_object in stix_objects + ] + return dict_to_stix2(bundle, allow_custom=True) + @classmethod def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): relationship = { @@ -453,3 +531,11 @@ def get_bundle_with_tool_galaxy(cls): @classmethod def get_bundle_with_vulnerability_galaxy(cls): return cls.__assemble_galaxy_bundle(*_VULNERABILITY_OBJECTS) + + ############################################################################ + # MISP OBJECTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_with_directory_objects(cls): + return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 06b93f85..190ffb55 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -4,6 +4,7 @@ from .test_external_stix21_bundles import TestExternalSTIX21Bundles from ._test_stix import TestSTIX21 from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import +from uuid import uuid5 class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): @@ -218,4 +219,77 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): self.assertEqual( meta['external_id'], attribute_vuln.external_references[0].external_id - ) \ No newline at end of file + ) + + ############################################################################ + # MISP OBJECTS IMPORT TESTS. # + ############################################################################ + + def _check_directory_object(self, misp_object, observed_data, directory): + self.assertEqual(misp_object.name, 'directory') + self._check_misp_object_fields(misp_object, observed_data, directory) + self.assertEqual(len(misp_object.attributes), 5) + atime, ctime, mtime, path, path_enc = misp_object.attributes + self._assert_multiple_equal( + atime.type, ctime.type, mtime.type, 'datetime' + ) + self.assertEqual(atime.object_relation, 'access-time') + self.assertEqual(atime.value, directory.atime) + self.assertEqual( + atime.uuid, uuid5(self._UUIDv4, f'{directory.id} - access-time - {atime.value}') + ) + self.assertEqual(ctime.object_relation, 'creation-time') + self.assertEqual(ctime.value, directory.ctime) + self.assertEqual( + ctime.uuid, uuid5(self._UUIDv4, f'{directory.id} - creation-time - {ctime.value}') + ) + self.assertEqual(mtime.object_relation, 'modification-time') + self.assertEqual(mtime.value, directory.mtime) + self.assertEqual( + mtime.uuid, uuid5(self._UUIDv4, f'{directory.id} - modification-time - {mtime.value}') + ) + self.assertEqual(path.object_relation, 'path') + self.assertEqual(path.value, directory.path) + self.assertEqual( + path.uuid, uuid5(self._UUIDv4, f'{directory.id} - path - {path.value}') + ) + self.assertEqual(path_enc.object_relation, 'path-encoding') + self.assertEqual(path_enc.value, directory.path_enc) + self.assertEqual( + path_enc.uuid, uuid5(self._UUIDv4, f'{directory.id} - path-encoding - {path_enc.value}') + ) + + def _check_misp_object_fields(self, misp_object, observed_data, observable_object): + self.assertEqual(misp_object.uuid, observable_object.id.split('--')[1]) + self.assertEqual(misp_object.comment, f'Observed Data ID: {observed_data.id}') + if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): + self.assertEqual(misp_object.first_seen, observed_data.first_observed) + self.assertEqual(misp_object.last_seen, observed_data.last_observed) + self.assertEqual(misp_object.timestamp, observed_data.modified) + + def test_stix21_bundle_with_directory_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_directory_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, directory1, directory2, directory3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + referenced_directory, directory, single_directory = misp_objects + self._check_directory_object(referenced_directory, od1, directory2) + self._check_directory_object(directory, od1, directory1) + self._check_directory_object(single_directory, od2, directory3) + reference1 = directory.references[0] + self._assert_multiple_equal( + reference1.referenced_uuid, + referenced_directory.uuid, + directory2.id.split('--')[1] + ) + self.assertEqual(reference1.relationship_type, 'contains') + reference2 = referenced_directory.references[0] + self._assert_multiple_equal( + reference2.referenced_uuid, + single_directory.uuid, + directory3.id.split('--')[1] + ) + self.assertEqual(reference2.relationship_type, 'contains') From 370d3c8f66867fc73cbc7388397aac488ec6f627 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 19:11:22 +0100 Subject: [PATCH 009/137] fix: [stix2 import] Using the AS value parsing method for an AS value that was missing it --- .../stix2misp/converters/stix2_observable_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 6a408048..2e80de1e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -533,7 +533,8 @@ def _parse_asn_observable( observed_data_id: Optional[str] = None) -> Iterator: object_id = getattr(observable, 'id', observed_data_id) yield from self._populate_object_attributes( - self._mapping.asn_attribute(), f'AS{observable.number}', object_id + self._mapping.asn_attribute(), + self._parse_AS_value(observable.number), object_id ) if hasattr(observable, 'name'): yield from self._populate_object_attributes( From 004437f78c23dcba242951f0f315ac07f7efb413 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 22:38:20 +0100 Subject: [PATCH 010/137] fix: [stix2 import] Reuse of the method parsing Directory observable objects with an `id` field --- .../converters/stix2_observed_data_converter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 20432664..cbf801dc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -127,12 +127,9 @@ def _parse_directory_observable_object( object_id: Optional[str] = None) -> MISPObject: directory = observed_data.objects[object_id] if directory.get('id') is not None: - misp_object = self._create_misp_object_from_observable_object_ref( - 'directory', directory, observed_data + return self._parse_directory_observable_object_ref( + observed_data, directory ) - for attribute in self._parse_directory_observable(directory): - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) object_id = f'{observed_data.id} - {object_id}' misp_object = self._create_misp_object_from_observable_object( 'directory', observed_data, object_id @@ -143,7 +140,7 @@ def _parse_directory_observable_object( return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_directory_observable_object_ref( - self, observed_data: ObservedData_v21, + self, observed_data: _OBSERVED_DATA_TYPING, directory: _DIRECTORY_TYPING) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( 'directory', directory, observed_data @@ -206,6 +203,10 @@ def _parse_directory_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING): if len(observed_data.objects) == 1: directory = next(iter(observed_data.objects.values())) + if directory.get('id') is not None: + return self._parse_directory_observable_object_ref( + observed_data, directory + ) misp_object = self._create_misp_object_from_observable_object( 'directory', observed_data ) From 1ef210252275b13c40c3c9f731fa94030be22d33 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 23:48:08 +0100 Subject: [PATCH 011/137] fix: [tests] Added missing tests for directory path attribute types --- tests/test_external_stix21_import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 190ffb55..b43b928d 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -248,11 +248,13 @@ def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual( mtime.uuid, uuid5(self._UUIDv4, f'{directory.id} - modification-time - {mtime.value}') ) + self.assertEqual(path.type, 'text') self.assertEqual(path.object_relation, 'path') self.assertEqual(path.value, directory.path) self.assertEqual( path.uuid, uuid5(self._UUIDv4, f'{directory.id} - path - {path.value}') ) + self.assertEqual(path_enc.type, 'text') self.assertEqual(path_enc.object_relation, 'path-encoding') self.assertEqual(path_enc.value, directory.path_enc) self.assertEqual( From e89a95b9ab9bd4da5464d5a765be5b376ccf458b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 11 Jan 2024 23:59:59 +0100 Subject: [PATCH 012/137] fix: [stix2 import] Fixed directory observable objects parsing method header - In this specific location, the `object_id` argument is not Optional --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index cbf801dc..390f4485 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -124,7 +124,7 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): def _parse_directory_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, - object_id: Optional[str] = None) -> MISPObject: + object_id: str) -> MISPObject: directory = observed_data.objects[object_id] if directory.get('id') is not None: return self._parse_directory_observable_object_ref( From 222ab5399fd5c1b252d17479b53d27f54ca136cd Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 12 Jan 2024 23:47:06 +0100 Subject: [PATCH 013/137] fix: [stix2 import] Removing unused variable in marking definitions parsing --- misp_stix_converter/stix2misp/stix2_to_misp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 24978c57..f8728fc8 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1168,7 +1168,6 @@ def _parse_marking_definition( return f"{definition_type}:{definition}" if hasattr(marking_definition, 'name'): # should be TLP 2.0 definition - name = marking_definition.name return marking_definition.name.lower() raise MarkingDefinitionLoadingError(marking_definition.id) From 7353b122d06638f4001536808a6fa26d01cd87c1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 15 Jan 2024 20:18:15 +0100 Subject: [PATCH 014/137] chg: [tests] Deduplicating existing tests for external directory observable objects --- tests/_test_stix_import.py | 40 ++++++++++++++++++++++++++++ tests/test_external_stix20_import.py | 39 +++++---------------------- tests/test_external_stix21_import.py | 36 ++++--------------------- 3 files changed, 52 insertions(+), 63 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 7e4f6d0d..73b30ab3 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -325,6 +325,10 @@ class TestExternalSTIX2Import(TestSTIX2Import): def setUp(self): self.parser = ExternalSTIX2toMISPParser() + ############################################################################ + # MISP GALAXIES CHECKING METHODS # + ############################################################################ + def _check_galaxy_features(self, galaxies, stix_object): self.assertEqual(len(galaxies), 1) galaxy = galaxies[0] @@ -350,6 +354,42 @@ def _check_galaxy_features(self, galaxies, stix_object): self.assertEqual(cluster.description, stix_object.description) return cluster.meta + ############################################################################ + # OBSERVED DATA OBJECTS CHECKING METHODS # + ############################################################################ + + def _check_directory_fields(self, misp_object, directory, object_id): + self.assertEqual(len(misp_object.attributes), 5) + accessed, created, modified, path, path_enc = misp_object.attributes + self._assert_multiple_equal( + accessed.type, created.type, modified.type, 'datetime' + ) + self.assertEqual(accessed.object_relation, 'access-time') + self.assertEqual( + accessed.uuid, uuid5(self._UUIDv4, f'{object_id} - access-time - {accessed.value}') + ) + self.assertEqual(created.object_relation, 'creation-time') + self.assertEqual( + created.uuid, uuid5(self._UUIDv4, f'{object_id} - creation-time - {created.value}') + ) + self.assertEqual(modified.object_relation, 'modification-time') + self.assertEqual( + modified.uuid, uuid5(self._UUIDv4, f'{object_id} - modification-time - {modified.value}') + ) + self.assertEqual(path.type, 'text') + self.assertEqual(path.object_relation, 'path') + self.assertEqual(path.value, directory.path) + self.assertEqual( + path.uuid, uuid5(self._UUIDv4, f'{object_id} - path - {path.value}') + ) + self.assertEqual(path_enc.type, 'text') + self.assertEqual(path_enc.object_relation, 'path-encoding') + self.assertEqual(path_enc.value, directory.path_enc) + self.assertEqual( + path_enc.uuid, uuid5(self._UUIDv4, f'{object_id} - path-encoding - {path_enc.value}') + ) + return accessed.value, created.value, modified.value + class TestInternalSTIX2Import(TestSTIX2Import): def setUp(self): diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index e5217ac6..c2ad946b 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -193,45 +193,20 @@ def _check_directory_object(self, misp_object, observed_data, identifier=None): else: object_id = f'{object_id} - {identifier}' directory = observed_data.objects[identifier] - self.assertEqual(len(misp_object.attributes), 5) - accessed, created, modified, path, path_enc = misp_object.attributes - self._assert_multiple_equal( - accessed.type, created.type, modified.type, 'datetime' - ) - self.assertEqual(accessed.object_relation, 'access-time') - self.assertEqual(accessed.value, directory.accessed) - self.assertEqual( - accessed.uuid, uuid5(self._UUIDv4, f'{object_id} - access-time - {accessed.value}') - ) - self.assertEqual(created.object_relation, 'creation-time') - self.assertEqual(created.value, directory.created) - self.assertEqual( - created.uuid, uuid5(self._UUIDv4, f'{object_id} - creation-time - {created.value}') - ) - self.assertEqual(modified.object_relation, 'modification-time') - self.assertEqual(modified.value, directory.modified) - self.assertEqual( - modified.uuid, uuid5(self._UUIDv4, f'{object_id} - modification-time - {modified.value}') - ) - self.assertEqual(path.type, 'text') - self.assertEqual(path.object_relation, 'path') - self.assertEqual(path.value, directory.path) - self.assertEqual( - path.uuid, uuid5(self._UUIDv4, f'{object_id} - path - {path.value}') - ) - self.assertEqual(path_enc.type, 'text') - self.assertEqual(path_enc.object_relation, 'path-encoding') - self.assertEqual(path_enc.value, directory.path_enc) - self.assertEqual( - path_enc.uuid, uuid5(self._UUIDv4, f'{object_id} - path-encoding - {path_enc.value}') + accessed, created, modified = self._check_directory_fields( + misp_object, directory, object_id ) + self.assertEqual(accessed, directory.accessed) + self.assertEqual(created, directory.created) + self.assertEqual(modified, directory.modified) def _check_misp_object_fields(self, misp_object, observed_data, identifier): if identifier is None: self.assertEqual(misp_object.uuid, observed_data.id.split('--')[1]) else: self.assertEqual( - misp_object.uuid, uuid5(self._UUIDv4, f'{observed_data.id} - {identifier}') + misp_object.uuid, + uuid5(self._UUIDv4, f'{observed_data.id} - {identifier}') ) if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): self.assertEqual(misp_object.first_seen, observed_data.first_observed) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index b43b928d..5a98178e 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -228,38 +228,12 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(misp_object.name, 'directory') self._check_misp_object_fields(misp_object, observed_data, directory) - self.assertEqual(len(misp_object.attributes), 5) - atime, ctime, mtime, path, path_enc = misp_object.attributes - self._assert_multiple_equal( - atime.type, ctime.type, mtime.type, 'datetime' - ) - self.assertEqual(atime.object_relation, 'access-time') - self.assertEqual(atime.value, directory.atime) - self.assertEqual( - atime.uuid, uuid5(self._UUIDv4, f'{directory.id} - access-time - {atime.value}') - ) - self.assertEqual(ctime.object_relation, 'creation-time') - self.assertEqual(ctime.value, directory.ctime) - self.assertEqual( - ctime.uuid, uuid5(self._UUIDv4, f'{directory.id} - creation-time - {ctime.value}') - ) - self.assertEqual(mtime.object_relation, 'modification-time') - self.assertEqual(mtime.value, directory.mtime) - self.assertEqual( - mtime.uuid, uuid5(self._UUIDv4, f'{directory.id} - modification-time - {mtime.value}') - ) - self.assertEqual(path.type, 'text') - self.assertEqual(path.object_relation, 'path') - self.assertEqual(path.value, directory.path) - self.assertEqual( - path.uuid, uuid5(self._UUIDv4, f'{directory.id} - path - {path.value}') - ) - self.assertEqual(path_enc.type, 'text') - self.assertEqual(path_enc.object_relation, 'path-encoding') - self.assertEqual(path_enc.value, directory.path_enc) - self.assertEqual( - path_enc.uuid, uuid5(self._UUIDv4, f'{directory.id} - path-encoding - {path_enc.value}') + atime, ctime, mtime = self._check_directory_fields( + misp_object, directory, directory.id ) + self.assertEqual(atime, directory.atime) + self.assertEqual(ctime, directory.ctime) + self.assertEqual(mtime, directory.mtime) def _check_misp_object_fields(self, misp_object, observed_data, observable_object): self.assertEqual(misp_object.uuid, observable_object.id.split('--')[1]) From 5006b3568b9b9d29b537f4e6d8cad84a312e30d5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 15 Jan 2024 20:22:05 +0100 Subject: [PATCH 015/137] add: [stix2 import] Parsing Observed Data with Autonomous System observable objects from converters --- .../stix2_observed_data_converter.py | 127 +++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 390f4485..860858a0 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -14,10 +14,10 @@ from collections import defaultdict from pymisp import MISPObject from stix2.v20.observables import ( - Directory as Directory_v20) + AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - Directory as Directory_v21) + AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Optional, TYPE_CHECKING, Union @@ -26,6 +26,7 @@ from ..internal_stix2_to_misp import InternalSTIX2toMISPParser _OBSERVABLE_OBJECTS_TYPING = Union[ + AutonomousSystem_v20, AutonomousSystem_v21, Directory_v20, Directory_v21 ] _OBSERVED_DATA_TYPING = Union[ @@ -122,6 +123,128 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ + def _parse_as_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, object_id: str): + autonomous_system = observed_data.objects[object_id] + if autonomous_system.get('id') is not None: + return self._parse_autonomous_system_observable_object_ref( + observed_data, autonomous_system + ) + object_id = f'{observed_data.id} - {object_id}' + AS_value = self._parse_AS_value(autonomous_system.number) + if hasattr(autonomous_system, 'name'): + misp_object = self._create_misp_object_from_observable_object( + 'asn', observed_data, object_id + ) + misp_object.add_attribute( + 'asn', AS_value, + uuid=self.main_parser._create_v5_uuid( + f'{object_id} - asn - {AS_value}' + ) + ) + description = autonomous_system.name + misp_object.add_attribute( + 'description', description, + uuid=self.main_parser._create_v5_uuid( + f'{object_id} - description - {description}' + ) + ) + return self.main_parser._add_misp_object(misp_object, observed_data) + return self.main_parser._add_misp_attribute( + { + 'type': 'AS', 'value': AS_value, + 'uuid': self.main_parser._create_v5_uuid(object_id), + **self._parse_timeline(observed_data) + }, + observed_data + ) + + def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + autonomous_system = observable['observable'] + if hasattr(autonomous_system, 'name'): + self._parse_autonomous_system_observable_object_ref( + observed_data, autonomous_system + ) + observable['used'][self.event_uuid] = True + continue + self.main_parser._add_misp_attribute( + { + 'type': 'AS', + 'value': self._parse_AS_value(autonomous_system.number), + 'comment': f'Observed Data ID: {observed_data.id}', + 'uuid': self._sanitise_uuid(autonomous_system.id), + **self._parse_timeline(observed_data) + }, + observed_data + ) + observable['used'][self.event_uuid] = True + + def _parse_as_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + autonomous_system = next(iter(observed_data.objects.values())) + if autonomous_system.get('id') is not None: + return self._parse_autonomous_system_observable_object_ref( + observed_data, autonomous_system + ) + AS_value = self._parse_AS_value(autonomous_system.number) + if hasattr(autonomous_system, 'name'): + misp_object = self._create_misp_object_from_observable_object( + 'asn', observed_data + ) + misp_object.add_attribute( + 'asn', AS_value, + uuid=self.main_parser._create_v5_uuid( + f'{observed_data.id} - asn - {AS_value}' + ) + ) + description = autonomous_system.name + misp_object.add_attribute( + 'description', description, + uuid=self.main_parser._create_v5_uuid( + f'{observed_data.id} - description - {description}' + ) + ) + return self.main_parser._add_misp_object( + misp_object, observed_data + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'AS', 'value': AS_value, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for object_id in observed_data.objects: + self._parse_as_observable_object(observed_data, object_id) + + def _parse_autonomous_system_observable_object_ref( + self, observed_data: ObservedData_v21, + autonomous_system: _AUTONOMOUS_SYSTEM_TYPING) -> MISPObject: + misp_object = self._create_misp_object_from_observable_object_ref( + 'asn', autonomous_system, observed_data + ) + AS_value = self._parse_AS_value(autonomous_system.number) + misp_object.add_attribute( + 'asn', AS_value, + uuid=self.main_parser._create_v5_uuid( + f'{autonomous_system.id} - asn - {AS_value}' + ) + ) + description = autonomous_system.name + misp_object.add_attribute( + 'description', description, + uuid=self.main_parser._create_v5_uuid( + f'{autonomous_system.id} - description - {description}' + ) + ) + return self.main_parser._add_misp_object(misp_object, observed_data) + def _parse_directory_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, object_id: str) -> MISPObject: From 3b3bd0409722355893b7baf508ca6345c55c246d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 15 Jan 2024 20:38:20 +0100 Subject: [PATCH 016/137] add: [tests] Tests for Autonomous System observable objects with observed data import from STIX 2.x --- tests/_test_stix_import.py | 12 +++++ tests/test_external_stix20_bundles.py | 66 ++++++++++++++++++++++- tests/test_external_stix20_import.py | 34 +++++++++++- tests/test_external_stix21_bundles.py | 77 ++++++++++++++++++++++++++- tests/test_external_stix21_import.py | 27 +++++++++- 5 files changed, 212 insertions(+), 4 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 73b30ab3..faa84b66 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -358,6 +358,18 @@ def _check_galaxy_features(self, galaxies, stix_object): # OBSERVED DATA OBJECTS CHECKING METHODS # ############################################################################ + def _check_as_fields(self, misp_object, autonomous_system, object_id): + self.assertEqual(len(misp_object.attributes), 2) + asn, name = misp_object.attributes + self.assertEqual(asn.type, 'AS') + self.assertEqual(asn.object_relation, 'asn') + self.assertEqual(asn.value, f'AS{autonomous_system.number}') + self.assertEqual(asn.uuid, uuid5(self._UUIDv4, f'{object_id} - asn - {asn.value}')) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'description') + self.assertEqual(name.value, autonomous_system.name) + self.assertEqual(name.uuid, uuid5(self._UUIDv4, f'{object_id} - description - {name.value}')) + def _check_directory_fields(self, misp_object, directory, object_id): self.assertEqual(len(misp_object.attributes), 5) accessed, created, modified, path, path_enc = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index b80db529..63425f2e 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -5,6 +5,66 @@ from stix2.parsing import dict_to_stix2 +_AS_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "autonomous-system", + "spec_version": "2.1", + "number": 666, + "name": "Satan autonomous system" + }, + "1": { + "type": "autonomous-system", + "spec_version": "2.1", + "number": 1234 + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "autonomous-system", + "spec_version": "2.1", + "number": 197869, + "name": "CIRCL" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--1bf81a4f-0e70-4a34-944b-7e46f67ff7a7", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "autonomous-system", + "spec_version": "2.1", + "number": 50588 + } + } + } +] _ATTACK_PATTERN_OBJECTS = [ { "type": "attack-pattern", @@ -471,9 +531,13 @@ def get_bundle_with_vulnerability_galaxy(cls): return cls.__assemble_galaxy_bundle(*_VULNERABILITY_OBJECTS) ############################################################################ - # MISP OBJECTS SAMPLES # + # OBSERVED DATA SAMPLES. # ############################################################################ + @classmethod + def get_bundle_with_as_objects(cls): + return cls.__assemble_bundle(*_AS_OBJECTS) + @classmethod def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index c2ad946b..9820f8e1 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -181,9 +181,27 @@ def test_stix20_bundle_with_vulnerability_galaxy(self): ) ############################################################################ - # MISP OBJECTS IMPORT TESTS. # + # OBSERVED DATA OBJECTS IMPORT TESTS # ############################################################################ + def _check_as_attribute(self, attribute, observed_data, identifier=None): + self.assertEqual(attribute.type, 'AS') + self._check_misp_object_fields(attribute, observed_data, identifier) + autonomous_system = observed_data.objects[identifier or '0'] + self.assertEqual(attribute.type, 'AS') + self.assertEqual(attribute.value, f'AS{autonomous_system.number}') + + def _check_as_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'asn') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + autonomous_system = observed_data.objects[identifier] + self._check_as_fields(misp_object, autonomous_system, object_id) + def _check_directory_object(self, misp_object, observed_data, identifier=None): self.assertEqual(misp_object.name, 'directory') self._check_misp_object_fields(misp_object, observed_data, identifier) @@ -213,6 +231,20 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier): self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def test_stix20_bundle_with_as_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_as_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2, observed_data3 = bundle.objects + misp_content = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_content), 4) + m_object, s_object, m_attribute, s_attribute = misp_content + self._check_as_object(m_object, observed_data1, '0') + self._check_as_object(s_object, observed_data2) + self._check_as_attribute(m_attribute, observed_data1, '1') + self._check_as_attribute(s_attribute, observed_data3) + def test_stix20_bundle_with_directory_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_directory_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 7abde011..da0b9e77 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -5,6 +5,77 @@ from stix2.parsing import dict_to_stix2 +_AS_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "autonomous-system--46f7e73f-36e5-40dd-9b27-735a0a6b44c2", + "autonomous-system--49713b77-b6ee-4069-a1b4-2a8ad49adf62" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "autonomous-system--81294150-8f7a-453d-a1d8-96c2cfe04efa" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--1bf81a4f-0e70-4a34-944b-7e46f67ff7a7", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "autonomous-system--cd890f31-5825-4fea-85ca-0b3ab3872926" + ] + }, + { + "type": "autonomous-system", + "spec_version": "2.1", + "id": "autonomous-system--46f7e73f-36e5-40dd-9b27-735a0a6b44c2", + "number": 666, + "name": "Satan autonomous system" + }, + { + "type": "autonomous-system", + "spec_version": "2.1", + "id": "autonomous-system--49713b77-b6ee-4069-a1b4-2a8ad49adf62", + "number": 1234 + }, + { + "type": "autonomous-system", + "spec_version": "2.1", + "id": "autonomous-system--81294150-8f7a-453d-a1d8-96c2cfe04efa", + "number": 197869, + "name": "CIRCL" + }, + { + "type": "autonomous-system", + "spec_version": "2.1", + "id": "autonomous-system--cd890f31-5825-4fea-85ca-0b3ab3872926", + "number": 50588 + } +] _ATTACK_PATTERN_OBJECTS = [ { "type": "attack-pattern", @@ -533,9 +604,13 @@ def get_bundle_with_vulnerability_galaxy(cls): return cls.__assemble_galaxy_bundle(*_VULNERABILITY_OBJECTS) ############################################################################ - # MISP OBJECTS SAMPLES # + # OBSERVED DATA SAMPLES. # ############################################################################ + @classmethod + def get_bundle_with_as_objects(cls): + return cls.__assemble_bundle(*_AS_OBJECTS) + @classmethod def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 5a98178e..dbe74fdb 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -222,9 +222,20 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): ) ############################################################################ - # MISP OBJECTS IMPORT TESTS. # + # OBSERVED DATA OBJECTS IMPORT TESTS # ############################################################################ + def _check_as_attribute(self, attribute, observed_data, autonomous_system): + self.assertEqual(attribute.type, 'AS') + self._check_misp_object_fields(attribute, observed_data, autonomous_system) + self.assertEqual(attribute.type, 'AS') + self.assertEqual(attribute.value, f'AS{autonomous_system.number}') + + def _check_as_object(self, misp_object, observed_data, autonomous_system): + self.assertEqual(misp_object.name, 'asn') + self._check_misp_object_fields(misp_object, observed_data, autonomous_system) + self._check_as_fields(misp_object, autonomous_system, autonomous_system.id) + def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(misp_object.name, 'directory') self._check_misp_object_fields(misp_object, observed_data, directory) @@ -243,6 +254,20 @@ def _check_misp_object_fields(self, misp_object, observed_data, observable_objec self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def test_stix21_bundle_with_as_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_as_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, od3, as1, as2, as3, as4 = bundle.objects + misp_content = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_content), 4) + m_object, s_object, m_attribute, s_attribute = misp_content + self._check_as_object(m_object, od1, as1) + self._check_as_object(s_object, od2, as3) + self._check_as_attribute(m_attribute, od1, as2) + self._check_as_attribute(s_attribute, od3, as4) + def test_stix21_bundle_with_directory_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_directory_objects() self.parser.load_stix_bundle(bundle) From f805f7384e92685d8e9e7fbbe6bcf623989d31ec Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 15 Jan 2024 20:39:29 +0100 Subject: [PATCH 017/137] fix: [stix2 import] Properly calling the UUID sanitation method --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 860858a0..d08cf8e7 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -174,7 +174,9 @@ def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): 'type': 'AS', 'value': self._parse_AS_value(autonomous_system.number), 'comment': f'Observed Data ID: {observed_data.id}', - 'uuid': self._sanitise_uuid(autonomous_system.id), + 'uuid': self.main_parser._sanitise_uuid( + autonomous_system.id + ), **self._parse_timeline(observed_data) }, observed_data From 3850ef2c4662d52e053d288aea90dda08c02e7ca Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 16 Jan 2024 15:20:17 +0100 Subject: [PATCH 018/137] wip: [stix2 import] Parsing email address observable objects in observed data from external STIX 2.x content, in converters --- .../stix2_observed_data_converter.py | 142 +++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index d08cf8e7..fda8414e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -8,7 +8,8 @@ ExternalSTIX2ObservableConverter, ExternalSTIX2ObservableMapping, InternalSTIX2ObservableConverter, InternalSTIX2ObservableMapping, STIX2ObservableConverter, _AUTONOMOUS_SYSTEM_TYPING, _DIRECTORY_TYPING, - _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) + _EMAIL_ADDRESS_TYPING, _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, + _PROCESS_TYPING) from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict @@ -372,6 +373,145 @@ def _parse_directory_observable_objects( contained_object.uuid, 'contains' ) + def _parse_email_address_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, identifier: str): + attribute = { + 'comment': f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + } + email_address = observed_data.objects[identifier] + object_id = f'{observed_data.id} - {identifier}' + if hasattr(email_address, 'display_name'): + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': email_address.value, + 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - email-dst - {email_address.value}' + ), + **attribute + }, + observed_data + ) + attr_type = 'email-dst-display-name' + display_name = email_address.display_name + self.main_parser._add_misp_attribute( + { + 'type': attr_type, 'value': display_name, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - {attr_type} - {display_name}' + ) + }, + observed_data + ) + else: + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': email_address.value, + 'uuid': self.main_parser._create_v5_uuid(object_id), + **attribute + }, + observed_data + ) + + def _parse_email_address_observable_object_ref( + self, observed_data: _OBSERVED_DATA_TYPING, + email_address: _EMAIL_ADDRESS_TYPING): + attribute = { + 'comment': f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + } + if hasattr(email_address, 'display_name'): + address = email_address.value + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': address, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f'{email_address.id} - email-dst - {address}' + ) + }, + observed_data + ) + attr_type = 'email-dst-display-name' + display_name = email_address.display_name + self.main_parser._add_misp_attribute( + { + 'type': attr_type, 'value': display_name, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f'{email_address.id} - {attr_type} - {display_name}' + ) + }, + observed_data + ) + else: + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': email_address.value, + 'uuid': self.main_parser._sanitise_uuid(email_address.id), + **attribute + }, + observed_data + ) + + def _parse_email_address_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + email_address = observable['observable'] + self._parse_email_address_observable_object_ref( + observed_data, email_address + ) + observable['used'][self.event_uuid] = True + + def _parse_email_address_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + email_address = next(iter(observed_data.objects.values())) + if email_address.get('id') is not None: + return self._parse_email_address_observable_object_ref( + observed_data, email_address + ) + attribute = { + 'comment': f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + } + address = email_address.value + if hasattr(email_address, 'display_name'): + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': address, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f'{observed_data.id} - email-dst - {address}' + ) + }, + observed_data + ) + attr_type = 'email-dst-display-name' + display_name = email_address.display_name + self.main_parser._add_misp_attribute( + { + 'type': attr_type, 'value': display_name, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f'{observed_data.id} - {attr_type} - {display_name}' + ) + }, + observed_data + ) + else: + self.main_parser._add_misp_attribute( + { + 'type': 'email-dst', 'value': address, **attribute, + 'uuid': self.main_parser._sanitise_uuid( + observed_data.id + ) + }, + observed_data + ) + else: + for identifier in observed_data.objects: + self._parse_email_address_observable_object( + observed_data, identifier + ) + ############################################################################ # UTILITY METHODS. # ############################################################################ From 87f546ba2c41ead8788b4e8fccde087435bbda23 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 16 Jan 2024 15:21:25 +0100 Subject: [PATCH 019/137] fix: [tests] Removed `spec_version` fields in STIX 2.0 samples --- tests/test_external_stix20_bundles.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 63425f2e..ec652f4f 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -18,13 +18,11 @@ "objects": { "0": { "type": "autonomous-system", - "spec_version": "2.1", "number": 666, "name": "Satan autonomous system" }, "1": { "type": "autonomous-system", - "spec_version": "2.1", "number": 1234 } } @@ -41,7 +39,6 @@ "objects": { "0": { "type": "autonomous-system", - "spec_version": "2.1", "number": 197869, "name": "CIRCL" } @@ -59,7 +56,6 @@ "objects": { "0": { "type": "autonomous-system", - "spec_version": "2.1", "number": 50588 } } From c339920034db677674a882907cfb86136f1603d6 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 16 Jan 2024 15:22:26 +0100 Subject: [PATCH 020/137] wip: [tests] Tests for email address observable objects in observed data import from external STIX 2.x content --- tests/test_external_stix20_bundles.py | 60 +++++++++++++++++++++ tests/test_external_stix20_import.py | 57 ++++++++++++++++++-- tests/test_external_stix21_bundles.py | 75 +++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 49 +++++++++++++++-- 4 files changed, 234 insertions(+), 7 deletions(-) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index ec652f4f..f9a0b9f0 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -217,6 +217,62 @@ } } ] +_EMAIL_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "email-addr", + "value": "john.doe@gmail.com", + "display_name": "John Doe" + }, + "1": { + "type": "email-addr", + "value": "john@doe.org" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "email-addr", + "value": "donald.duck@disney.com", + "display_name": "Donald Duck" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--1bf81a4f-0e70-4a34-944b-7e46f67ff7a7", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "email-addr", + "value": "donald.duck@gmail.com" + } + } + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -537,3 +593,7 @@ def get_bundle_with_as_objects(cls): @classmethod def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) + + @classmethod + def get_bundle_with_email_address_attributes(cls): + return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 9820f8e1..54094036 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -8,7 +8,7 @@ class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): - + ############################################################################ # MISP GALAXIES IMPORT TESTS # ############################################################################ @@ -185,7 +185,6 @@ def test_stix20_bundle_with_vulnerability_galaxy(self): ############################################################################ def _check_as_attribute(self, attribute, observed_data, identifier=None): - self.assertEqual(attribute.type, 'AS') self._check_misp_object_fields(attribute, observed_data, identifier) autonomous_system = observed_data.objects[identifier or '0'] self.assertEqual(attribute.type, 'AS') @@ -218,6 +217,40 @@ def _check_directory_object(self, misp_object, observed_data, identifier=None): self.assertEqual(created, directory.created) self.assertEqual(modified, directory.modified) + def _check_email_address_attribute( + self, observed_data, address, identifier=None): + email_address = observed_data.objects[identifier or '0'] + self._check_misp_object_fields(address, observed_data, identifier) + self.assertEqual(address.type, 'email-dst') + self.assertEqual(address.value, email_address.value) + + def _check_email_address_attribute_with_display_name( + self, observed_data, address, display_name, identifier=None): + if identifier is None: + email_address = observed_data.objects['0'] + self._check_misp_object_fields( + address, observed_data, f'email-dst - {email_address.value}' + ) + self._check_misp_object_fields( + display_name, observed_data, + f'email-dst-display-name - {email_address.display_name}' + ) + else: + email_address = observed_data.objects[identifier] + self._check_misp_object_fields( + address, observed_data, + f'{identifier} - email-dst - {email_address.value}' + ) + self._check_misp_object_fields( + display_name, observed_data, + f'{identifier} - email-dst-display-name - {email_address.display_name}' + ) + self.assertEqual(address.type, 'email-dst') + self.assertEqual(address.value, email_address.value) + self.assertEqual(display_name.type, 'email-dst-display-name') + self.assertEqual(display_name.value, email_address.display_name) + + def _check_misp_object_fields(self, misp_object, observed_data, identifier): if identifier is None: self.assertEqual(misp_object.uuid, observed_data.id.split('--')[1]) @@ -263,4 +296,22 @@ def test_stix20_bundle_with_directory_objects(self): referenced_directory.uuid, uuid5(self._UUIDv4, f'{observed_data2.id} - 1') ) - self.assertEqual(reference.relationship_type, 'contains') \ No newline at end of file + self.assertEqual(reference.relationship_type, 'contains') + + def test_stix20_bundle_with_email_address_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_email_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2, observed_data3 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 6) + mm_address, mm_display_name, ms_address, sm_address, sm_display_name, ss_address = attributes + self._check_email_address_attribute_with_display_name( + observed_data1, mm_address, mm_display_name, '0' + ) + self._check_email_address_attribute(observed_data1, ms_address, '1') + self._check_email_address_attribute_with_display_name( + observed_data2, sm_address, sm_display_name + ) + self._check_email_address_attribute(observed_data3, ss_address) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index da0b9e77..8131e038 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -252,6 +252,77 @@ "atime": "2023-12-12T11:24:30Z" } ] +_EMAIL_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "email-addr--46f7e73f-36e5-40dd-9b27-735a0a6b44c2", + "email-addr--49713b77-b6ee-4069-a1b4-2a8ad49adf62", + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "email-addr--81294150-8f7a-453d-a1d8-96c2cfe04efa" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--1bf81a4f-0e70-4a34-944b-7e46f67ff7a7", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "email-addr--cd890f31-5825-4fea-85ca-0b3ab3872926" + ] + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--46f7e73f-36e5-40dd-9b27-735a0a6b44c2", + "value": "john.doe@gmail.com", + "display_name": "John Doe" + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--49713b77-b6ee-4069-a1b4-2a8ad49adf62", + "value": "john@doe.org" + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--81294150-8f7a-453d-a1d8-96c2cfe04efa", + "value": "donald.duck@disney.com", + "display_name": "Donald Duck" + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--cd890f31-5825-4fea-85ca-0b3ab3872926", + "value": "donald.duck@gmail.com" + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -614,3 +685,7 @@ def get_bundle_with_as_objects(cls): @classmethod def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) + + @classmethod + def get_bundle_with_email_address_attributes(cls): + return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index dbe74fdb..980cbaa7 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -8,7 +8,7 @@ class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): - + ################################################################################ # MISP GALAXIES IMPORT TESTS # ################################################################################ @@ -226,7 +226,6 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): ############################################################################ def _check_as_attribute(self, attribute, observed_data, autonomous_system): - self.assertEqual(attribute.type, 'AS') self._check_misp_object_fields(attribute, observed_data, autonomous_system) self.assertEqual(attribute.type, 'AS') self.assertEqual(attribute.value, f'AS{autonomous_system.number}') @@ -246,8 +245,32 @@ def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(ctime, directory.ctime) self.assertEqual(mtime, directory.mtime) - def _check_misp_object_fields(self, misp_object, observed_data, observable_object): - self.assertEqual(misp_object.uuid, observable_object.id.split('--')[1]) + def _check_email_address_attribute(self, observed_data, address, email_address): + self._check_misp_object_fields(address, observed_data, email_address) + self.assertEqual(address.type, 'email-dst') + self.assertEqual(address.value, email_address.value) + + def _check_email_address_attribute_with_display_name( + self, observed_data, address, display_name, email_address): + self._check_misp_object_fields( + address, observed_data, email_address, + f'{email_address.id} - email-dst - {email_address.value}' + ) + self.assertEqual(address.type, 'email-dst') + self.assertEqual(address.value, email_address.value) + self._check_misp_object_fields( + display_name, observed_data, email_address, + f'{email_address.id} - email-dst-display-name - {email_address.display_name}' + ) + self.assertEqual(display_name.type, 'email-dst-display-name') + self.assertEqual(display_name.value, email_address.display_name) + + def _check_misp_object_fields( + self, misp_object, observed_data, observable_object, identifier=None): + if identifier is None: + self.assertEqual(misp_object.uuid, observable_object.id.split('--')[1]) + else: + self.assertEqual(misp_object.uuid, uuid5(self._UUIDv4, identifier)) self.assertEqual(misp_object.comment, f'Observed Data ID: {observed_data.id}') if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): self.assertEqual(misp_object.first_seen, observed_data.first_observed) @@ -294,3 +317,21 @@ def test_stix21_bundle_with_directory_objects(self): directory3.id.split('--')[1] ) self.assertEqual(reference2.relationship_type, 'contains') + + def test_stix21_bundle_with_email_address_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_email_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, od3, ea1, ea2, ea3, ea4 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 6) + mm_address, mm_display_name, ms_address, sm_address, sm_display_name, ss_address = attributes + self._check_email_address_attribute_with_display_name( + od1, mm_address, mm_display_name, ea1 + ) + self._check_email_address_attribute(od1, ms_address, ea2) + self._check_email_address_attribute_with_display_name( + od2, sm_address, sm_display_name, ea3 + ) + self._check_email_address_attribute(od3, ss_address, ea4) From b1536f03583d7bac7ff837ea1c6a40c55bc59376 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 17 Jan 2024 12:59:22 +0100 Subject: [PATCH 021/137] fix: [stix2 import] Better overall UUID sanitation & comments handling for MISP attributes creation --- .../stix2_observable_objects_converter.py | 6 ++- .../stix2_observed_data_converter.py | 38 ++++++++++--------- .../stix2misp/converters/stix2converter.py | 4 +- misp_stix_converter/stix2misp/importparser.py | 13 ++++--- 4 files changed, 34 insertions(+), 27 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index beabedce..7d30ab02 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -41,11 +41,13 @@ def __init__(self, main: 'ExternalSTIX2toMISPParser'): def _create_misp_attribute( self, attribute_type: str, observable: DomainName, - feature: Optional[str] = 'value', + feature: Optional[str] = 'value', comment: Optional[str] = None, **kwargs: Dict[str, str]) -> MISPAttribute: return { 'value': getattr(observable, feature), 'type': attribute_type, - **kwargs, **self.main_parser._sanitise_attribute_uuid(observable.id) + **kwargs, **self.main_parser._sanitise_attribute_uuid( + observable.id, comment=comment + ) } def _create_misp_object_from_observable_object( diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index fda8414e..a4b377fd 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -174,11 +174,11 @@ def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): { 'type': 'AS', 'value': self._parse_AS_value(autonomous_system.number), - 'comment': f'Observed Data ID: {observed_data.id}', - 'uuid': self.main_parser._sanitise_uuid( - autonomous_system.id - ), - **self._parse_timeline(observed_data) + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + autonomous_system.id, + comment=f'Observed Data ID: {observed_data.id}' + ) }, observed_data ) @@ -416,11 +416,11 @@ def _parse_email_address_observable_object( def _parse_email_address_observable_object_ref( self, observed_data: _OBSERVED_DATA_TYPING, email_address: _EMAIL_ADDRESS_TYPING): - attribute = { - 'comment': f'Observed Data ID: {observed_data.id}', - **self._parse_timeline(observed_data) - } if hasattr(email_address, 'display_name'): + attribute = { + 'comment': f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + } address = email_address.value self.main_parser._add_misp_attribute( { @@ -446,8 +446,11 @@ def _parse_email_address_observable_object_ref( self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': email_address.value, - 'uuid': self.main_parser._sanitise_uuid(email_address.id), - **attribute + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + email_address.id, + comment=f'Observed Data ID: {observed_data.id}' + ) }, observed_data ) @@ -470,12 +473,12 @@ def _parse_email_address_observable_objects( return self._parse_email_address_observable_object_ref( observed_data, email_address ) - attribute = { - 'comment': f'Observed Data ID: {observed_data.id}', - **self._parse_timeline(observed_data) - } address = email_address.value if hasattr(email_address, 'display_name'): + attribute = { + 'comment': f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + } self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': address, **attribute, @@ -499,8 +502,9 @@ def _parse_email_address_observable_objects( else: self.main_parser._add_misp_attribute( { - 'type': 'email-dst', 'value': address, **attribute, - 'uuid': self.main_parser._sanitise_uuid( + 'type': 'email-dst', 'value': address, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( observed_data.id ) }, diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index d0adedbb..2c15e775 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -50,11 +50,9 @@ def main_parser(self) -> _MAIN_PARSER_TYPING: def _create_attribute_dict(self, stix_object: _SDO_TYPING) -> dict: attribute = self._parse_timeline(stix_object) - if hasattr(stix_object, 'description') and stix_object.description: - attribute['comment'] = stix_object.description attribute.update( self.main_parser._sanitise_attribute_uuid( - stix_object.id, comment=attribute.get('comment') + stix_object.id, comment=stix_object.get('description') ) ) return attribute diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 6eb4df05..796a6179 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -487,8 +487,8 @@ def _sanitise_attribute_uuid( return { 'uuid': self.replacement_uuids[attribute_uuid], 'comment': ( - f'{comment} - {attribute_comment}' - if comment else attribute_comment + attribute_comment if comment is None + else f'{comment} - {attribute_comment}' ) } if UUID(attribute_uuid).version not in _RFC_VERSIONS: @@ -497,11 +497,14 @@ def _sanitise_attribute_uuid( return { 'uuid': sanitised_uuid, 'comment': ( - f'{comment} - {attribute_comment}' - if comment else attribute_comment + attribute_comment if comment is None + else f'{comment} - {attribute_comment}' ) } - return {'uuid': attribute_uuid} + attribute = {'uuid': attribute_uuid} + if comment is not None: + attribute['comment'] = comment + return attribute def _sanitise_object_uuid( self, misp_object: Union[MISPEvent, MISPObject], object_id: str): From bc4535bd274e89d48a72b75d26c6b03dc1305be5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 17 Jan 2024 15:38:47 +0100 Subject: [PATCH 022/137] fix: [stix2 import] Fixed Observable objects types mapping - Considering the possibility to have both types of IP addresses wihtin the Observed Data list of obervable objects --- .../stix2misp/converters/stix2_observable_converter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 2e80de1e..6a5fcd08 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -466,6 +466,7 @@ class ExternalSTIX2ObservableMapping( **dict.fromkeys( ( 'ipv4-addr', + 'ipv4-addr_ipv6-addr', 'ipv6-addr' ), 'ip_address' From 63e8f948188f3c1f649eeed0f77e8d5324ba8f4f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 17 Jan 2024 17:34:10 +0100 Subject: [PATCH 023/137] wip: [stix2 import] Parsing some Observable objects - converted to MISP attributes - in a generic way, from Observed Data converter --- .../stix2_observed_data_converter.py | 252 ++++++++++++++++-- 1 file changed, 234 insertions(+), 18 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index a4b377fd..472a27dc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -15,10 +15,16 @@ from collections import defaultdict from pymisp import MISPObject from stix2.v20.observables import ( - AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20) + AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, + DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, + IPv6Address as IPv6Address_v20, MACAddress as MACAddress_v20, + Mutex as Mutex_v20, URL as URL_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21) + AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, + DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, + IPv6Address as IPv6Address_v21, MACAddress as MACAddress_v21, + Mutex as Mutex_v21, URL as URL_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Optional, TYPE_CHECKING, Union @@ -26,6 +32,14 @@ from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser +_GENERIC_OBSERVABLE_OBJECTS_TYPING = Union[ + DomainName_v20, DomainName_v21, + IPv4Address_v20, IPv4Address_v21, + IPv6Address_v20, IPv6Address_v21, + MACAddress_v20, MACAddress_v21, + Mutex_v20, Mutex_v21, + URL_v20, URL_v21 +] _OBSERVABLE_OBJECTS_TYPING = Union[ AutonomousSystem_v20, AutonomousSystem_v21, Directory_v20, Directory_v21 @@ -129,7 +143,7 @@ def _parse_as_observable_object( autonomous_system = observed_data.objects[object_id] if autonomous_system.get('id') is not None: return self._parse_autonomous_system_observable_object_ref( - observed_data, autonomous_system + autonomous_system, observed_data ) object_id = f'{observed_data.id} - {object_id}' AS_value = self._parse_AS_value(autonomous_system.number) @@ -155,6 +169,7 @@ def _parse_as_observable_object( { 'type': 'AS', 'value': AS_value, 'uuid': self.main_parser._create_v5_uuid(object_id), + 'comment': f'Observed Data ID: {observed_data.id}', **self._parse_timeline(observed_data) }, observed_data @@ -166,7 +181,7 @@ def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): autonomous_system = observable['observable'] if hasattr(autonomous_system, 'name'): self._parse_autonomous_system_observable_object_ref( - observed_data, autonomous_system + autonomous_system, observed_data ) observable['used'][self.event_uuid] = True continue @@ -190,7 +205,7 @@ def _parse_as_observable_objects( autonomous_system = next(iter(observed_data.objects.values())) if autonomous_system.get('id') is not None: return self._parse_autonomous_system_observable_object_ref( - observed_data, autonomous_system + autonomous_system, observed_data ) AS_value = self._parse_AS_value(autonomous_system.number) if hasattr(autonomous_system, 'name'): @@ -227,8 +242,8 @@ def _parse_as_observable_objects( self._parse_as_observable_object(observed_data, object_id) def _parse_autonomous_system_observable_object_ref( - self, observed_data: ObservedData_v21, - autonomous_system: _AUTONOMOUS_SYSTEM_TYPING) -> MISPObject: + self, autonomous_system: _AUTONOMOUS_SYSTEM_TYPING, + observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( 'asn', autonomous_system, observed_data ) @@ -254,7 +269,7 @@ def _parse_directory_observable_object( directory = observed_data.objects[object_id] if directory.get('id') is not None: return self._parse_directory_observable_object_ref( - observed_data, directory + directory, observed_data ) object_id = f'{observed_data.id} - {object_id}' misp_object = self._create_misp_object_from_observable_object( @@ -266,8 +281,8 @@ def _parse_directory_observable_object( return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_directory_observable_object_ref( - self, observed_data: _OBSERVED_DATA_TYPING, - directory: _DIRECTORY_TYPING) -> MISPObject: + self, directory: _DIRECTORY_TYPING, + observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( 'directory', directory, observed_data ) @@ -284,7 +299,7 @@ def _parse_directory_observable_object_refs( if observable['used'].get(self.event_uuid, False): continue misp_object = self._parse_directory_observable_object_ref( - observed_data, directory + directory, observed_data ) observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True @@ -293,7 +308,7 @@ def _parse_directory_observable_object_refs( observable['misp_object'] if observable['used'].get(self.event_uuid, False) else self._parse_directory_observable_object_ref( - observed_data, directory + directory, observed_data ) ) observable['misp_object'] = misp_object @@ -319,7 +334,7 @@ def _parse_directory_observable_object_refs( ) continue contained_object = self._parse_directory_observable_object_ref( - observed_data, contained['observable'] + contained['observable'], observed_data ) contained['misp_object'] = contained_object contained['used'][self.event_uuid] = True @@ -331,7 +346,7 @@ def _parse_directory_observable_objects( directory = next(iter(observed_data.objects.values())) if directory.get('id') is not None: return self._parse_directory_observable_object_ref( - observed_data, directory + directory, observed_data ) misp_object = self._create_misp_object_from_observable_object( 'directory', observed_data @@ -373,6 +388,39 @@ def _parse_directory_observable_objects( contained_object.uuid, 'contains' ) + def _parse_domain_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + domain = observable['observable'] + self._parse_generic_observable_object_ref( + domain, observed_data, 'domain' + ) + observable['used'][self.event_uuid] = True + + def _parse_domain_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + domain = next(iter(observed_data.objects.values())) + if domain.get('id') is not None: + return self._parse_generic_observable_object_ref( + domain, observed_data, 'domain' + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'domain', 'value': domain.value, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'domain' + ) + def _parse_email_address_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, identifier: str): attribute = { @@ -414,8 +462,8 @@ def _parse_email_address_observable_object( ) def _parse_email_address_observable_object_ref( - self, observed_data: _OBSERVED_DATA_TYPING, - email_address: _EMAIL_ADDRESS_TYPING): + self, email_address: _EMAIL_ADDRESS_TYPING, + observed_data: _OBSERVED_DATA_TYPING): if hasattr(email_address, 'display_name'): attribute = { 'comment': f'Observed Data ID: {observed_data.id}', @@ -461,7 +509,7 @@ def _parse_email_address_observable_object_refs( observable = self._fetch_observables(object_ref) email_address = observable['observable'] self._parse_email_address_observable_object_ref( - observed_data, email_address + email_address, observed_data ) observable['used'][self.event_uuid] = True @@ -471,7 +519,7 @@ def _parse_email_address_observable_objects( email_address = next(iter(observed_data.objects.values())) if email_address.get('id') is not None: return self._parse_email_address_observable_object_ref( - observed_data, email_address + email_address, observed_data ) address = email_address.value if hasattr(email_address, 'display_name'): @@ -516,6 +564,174 @@ def _parse_email_address_observable_objects( observed_data, identifier ) + def _parse_generic_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, identifier: str, + attribute_type: str, feature: Optional[str] = 'value'): + observable_object = observed_data.objects[identifier] + if hasattr(observable_object, 'id'): + return self._parse_generic_observable_object_ref( + observable_object, observed_data, attribute_type, feature + ) + self.main_parser._add_misp_attribute( + { + 'type': attribute_type, + 'value': getattr(observable_object, feature), + 'comment': f'Observed Data ID: {observed_data.id}', + 'uuid': self.main_parser._create_v5_uuid( + f'{observed_data.id} - {identifier}' + ), + **self._parse_timeline(observed_data) + }, + observed_data + ) + + def _parse_generic_observable_object_ref( + self, observable_object: _GENERIC_OBSERVABLE_OBJECTS_TYPING, + observed_data: _OBSERVED_DATA_TYPING, attribute_type: str, + feature: Optional[str] = 'value'): + self.main_parser._add_misp_attribute( + { + 'type': attribute_type, + 'value': getattr(observable_object, feature), + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observable_object.id, + comment=f'Observed Data ID: {observed_data.id}' + ), + }, + observed_data + ) + + def _parse_ip_address_observable_object_refs( + self, observed_data: _OBSERVED_DATA_TYPING): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + ip_address = observable['observable'] + self._parse_generic_observable_object_ref( + ip_address, observed_data, 'ip-dst' + ) + observable['used'][self.event_uuid] = True + + def _parse_ip_address_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + ip_address = next(iter(observed_data.objects.values())) + if ip_address.get('id') is not None: + return self._parse_generic_observable_object_ref( + ip_address, observed_data, 'ip-dst' + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'ip-dst', 'value': ip_address.value, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'ip-dst' + ) + + def _parse_mac_address_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + mac_address = observable['observable'] + self._parse_generic_observable_object_ref( + mac_address, observed_data, 'mac-address' + ) + observable['used'][self.event_uuid] = True + + def _parse_mac_address_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + mac_address = next(iter(observed_data.objects.values())) + if mac_address.get('id') is not None: + return self._parse_generic_observable_object_ref( + mac_address, observed_data, 'mac-address' + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'mac-address', 'value': mac_address.value, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'mac-address' + ) + + def _parse_mutex_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + mutex = observable['observable'] + self._parse_generic_observable_object_ref( + mutex, observed_data, 'mutex', feature='name' + ) + observable['used'][self.event_uuid] = True + + def _parse_mutex_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + mutex = next(iter(observed_data.objects.values())) + if mutex.get('id') is not None: + return self._parse_generic_observable_object_ref( + mutex, observed_data, 'mutex', feature='name' + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'mutex', 'value': mutex.name, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'mutex', feature='name' + ) + + def _parse_url_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + url = observable['observable'] + self._parse_generic_observable_object_ref(url, observed_data, 'url') + observable['used'][self.event_uuid] = True + + def _parse_url_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + url = next(iter(observed_data.objects.values())) + if url.get('id') is not None: + return self._parse_generic_observable_object_ref( + url, observed_data, 'url' + ) + return self.main_parser._add_misp_attribute( + { + 'type': 'url', 'value': url.value, + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + observed_data.id + ) + }, + observed_data + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'url' + ) + ############################################################################ # UTILITY METHODS. # ############################################################################ From c6494c02f74b2ce8bd10161fd5faa6353b8520ef Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 17 Jan 2024 17:36:29 +0100 Subject: [PATCH 024/137] wip: [tests] Tests for Observable objects converted in a generic way to MISP attributes --- tests/test_external_stix20_bundles.py | 210 ++++++++++++++++++++ tests/test_external_stix20_import.py | 102 ++++++++++ tests/test_external_stix21_bundles.py | 265 ++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 72 +++++++ 4 files changed, 649 insertions(+) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index f9a0b9f0..0d9e4828 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -217,6 +217,44 @@ } } ] +_DOMAIN_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "domain-name", + "value": "circl.lu" + }, + "1": { + "type": "domain-name", + "value": "lhc.lu" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "domain-name", + "value": "misp-project.org" + } + } + } +] _EMAIL_ADDRESS_ATTRIBUTES = [ { "type": "observed-data", @@ -310,6 +348,82 @@ ] } ] +_IP_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "ipv4-addr", + "value": "8.8.8.8" + }, + "1": { + "type": "ipv6-addr", + "value": "2001:4860:4860::8888" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "ipv4-addr", + "value": "185.194.93.14" + } + } + } +] +_MAC_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "mac-addr", + "value": "d2:fb:49:24:37:18" + }, + "1": { + "type": "mac-addr", + "value": "62:3e:5f:53:ac:68" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "mac-addr", + "value": "ae:49:db:d4:d9:cf" + } + } + } +] _MALWARE_OBJECTS = [ { "type": "malware", @@ -339,6 +453,44 @@ "description": "A WEBC2 backdoor is designed to retrieve a Web page from a C2 server. It expects the page to contain special HTML tags; the backdoor will attempt to interpret the data between the tags as commands." } ] +_MUTEX_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "mutex", + "name": "shared_resource_lock" + }, + "1": { + "type": "mutex", + "name": "thread_synchronization_lock" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "mutex", + "name": "sensitive_resource_lock" + } + } + } +] _THREAT_ACTOR_OBJECTS = [ { "type": "threat-actor", @@ -432,6 +584,44 @@ ] } ] +_URL_ATTRIBUTES = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "url", + "value": "https://circl.lu/team/" + }, + "1": { + "type": "url", + "value": "https://cybersecurity.lu/entity/324" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "url", + "value": "https://misp-project.org/blog/" + } + } + } +] _VULNERABILITY_OBJECTS = [ { "type": "vulnerability", @@ -594,6 +784,26 @@ def get_bundle_with_as_objects(cls): def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) + @classmethod + def get_bundle_with_domain_attributes(cls): + return cls.__assemble_bundle(*_DOMAIN_ATTRIBUTES) + @classmethod def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_ip_address_attributes(cls): + return cls.__assemble_bundle(*_IP_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_mac_address_attributes(cls): + return cls.__assemble_bundle(*_MAC_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_mutex_attributes(cls): + return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + + @classmethod + def get_bundle_with_url_attributes(cls): + return cls.__assemble_bundle(*_URL_ATTRIBUTES) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 54094036..98993c0d 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -250,6 +250,13 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_generic_attribute( + self, observed_data, attribute, attribute_type, + identifier=None, feature='value'): + observable_object = observed_data.objects[identifier or '0'] + self._check_misp_object_fields(attribute, observed_data, identifier) + self.assertEqual(attribute.type, attribute_type) + self.assertEqual(attribute.value, getattr(observable_object, feature)) def _check_misp_object_fields(self, misp_object, observed_data, identifier): if identifier is None: @@ -298,6 +305,25 @@ def test_stix20_bundle_with_directory_objects(self): ) self.assertEqual(reference.relationship_type, 'contains') + def test_stix20_bundle_with_domain_attributes(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_domain_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(bundle) + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 3) + m_domain1, m_domain2, s_domain = attributes + self._check_generic_attribute( + observed_data1, m_domain1, 'domain', '0' + ) + self._check_generic_attribute( + observed_data1, m_domain2, 'domain', '1' + ) + self._check_generic_attribute( + observed_data2, s_domain, 'domain' + ) + def test_stix20_bundle_with_email_address_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_email_address_attributes() self.parser.load_stix_bundle(bundle) @@ -315,3 +341,79 @@ def test_stix20_bundle_with_email_address_objects(self): observed_data2, sm_address, sm_display_name ) self._check_email_address_attribute(observed_data3, ss_address) + + def test_stix20_bundle_with_ip_address_attributes(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_ip_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(bundle) + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 3) + m_ip1, m_ip2, s_ip = attributes + self._check_generic_attribute( + observed_data1, m_ip1, 'ip-dst', '0' + ) + self._check_generic_attribute( + observed_data1, m_ip2, 'ip-dst', '1' + ) + self._check_generic_attribute( + observed_data2, s_ip, 'ip-dst' + ) + + def test_stix20_bundle_with_mac_address_attributes(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_mac_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(bundle) + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 3) + m_mac1, m_mac2, s_mac = attributes + self._check_generic_attribute( + observed_data1, m_mac1, 'mac-address', '0' + ) + self._check_generic_attribute( + observed_data1, m_mac2, 'mac-address', '1' + ) + self._check_generic_attribute( + observed_data2, s_mac, 'mac-address' + ) + + def test_stix20_bundle_with_mutex_attributes(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_mutex_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(bundle) + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 3) + m_mutex1, m_mutex2, s_mutex = attributes + self._check_generic_attribute( + observed_data1, m_mutex1, 'mutex', '0', 'name' + ) + self._check_generic_attribute( + observed_data1, m_mutex2, 'mutex', '1', 'name' + ) + self._check_generic_attribute( + observed_data2, s_mutex, 'mutex', feature='name' + ) + + def test_stix20_bundle_with_url_attributes(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_url_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle(bundle) + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + attributes = self._check_misp_event_features(event, report) + self.assertEqual(len(attributes), 3) + m_url1, m_url2, s_url = attributes + self._check_generic_attribute( + observed_data1, m_url1, 'url', '0' + ) + self._check_generic_attribute( + observed_data1, m_url2, 'url', '1' + ) + self._check_generic_attribute( + observed_data2, s_url, 'url' + ) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 8131e038..5c27f3f6 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -252,6 +252,55 @@ "atime": "2023-12-12T11:24:30Z" } ] +_DOMAIN_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "domain-name--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "domain-name--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "domain-name--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "value": "circl.lu" + }, + { + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--5e384ae7-672c-4250-9cda-3b4da964451a", + "value": "lhc.lu" + }, + { + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--f93cb275-0366-4ecc-abf0-a17928d1e177", + "value": "misp-project.org" + } +] _EMAIL_ADDRESS_ATTRIBUTES = [ { "type": "observed-data", @@ -362,6 +411,55 @@ ] } ] +_IP_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "ipv4-addr--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "ipv6-addr--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "ipv4-addr--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "value": "8.8.8.8" + }, + { + "type": "ipv6-addr", + "spec_version": "2.1", + "id": "ipv6-addr--5e384ae7-672c-4250-9cda-3b4da964451a", + "value": "2001:4860:4860::8888" + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--f93cb275-0366-4ecc-abf0-a17928d1e177", + "value": "185.194.93.14" + } +] _LOCATION_OBJECTS = [ { "type": "location", @@ -384,6 +482,55 @@ "region": "northern-europe" } ] +_MAC_ADDRESS_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "mac-addr--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "mac-addr--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "mac-addr--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "mac-addr", + "spec_version": "2.1", + "id": "mac-addr--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "value": "d2:fb:49:24:37:18" + }, + { + "type": "mac-addr", + "spec_version": "2.1", + "id": "mac-addr--5e384ae7-672c-4250-9cda-3b4da964451a", + "value": "62:3e:5f:53:ac:68" + }, + { + "type": "mac-addr", + "spec_version": "2.1", + "id": "mac-addr--f93cb275-0366-4ecc-abf0-a17928d1e177", + "value": "ae:49:db:d4:d9:cf" + } +] _MALWARE_OBJECTS = [ { "type": "malware", @@ -417,6 +564,55 @@ "description": "A WEBC2 backdoor is designed to retrieve a Web page from a C2 server. It expects the page to contain special HTML tags; the backdoor will attempt to interpret the data between the tags as commands." } ] +_MUTEX_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "mutex--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "mutex--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "mutex--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "mutex", + "spec_version": "2.1", + "id": "mutex--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "name": "shared_resource_lock" + }, + { + "type": "mutex", + "spec_version": "2.1", + "id": "mutex--5e384ae7-672c-4250-9cda-3b4da964451a", + "name": "thread_synchronization_lock" + }, + { + "type": "mutex", + "spec_version": "2.1", + "id": "mutex--f93cb275-0366-4ecc-abf0-a17928d1e177", + "name": "sensitive_resource_lock" + } +] _THREAT_ACTOR_OBJECTS = [ { "type": "threat-actor", @@ -468,6 +664,55 @@ "primary_motivation": "organizational-gain" } ] +_URL_ATTRIBUTES = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "url--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "url--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "url--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "url", + "spec_version": "2.1", + "id": "url--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "value": "https://circl.lu/team/" + }, + { + "type": "url", + "spec_version": "2.1", + "id": "url--5e384ae7-672c-4250-9cda-3b4da964451a", + "value": "https://cybersecurity.lu/entity/324" + }, + { + "type": "url", + "spec_version": "2.1", + "id": "url--f93cb275-0366-4ecc-abf0-a17928d1e177", + "value": "https://misp-project.org/blog/" + } +] _TOOL_OBJECTS = [ { "type": "tool", @@ -686,6 +931,26 @@ def get_bundle_with_as_objects(cls): def get_bundle_with_directory_objects(cls): return cls.__assemble_bundle(*_DIRECTORY_OBJECTS) + @classmethod + def get_bundle_with_domain_attributes(cls): + return cls.__assemble_bundle(*_DOMAIN_ATTRIBUTES) + @classmethod def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_ip_address_attributes(cls): + return cls.__assemble_bundle(*_IP_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_mac_address_attributes(cls): + return cls.__assemble_bundle(*_MAC_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_mutex_attributes(cls): + return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + + @classmethod + def get_bundle_with_url_attributes(cls): + return cls.__assemble_bundle(*_URL_ATTRIBUTES) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 980cbaa7..26e16b00 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -265,6 +265,13 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_generic_attribute( + self, observed_data, observable_object, attribute, + attribute_type, feature='value'): + self._check_misp_object_fields(attribute, observed_data, observable_object) + self.assertEqual(attribute.type, attribute_type) + self.assertEqual(attribute.value, getattr(observable_object, feature)) + def _check_misp_object_fields( self, misp_object, observed_data, observable_object, identifier=None): if identifier is None: @@ -318,6 +325,19 @@ def test_stix21_bundle_with_directory_objects(self): ) self.assertEqual(reference2.relationship_type, 'contains') + def test_stix21_bundle_with_domain_attributes(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_domain_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, domain_1, domain_2, domain_3 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 3) + m_domain1, m_domain2, s_domain = attributes + self._check_generic_attribute(od1, domain_1, m_domain1, 'domain') + self._check_generic_attribute(od1, domain_2, m_domain2, 'domain') + self._check_generic_attribute(od2, domain_3, s_domain, 'domain') + def test_stix21_bundle_with_email_address_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_email_address_attributes() self.parser.load_stix_bundle(bundle) @@ -335,3 +355,55 @@ def test_stix21_bundle_with_email_address_objects(self): od2, sm_address, sm_display_name, ea3 ) self._check_email_address_attribute(od3, ss_address, ea4) + + def test_stix21_bundle_with_ip_address_attributes(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_ip_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, address_1, address_2, address_3 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 3) + m_ip1, m_ip2, s_ip = attributes + self._check_generic_attribute(od1, address_1, m_ip1, 'ip-dst') + self._check_generic_attribute(od1, address_2, m_ip2, 'ip-dst') + self._check_generic_attribute(od2, address_3, s_ip, 'ip-dst') + + def test_stix21_bundle_with_mac_address_attributes(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_mac_address_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, address_1, address_2, address_3 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 3) + m_mac1, m_mac2, s_mac = attributes + self._check_generic_attribute(od1, address_1, m_mac1, 'mac-address') + self._check_generic_attribute(od1, address_2, m_mac2, 'mac-address') + self._check_generic_attribute(od2, address_3, s_mac, 'mac-address') + + def test_stix21_bundle_with_mutex_attributes(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_mutex_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, mutex_1, mutex_2, mutex_3 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 3) + m_mutex1, m_mutex2, s_mutex = attributes + self._check_generic_attribute(od1, mutex_1, m_mutex1, 'mutex', 'name') + self._check_generic_attribute(od1, mutex_2, m_mutex2, 'mutex', 'name') + self._check_generic_attribute(od2, mutex_3, s_mutex, 'mutex', 'name') + + def test_stix21_bundle_with_url_attributes(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_url_attributes() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, url_1, url_2, url_3 = bundle.objects + attributes = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(attributes), 3) + m_url1, m_url2, s_url = attributes + self._check_generic_attribute(od1, url_1, m_url1, 'url') + self._check_generic_attribute(od1, url_2, m_url2, 'url') + self._check_generic_attribute(od2, url_3, s_url, 'url') From c0423d0350cb2e971721f72f6abbb7a92e2350f6 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 18 Jan 2024 23:48:52 +0100 Subject: [PATCH 025/137] wip: [stix2 import] Handling the observable relationships after the observed data objects are all parsed --- misp_stix_converter/stix2misp/stix2_to_misp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index f8728fc8..8aba5fba 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -554,6 +554,9 @@ def _get_stix_object(self, object_ref: str): raise ObjectRefLoadingError(object_ref) def _handle_unparsed_content(self): + if hasattr(self, '_observed_data_parser'): + if hasattr(self.observed_data_parser, '_observable_relationships'): + self.observed_data_parser.parse_relationships() if hasattr(self, '_relationship'): if hasattr(self, '_sighting'): self._parse_relationships_and_sightings() From 6e666cb03e0b2c269f0b33cfd9d376b5eacfcbc5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 22 Jan 2024 11:37:23 +0100 Subject: [PATCH 026/137] fix: [stix2 import] Quick clean-up on some observed data method arguments --- .../converters/stix2_observed_data_converter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 472a27dc..e961f3f9 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -139,7 +139,7 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): ############################################################################ def _parse_as_observable_object( - self, observed_data: _OBSERVED_DATA_TYPING, object_id: str): + self, observed_data: ObservedData_v20, object_id: str): autonomous_system = observed_data.objects[object_id] if autonomous_system.get('id') is not None: return self._parse_autonomous_system_observable_object_ref( @@ -264,7 +264,7 @@ def _parse_autonomous_system_observable_object_ref( return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_directory_observable_object( - self, observed_data: _OBSERVED_DATA_TYPING, + self, observed_data: ObservedData_v20, object_id: str) -> MISPObject: directory = observed_data.objects[object_id] if directory.get('id') is not None: @@ -275,8 +275,7 @@ def _parse_directory_observable_object( misp_object = self._create_misp_object_from_observable_object( 'directory', observed_data, object_id ) - attributes = self._parse_directory_observable(directory, object_id) - for attribute in attributes: + for attribute in self._parse_directory_observable(directory, object_id): misp_object.add_attribute(**attribute) return self.main_parser._add_misp_object(misp_object, observed_data) @@ -422,7 +421,7 @@ def _parse_domain_observable_objects( ) def _parse_email_address_observable_object( - self, observed_data: _OBSERVED_DATA_TYPING, identifier: str): + self, observed_data: ObservedData_v20, identifier: str): attribute = { 'comment': f'Observed Data ID: {observed_data.id}', **self._parse_timeline(observed_data) @@ -565,7 +564,7 @@ def _parse_email_address_observable_objects( ) def _parse_generic_observable_object( - self, observed_data: _OBSERVED_DATA_TYPING, identifier: str, + self, observed_data: ObservedData_v20, identifier: str, attribute_type: str, feature: Optional[str] = 'value'): observable_object = observed_data.objects[identifier] if hasattr(observable_object, 'id'): From 677dcc39fe871bf3d19ae6b355fef81585f24055 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 22 Jan 2024 11:41:35 +0100 Subject: [PATCH 027/137] wip: [stix2 import] Parsing external STIX 2.x Observed data with artifact observable objects, from converters --- .../converters/stix2_observable_converter.py | 19 +++-- .../stix2_observed_data_converter.py | 75 ++++++++++++++++--- .../stix2misp/converters/stix2mapping.py | 7 ++ 3 files changed, 84 insertions(+), 17 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 6a5fcd08..f0279431 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -114,7 +114,6 @@ class STIX2ObservableMapping(STIX2Mapping, metaclass=ABCMeta): 'type': 'text', 'object_relation': 'encryption_algorithm' }, mime_type={'type': 'mime-type', 'object_relation': 'mime_type'}, - payload_bin={'type': 'text', 'object_relation': 'payload_bin'}, url=STIX2Mapping.url_attribute() ) __email_additional_header_fields_mapping = Mapping( @@ -415,6 +414,7 @@ class ExternalSTIX2ObservableMapping( STIX2ObservableMapping, ExternalSTIX2Mapping): __observable_mapping = Mapping( **{ + 'artifact': 'artifact', 'autonomous-system': 'as', 'directory': 'directory', 'domain-name': 'domain', @@ -510,10 +510,17 @@ def observable_mapping(cls, field: str) -> Union[str, None]: class ExternalSTIX2ObservableConverter( STIX2ObservableConverter, ExternalSTIX2Converter): def _parse_artifact_observable( - self, observable: Artifact_v21, + self, observable: _ARTIFACT_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: if object_id is None: object_id = observable.id + if hasattr(observable, 'payload_bin'): + value = getattr(observable, 'id', object_id.split(' - ')[0]) + yield from self._populate_object_attributes_with_data( + {'type': 'attachment', 'object_relation': 'payload_bin'}, + {'data': observable.payload_bin, 'value': value.split('--')[1]}, + object_id + ) if hasattr(observable, 'hashes'): for hash_type, value in observable.hashes.items(): attribute = self._mapping.file_hashes_mapping(hash_type) @@ -808,17 +815,17 @@ def _parse_email_body_observable( 'value': value.split('=').strip("'"), 'data': observable.payload_bin } - mapping = getattr(self._mapping, f'{feature}_attribute') + mapping = f'{feature}_attribute' if hasattr(observable, 'id'): yield { - **content, **mapping(), + **content, **getattr(self._mapping, mapping)(), 'uuid': self.main_parser._sanitise_attribute_uuid( observable.id ) } else: - yield from self._populate_object_attribute_with_data( - mapping(), content, object_id + yield from self._populate_object_attributes_with_data( + getattr(self._mapping, mapping)(), content, object_id ) def _parse_file_parent_observable(self, observable: _DIRECTORY_TYPING, diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index e961f3f9..b915a74f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -7,24 +7,24 @@ from .stix2_observable_converter import ( ExternalSTIX2ObservableConverter, ExternalSTIX2ObservableMapping, InternalSTIX2ObservableConverter, InternalSTIX2ObservableMapping, - STIX2ObservableConverter, _AUTONOMOUS_SYSTEM_TYPING, _DIRECTORY_TYPING, - _EMAIL_ADDRESS_TYPING, _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, - _PROCESS_TYPING) + STIX2ObservableConverter, _ARTIFACT_TYPING, _AUTONOMOUS_SYSTEM_TYPING, + _DIRECTORY_TYPING, _EMAIL_ADDRESS_TYPING, _EXTENSION_TYPING, + _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject from stix2.v20.observables import ( - AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, - DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, - IPv6Address as IPv6Address_v20, MACAddress as MACAddress_v20, - Mutex as Mutex_v20, URL as URL_v20) + Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, + Directory as Directory_v20, DomainName as DomainName_v20, + IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, + MACAddress as MACAddress_v20, Mutex as Mutex_v20, URL as URL_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, - DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, - IPv6Address as IPv6Address_v21, MACAddress as MACAddress_v21, - Mutex as Mutex_v21, URL as URL_v21) + Artifact as Artifact_v21, AutonomousSystem as AutonomousSystem_v21, + Directory as Directory_v21, DomainName as DomainName_v21, + IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, + MACAddress as MACAddress_v21, Mutex as Mutex_v21, URL as URL_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Optional, TYPE_CHECKING, Union @@ -41,6 +41,7 @@ URL_v20, URL_v21 ] _OBSERVABLE_OBJECTS_TYPING = Union[ + Artifact_v20, Artifact_v21, AutonomousSystem_v20, AutonomousSystem_v21, Directory_v20, Directory_v21 ] @@ -138,6 +139,58 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ + def _parse_artifact_observable_object(self, observed_data: ObservedData_v20, + identifier: str) -> MISPObject: + artifact = observed_data.objects[identifier] + if artifact.get('id') is not None: + return self._parse_artifact_observable_object_ref( + artifact, observed_data + ) + object_id = f'{observed_data.id} - {identifier}' + misp_object = self._create_misp_object_from_observable_object( + 'artifact', observed_data, object_id + ) + for attribute in self._parse_artifact_observable(artifact, object_id): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_artifact_observable_object_ref( + self, artifact: _ARTIFACT_TYPING, + observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: + misp_object = self._create_misp_object_from_observable_object_ref( + 'artifact', artifact, observed_data + ) + for attribute in self._parse_artifact_observable(artifact): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_artifact_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + artifact = observable['observable'] + self._parse_artifact_observable_object_ref(artifact, observed_data) + observable['used'][self.event_uuid] = True + + def _parse_artifact_observable_objects(self, observed_data): + if len(observed_data.objects) == 1: + artifact = next(iter(observed_data.objects.values())) + if artifact.get('id') is not None: + return self._parse_artifact_observable_object_ref( + artifact, observed_data + ) + misp_object = self._create_misp_object_from_observable_object( + 'artifact', observed_data + ) + attributes = self._parse_artifact_observable( + artifact, observed_data.id + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + for identifier in observed_data.objects: + self._parse_artifact_observable_object(observed_data, identifier) + def _parse_as_observable_object( self, observed_data: ObservedData_v20, object_id: str): autonomous_system = observed_data.objects[object_id] diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 64a372df..61f248dc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -161,6 +161,9 @@ class STIX2Mapping: __path_attribute = Mapping( **{'type': 'text', 'object_relation': 'path'} ) + __payload_bin_attribute = Mapping( + **{'type': 'attachment', 'object_relation': 'payload_bin'} + ) __pid_attribute = Mapping( **{'type': 'text', 'object_relation': 'pid'} ) @@ -605,6 +608,10 @@ def password_attribute(cls) -> dict: def path_attribute(cls) -> dict: return cls.__path_attribute + @staticmethod + def payload_bin_attribute(cls) -> dict: + return cls.__payload_bin_attribute + @classmethod def pe_object_mapping(cls) -> dict: return cls.__pe_object_mapping From 5da13af18b0681f2194047ded944a901151b8bcf Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 22 Jan 2024 11:43:34 +0100 Subject: [PATCH 028/137] wip: [tests] Tests for external STIX 2.x Observed Data with artifact observable objects import to MISP --- tests/_test_stix_import.py | 83 +++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 60 +++++++++++++++++++ tests/test_external_stix20_import.py | 30 +++++++++- tests/test_external_stix21_bundles.py | 72 +++++++++++++++++++++++ tests/test_external_stix21_import.py | 19 ++++++ 5 files changed, 263 insertions(+), 1 deletion(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index faa84b66..fb154097 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -358,6 +358,89 @@ def _check_galaxy_features(self, galaxies, stix_object): # OBSERVED DATA OBJECTS CHECKING METHODS # ############################################################################ + def _check_artifact_fields(self, misp_object, artifact, object_id): + self.assertEqual(len(misp_object.attributes), 6) + payload_bin, md5, sha1, sha256, decryption, mime_type = misp_object.attributes + self.assertEqual(payload_bin.type, 'attachment') + self.assertEqual(payload_bin.object_relation, 'payload_bin') + self.assertEqual( + payload_bin.value, + getattr(artifact, 'id', object_id.split(' - ')[0]).split('--')[1] + ) + self.assertEqual(self._get_data_value(payload_bin.data), artifact.payload_bin) + self.assertEqual( + payload_bin.uuid, + uuid5( + self._UUIDv4, f'{object_id} - payload_bin - {payload_bin.value}' + ) + ) + hashes = artifact.hashes + self.assertEqual(md5.type, 'md5') + self.assertEqual(md5.object_relation, 'md5') + self.assertEqual(md5.value, hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + self.assertEqual(sha1.type, 'sha1') + self.assertEqual(sha1.object_relation, 'sha1') + self.assertEqual(sha1.value, hashes['SHA-1']) + self.assertEqual( + sha1.uuid, uuid5(self._UUIDv4, f'{object_id} - sha1 - {sha1.value}') + ) + self.assertEqual(sha256.type, 'sha256') + self.assertEqual(sha256.object_relation, 'sha256') + self.assertEqual(sha256.value, hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5( + self._UUIDv4, f'{object_id} - sha256 - {sha256.value}' + ) + ) + self.assertEqual(decryption.type, 'text') + self.assertEqual(decryption.object_relation, 'decryption_key') + self.assertEqual(decryption.value, artifact.decryption_key) + self.assertEqual( + decryption.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - decryption_key - {decryption.value}' + ) + ) + self.assertEqual(mime_type.type, 'mime-type') + self.assertEqual(mime_type.object_relation, 'mime_type') + self.assertEqual(mime_type.value, artifact.mime_type) + self.assertEqual( + mime_type.uuid, + uuid5( + self._UUIDv4, f'{object_id} - mime_type - {mime_type.value}' + ) + ) + + def _check_artifact_with_url_fields(self, misp_object, artifact, object_id): + self.assertEqual(misp_object.name, 'artifact') + self.assertEqual(len(misp_object.attributes), 3) + hashes = artifact.hashes + md5, sha256, url = misp_object.attributes + self.assertEqual(md5.type, 'md5') + self.assertEqual(md5.object_relation, 'md5') + self.assertEqual(md5.value, hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + self.assertEqual(sha256.type, 'sha256') + self.assertEqual(sha256.object_relation, 'sha256') + self.assertEqual(sha256.value, hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha256 - {sha256.value}') + ) + self.assertEqual(url.type, 'url') + self.assertEqual(url.object_relation, 'url') + self.assertEqual(url.value, artifact.url) + self.assertEqual( + url.uuid, uuid5(self._UUIDv4, f'{object_id} - url - {url.value}') + ) + def _check_as_fields(self, misp_object, autonomous_system, object_id): self.assertEqual(len(misp_object.attributes), 2) asn, name = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 0d9e4828..ae478800 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -5,6 +5,62 @@ from stix2.parsing import dict_to_stix2 +_ARTIFACT_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "artifact", + "mime_type": "application/zip", + "payload_bin": "UEsDBAoACQAAAKBINlgCq9FEEAAAAAQAAAADABwAb3VpVVQJAAOrIa5lrSGuZXV4CwABBPUBAAAEFAAAAOLQGBmrTcdmURq/qqA1qFFQSwcIAqvRRBAAAAAEAAAAUEsBAh4DCgAJAAAAoEg2WAKr0UQQAAAABAAAAAMAGAAAAAAAAQAAAKSBAAAAAG91aVVUBQADqyGuZXV4CwABBPUBAAAEFAAAAFBLBQYAAAAAAQABAEkAAABdAAAAAAA=", + "hashes": { + "MD5": "bc590af5f7b16b890860248dc0d4c68f", + "SHA-1": "003d59659a3e28781aaf03da1ac1cb0e326ed65e", + "SHA-256": "2dd39c08867f34010fd9ea1833aa549a02da16950dda4a8ef922113a9eccd963" + }, + "decryption_key": "infected", + }, + "1": { + "type": "artifact", + "url": "https://files.pythonhosted.org/packages/1a/62/29f55ef42483c30281fab9d3282ac467f215501826f3251678d8ec2da2e1/misp_stix-2.4.183.tar.gz", + "hashes": { + "MD5": "b3982699c1b9a25346cc8498f483b150", + "SHA-256": "836f395a4f86e9d1b2f528756c248e76665c02c5d0fc89f9b26136db5ac7f7ae" + } + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "artifact", + "mime_type": "application/zip", + "payload_bin": "UEsDBAoACQAAANVUNlgGfJ2iEAAAAAQAAAADABwAbm9uVVQJAAOhN65lozeuZXV4CwABBPUBAAAEFAAAAE7nhRTz5ElBwvqrXUHYVMlQSwcIBnydohAAAAAEAAAAUEsBAh4DCgAJAAAA1VQ2WAZ8naIQAAAABAAAAAMAGAAAAAAAAQAAAKSBAAAAAG5vblVUBQADoTeuZXV4CwABBPUBAAAEFAAAAFBLBQYAAAAAAQABAEkAAABdAAAAAAA=", + "hashes": { + "MD5": "5bfd0814254d0ff993a83560cb740042", + "SHA-1": "5ec1405887e5a74bf2cb97a8d64481194dc13fdc", + "SHA-256": "367e474683cb1f61aae1f963aa9a17446afb5f71a8a03dae7203ac84765a5efa" + }, + "decryption_key": "clear", + } + } + } +] _AS_OBJECTS = [ { "type": "observed-data", @@ -776,6 +832,10 @@ def get_bundle_with_vulnerability_galaxy(cls): # OBSERVED DATA SAMPLES. # ############################################################################ + @classmethod + def get_bundle_with_artifact_objects(cls): + return cls.__assemble_bundle(*_ARTIFACT_OBJECTS) + @classmethod def get_bundle_with_as_objects(cls): return cls.__assemble_bundle(*_AS_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 98993c0d..29a44361 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -184,6 +184,18 @@ def test_stix20_bundle_with_vulnerability_galaxy(self): # OBSERVED DATA OBJECTS IMPORT TESTS # ############################################################################ + def _check_artifact_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + self._check_artifact_fields( + misp_object, observed_data.objects[identifier], object_id + ) + def _check_as_attribute(self, attribute, observed_data, identifier=None): self._check_misp_object_fields(attribute, observed_data, identifier) autonomous_system = observed_data.objects[identifier or '0'] @@ -258,7 +270,7 @@ def _check_generic_attribute( self.assertEqual(attribute.type, attribute_type) self.assertEqual(attribute.value, getattr(observable_object, feature)) - def _check_misp_object_fields(self, misp_object, observed_data, identifier): + def _check_misp_object_fields(self, misp_object, observed_data, identifier=None): if identifier is None: self.assertEqual(misp_object.uuid, observed_data.id.split('--')[1]) else: @@ -271,6 +283,22 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier): self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def test_stix20_bundle_with_artifact_object(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_artifact_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_artifact_object(multiple1, observed_data1, '0') + self._check_misp_object_fields(multiple2, observed_data1, '1') + self._check_artifact_with_url_fields( + multiple2, observed_data1.objects['1'], f'{observed_data1.id} - 1' + ) + self._check_artifact_object(single, observed_data2) + def test_stix20_bundle_with_as_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_as_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 5c27f3f6..9a7929ff 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -5,6 +5,74 @@ from stix2.parsing import dict_to_stix2 +_ARTIFACT_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "artifact--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "artifact--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "artifact", + "spec_version": "2.1", + "id": "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "mime_type": "application/zip", + "payload_bin": "UEsDBAoACQAAAKBINlgCq9FEEAAAAAQAAAADABwAb3VpVVQJAAOrIa5lrSGuZXV4CwABBPUBAAAEFAAAAOLQGBmrTcdmURq/qqA1qFFQSwcIAqvRRBAAAAAEAAAAUEsBAh4DCgAJAAAAoEg2WAKr0UQQAAAABAAAAAMAGAAAAAAAAQAAAKSBAAAAAG91aVVUBQADqyGuZXV4CwABBPUBAAAEFAAAAFBLBQYAAAAAAQABAEkAAABdAAAAAAA=", + "hashes": { + "MD5": "bc590af5f7b16b890860248dc0d4c68f", + "SHA-1": "003d59659a3e28781aaf03da1ac1cb0e326ed65e", + "SHA-256": "2dd39c08867f34010fd9ea1833aa549a02da16950dda4a8ef922113a9eccd963" + }, + "decryption_key": "infected", + }, + { + "type": "artifact", + "spec_version": "2.1", + "id": "artifact--5e384ae7-672c-4250-9cda-3b4da964451a", + "url": "https://files.pythonhosted.org/packages/1a/62/29f55ef42483c30281fab9d3282ac467f215501826f3251678d8ec2da2e1/misp_stix-2.4.183.tar.gz", + "hashes": { + "MD5": "b3982699c1b9a25346cc8498f483b150", + "SHA-256": "836f395a4f86e9d1b2f528756c248e76665c02c5d0fc89f9b26136db5ac7f7ae" + } + }, + { + "type": "artifact", + "spec_version": "2.1", + "id": "artifact--f93cb275-0366-4ecc-abf0-a17928d1e177", + "mime_type": "application/zip", + "payload_bin": "UEsDBAoACQAAANVUNlgGfJ2iEAAAAAQAAAADABwAbm9uVVQJAAOhN65lozeuZXV4CwABBPUBAAAEFAAAAE7nhRTz5ElBwvqrXUHYVMlQSwcIBnydohAAAAAEAAAAUEsBAh4DCgAJAAAA1VQ2WAZ8naIQAAAABAAAAAMAGAAAAAAAAQAAAKSBAAAAAG5vblVUBQADoTeuZXV4CwABBPUBAAAEFAAAAFBLBQYAAAAAAQABAEkAAABdAAAAAAA=", + "hashes": { + "MD5": "5bfd0814254d0ff993a83560cb740042", + "SHA-1": "5ec1405887e5a74bf2cb97a8d64481194dc13fdc", + "SHA-256": "367e474683cb1f61aae1f963aa9a17446afb5f71a8a03dae7203ac84765a5efa" + }, + "decryption_key": "clear", + } +] + _AS_OBJECTS = [ { "type": "observed-data", @@ -923,6 +991,10 @@ def get_bundle_with_vulnerability_galaxy(cls): # OBSERVED DATA SAMPLES. # ############################################################################ + @classmethod + def get_bundle_with_artifact_objects(cls): + return cls.__assemble_bundle(*_ARTIFACT_OBJECTS) + @classmethod def get_bundle_with_as_objects(cls): return cls.__assemble_bundle(*_AS_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 26e16b00..b37f16d4 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -225,6 +225,11 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): # OBSERVED DATA OBJECTS IMPORT TESTS # ############################################################################ + def _check_artifact_object(self, misp_object, observed_data, artifact): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, artifact) + self._check_artifact_fields(misp_object, artifact, artifact.id) + def _check_as_attribute(self, attribute, observed_data, autonomous_system): self._check_misp_object_fields(attribute, observed_data, autonomous_system) self.assertEqual(attribute.type, 'AS') @@ -284,6 +289,20 @@ def _check_misp_object_fields( self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def test_stix21_bundle_with_artifact_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_artifact_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, artifact1, artifact2, artifact3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_artifact_object(multiple1, od1, artifact1) + self._check_misp_object_fields(multiple2, od1, artifact2) + self._check_artifact_with_url_fields(multiple2, artifact2, artifact2.id) + self._check_artifact_object(single, od2, artifact3) + def test_stix21_bundle_with_as_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_as_objects() self.parser.load_stix_bundle(bundle) From 0f10c865c21fe5147b4a3f2fb255331b41aa0fac Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 22 Jan 2024 15:31:06 +0100 Subject: [PATCH 029/137] chg: [stix2 import] Added generic conversion methods for observable objects associated to observed data objects imported as MISP objects --- .../stix2_observed_data_converter.py | 208 ++++++++---------- 1 file changed, 92 insertions(+), 116 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index b915a74f..568122fd 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -32,7 +32,11 @@ from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser -_GENERIC_OBSERVABLE_OBJECTS_TYPING = Union[ +_GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ + Artifact_v20, Artifact_v21, + Directory_v20, Directory_v21 +] +_GENERIC_OBSERVABLE_TYPING = Union[ DomainName_v20, DomainName_v21, IPv4Address_v20, IPv4Address_v21, IPv6Address_v20, IPv6Address_v21, @@ -139,57 +143,26 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ - def _parse_artifact_observable_object(self, observed_data: ObservedData_v20, - identifier: str) -> MISPObject: - artifact = observed_data.objects[identifier] - if artifact.get('id') is not None: - return self._parse_artifact_observable_object_ref( - artifact, observed_data - ) - object_id = f'{observed_data.id} - {identifier}' - misp_object = self._create_misp_object_from_observable_object( - 'artifact', observed_data, object_id - ) - for attribute in self._parse_artifact_observable(artifact, object_id): - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) - - def _parse_artifact_observable_object_ref( - self, artifact: _ARTIFACT_TYPING, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object_from_observable_object_ref( - 'artifact', artifact, observed_data - ) - for attribute in self._parse_artifact_observable(artifact): - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) - def _parse_artifact_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) artifact = observable['observable'] - self._parse_artifact_observable_object_ref(artifact, observed_data) + self._parse_generic_observable_object_ref( + artifact, observed_data, 'artifact' + ) observable['used'][self.event_uuid] = True - def _parse_artifact_observable_objects(self, observed_data): + def _parse_artifact_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): if len(observed_data.objects) == 1: - artifact = next(iter(observed_data.objects.values())) - if artifact.get('id') is not None: - return self._parse_artifact_observable_object_ref( - artifact, observed_data - ) - misp_object = self._create_misp_object_from_observable_object( - 'artifact', observed_data - ) - attributes = self._parse_artifact_observable( - artifact, observed_data.id + return self._parse_generic_single_observable_object( + observed_data, 'artifact' ) - for attribute in attributes: - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) for identifier in observed_data.objects: - self._parse_artifact_observable_object(observed_data, identifier) + self._parse_generic_observable_object( + observed_data, identifier, 'artifact' + ) def _parse_as_observable_object( self, observed_data: ObservedData_v20, object_id: str): @@ -316,55 +289,22 @@ def _parse_autonomous_system_observable_object_ref( ) return self.main_parser._add_misp_object(misp_object, observed_data) - def _parse_directory_observable_object( - self, observed_data: ObservedData_v20, - object_id: str) -> MISPObject: - directory = observed_data.objects[object_id] - if directory.get('id') is not None: - return self._parse_directory_observable_object_ref( - directory, observed_data - ) - object_id = f'{observed_data.id} - {object_id}' - misp_object = self._create_misp_object_from_observable_object( - 'directory', observed_data, object_id - ) - for attribute in self._parse_directory_observable(directory, object_id): - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) - - def _parse_directory_observable_object_ref( - self, directory: _DIRECTORY_TYPING, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object_from_observable_object_ref( - 'directory', directory, observed_data - ) - for attribute in self._parse_directory_observable(directory): - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) - def _parse_directory_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) directory = observable['observable'] - if not hasattr(directory, 'contains_refs'): - if observable['used'].get(self.event_uuid, False): - continue - misp_object = self._parse_directory_observable_object_ref( - directory, observed_data - ) - observable['misp_object'] = misp_object - observable['used'][self.event_uuid] = True - continue misp_object = ( observable['misp_object'] if observable['used'].get(self.event_uuid, False) else - self._parse_directory_observable_object_ref( - directory, observed_data + self._parse_generic_observable_object_ref( + directory, observed_data, 'directory' ) ) observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True + if not hasattr(directory, 'contains_refs'): + continue for contained_ref in directory.contains_refs: contained = self._fetch_observables(contained_ref) if contained_ref not in observed_data.object_refs: @@ -385,8 +325,8 @@ def _parse_directory_observable_object_refs( contained['misp_object'].uuid, 'contains' ) continue - contained_object = self._parse_directory_observable_object_ref( - contained['observable'], observed_data + contained_object = self._parse_generic_observable_object_ref( + contained['observable'], observed_data, 'directory' ) contained['misp_object'] = contained_object contained['used'][self.event_uuid] = True @@ -395,20 +335,9 @@ def _parse_directory_observable_object_refs( def _parse_directory_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING): if len(observed_data.objects) == 1: - directory = next(iter(observed_data.objects.values())) - if directory.get('id') is not None: - return self._parse_directory_observable_object_ref( - directory, observed_data - ) - misp_object = self._create_misp_object_from_observable_object( - 'directory', observed_data + return self._parse_generic_single_observable_object( + observed_data, 'directory' ) - attributes = self._parse_directory_observable( - directory, observed_data.id - ) - for attribute in attributes: - misp_object.add_attribute(**attribute) - return self.main_parser._add_misp_object(misp_object, observed_data) observable_objects = { object_id: {'used': False, 'observable': observable} for object_id, observable in observed_data.objects.items() @@ -417,8 +346,8 @@ def _parse_directory_observable_objects( directory = observable['observable'] misp_object = ( observable['misp_object'] if observable['used'] else - self._parse_directory_observable_object( - observed_data, object_id + self._parse_generic_observable_object( + observed_data, object_id, 'directory' ) ) observable['misp_object'] = misp_object @@ -431,8 +360,8 @@ def _parse_directory_observable_objects( contained['misp_object'].uuid, 'contains' ) continue - contained_object = self._parse_directory_observable_object( - observed_data, contained_ref + contained_object = self._parse_generic_observable_object( + observed_data, contained_ref, 'directory' ) contained['misp_object'] = contained_object contained['used'] = True @@ -445,7 +374,7 @@ def _parse_domain_observable_object_refs( for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) domain = observable['observable'] - self._parse_generic_observable_object_ref( + self._parse_generic_observable_object_ref_as_attribute( domain, observed_data, 'domain' ) observable['used'][self.event_uuid] = True @@ -455,7 +384,7 @@ def _parse_domain_observable_objects( if len(observed_data.objects) == 1: domain = next(iter(observed_data.objects.values())) if domain.get('id') is not None: - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( domain, observed_data, 'domain' ) return self.main_parser._add_misp_attribute( @@ -469,7 +398,7 @@ def _parse_domain_observable_objects( observed_data ) for identifier in observed_data.objects: - self._parse_generic_observable_object( + self._parse_generic_observable_object_as_attribute( observed_data, identifier, 'domain' ) @@ -617,11 +546,28 @@ def _parse_email_address_observable_objects( ) def _parse_generic_observable_object( - self, observed_data: ObservedData_v20, identifier: str, + self, observed_data: _OBSERVED_DATA_TYPING, object_id: str, + name: str, feature: Optional[str] = None) -> MISPObject: + observable_object = observed_data.objects[object_id] + if observable_object.get('id') is not None: + return self._parse_generic_observable_object_ref( + observable_object, observed_data, name, feature + ) + object_id = f'{observed_data.id} - {object_id}' + misp_object = self._create_misp_object_from_observable_object( + name, observed_data, object_id + ) + to_call = getattr(self, f'_parse_{feature or name}_observable') + for attribute in to_call(observable_object, object_id): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_generic_observable_object_as_attribute( + self, observed_data: _OBSERVED_DATA_TYPING, identifier: str, attribute_type: str, feature: Optional[str] = 'value'): observable_object = observed_data.objects[identifier] if hasattr(observable_object, 'id'): - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( observable_object, observed_data, attribute_type, feature ) self.main_parser._add_misp_attribute( @@ -638,7 +584,19 @@ def _parse_generic_observable_object( ) def _parse_generic_observable_object_ref( - self, observable_object: _GENERIC_OBSERVABLE_OBJECTS_TYPING, + self, observable_object: _GENERIC_OBSERVABLE_OBJECT_TYPING, + observed_data: ObservedData_v21, name: str, + feature: Optional[str] = None) -> MISPObject: + misp_object = self._create_misp_object_from_observable_object_ref( + name, observable_object, observed_data + ) + to_call = getattr(self, f'_parse_{feature or name}_observable') + for attribute in to_call(observable_object): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + + def _parse_generic_observable_object_ref_as_attribute( + self, observable_object: _GENERIC_OBSERVABLE_TYPING, observed_data: _OBSERVED_DATA_TYPING, attribute_type: str, feature: Optional[str] = 'value'): self.main_parser._add_misp_attribute( @@ -654,12 +612,28 @@ def _parse_generic_observable_object_ref( observed_data ) + def _parse_generic_single_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, name: str, + feature: Optional[str] = None) -> MISPObject: + observable_object = next(iter(observed_data.objects.values())) + if observable_object.get('id') is not None: + return self._parse_generic_observable_object_ref( + observable_object, observed_data, name, feature + ) + misp_object = self._create_misp_object_from_observable_object( + name, observed_data + ) + to_call = getattr(self, f'_parse_{feature or name}_observable') + for attribute in to_call(observable_object, observed_data.id): + misp_object.add_attribute(**attribute) + return self.main_parser._add_misp_object(misp_object, observed_data) + def _parse_ip_address_observable_object_refs( self, observed_data: _OBSERVED_DATA_TYPING): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) ip_address = observable['observable'] - self._parse_generic_observable_object_ref( + self._parse_generic_observable_object_ref_as_attribute( ip_address, observed_data, 'ip-dst' ) observable['used'][self.event_uuid] = True @@ -669,7 +643,7 @@ def _parse_ip_address_observable_objects( if len(observed_data.objects) == 1: ip_address = next(iter(observed_data.objects.values())) if ip_address.get('id') is not None: - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( ip_address, observed_data, 'ip-dst' ) return self.main_parser._add_misp_attribute( @@ -683,7 +657,7 @@ def _parse_ip_address_observable_objects( observed_data ) for identifier in observed_data.objects: - self._parse_generic_observable_object( + self._parse_generic_observable_object_as_attribute( observed_data, identifier, 'ip-dst' ) @@ -692,7 +666,7 @@ def _parse_mac_address_observable_object_refs( for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) mac_address = observable['observable'] - self._parse_generic_observable_object_ref( + self._parse_generic_observable_object_ref_as_attribute( mac_address, observed_data, 'mac-address' ) observable['used'][self.event_uuid] = True @@ -702,7 +676,7 @@ def _parse_mac_address_observable_objects( if len(observed_data.objects) == 1: mac_address = next(iter(observed_data.objects.values())) if mac_address.get('id') is not None: - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( mac_address, observed_data, 'mac-address' ) return self.main_parser._add_misp_attribute( @@ -716,7 +690,7 @@ def _parse_mac_address_observable_objects( observed_data ) for identifier in observed_data.objects: - self._parse_generic_observable_object( + self._parse_generic_observable_object_as_attribute( observed_data, identifier, 'mac-address' ) @@ -725,7 +699,7 @@ def _parse_mutex_observable_object_refs( for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) mutex = observable['observable'] - self._parse_generic_observable_object_ref( + self._parse_generic_observable_object_ref_as_attribute( mutex, observed_data, 'mutex', feature='name' ) observable['used'][self.event_uuid] = True @@ -735,7 +709,7 @@ def _parse_mutex_observable_objects( if len(observed_data.objects) == 1: mutex = next(iter(observed_data.objects.values())) if mutex.get('id') is not None: - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( mutex, observed_data, 'mutex', feature='name' ) return self.main_parser._add_misp_attribute( @@ -749,7 +723,7 @@ def _parse_mutex_observable_objects( observed_data ) for identifier in observed_data.objects: - self._parse_generic_observable_object( + self._parse_generic_observable_object_as_attribute( observed_data, identifier, 'mutex', feature='name' ) @@ -758,7 +732,9 @@ def _parse_url_observable_object_refs( for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) url = observable['observable'] - self._parse_generic_observable_object_ref(url, observed_data, 'url') + self._parse_generic_observable_object_ref_as_attribute( + url, observed_data, 'url' + ) observable['used'][self.event_uuid] = True def _parse_url_observable_objects( @@ -766,7 +742,7 @@ def _parse_url_observable_objects( if len(observed_data.objects) == 1: url = next(iter(observed_data.objects.values())) if url.get('id') is not None: - return self._parse_generic_observable_object_ref( + return self._parse_generic_observable_object_ref_as_attribute( url, observed_data, 'url' ) return self.main_parser._add_misp_attribute( @@ -780,7 +756,7 @@ def _parse_url_observable_objects( observed_data ) for identifier in observed_data.objects: - self._parse_generic_observable_object( + self._parse_generic_observable_object_as_attribute( observed_data, identifier, 'url' ) From 1535e9381f57ee5d1639a8fb4be41005b09f63c1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 23 Jan 2024 16:51:39 +0100 Subject: [PATCH 030/137] fix: [stix2 import] Better handling of generic observable object parsers --- .../converters/stix2_observable_converter.py | 11 ----- .../stix2_observable_objects_converter.py | 4 +- .../stix2_observed_data_converter.py | 46 +++++++++++++------ 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index f0279431..d6f16559 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -550,17 +550,6 @@ def _parse_asn_observable( object_id ) - def _parse_directory_observable( - self, observable: _DIRECTORY_TYPING, - object_id: Optional[str] = None) -> Iterator[dict]: - if object_id is None: - object_id = observable.id - for field, mapping in self._mapping.directory_object_mapping().items(): - if hasattr(observable, field): - yield from self._populate_object_attributes( - mapping, getattr(observable, field), object_id - ) - def _parse_domain_observable(self, observable: _DOMAIN_TYPING, object_id: Optional[str] = None) -> dict: attribute = { diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 7d30ab02..7139bd90 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -125,7 +125,9 @@ def _parse_directory_observable_object( if observable['used'].get(self.event_uuid, False): return observable.get('misp_object', observable['misp_attribute']) directory = observable['observable'] - attributes = tuple(self._parse_directory_observable(directory)) + attributes = tuple( + self._parse_generic_observable(directory, 'directory') + ) observable['used'][self.event_uuid] = True force_object = ( len(attributes) > 1 or any( diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 568122fd..4ce92ce4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -26,7 +26,7 @@ IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, MACAddress as MACAddress_v21, Mutex as Mutex_v21, URL as URL_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 -from typing import Optional, TYPE_CHECKING, Union +from typing import Iterator, Optional, TYPE_CHECKING, Union if TYPE_CHECKING: from ..external_stix2_to_misp import ExternalSTIX2toMISPParser @@ -149,7 +149,7 @@ def _parse_artifact_observable_object_refs( observable = self._fetch_observables(object_ref) artifact = observable['observable'] self._parse_generic_observable_object_ref( - artifact, observed_data, 'artifact' + artifact, observed_data, 'artifact', False ) observable['used'][self.event_uuid] = True @@ -157,11 +157,11 @@ def _parse_artifact_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING): if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( - observed_data, 'artifact' + observed_data, 'artifact', False ) for identifier in observed_data.objects: self._parse_generic_observable_object( - observed_data, identifier, 'artifact' + observed_data, identifier, 'artifact', False ) def _parse_as_observable_object( @@ -547,18 +547,23 @@ def _parse_email_address_observable_objects( def _parse_generic_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, object_id: str, - name: str, feature: Optional[str] = None) -> MISPObject: + name: str, generic: Optional[bool] = True) -> MISPObject: observable_object = observed_data.objects[object_id] if observable_object.get('id') is not None: return self._parse_generic_observable_object_ref( - observable_object, observed_data, name, feature + observable_object, observed_data, name, generic ) object_id = f'{observed_data.id} - {object_id}' misp_object = self._create_misp_object_from_observable_object( name, observed_data, object_id ) - to_call = getattr(self, f'_parse_{feature or name}_observable') - for attribute in to_call(observable_object, object_id): + attributes = ( + self._parse_generic_observable(observable_object, name, object_id) + if generic else getattr(self, f'_parse_{name}_observable')( + observable_object, object_id + ) + ) + for attribute in attributes: misp_object.add_attribute(**attribute) return self.main_parser._add_misp_object(misp_object, observed_data) @@ -586,12 +591,17 @@ def _parse_generic_observable_object_as_attribute( def _parse_generic_observable_object_ref( self, observable_object: _GENERIC_OBSERVABLE_OBJECT_TYPING, observed_data: ObservedData_v21, name: str, - feature: Optional[str] = None) -> MISPObject: + generic: Optional[bool] = True) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( name, observable_object, observed_data ) - to_call = getattr(self, f'_parse_{feature or name}_observable') - for attribute in to_call(observable_object): + attributes = ( + self._parse_generic_observable(observable_object, name) + if generic else getattr(self, f'_parse_{name}_observable')( + observable_object + ) + ) + for attribute in attributes: misp_object.add_attribute(**attribute) return self.main_parser._add_misp_object(misp_object, observed_data) @@ -614,17 +624,23 @@ def _parse_generic_observable_object_ref_as_attribute( def _parse_generic_single_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, name: str, - feature: Optional[str] = None) -> MISPObject: + generic: Optional[bool] = True) -> MISPObject: observable_object = next(iter(observed_data.objects.values())) if observable_object.get('id') is not None: return self._parse_generic_observable_object_ref( - observable_object, observed_data, name, feature + observable_object, observed_data, name, generic ) misp_object = self._create_misp_object_from_observable_object( name, observed_data ) - to_call = getattr(self, f'_parse_{feature or name}_observable') - for attribute in to_call(observable_object, observed_data.id): + object_id = observed_data.id + attributes = ( + self._parse_generic_observable(observable_object, name, object_id) + if generic else getattr(self, f'_parse_{name}_observable')( + observable_object, object_id + ) + ) + for attribute in attributes: misp_object.add_attribute(**attribute) return self.main_parser._add_misp_object(misp_object, observed_data) From df42b914b54f9394b4f4aef72296ed8fb59f3407 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 23 Jan 2024 23:30:51 +0100 Subject: [PATCH 031/137] wip: [stix2 import] Reusing the generic observed data parsing methods to support Software observable objects conversion from the converters --- .../stix2_observed_data_converter.py | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 4ce92ce4..7baf9a2d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -18,13 +18,15 @@ Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, - MACAddress as MACAddress_v20, Mutex as Mutex_v20, URL as URL_v20) + MACAddress as MACAddress_v20, Mutex as Mutex_v20, Software as Software_v20, + URL as URL_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( Artifact as Artifact_v21, AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, - MACAddress as MACAddress_v21, Mutex as Mutex_v21, URL as URL_v21) + MACAddress as MACAddress_v21, Mutex as Mutex_v21, Software as Software_v21, + URL as URL_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Iterator, Optional, TYPE_CHECKING, Union @@ -34,7 +36,8 @@ _GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ Artifact_v20, Artifact_v21, - Directory_v20, Directory_v21 + Directory_v20, Directory_v21, + Software_v20, Software_v21 ] _GENERIC_OBSERVABLE_TYPING = Union[ DomainName_v20, DomainName_v21, @@ -47,7 +50,8 @@ _OBSERVABLE_OBJECTS_TYPING = Union[ Artifact_v20, Artifact_v21, AutonomousSystem_v20, AutonomousSystem_v21, - Directory_v20, Directory_v21 + Directory_v20, Directory_v21, + Software_v20, Software_v21 ] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 @@ -743,6 +747,27 @@ def _parse_mutex_observable_objects( observed_data, identifier, 'mutex', feature='name' ) + def _parse_software_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + software = observable['observable'] + self._parse_generic_observable_object_ref( + software, observed_data, 'software' + ) + observable['used'][self.event_uuid] = True + + def _parse_software_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + return self._parse_generic_single_observable_object( + observed_data, 'software' + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'software' + ) + def _parse_url_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: From 088896fc278e106a76300aa9f548b185c971b9c1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 23 Jan 2024 23:33:06 +0100 Subject: [PATCH 032/137] wip: [tests] Tests for external Software Observable objects - within or referenced by Observed data objects - import to MISP objects --- tests/_test_stix_import.py | 91 +++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 61 ++++++++++++++++++ tests/test_external_stix20_import.py | 24 +++++++ tests/test_external_stix21_bundles.py | 74 +++++++++++++++++++++- tests/test_external_stix21_import.py | 19 ++++++ 5 files changed, 268 insertions(+), 1 deletion(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index fb154097..f45a5c43 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -485,6 +485,97 @@ def _check_directory_fields(self, misp_object, directory, object_id): ) return accessed.value, created.value, modified.value + def _check_software_fields(self, misp_object, software, object_id): + self.assertEqual(len(misp_object.attributes), 4) + language, name, vendor, version = misp_object.attributes + self.assertEqual(language.type, 'text') + self.assertEqual(language.object_relation, 'language') + self.assertEqual(language.value, software.languages[0]) + self.assertEqual( + language.uuid, + uuid5(self._UUIDv4, f'{object_id} - language - {language.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, software.name) + self.assertEqual( + name.uuid, + uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(vendor.type, 'text') + self.assertEqual(vendor.object_relation, 'vendor') + self.assertEqual(vendor.value, software.vendor) + self.assertEqual( + vendor.uuid, + uuid5(self._UUIDv4, f'{object_id} - vendor - {vendor.value}') + ) + self.assertEqual(version.type, 'text') + self.assertEqual(version.object_relation, 'version') + self.assertEqual(version.value, software.version) + self.assertEqual( + version.uuid, + uuid5(self._UUIDv4, f'{object_id} - version - {version.value}') + ) + + def _check_software_with_swid_fields(self, misp_object, software, object_id): + self.assertEqual(misp_object.name, 'software') + self.assertEqual(len(misp_object.attributes), 8) + cpe, lang1, lang2, lang3, name, swid, vendor, version = misp_object.attributes + self._assert_multiple_equal(cpe.type, cpe.object_relation, 'cpe') + self.assertEqual(cpe.value, software.cpe) + self.assertEqual( + cpe.uuid, uuid5(self._UUIDv4, f'{object_id} - cpe - {cpe.value}') + ) + language1, language2, language3 = software.languages + self._assert_multiple_equal(lang1.type, lang2.type, lang3.type, 'text') + self._assert_multiple_equal( + lang1.object_relation, lang2.object_relation, + lang3.object_relation, 'language' + ) + self.assertEqual(lang1.value, language1) + self.assertEqual( + lang1.uuid, + uuid5(self._UUIDv4, f'{object_id} - language - {lang1.value}') + ) + self.assertEqual(lang2.value, language2) + self.assertEqual( + lang2.uuid, + uuid5(self._UUIDv4, f'{object_id} - language - {lang2.value}') + ) + self.assertEqual(lang3.value, language3) + self.assertEqual( + lang3.uuid, + uuid5(self._UUIDv4, f'{object_id} - language - {lang3.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, software.name) + self.assertEqual( + name.uuid, + uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(swid.type, 'text') + self.assertEqual(swid.object_relation, 'swid') + self.assertEqual(swid.value, software.swid) + self.assertEqual( + swid.uuid, + uuid5(self._UUIDv4, f'{object_id} - swid - {swid.value}') + ) + self.assertEqual(vendor.type, 'text') + self.assertEqual(vendor.object_relation, 'vendor') + self.assertEqual(vendor.value, software.vendor) + self.assertEqual( + vendor.uuid, + uuid5(self._UUIDv4, f'{object_id} - vendor - {vendor.value}') + ) + self.assertEqual(version.type, 'text') + self.assertEqual(version.object_relation, 'version') + self.assertEqual(version.value, software.version) + self.assertEqual( + version.uuid, + uuid5(self._UUIDv4, f'{object_id} - version - {version.value}') + ) + class TestInternalSTIX2Import(TestSTIX2Import): def setUp(self): diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index ae478800..9f325045 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -547,6 +547,63 @@ } } ] +_SOFTWARE_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "software", + "name": "MISP", + "languages": [ + "PHP" + ], + "vendor": "MISP Project", + "version": "2.4.183" + }, + "1": { + "type": "software", + "name": "misp-stix", + "languages": [ + "Python" + ], + "vendor": "CIRCL", + "version": "2.4.183" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "software", + "name": "Acrobat X Pro", + "cpe": "cpe:2.3:a:adobe:acrobat:10.0:-:pro:*:*:*:*:*", + "swid": "trueAcrobat X Pro10.010000Adobe Inc.regid.1986-12.com.adobeAdobe Inc.regid.1986-12.com.adobeAcrobatPro-AS1-Win-GM-MULregid.1986-12.com.adobeAdobe Inc.regid.1986-12.com.adobeunlicensedVOLUMEVOLUME970787034620329571838915", + "languages": [ + "C#", + "Javascript", + "Postscript" + ], + "vendor": "Adobe Inc.", + "version": "10.0" + } + } + } +] _THREAT_ACTOR_OBJECTS = [ { "type": "threat-actor", @@ -864,6 +921,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_software_objects(cls): + return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) + @classmethod def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 29a44361..fb4c5d3f 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -283,6 +283,14 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier=None) self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_software_object(self, misp_object, observed_data, identifier): + self.assertEqual(misp_object.name, 'software') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = f'{observed_data.id} - {identifier}' + self._check_software_fields( + misp_object, observed_data.objects[identifier], object_id + ) + def test_stix20_bundle_with_artifact_object(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_artifact_objects() self.parser.load_stix_bundle(bundle) @@ -427,6 +435,22 @@ def test_stix20_bundle_with_mutex_attributes(self): observed_data2, s_mutex, 'mutex', feature='name' ) + def test_stix20_bundle_with_software_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_software_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_software_object(multiple1, observed_data1, '0') + self._check_software_object(multiple2, observed_data1, '1') + self._check_misp_object_fields(single, observed_data2) + self._check_software_with_swid_fields( + single, observed_data2.objects['0'], observed_data2.id + ) + def test_stix20_bundle_with_url_attributes(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_url_attributes() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 9a7929ff..978e7afe 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -72,7 +72,6 @@ "decryption_key": "clear", } ] - _AS_OBJECTS = [ { "type": "observed-data", @@ -681,6 +680,75 @@ "name": "sensitive_resource_lock" } ] +_SOFTWARE_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "software--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "software--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "software--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "software", + "spec_version": "2.1", + "id": "software--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "name": "MISP", + "languages": [ + "PHP" + ], + "vendor": "MISP Project", + "version": "2.4.183" + }, + { + "type": "software", + "spec_version": "2.1", + "id": "software--5e384ae7-672c-4250-9cda-3b4da964451a", + "name": "misp-stix", + "languages": [ + "Python" + ], + "vendor": "CIRCL", + "spec_version": "2.1", + "version": "2.4.183" + }, + { + "type": "software", + "id": "software--f93cb275-0366-4ecc-abf0-a17928d1e177", + "spec_version": "2.1", + "name": "Acrobat X Pro", + "cpe": "cpe:2.3:a:adobe:acrobat:10.0:-:pro:*:*:*:*:*", + "swid": "trueAcrobat X Pro10.010000Adobe Inc.regid.1986-12.com.adobeAdobe Inc.regid.1986-12.com.adobeAcrobatPro-AS1-Win-GM-MULregid.1986-12.com.adobeAdobe Inc.regid.1986-12.com.adobeunlicensedVOLUMEVOLUME970787034620329571838915", + "languages": [ + "C#", + "Javascript", + "Postscript" + ], + "vendor": "Adobe Inc.", + "version": "10.0" + } +] _THREAT_ACTOR_OBJECTS = [ { "type": "threat-actor", @@ -1023,6 +1091,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_software_objects(cls): + return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) + @classmethod def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index b37f16d4..e09f247d 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -289,6 +289,11 @@ def _check_misp_object_fields( self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_software_object(self, misp_object, observed_data, software): + self.assertEqual(misp_object.name, 'software') + self._check_misp_object_fields(misp_object, observed_data, software) + self._check_software_fields(misp_object, software, software.id) + def test_stix21_bundle_with_artifact_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_artifact_objects() self.parser.load_stix_bundle(bundle) @@ -414,6 +419,20 @@ def test_stix21_bundle_with_mutex_attributes(self): self._check_generic_attribute(od1, mutex_2, m_mutex2, 'mutex', 'name') self._check_generic_attribute(od2, mutex_3, s_mutex, 'mutex', 'name') + def test_stix21_bundle_with_software_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_software_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, software1, software2, software3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_software_object(multiple1, od1, software1) + self._check_software_object(multiple2, od1, software2) + self._check_misp_object_fields(single, od2, software3) + self._check_software_with_swid_fields(single, software3, software3.id) + def test_stix21_bundle_with_url_attributes(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_url_attributes() self.parser.load_stix_bundle(bundle) From 38ba26ad19602d3243bc58ff99267579e8566310 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 24 Jan 2024 23:42:06 +0100 Subject: [PATCH 033/137] wip: [stix2 import] Reusing the generic observed data parsing methods to support X509 observable objects conversion from the converters --- .../stix2_observed_data_converter.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 7baf9a2d..91697487 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -801,6 +801,27 @@ def _parse_url_observable_objects( observed_data, identifier, 'url' ) + def _parse_x509_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + x509 = observable['observable'] + self._parse_generic_observable_object( + x509, observed_data, 'x509', False + ) + observable['used'][self.event_uuid] = True + + def _parse_x509_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + return self._parse_generic_single_observable_object( + observed_data, 'x509', False + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'x509', False + ) + ############################################################################ # UTILITY METHODS. # ############################################################################ From 5fc6a9d0f1bf13272a9c2f828dcd3d871fbfe7f1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 26 Jan 2024 23:56:02 +0100 Subject: [PATCH 034/137] fix: [stix2 import] Removed duplicated MISP Attribute dict creation methods --- .../stix2misp/converters/stix2converter.py | 3 --- misp_stix_converter/stix2misp/stix2_to_misp.py | 11 ----------- 2 files changed, 14 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index 2c15e775..3fb44540 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -242,9 +242,6 @@ def parse(self, stix_object_ref: str): stix_object = self.main_parser._get_stix_object(stix_object_ref) self._parse_galaxy(stix_object) - def _create_attribute_dict(self, stix_object: _SDO_TYPING) -> dict: - return super()._create_attribute_dict(stix_object) - ############################################################################ # GALAXIES PARSING METHODS # ############################################################################ diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 8aba5fba..a79e8eb1 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1061,17 +1061,6 @@ def _add_misp_object(self, misp_object: MISPObject, attribute.add_tag(tag) return self.misp_event.add_object(misp_object) - def _create_attribute_dict(self, stix_object: _SDO_TYPING) -> dict: - attribute = self._parse_timeline(stix_object) - if hasattr(stix_object, 'description') and stix_object.description: - attribute['comment'] = stix_object.description - attribute.update( - self._sanitise_attribute_uuid( - stix_object.id, comment=attribute.get('comment') - ) - ) - return attribute - def _create_generic_event(self) -> MISPEvent: misp_event = MISPEvent() event_args = { From 645dc73380543a9deea1c08800e127baf5366748 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Sat, 27 Jan 2024 21:22:29 +0100 Subject: [PATCH 035/137] fix: [stix2 import] Deduplication in the STIX 2.1 Directory objects parsing --- .../converters/stix2_observed_data_converter.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 91697487..d00e4c84 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -311,12 +311,12 @@ def _parse_directory_observable_object_refs( continue for contained_ref in directory.contains_refs: contained = self._fetch_observables(contained_ref) + if contained['used'].get(self.event_uuid, False): + misp_object.add_reference( + contained['misp_object'].uuid, 'contains' + ) + continue if contained_ref not in observed_data.object_refs: - if contained['used'].get(self.event_uuid, False): - misp_object.add_reference( - contained['misp_object'].uuid, 'contains' - ) - continue self.observable_relationships[misp_object.uuid].add( ( self.main_parser._sanitise_uuid(contained_ref), @@ -324,11 +324,6 @@ def _parse_directory_observable_object_refs( ) ) continue - if contained['used'].get(self.event_uuid, False): - misp_object.add_reference( - contained['misp_object'].uuid, 'contains' - ) - continue contained_object = self._parse_generic_observable_object_ref( contained['observable'], observed_data, 'directory' ) From 7bf05832f4f1552930166fba08d0a7d7f89a5eac Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 29 Jan 2024 14:04:21 +0100 Subject: [PATCH 036/137] fix: [stix2 import] Typo on the generic observable object parsing method to call --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index d00e4c84..b1dcc043 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -801,7 +801,7 @@ def _parse_x509_observable_object_refs( for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) x509 = observable['observable'] - self._parse_generic_observable_object( + self._parse_generic_observable_object_ref( x509, observed_data, 'x509', False ) observable['used'][self.event_uuid] = True From 05948b5156c98aac7bc124cdcacc6d8d85c1d37b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 29 Jan 2024 14:10:50 +0100 Subject: [PATCH 037/137] fix: [stix2 import] Updated typings --- .../converters/stix2_observed_data_converter.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index b1dcc043..3892978e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -19,14 +19,14 @@ Directory as Directory_v20, DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, MACAddress as MACAddress_v20, Mutex as Mutex_v20, Software as Software_v20, - URL as URL_v20) + URL as URL_v20, X509Certificate as X509Certificate_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( Artifact as Artifact_v21, AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, MACAddress as MACAddress_v21, Mutex as Mutex_v21, Software as Software_v21, - URL as URL_v21) + URL as URL_v21, X509Certificate as X509Certificate_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Iterator, Optional, TYPE_CHECKING, Union @@ -37,7 +37,8 @@ _GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ Artifact_v20, Artifact_v21, Directory_v20, Directory_v21, - Software_v20, Software_v21 + Software_v20, Software_v21, + X509Certificate_v20, X509Certificate_v21 ] _GENERIC_OBSERVABLE_TYPING = Union[ DomainName_v20, DomainName_v21, @@ -51,7 +52,8 @@ Artifact_v20, Artifact_v21, AutonomousSystem_v20, AutonomousSystem_v21, Directory_v20, Directory_v21, - Software_v20, Software_v21 + Software_v20, Software_v21, + X509Certificate_v20, X509Certificate_v21 ] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 From ef9f86cd0c7f0e016b2fbcc2425d9dbe62bbe29d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 29 Jan 2024 14:12:15 +0100 Subject: [PATCH 038/137] wip: [tests] Tests for X509 Certificate objects import from STIX 2.x --- tests/_test_stix_import.py | 137 ++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 87 ++++++++++++++++ tests/test_external_stix20_import.py | 25 +++++ tests/test_external_stix21_bundles.py | 98 ++++++++++++++++++ tests/test_external_stix21_import.py | 18 ++++ 5 files changed, 365 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index f45a5c43..65b19b56 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -576,6 +576,143 @@ def _check_software_with_swid_fields(self, misp_object, software, object_id): uuid5(self._UUIDv4, f'{object_id} - version - {version.value}') ) + def _check_x509_fields(self, misp_object, x509, object_id): + self.assertEqual(len(misp_object.attributes), 14) + (md5, sha1, sha256, signed, issuer, serial_number, signature, + subject, key_algo, key_exponent, key_modulus, not_after, + not_before, version) = misp_object.attributes + hashes = x509.hashes + self._assert_multiple_equal( + md5.type, md5.object_relation, 'x509-fingerprint-md5' + ) + self.assertEqual(md5.value, hashes['MD5']) + self.assertEqual( + md5.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - x509-fingerprint-md5 - {md5.value}' + ) + ) + self._assert_multiple_equal( + sha1.type, sha1.object_relation, 'x509-fingerprint-sha1' + ) + self.assertEqual(sha1.value, hashes['SHA-1']) + self.assertEqual( + sha1.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - x509-fingerprint-sha1 - {sha1.value}' + ) + ) + self._assert_multiple_equal( + sha256.type, sha256.object_relation, 'x509-fingerprint-sha256' + ) + self.assertEqual(sha256.value, hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - x509-fingerprint-sha256 - {sha256.value}' + ) + ) + self.assertEqual(signed.type, 'boolean') + self.assertEqual(signed.object_relation, 'self_signed') + self.assertEqual(signed.value, x509.is_self_signed) + self.assertEqual( + signed.uuid, + uuid5(self._UUIDv4, f'{object_id} - self_signed - {signed.value}') + ) + self.assertEqual(issuer.type, 'text') + self.assertEqual(issuer.object_relation, 'issuer') + self.assertEqual(issuer.value, x509.issuer) + self.assertEqual( + issuer.uuid, + uuid5(self._UUIDv4, f'{object_id} - issuer - {issuer.value}') + ) + self.assertEqual(serial_number.type, 'text') + self.assertEqual(serial_number.object_relation, 'serial-number') + self.assertEqual(serial_number.value, x509.serial_number) + self.assertEqual( + serial_number.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - serial-number - {serial_number.value}' + ) + ) + self.assertEqual(signature.type, 'text') + self.assertEqual(signature.object_relation, 'signature_algorithm') + self.assertEqual(signature.value, x509.signature_algorithm) + self.assertEqual( + signature.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - signature_algorithm - {signature.value}' + ) + ) + self.assertEqual(subject.type, 'text') + self.assertEqual(subject.object_relation, 'subject') + self.assertEqual(subject.value, x509.subject) + self.assertEqual( + subject.uuid, + uuid5(self._UUIDv4, f'{object_id} - subject - {subject.value}') + ) + self.assertEqual(key_algo.type, 'text') + self.assertEqual(key_algo.object_relation, 'pubkey-info-algorithm') + self.assertEqual(key_algo.value, x509.subject_public_key_algorithm) + self.assertEqual( + key_algo.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - pubkey-info-algorithm - {key_algo.value}' + ) + ) + self.assertEqual(key_exponent.type, 'text') + self.assertEqual(key_exponent.object_relation, 'pubkey-info-exponent') + self.assertEqual(key_exponent.value, x509.subject_public_key_exponent) + self.assertEqual( + key_exponent.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - pubkey-info-exponent - {key_exponent.value}' + ) + ) + self.assertEqual(key_modulus.type, 'text') + self.assertEqual(key_modulus.object_relation, 'pubkey-info-modulus') + self.assertEqual(key_modulus.value, x509.subject_public_key_modulus) + self.assertEqual( + key_modulus.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - pubkey-info-modulus - {key_modulus.value}' + ) + ) + self.assertEqual(not_before.type, 'datetime') + self.assertEqual(not_before.object_relation, 'validity-not-before') + self.assertEqual(not_before.value, x509.validity_not_before) + self.assertEqual( + not_before.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - validity-not-before - {not_before.value}' + ) + ) + self.assertEqual(not_after.type, 'datetime') + self.assertEqual(not_after.object_relation, 'validity-not-after') + self.assertEqual(not_after.value, x509.validity_not_after) + self.assertEqual( + not_after.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - validity-not-after - {not_after.value}' + ) + ) + self.assertEqual(version.type, 'text') + self.assertEqual(version.object_relation, 'version') + self.assertEqual(version.value, x509.version) + self.assertEqual( + version.uuid, + uuid5(self._UUIDv4, f'{object_id} - version - {version.value}') + ) class TestInternalSTIX2Import(TestSTIX2Import): def setUp(self): diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 9f325045..47e3a899 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -767,6 +767,89 @@ ] } ] +_X509_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "x509-certificate", + "is_self_signed": False, + "hashes": { + "MD5": "a39a417abcbe62460665c9da765aefbe", + "SHA-1": "bffc1a508d3c02d4a3f86941d3a99f7bf9ec3895", + "SHA-256": "caf71b19bf181230a3b203a6c3beaceb4d261409a8dbeef2e1d9eb4a5e0182c6" + }, + "version": "3", + "serial_number": "00:bb:ae:27:7a:c3:d9:cf:3f:85:00:86:a3:14:e7:0a:d7", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO RSA Code Signing CA", + "validity_not_before": "2017-11-12T00:00:00Z", + "validity_not_after": "2018-09-12T23:59:59Z", + "subject": "C=GB/postalCode=EC1V 2NX, ST=London, L=London/street=Kemp House, 160 City Road, O=CYBASICS LTD, CN=CYBASICS LTD", + "subject_public_key_algorithm": "rsaEncryption", + "subject_public_key_modulus": "c7:97:8a:4c:a0:6b:9c:91:d0:ed:7e:74:ca:8c:48:41:84:cf:fa:f1:07:ae:51:3f:d1:cb:3c:2e:43:1e:c3:dc:c2:e7:fa:60:cd:c7:25:2c:c4:2e:1c:e0:c2:a2:63:8b:df:f7:1a:55:c8:66:0d:eb:a9:a7:9e:f6:89:6e:ca:63:be:b8:75:18:56:6d:53:c1:8b:b4:f6:b5:04:6d:cc:0f:17:e0:b5:12:70:d6:b5:55:77:76:98:de:84:44:55:6d:5f:8a:a6:1e:a8:62:47:22:96:3d:5a:85:c9:9f:00:f3:3b:c6:ec:cb:68:ff:34:ab:73:d7:02:b6:29:aa:ff:87:1b:39:87:e5:0f:fd:f0:6a:d6:de:81:a3:e6:05:61:5d:84:6c:1f:5e:20:ae:c1:93:56:45:37:b7:c0:d6:6d:ab:27:f6:98:70:cf:a2:9b:c8:4a:04:2d:dc:01:fb:1a:f1:dc:8f:4c:31:7c:c4:71:4a:1c:d7:81:ed:1a:04:cb:4d:aa:3b:37:94:d3:7d:14:c4:4c:0e:8d:eb:75:1c:26:46:35:1c:83:2a:09:cf:41:c9:cb:c6:8c:a6:db:28:90:48:17:92:ff:70:db:5c:4f:d2:27:1a:51:2b:1f:12:f8:f6:ee:8a:88:15:fd:68:13:f0:7a:50:4f:8e:23:d5:4d:51", + "subject_public_key_exponent": 65537 + }, + "1": { + "type": "x509-certificate", + "is_self_signed": False, + "hashes": { + "MD5": "219794f8f6128c731f476d11e7fa5d4f", + "SHA-1": "d02be9aa68a05fdf7e99899a9719f275db5e6b2f", + "SHA-256": "0adb35fcd170c6da0e45a00c9b36533b21dc2bcf793e6facf0eb30829cbcc5fb" + }, + "version": "3", + "serial_number": "00:bc:b4:e7:32:76:0e:ca:64:31:8e:17:6c:fd:4a:ef:30", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Code Signing CA", + "validity_not_before": "2015-12-08T00:00:00Z", + "validity_not_after": "2016-12-07T23:59:59Z", + "subject": "/C=GB/postalCode=RG12 2LS/ST=Berkshire/L=Bracknell/street=15 Shepherds Hill/postOfficeBox=RG12 2LS/O=Network Software Ltd/CN=Network Software Ltd", + "subject_public_key_algorithm": "sha256WithRSAEncryption", + "subject_public_key_modulus": "00:ae:29:f8:d7:56:2f:fd:61:40:89:6f:cc:a3:1c:e0:49:0c:21:9f:5e:60:0c:a9:dc:cf:5f:79:83:fd:12:8f:f3:fc:c1:49:a3:e2:9c:a8:e9:d2:88:44:16:bd:39:2e:23:5b:84:e9:54:70:4b:ce:e3:c2:19:fd:a4:8b:45:ca:ad:aa:08:ae:cc:ab:8f:eb:60:74:fa:e0:2b:e5:d1:7b:5d:87:43:26:71:96:d1:ec:5f:23:15:40:37:0e:cc:b1:e1:5a:57:f1:24:58:2c:d6:04:f3:8e:34:9a:ea:bb:88:d5:9b:c3:38:8d:e4:90:7b:e7:ef:89:ea:31:92:97:46:80:f9:f8:b2:78:53:19:b8:66:15:37:af:32:08:58:3f:42:1a:67:f5:9a:40:b7:25:75:dc:3c:5f:b1:7c:12:63:f8:2b:60:93:b5:04:c4:10:9c:2d:1f:aa:9f:af:b1:e9:ee:70:21:fb:7e:aa:b3:1a:8e:e4:4c:18:6e:6a:5d:c4:61:e3:bd:83:d2:af:c6:ce:bc:f8:b8:0f:db:e0:9e:ec:f4:e2:61:99:ee:81:63:d1:71:e4:a7:2b:de:5c:0a:6d:2e:33:94:50:1f:33:e9:bb:1c:eb:e6:d2:18:3d:4f:02:02:dc:30:2e:52:19:4f:9c:0d:15:9d:56:f1:cb:30:59:57", + "subject_public_key_exponent": 65537 + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "x509-certificate", + "is_self_signed": False, + "hashes": { + "MD5": "09716af84e900e403494c28ad8c5869c", + "SHA-1": "1456d8a00d8be963e2224d845b12e5084ea0b707", + "SHA-256": "2d23636c25eb5c1b473e0ae66fdb076687b40bd080f161c79663572f171d5598" + }, + "version": "3", + "serial_number": "5e:15:20:5f:18:04:42:cc:6c:3c:0f:03:e1:a3:3d:9f", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "C=US, O=thawte, Inc., CN=thawte SHA256 Code Signing CA", + "validity_not_before": "2017-07-09T00:00:00Z", + "validity_not_after": "2018-07-09T23:59:59Z", + "subject": "C=GB, ST=London, L=London, O=Ziber Ltd, CN=Ziber Ltd", + "subject_public_key_algorithm": "rsaEncryption", + "subject_public_key_modulus": "00:e2:12:e4:5c:44:90:fa:0f:75:77:c8:88:51:21:1d:ce:b8:0e:f2:73:d5:68:79:02:50:51:5f:2c:a3:82:d1:48:60:f8:fa:c7:75:72:12:bc:b9:7c:d9:12:a8:1a:18:3a:f9:1d:a9:18:04:59:cd:8a:81:03:f7:0a:3d:22:6e:7d:63:65:d7:4d:c5:65:0e:fc:4f:97:9c:e0:3d:52:a4:d9:0b:d9:04:c3:f3:52:2a:a3:cc:e2:82:2c:2b:b8:54:1b:cc:41:2b:1b:76:d0:2a:fd:65:c4:3f:a2:4b:36:5f:5a:79:28:4b:98:1e:38:6c:b6:33:d2:3d:db:53:9c:0b:3f:2b:ab:87:2e:94:47:72:4f:27:58:8d:b0:b2:38:5f:1d:e0:67:53:6e:38:c7:ac:24:49:c9:b6:81:42:e0:06:95:26:c0:c9:bf:5e:7f:1b:92:f5:58:8e:8a:70:88:a9:e5:82:5c:5c:71:54:e0:74:1b:a9:33:1a:f2:3d:bf:9d:1b:45:1a:0e:02:d8:a3:d8:db:64:a9:f8:28:16:7f:4e:c3:ee:33:a1:be:18:72:e3:bd:79:12:54:ea:b9:77:9b:d0:d0:b0:2d:75:af:4d:47:4e:c1:16:84:a2:88:65:ef:18:ff:33:2a:ab:83:7c:43:14:ad:b8:cd:f0:b9:7c:c1:23", + "subject_public_key_exponent": 65537 + } + } + } +] class TestExternalSTIX20Bundles: @@ -928,3 +1011,7 @@ def get_bundle_with_software_objects(cls): @classmethod def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) + + @classmethod + def get_bundle_with_x509_objects(cls): + return cls.__assemble_bundle(*_X509_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index fb4c5d3f..a8190495 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -291,6 +291,18 @@ def _check_software_object(self, misp_object, observed_data, identifier): misp_object, observed_data.objects[identifier], object_id ) + def _check_x509_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'x509') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + self._check_x509_fields( + misp_object, observed_data.objects[identifier], object_id + ) + def test_stix20_bundle_with_artifact_object(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_artifact_objects() self.parser.load_stix_bundle(bundle) @@ -469,3 +481,16 @@ def test_stix20_bundle_with_url_attributes(self): self._check_generic_attribute( observed_data2, s_url, 'url' ) + + def test_stix20_bundle_with_x509_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_x509_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_x509_object(multiple1, observed_data1, '0') + self._check_x509_object(multiple2, observed_data1, '1') + self._check_x509_object(single, observed_data2) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 978e7afe..4239a570 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -929,6 +929,100 @@ ] } ] +_X509_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "x509-certificate--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "x509-certificate--5e384ae7-672c-4250-9cda-3b4da964451a" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "x509-certificate--f93cb275-0366-4ecc-abf0-a17928d1e177" + ] + }, + { + "type": "x509-certificate", + "spec_version": "2.1", + "id": "x509-certificate--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "is_self_signed": False, + "hashes": { + "MD5": "a39a417abcbe62460665c9da765aefbe", + "SHA-1": "bffc1a508d3c02d4a3f86941d3a99f7bf9ec3895", + "SHA-256": "caf71b19bf181230a3b203a6c3beaceb4d261409a8dbeef2e1d9eb4a5e0182c6" + }, + "version": "3", + "serial_number": "00:bb:ae:27:7a:c3:d9:cf:3f:85:00:86:a3:14:e7:0a:d7", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO RSA Code Signing CA", + "validity_not_before": "2017-11-12T00:00:00Z", + "validity_not_after": "2018-09-12T23:59:59Z", + "subject": "C=GB/postalCode=EC1V 2NX, ST=London, L=London/street=Kemp House, 160 City Road, O=CYBASICS LTD, CN=CYBASICS LTD", + "subject_public_key_algorithm": "rsaEncryption", + "subject_public_key_modulus": "c7:97:8a:4c:a0:6b:9c:91:d0:ed:7e:74:ca:8c:48:41:84:cf:fa:f1:07:ae:51:3f:d1:cb:3c:2e:43:1e:c3:dc:c2:e7:fa:60:cd:c7:25:2c:c4:2e:1c:e0:c2:a2:63:8b:df:f7:1a:55:c8:66:0d:eb:a9:a7:9e:f6:89:6e:ca:63:be:b8:75:18:56:6d:53:c1:8b:b4:f6:b5:04:6d:cc:0f:17:e0:b5:12:70:d6:b5:55:77:76:98:de:84:44:55:6d:5f:8a:a6:1e:a8:62:47:22:96:3d:5a:85:c9:9f:00:f3:3b:c6:ec:cb:68:ff:34:ab:73:d7:02:b6:29:aa:ff:87:1b:39:87:e5:0f:fd:f0:6a:d6:de:81:a3:e6:05:61:5d:84:6c:1f:5e:20:ae:c1:93:56:45:37:b7:c0:d6:6d:ab:27:f6:98:70:cf:a2:9b:c8:4a:04:2d:dc:01:fb:1a:f1:dc:8f:4c:31:7c:c4:71:4a:1c:d7:81:ed:1a:04:cb:4d:aa:3b:37:94:d3:7d:14:c4:4c:0e:8d:eb:75:1c:26:46:35:1c:83:2a:09:cf:41:c9:cb:c6:8c:a6:db:28:90:48:17:92:ff:70:db:5c:4f:d2:27:1a:51:2b:1f:12:f8:f6:ee:8a:88:15:fd:68:13:f0:7a:50:4f:8e:23:d5:4d:51", + "subject_public_key_exponent": 65537 + }, + { + "type": "x509-certificate", + "spec_version": "2.1", + "id": "x509-certificate--5e384ae7-672c-4250-9cda-3b4da964451a", + "is_self_signed": False, + "hashes": { + "MD5": "219794f8f6128c731f476d11e7fa5d4f", + "SHA-1": "d02be9aa68a05fdf7e99899a9719f275db5e6b2f", + "SHA-256": "0adb35fcd170c6da0e45a00c9b36533b21dc2bcf793e6facf0eb30829cbcc5fb" + }, + "version": "3", + "serial_number": "00:bc:b4:e7:32:76:0e:ca:64:31:8e:17:6c:fd:4a:ef:30", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Code Signing CA", + "validity_not_before": "2015-12-08T00:00:00Z", + "validity_not_after": "2016-12-07T23:59:59Z", + "subject": "/C=GB/postalCode=RG12 2LS/ST=Berkshire/L=Bracknell/street=15 Shepherds Hill/postOfficeBox=RG12 2LS/O=Network Software Ltd/CN=Network Software Ltd", + "subject_public_key_algorithm": "sha256WithRSAEncryption", + "subject_public_key_modulus": "00:ae:29:f8:d7:56:2f:fd:61:40:89:6f:cc:a3:1c:e0:49:0c:21:9f:5e:60:0c:a9:dc:cf:5f:79:83:fd:12:8f:f3:fc:c1:49:a3:e2:9c:a8:e9:d2:88:44:16:bd:39:2e:23:5b:84:e9:54:70:4b:ce:e3:c2:19:fd:a4:8b:45:ca:ad:aa:08:ae:cc:ab:8f:eb:60:74:fa:e0:2b:e5:d1:7b:5d:87:43:26:71:96:d1:ec:5f:23:15:40:37:0e:cc:b1:e1:5a:57:f1:24:58:2c:d6:04:f3:8e:34:9a:ea:bb:88:d5:9b:c3:38:8d:e4:90:7b:e7:ef:89:ea:31:92:97:46:80:f9:f8:b2:78:53:19:b8:66:15:37:af:32:08:58:3f:42:1a:67:f5:9a:40:b7:25:75:dc:3c:5f:b1:7c:12:63:f8:2b:60:93:b5:04:c4:10:9c:2d:1f:aa:9f:af:b1:e9:ee:70:21:fb:7e:aa:b3:1a:8e:e4:4c:18:6e:6a:5d:c4:61:e3:bd:83:d2:af:c6:ce:bc:f8:b8:0f:db:e0:9e:ec:f4:e2:61:99:ee:81:63:d1:71:e4:a7:2b:de:5c:0a:6d:2e:33:94:50:1f:33:e9:bb:1c:eb:e6:d2:18:3d:4f:02:02:dc:30:2e:52:19:4f:9c:0d:15:9d:56:f1:cb:30:59:57", + "subject_public_key_exponent": 65537 + }, + { + "type": "x509-certificate", + "spec_version": "2.1", + "id": "x509-certificate--f93cb275-0366-4ecc-abf0-a17928d1e177", + "is_self_signed": False, + "hashes": { + "MD5": "09716af84e900e403494c28ad8c5869c", + "SHA-1": "1456d8a00d8be963e2224d845b12e5084ea0b707", + "SHA-256": "2d23636c25eb5c1b473e0ae66fdb076687b40bd080f161c79663572f171d5598" + }, + "version": "3", + "serial_number": "5e:15:20:5f:18:04:42:cc:6c:3c:0f:03:e1:a3:3d:9f", + "signature_algorithm": "sha256WithRSAEncryption", + "issuer": "C=US, O=thawte, Inc., CN=thawte SHA256 Code Signing CA", + "validity_not_before": "2017-07-09T00:00:00Z", + "validity_not_after": "2018-07-09T23:59:59Z", + "subject": "C=GB, ST=London, L=London, O=Ziber Ltd, CN=Ziber Ltd", + "subject_public_key_algorithm": "rsaEncryption", + "subject_public_key_modulus": "00:e2:12:e4:5c:44:90:fa:0f:75:77:c8:88:51:21:1d:ce:b8:0e:f2:73:d5:68:79:02:50:51:5f:2c:a3:82:d1:48:60:f8:fa:c7:75:72:12:bc:b9:7c:d9:12:a8:1a:18:3a:f9:1d:a9:18:04:59:cd:8a:81:03:f7:0a:3d:22:6e:7d:63:65:d7:4d:c5:65:0e:fc:4f:97:9c:e0:3d:52:a4:d9:0b:d9:04:c3:f3:52:2a:a3:cc:e2:82:2c:2b:b8:54:1b:cc:41:2b:1b:76:d0:2a:fd:65:c4:3f:a2:4b:36:5f:5a:79:28:4b:98:1e:38:6c:b6:33:d2:3d:db:53:9c:0b:3f:2b:ab:87:2e:94:47:72:4f:27:58:8d:b0:b2:38:5f:1d:e0:67:53:6e:38:c7:ac:24:49:c9:b6:81:42:e0:06:95:26:c0:c9:bf:5e:7f:1b:92:f5:58:8e:8a:70:88:a9:e5:82:5c:5c:71:54:e0:74:1b:a9:33:1a:f2:3d:bf:9d:1b:45:1a:0e:02:d8:a3:d8:db:64:a9:f8:28:16:7f:4e:c3:ee:33:a1:be:18:72:e3:bd:79:12:54:ea:b9:77:9b:d0:d0:b0:2d:75:af:4d:47:4e:c1:16:84:a2:88:65:ef:18:ff:33:2a:ab:83:7c:43:14:ad:b8:cd:f0:b9:7c:c1:23", + "subject_public_key_exponent": 65537 + } +] class TestExternalSTIX21Bundles: @@ -1098,3 +1192,7 @@ def get_bundle_with_software_objects(cls): @classmethod def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) + + @classmethod + def get_bundle_with_x509_objects(cls): + return cls.__assemble_bundle(*_X509_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index e09f247d..0a4644ed 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -294,6 +294,11 @@ def _check_software_object(self, misp_object, observed_data, software): self._check_misp_object_fields(misp_object, observed_data, software) self._check_software_fields(misp_object, software, software.id) + def _check_x509_object(self, misp_object, observed_data, x509): + self.assertEqual(misp_object.name, 'x509') + self._check_misp_object_fields(misp_object, observed_data, x509) + self._check_x509_fields(misp_object, x509, x509.id) + def test_stix21_bundle_with_artifact_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_artifact_objects() self.parser.load_stix_bundle(bundle) @@ -445,3 +450,16 @@ def test_stix21_bundle_with_url_attributes(self): self._check_generic_attribute(od1, url_1, m_url1, 'url') self._check_generic_attribute(od1, url_2, m_url2, 'url') self._check_generic_attribute(od2, url_3, s_url, 'url') + + def test_stix21_bundle_with_x509_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_x509_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, cert1, cert2, cert3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._check_x509_object(multiple1, od1, cert1) + self._check_x509_object(multiple2, od1, cert2) + self._check_x509_object(single, od2, cert3) \ No newline at end of file From eaece3ca28fe3aca078f5d4fc87c78b59397a552 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 30 Jan 2024 16:01:23 +0100 Subject: [PATCH 039/137] fix: [stix2 import] Utilising the newly added `environment-variables` attribute to properly import the environment variables & arguments of a STIX 2.x process object --- .../converters/stix2_observable_converter.py | 30 +++++++++++-------- .../stix2misp/converters/stix2mapping.py | 7 +++++ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index d6f16559..262d64e7 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -357,19 +357,25 @@ def _parse_process_observable( yield from self._populate_object_attributes( mapping, getattr(observable, field), object_id ) - for feature in ('environment_variables', 'arguments'): - if hasattr(observable, feature): - value = ' '.join( - f'{key} {value}' for key, value in - getattr(observable, feature).items() + if hasattr(observable, 'arguments'): + value = ' '.join(observable.arguments) + yield { + **self._mappping.args_attribute(), + 'value': value, 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - args - {value}' ) - yield { - **self._mapping.args_attribute(), - 'value': value, 'uuid': self.main_parser._create_v5_uuid( - f'{object_id} - args - {value}' - ) - } - break + } + if hasattr(observable, 'environment_variables'): + value = ' - '.join( + f'{key}: {value}' for key, value in + observable.environment_variables.items() + ) + yield { + **self._mapping.environment_variables_attribute(), + 'value': value, 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - environment-variables - {value}' + ) + } def _parse_registry_key_observable( self, observable: _REGISTRY_KEY_TYPING, diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 61f248dc..f3e7b80b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -92,6 +92,9 @@ class STIX2Mapping: __entrypoint_address_attribute = Mapping( **{'type': 'text', 'object_relation': 'entrypoint-address'} ) + __environment_variables_attribute = Mapping( + **{"type": "text", "object_relation": "environment-variables"} + ) __file_encoding_attribute = Mapping( **{'type': 'text', 'object_relation': 'file-encoding'} ) @@ -504,6 +507,10 @@ def email_subject_attribute(cls) -> dict: def entropy_attribute(cls) -> dict: return cls.__entropy_attribute + @classmethod + def environment_variables_attribute(cls) -> dict: + return cls.__environment_variables_attribute + @classmethod def entrypoint_address_attribute(cls) -> dict: return cls.__entrypoint_address_attribute From b3a47115375c9072c155fc695b607df992fb7b99 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 30 Jan 2024 23:27:07 +0100 Subject: [PATCH 040/137] wip: [stix2 import] Parsing Process observable objects from converters --- .../stix2_observed_data_converter.py | 138 +++++++++++++++++- 1 file changed, 134 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 3892978e..9b224c4a 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -18,15 +18,17 @@ Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, - MACAddress as MACAddress_v20, Mutex as Mutex_v20, Software as Software_v20, - URL as URL_v20, X509Certificate as X509Certificate_v20) + MACAddress as MACAddress_v20, Mutex as Mutex_v20, Process as Process_v20, + Software as Software_v20, URL as URL_v20, + X509Certificate as X509Certificate_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( Artifact as Artifact_v21, AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, - MACAddress as MACAddress_v21, Mutex as Mutex_v21, Software as Software_v21, - URL as URL_v21, X509Certificate as X509Certificate_v21) + MACAddress as MACAddress_v21, Mutex as Mutex_v21, Process as Process_v21, + Software as Software_v21, URL as URL_v21, + X509Certificate as X509Certificate_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Iterator, Optional, TYPE_CHECKING, Union @@ -38,6 +40,7 @@ Artifact_v20, Artifact_v21, Directory_v20, Directory_v21, Software_v20, Software_v21, + Process_v20, Process_v21, X509Certificate_v20, X509Certificate_v21 ] _GENERIC_OBSERVABLE_TYPING = Union[ @@ -52,6 +55,7 @@ Artifact_v20, Artifact_v21, AutonomousSystem_v20, AutonomousSystem_v21, Directory_v20, Directory_v21, + Process_v20, Process_v21, Software_v20, Software_v21, X509Certificate_v20, X509Certificate_v21 ] @@ -744,6 +748,132 @@ def _parse_mutex_observable_objects( observed_data, identifier, 'mutex', feature='name' ) + def _parse_process_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.objects: + observable = self._fetch_observables(object_ref) + if object_ref.startswith('file--'): + if not observable['used'].get(self.event_uuid, False): + misp_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, 'file', False + ) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + continue + process = observable['observable'] + misp_object = ( + observable['misp_object'] if + observable['used'][self.event_uuid] else + self._parse_generic_observable_object_ref( + process, observed_data, 'process', False + ) + ) + observable['used'][self.event_uuid] = True + if hasattr(process, 'parent_ref'): + self._parse_process_reference_observable_object_ref( + observed_data, misp_object, process.parent_ref, 'child-of' + ) + if hasattr(process, 'child_refs'): + for child_ref in process.child_refs: + self._parse_process_reference_observable_object_ref( + observed_data, misp_object, child_ref, 'parent-of' + ) + if hasattr(process, 'image_ref'): + self._parse_process_reference_observable_object_ref( + observed_data, misp_object, process.image_ref, + 'executes', name='file' + ) + + def _parse_process_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + return self._parse_generic_single_observable_object( + observed_data, 'process' + ) + observable_objects = { + object_id: {'used': False, 'observable': observable} + for object_id, observable in observed_data.objects.items() + } + for object_id, observable in observable_objects.items(): + if observable['observable'].type == 'file': + if not observable['used']: + misp_object = self._parse_generic_observable_object( + observed_data, object_id, 'file', False + ) + observable['used'] = True + observable['misp_object'] = misp_object + continue + process = observable['observable'] + misp_object = ( + observable['misp_object'] if observable['used'] else + self._parse_generic_observable_object( + observed_data, object_id, 'process', False + ) + ) + observable['misp_object'] = misp_object + observable['used'] = True + if hasattr(process, 'parent_ref'): + self._parse_process_reference_observable_object( + observed_data, misp_object, + observable_objects[process.parent_ref], + process.parent_ref, 'child-of' + ) + if hasattr(process, 'child_refs'): + for child_ref in process.child_refs: + self._parse_process_reference_observable_object( + observed_data, misp_object, + observable_objects[child_ref], + child_ref, 'parent-of' + ) + for feature in ('binary', 'image'): + if hasattr(process, f'{feature}_ref'): + reference = getattr(process, f'{feature}_ref') + self._parse_process_reference_observable_object( + observed_data, misp_object, + observable_objects[reference], + reference, 'executes', name='file' + ) + + def _parse_process_reference_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, misp_object: MISPObject, + observable: dict, reference: str, relationship_type: str, + name: Optional[str] = 'process'): + if observable['used']: + misp_object.add_reference( + observable['misp_object'].uuid, relationship_type + ) + return + referenced_object = self._parse_generic_observable_object( + observed_data, reference, name, False + ) + misp_object.add_reference(referenced_object.uuid, relationship_type) + observable.update({'used': True, 'misp_object': referenced_object}) + + def _parse_process_reference_observable_object_ref( + self, observed_data: _OBSERVED_DATA_TYPING, misp_object: MISPObject, + reference: str, relationship_type: str, + name: Optional[str] = 'process'): + observable = self._fetch_observables(reference) + if observable['used'].get(self.event_uuid, False): + misp_object.add_reference( + observable['misp_object'].uuid, relationship_type + ) + return + if reference in observed_data.object_refs: + referenced_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, name, False + ) + observable['misp_object'] = referenced_object + observable['used'][self.event_uuid] = True + misp_object.add_reference(referenced_object.uuid, relationship_type) + else: + self.observable_relationships[misp_object.uuid].add( + ( + self.main_parser._sanitise_uuid(reference), + relationship_type + ) + ) + def _parse_software_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: From be6bfed5f9d820b2daf4bc18a7f77fb8fa920189 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 31 Jan 2024 13:36:29 +0100 Subject: [PATCH 041/137] fix: [stix2 import] Quick fix in the Process observable objects associated with Observed Data objects conversion method --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9b224c4a..f0337eeb 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -750,7 +750,7 @@ def _parse_mutex_observable_objects( def _parse_process_observable_object_refs( self, observed_data: ObservedData_v21): - for object_ref in observed_data.objects: + for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) if object_ref.startswith('file--'): if not observable['used'].get(self.event_uuid, False): @@ -763,7 +763,7 @@ def _parse_process_observable_object_refs( process = observable['observable'] misp_object = ( observable['misp_object'] if - observable['used'][self.event_uuid] else + observable['used'].get(self.event_uuid, False) else self._parse_generic_observable_object_ref( process, observed_data, 'process', False ) From 1a0cb5eb60c31062c0324dfec472cf3aeb4cffaa Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 31 Jan 2024 13:37:57 +0100 Subject: [PATCH 042/137] wip: [tests] Tests for external STIX 2.x Process observable objects associated with Observed Data object import as MISP `process` objects --- tests/_test_stix_import.py | 166 ++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 79 ++++++++++++ tests/test_external_stix20_import.py | 56 +++++++++ tests/test_external_stix21_bundles.py | 99 +++++++++++++++ tests/test_external_stix21_import.py | 44 +++++++ 5 files changed, 444 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 65b19b56..e30ccead 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -485,6 +485,172 @@ def _check_directory_fields(self, misp_object, directory, object_id): ) return accessed.value, created.value, modified.value + def _check_process_child_fields(self, misp_object, process, object_id): + self.assertEqual(len(misp_object.attributes), 2) + name, pid = misp_object.attributes + self._assert_multiple_equal(name.type, pid.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, process.name) + self.assertEqual( + name.uuid, uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(pid.object_relation, 'pid') + self.assertEqual(pid.value, process.pid) + self.assertEqual( + pid.uuid, uuid5(self._UUIDv4, f'{object_id} - pid - {pid.value}') + ) + + def _check_process_image_reference_fields(self, misp_object, file_object, object_id): + self.assertEqual(len(misp_object.attributes), 4) + mimetype, filename, encoding, size = misp_object.attributes + self.assertEqual(mimetype.type, 'mime-type') + self.assertEqual(mimetype.object_relation, 'mimetype') + self.assertEqual(mimetype.value, file_object.mime_type) + self.assertEqual( + mimetype.uuid, + uuid5(self._UUIDv4, f'{object_id} - mimetype - {mimetype.value}') + ) + self._assert_multiple_equal( + filename.type, filename.object_relation, 'filename' + ) + self.assertEqual(filename.value, file_object.name) + self.assertEqual( + filename.uuid, + uuid5(self._UUIDv4, f'{object_id} - filename - {filename.value}') + ) + self.assertEqual(encoding.type, 'text') + self.assertEqual(encoding.object_relation, 'file-encoding') + self.assertEqual(encoding.value, file_object.name_enc) + self.assertEqual( + encoding.uuid, + uuid5( + self._UUIDv4, f'{object_id} - file-encoding - {encoding.value}' + ) + ) + self._assert_multiple_equal( + size.type, size.object_relation, 'size-in-bytes' + ) + self.assertEqual(size.value, file_object.size) + self.assertEqual( + size.uuid, + uuid5(self._UUIDv4, f'{object_id} - size-in-bytes - {size.value}') + ) + + def _check_process_multiple_fields(self, misp_object, process, object_id): + self.assertEqual(len(misp_object.attributes), 3) + hidden, name, pid = misp_object.attributes + self.assertEqual(hidden.type, 'boolean') + self.assertEqual(hidden.object_relation, 'hidden') + self.assertEqual(hidden.value, process.is_hidden) + self.assertEqual( + hidden.uuid, + uuid5(self._UUIDv4, f'{object_id} - hidden - {hidden.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, process.name) + self.assertEqual( + name.uuid, + uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(pid.type, 'text') + self.assertEqual(pid.object_relation, 'pid') + self.assertEqual(pid.value, process.pid) + self.assertEqual( + pid.uuid, + uuid5(self._UUIDv4, f'{object_id} - pid - {pid.value}') + ) + + def _check_process_parent_fields(self, misp_object, process, object_id): + self.assertEqual(len(misp_object.attributes), 6) + command_line, creation_time, directory, name, pid, variables = misp_object.attributes + self._assert_multiple_equal( + command_line.type, directory.type, name.type, pid.type, + variables.type, 'text' + ) + self.assertEqual(command_line.object_relation, 'command-line') + self.assertEqual(command_line.value, process.command_line) + self.assertEqual( + command_line.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - command-line - {command_line.value}' + ) + ) + self.assertEqual(creation_time.type, 'datetime') + self.assertEqual(creation_time.object_relation, 'creation-time') + self.assertEqual( + creation_time.value, + getattr(process, 'created', process.created_time) + ) + self.assertEqual( + creation_time.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - creation-time - {creation_time.value}' + ) + ) + self.assertEqual(directory.object_relation, 'current-directory') + self.assertEqual(directory.value, process.cwd) + self.assertEqual( + directory.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - current-directory - {directory.value}' + ) + ) + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, process.name) + self.assertEqual( + name.uuid, uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(pid.object_relation, 'pid') + self.assertEqual(pid.value, process.pid) + self.assertEqual( + pid.uuid, uuid5(self._UUIDv4, f'{object_id} - pid - {pid.value}') + ) + self.assertEqual(variables.object_relation, 'environment-variables') + self.assertEqual( + variables.value, + ' - '.join( + f'{key}: {value}' for key, value in + process.environment_variables.items() + ) + ) + self.assertEqual( + variables.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - environment-variables - {variables.value}' + ) + ) + + def _check_process_single_fields(self, misp_object, process, object_id): + self.assertEqual(len(misp_object.attributes), 3) + command_line, name, pid = misp_object.attributes + self._assert_multiple_equal(command_line.type, name.type, pid.type, 'text') + self.assertEqual(command_line.object_relation, 'command-line') + self.assertEqual(command_line.value, process.command_line) + self.assertEqual( + command_line.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - command-line - {command_line.value}' + ) + ) + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, process.name) + self.assertEqual( + name.uuid, + uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self.assertEqual(pid.object_relation, 'pid') + self.assertEqual(pid.value, process.pid) + self.assertEqual( + pid.uuid, + uuid5(self._UUIDv4, f'{object_id} - pid - {pid.value}') + ) + def _check_software_fields(self, misp_object, software, object_id): self.assertEqual(len(misp_object.attributes), 4) language, name, vendor, version = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 47e3a899..9c8ef818 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -547,6 +547,81 @@ } } ] +_PROCESS_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "process", + "pid": 2510, + "name": "TestProcess", + "binary_ref": "4", + "parent_ref": "1", + "child_refs": [ + "3" + ], + "is_hidden": True + }, + "1": { + "type": "process", + "pid": 2107, + "name": "Friends_From_H", + "cwd": "/home/viktor", + "created": "2017-05-01T08:00:00Z", + "command_line": "grep -nrG iglocska ${HOME}/friends.txt", + "environment_variables": { + "HOME": "/home/viktor", + "USER": "viktor" + }, + "binary_ref": "2" + }, + "2": { + "type": "file", + "name": "parent_process.exe", + "size": 12367, + "name_enc": "UTF-8", + "mime_type": "application/exe" + }, + "3": { + "type": "process", + "pid": 1401, + "name": "ChildProcess" + }, + "4": { + "type": "file", + "name": "test_process.exe", + "size": 82639, + "name_enc": "UTF-8", + "mime_type": "application/exe" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "process", + "pid": 666, + "name": "SatanProcess", + "command_line": "rm -rf *" + } + } + } +] _SOFTWARE_OBJECTS = [ { "type": "observed-data", @@ -1004,6 +1079,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_process_objects(cls): + return cls.__assemble_bundle(*_PROCESS_OBJECTS) + @classmethod def get_bundle_with_software_objects(cls): return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index a8190495..6770bd1b 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -447,6 +447,62 @@ def test_stix20_bundle_with_mutex_attributes(self): observed_data2, s_mutex, 'mutex', feature='name' ) + def test_stix20_bundle_with_process_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_process_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 6) + multiple, parent, child, image1, image2, single = misp_objects + self._assert_multiple_equal( + multiple.name, parent.name, child.name, single.name, 'process' + ) + self._assert_multiple_equal(image1.name, image2.name, 'file') + + self._check_misp_object_fields(multiple, observed_data1, '0') + self._check_process_multiple_fields( + multiple, observed_data1.objects['0'], f'{observed_data1.id} - 0' + ) + self.assertEqual(len(multiple.references), 3) + child_ref, parent_ref, binary_ref = multiple.references + self.assertEqual(child_ref.referenced_uuid, parent.uuid) + self.assertEqual(child_ref.relationship_type, 'child-of') + self.assertEqual(parent_ref.referenced_uuid, child.uuid) + self.assertEqual(parent_ref.relationship_type, 'parent-of') + self.assertEqual(binary_ref.referenced_uuid, image1.uuid) + self.assertEqual(binary_ref.relationship_type, 'executes') + + self._check_misp_object_fields(parent, observed_data1, '1') + self._check_process_parent_fields( + parent, observed_data1.objects['1'], f'{observed_data1.id} - 1' + ) + self.assertEqual(len(parent.references), 1) + reference = parent.references[0] + self.assertEqual(reference.referenced_uuid, image2.uuid) + self.assertEqual(reference.relationship_type, 'executes') + + self._check_misp_object_fields(child, observed_data1, '3') + self._check_process_child_fields( + child, observed_data1.objects['3'], f'{observed_data1.id} - 3' + ) + + self._check_misp_object_fields(image2, observed_data1, '2') + self._check_process_image_reference_fields( + image2, observed_data1.objects['2'], f'{observed_data1.id} - 2' + ) + + self._check_misp_object_fields(image1, observed_data1, '4') + self._check_process_image_reference_fields( + image1, observed_data1.objects['4'], f'{observed_data1.id} - 4' + ) + + self._check_misp_object_fields(single, observed_data2) + self._check_process_single_fields( + single, observed_data2.objects['0'], observed_data2.id + ) + def test_stix20_bundle_with_software_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_software_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 4239a570..198908a5 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -680,6 +680,101 @@ "name": "sensitive_resource_lock" } ] +_PROCESS_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "process--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "process--5e384ae7-672c-4250-9cda-3b4da964451a", + "file--d1385ba1-69de-4774-879c-f2c4771b369d", + "process--f93cb275-0366-4ecc-abf0-a17928d1e177", + "file--43fdb7b9-a771-4b10-ab74-2bac893daf0d" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "process--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0" + ] + }, + { + "type": "process", + "spec_version": "2.1", + "id": "process--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "pid": 2510, + "name": "TestProcess", + "image_ref": "file--43fdb7b9-a771-4b10-ab74-2bac893daf0d", + "parent_ref": "process--5e384ae7-672c-4250-9cda-3b4da964451a", + "child_refs": [ + "process--f93cb275-0366-4ecc-abf0-a17928d1e177" + ], + "is_hidden": True + }, + { + "type": "process", + "spec_version": "2.1", + "id": "process--5e384ae7-672c-4250-9cda-3b4da964451a", + "pid": 2107, + "name": "Friends_From_H", + "cwd": "/home/viktor", + "created_time": "2017-05-01T08:00:00Z", + "command_line": "grep -nrG iglocska ${HOME}/friends.txt", + "environment_variables": { + "HOME": "/home/viktor", + "USER": "viktor" + }, + "image_ref": "file--d1385ba1-69de-4774-879c-f2c4771b369d" + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--d1385ba1-69de-4774-879c-f2c4771b369d", + "name": "parent_process.exe", + "size": 12367, + "name_enc": "UTF-8", + "mime_type": "application/exe" + }, + { + "type": "process", + "spec_version": "2.1", + "id": "process--f93cb275-0366-4ecc-abf0-a17928d1e177", + "pid": 1401, + "name": "ChildProcess" + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--43fdb7b9-a771-4b10-ab74-2bac893daf0d", + "name": "test_process.exe", + "size": 82639, + "name_enc": "UTF-8", + "mime_type": "application/exe" + }, + { + "type": "process", + "spec_version": "2.1", + "id": "process--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0", + "pid": 666, + "name": "SatanProcess", + "command_line": "rm -rf *" + } +] _SOFTWARE_OBJECTS = [ { "type": "observed-data", @@ -1185,6 +1280,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_process_objects(cls): + return cls.__assemble_bundle(*_PROCESS_OBJECTS) + @classmethod def get_bundle_with_software_objects(cls): return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 0a4644ed..d6f38d75 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -424,6 +424,50 @@ def test_stix21_bundle_with_mutex_attributes(self): self._check_generic_attribute(od1, mutex_2, m_mutex2, 'mutex', 'name') self._check_generic_attribute(od2, mutex_3, s_mutex, 'mutex', 'name') + def test_stix21_bundle_with_process_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_process_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, process1, process2, file1, process3, file2, process4 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 6) + multiple, parent, child, image1, image2, single = misp_objects + self._assert_multiple_equal( + multiple.name, parent.name, child.name, single.name, 'process' + ) + self._assert_multiple_equal(image1.name, image2.name, 'file') + + self._check_misp_object_fields(multiple, od1, process1) + self._check_process_multiple_fields(multiple, process1, process1.id) + self.assertEqual(len(multiple.references), 3) + child_ref, parent_ref, binary_ref = multiple.references + self.assertEqual(child_ref.referenced_uuid, parent.uuid) + self.assertEqual(child_ref.relationship_type, 'child-of') + self.assertEqual(parent_ref.referenced_uuid, child.uuid) + self.assertEqual(parent_ref.relationship_type, 'parent-of') + self.assertEqual(binary_ref.referenced_uuid, image1.uuid) + self.assertEqual(binary_ref.relationship_type, 'executes') + + self._check_misp_object_fields(parent, od1, process2) + self._check_process_parent_fields(parent, process2, process2.id) + self.assertEqual(len(parent.references), 1) + reference = parent.references[0] + self.assertEqual(reference.referenced_uuid, image2.uuid) + self.assertEqual(reference.relationship_type, 'executes') + + self._check_misp_object_fields(child, od1, process3) + self._check_process_child_fields(child, process3, process3.id) + + self._check_misp_object_fields(image1, od1, file2) + self._check_process_image_reference_fields(image1, file2, file2.id) + + self._check_misp_object_fields(image2, od1, file1) + self._check_process_image_reference_fields(image2, file1, file1.id) + + self._check_misp_object_fields(single, od2, process4) + self._check_process_single_fields(single, process4, process4.id) + def test_stix21_bundle_with_software_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_software_objects() self.parser.load_stix_bundle(bundle) From 4df743eea316c9b67bef671008334820fe95c3bd Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 1 Feb 2024 11:38:54 +0100 Subject: [PATCH 043/137] fix: [stix2 import] Avoid future potential issues with object names in generic conversion methods - When an object name has at least one `-` and we want to use the related mapping, we need to `replace('-', '_')` to avoid issues with mapping names --- .../converters/stix2_observed_data_converter.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index f0337eeb..5f80d244 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -562,9 +562,10 @@ def _parse_generic_observable_object( misp_object = self._create_misp_object_from_observable_object( name, observed_data, object_id ) + _name = name.replace('-', '_') attributes = ( - self._parse_generic_observable(observable_object, name, object_id) - if generic else getattr(self, f'_parse_{name}_observable')( + self._parse_generic_observable(observable_object, _name, object_id) + if generic else getattr(self, f'_parse_{_name}_observable')( observable_object, object_id ) ) @@ -600,9 +601,10 @@ def _parse_generic_observable_object_ref( misp_object = self._create_misp_object_from_observable_object_ref( name, observable_object, observed_data ) + _name = name.replace('-', '_') attributes = ( - self._parse_generic_observable(observable_object, name) - if generic else getattr(self, f'_parse_{name}_observable')( + self._parse_generic_observable(observable_object, _name) + if generic else getattr(self, f'_parse_{_name}_observable')( observable_object ) ) @@ -638,10 +640,11 @@ def _parse_generic_single_observable_object( misp_object = self._create_misp_object_from_observable_object( name, observed_data ) + _name = name.replace('-', '_') object_id = observed_data.id attributes = ( - self._parse_generic_observable(observable_object, name, object_id) - if generic else getattr(self, f'_parse_{name}_observable')( + self._parse_generic_observable(observable_object, _name, object_id) + if generic else getattr(self, f'_parse_{_name}_observable')( observable_object, object_id ) ) From 3b38a218ec855cdfff608f55b7ec462e12af5580 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 1 Feb 2024 12:01:26 +0100 Subject: [PATCH 044/137] fix: [tests] Quick fix on the `created` or `created_time` field from a process observable object --- tests/_test_stix_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index e30ccead..e12e8a0c 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -581,7 +581,7 @@ def _check_process_parent_fields(self, misp_object, process, object_id): self.assertEqual(creation_time.object_relation, 'creation-time') self.assertEqual( creation_time.value, - getattr(process, 'created', process.created_time) + process.created if hasattr(process, 'created') else process.created_time ) self.assertEqual( creation_time.uuid, From 64d31c85780903697b9fa087ecae93cbe0a13867 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 1 Feb 2024 23:54:24 +0100 Subject: [PATCH 045/137] wip: [stix2 import] Parsing external STIX 2.x User Account observable objects from converters --- .../converters/stix2_observable_converter.py | 11 +++++--- .../stix2_observed_data_converter.py | 27 +++++++++++++++++-- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 262d64e7..7d51d9ef 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -613,20 +613,23 @@ def _parse_url_observable(self, observable: _URL_TYPING, ) def _parse_user_account_observable( - self, observable: _USER_ACCOUNT_TYPING) -> Iterator[dict]: + self, observable: _USER_ACCOUNT_TYPING, + object_id: Optional[str] = None) -> Iterator[dict]: + if object_id is None: + object_id = observable.id user_account_mapping = self._mapping.user_account_object_mapping for field, attribute in user_account_mapping().items(): if hasattr(observable, field): yield from self._populate_object_attributes( - attribute, getattr(observable, field), observable.id + attribute, getattr(observable, field), object_id ) if 'unix-account-ext' in getattr(observable, 'extensions', {}): extension = observable.extensions['unix-account-ext'] - mapping = self._mapping.unix_user_account_extension_mapping + mapping = self._mapping.unix_user_account_extension_object_mapping for field, attribute in mapping().items(): if hasattr(extension, field): yield from self._populate_object_attributes( - attribute, getattr(extension, field), observable.id + attribute, getattr(extension, field), object_id ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 5f80d244..b709fce4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -19,7 +19,7 @@ Directory as Directory_v20, DomainName as DomainName_v20, IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, MACAddress as MACAddress_v20, Mutex as Mutex_v20, Process as Process_v20, - Software as Software_v20, URL as URL_v20, + Software as Software_v20, URL as URL_v20, UserAccount as UserAccount_v20, X509Certificate as X509Certificate_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( @@ -27,7 +27,7 @@ Directory as Directory_v21, DomainName as DomainName_v21, IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, MACAddress as MACAddress_v21, Mutex as Mutex_v21, Process as Process_v21, - Software as Software_v21, URL as URL_v21, + Software as Software_v21, URL as URL_v21, UserAccount as UserAccount_v21, X509Certificate as X509Certificate_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Iterator, Optional, TYPE_CHECKING, Union @@ -41,6 +41,7 @@ Directory_v20, Directory_v21, Software_v20, Software_v21, Process_v20, Process_v21, + UserAccount_v20, UserAccount_v21, X509Certificate_v20, X509Certificate_v21 ] _GENERIC_OBSERVABLE_TYPING = Union[ @@ -57,6 +58,7 @@ Directory_v20, Directory_v21, Process_v20, Process_v21, Software_v20, Software_v21, + UserAccount_v20, UserAccount_v21, X509Certificate_v20, X509Certificate_v21 ] _OBSERVED_DATA_TYPING = Union[ @@ -931,6 +933,27 @@ def _parse_url_observable_objects( observed_data, identifier, 'url' ) + def _parse_user_account_observable_objects( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + user_account = observable['observable'] + self._parse_generic_observable_object( + user_account, observed_data, 'user-account', False + ) + observable['used'][self.event_uuid] = True + + def _parse_user_account_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + return self._parse_generic_single_observable_object( + observed_data, 'user-account', False + ) + for identifier in observed_data.objects: + self._parse_generic_observable_object( + observed_data, identifier, 'user-account', False + ) + def _parse_x509_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: From 406343b87777d5becc767e270c1daa6075a99ae0 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 2 Feb 2024 16:15:40 +0100 Subject: [PATCH 046/137] fix: [stix2 import] Copy-paste typo --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index b709fce4..38b7b8ff 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -933,12 +933,12 @@ def _parse_url_observable_objects( observed_data, identifier, 'url' ) - def _parse_user_account_observable_objects( + def _parse_user_account_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) user_account = observable['observable'] - self._parse_generic_observable_object( + self._parse_generic_observable_object_ref( user_account, observed_data, 'user-account', False ) observable['used'][self.event_uuid] = True From da8b36db39a0b4fa0e93be308ee29000a3d21ce9 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 2 Feb 2024 16:25:49 +0100 Subject: [PATCH 047/137] wip: [tests] Tests for External STIX 2.x User Account observable objects import as MISP objects --- tests/_test_stix_import.py | 164 ++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 69 +++++++++++ tests/test_external_stix20_import.py | 52 ++++++++ tests/test_external_stix21_bundles.py | 80 +++++++++++++ tests/test_external_stix21_import.py | 45 +++++++ 5 files changed, 410 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index e12e8a0c..cad3eacb 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -742,6 +742,170 @@ def _check_software_with_swid_fields(self, misp_object, software, object_id): uuid5(self._UUIDv4, f'{object_id} - version - {version.value}') ) + def _check_user_account_extension_fields(self, attributes, extension, object_id): + group_id, group, home_dir, shell = attributes + self._assert_multiple_equal( + group_id.type, group.type, home_dir.type, shell.type, 'text' + ) + self.assertEqual(group_id.object_relation, 'group-id') + self.assertEqual(group_id.value, extension.gid) + self.assertEqual( + group_id.uuid, + uuid5(self._UUIDv4, f'{object_id} - group-id - {group_id.value}') + ) + self.assertEqual(group.object_relation, 'group') + self.assertEqual(group.value, extension.groups[0]) + self.assertEqual( + group.uuid, + uuid5(self._UUIDv4, f'{object_id} - group - {group.value}') + ) + self.assertEqual(home_dir.object_relation, 'home_dir') + self.assertEqual(home_dir.value, extension.home_dir) + self.assertEqual( + home_dir.uuid, + uuid5(self._UUIDv4, f'{object_id} - home_dir - {home_dir.value}') + ) + self.assertEqual(shell.object_relation, 'shell') + self.assertEqual(shell.value, extension.shell) + self.assertEqual( + shell.uuid, + uuid5(self._UUIDv4, f'{object_id} - shell - {shell.value}') + ) + + def _check_user_account_fields(self, attributes, user_account, object_id): + username, account_type, escalate, display_name, privileged, service, user_id = attributes + self._assert_multiple_equal( + username.type, account_type.type, display_name.type, user_id.type, 'text' + ) + self._assert_multiple_equal( + escalate.type, privileged.type, service.type, 'boolean' + ) + self.assertEqual(username.object_relation, 'username') + self.assertEqual(username.value, user_account.account_login) + self.assertEqual( + username.uuid, + uuid5(self._UUIDv4, f'{object_id} - username - {username.value}') + ) + self.assertEqual(account_type.object_relation, 'account-type') + self.assertEqual(account_type.value, user_account.account_type) + self.assertEqual( + account_type.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - account-type - {account_type.value}' + ) + ) + self.assertEqual(escalate.object_relation, 'can_escalate_privs') + self.assertEqual(escalate.value, user_account.can_escalate_privs) + self.assertEqual( + escalate.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - can_escalate_privs - {escalate.value}' + ) + ) + self.assertEqual(display_name.object_relation, 'display-name') + self.assertEqual(display_name.value, user_account.display_name) + self.assertEqual( + display_name.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - display-name - {display_name.value}' + ) + ) + self.assertEqual(privileged.object_relation, 'privileged') + self.assertEqual(privileged.value, user_account.is_privileged) + self.assertEqual( + privileged.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - privileged - {privileged.value}' + ) + ) + self.assertEqual(service.object_relation, 'is_service_account') + self.assertEqual(service.value, user_account.is_service_account) + self.assertEqual( + service.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - is_service_account - {service.value}' + ) + ) + self.assertEqual(user_id.object_relation, 'user-id') + self.assertEqual(user_id.value, user_account.user_id) + self.assertEqual( + user_id.uuid, + uuid5(self._UUIDv4, f'{object_id} - user-id - {user_id.value}') + ) + + def _check_user_account_timeline_fields(self, attributes, user_account, object_id): + created, first_login, last_login = attributes + self._assert_multiple_equal( + created.type, first_login.type, last_login.type, 'datetime' + ) + self.assertEqual(created.object_relation, 'created') + self.assertEqual(created.value, user_account.account_created) + self.assertEqual( + created.uuid, + uuid5(self._UUIDv4, f'{object_id} - created - {created.value}') + ) + self.assertEqual(first_login.object_relation, 'first_login') + self.assertEqual(first_login.value, user_account.account_first_login) + self.assertEqual( + first_login.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - first_login - {first_login.value}' + ) + ) + self.assertEqual(last_login.object_relation, 'last_login') + self.assertEqual(last_login.value, user_account.account_last_login) + self.assertEqual( + last_login.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - last_login - {last_login.value}' + ) + ) + + def _check_user_account_twitter_fields(self, misp_object, user_account, object_id): + self.assertEqual(len(misp_object.attributes), 4) + username, account_type, display_name, user_id = misp_object.attributes + self._assert_multiple_equal( + username.type, account_type.type, display_name.type, + user_id.type, 'text' + ) + self.assertEqual(username.object_relation, 'username') + self.assertEqual(username.value, user_account.account_login) + self.assertEqual( + username.uuid, + uuid5(self._UUIDv4, f'{object_id} - username - {username.value}') + ) + self.assertEqual(account_type.object_relation, 'account-type') + self.assertEqual(account_type.value, user_account.account_type) + self.assertEqual( + account_type.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - account-type - {account_type.value}' + ) + ) + self.assertEqual(display_name.object_relation, 'display-name') + self.assertEqual(display_name.value, user_account.display_name) + self.assertEqual( + display_name.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - display-name - {display_name.value}' + ) + ) + self.assertEqual(user_id.object_relation, 'user-id') + self.assertEqual(user_id.value, user_account.user_id) + self.assertEqual( + user_id.uuid, + uuid5(self._UUIDv4, f'{object_id} - user-id - {user_id.value}') + ) + def _check_x509_fields(self, misp_object, x509, object_id): self.assertEqual(len(misp_object.attributes), 14) (md5, sha1, sha256, signed, issuer, serial_number, signature, diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 9c8ef818..1f9e2a5d 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -810,6 +810,71 @@ } } ] +_USER_ACCOUNT_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "user-account", + "user_id": "1001", + "account_login": "jdoe", + "account_type": "unix", + "display_name": "John Doe", + "is_service_account": False, + "is_privileged": False, + "can_escalate_privs": True, + "account_created": "2016-01-20T12:31:12Z", + "password_last_changed": "2016-01-20T14:27:43Z", + "account_first_login": "2016-01-20T14:26:07Z", + "account_last_login": "2016-07-22T16:08:28Z" + }, + "1": { + "type": "user-account", + "user_id": "thegrugq_ebooks", + "account_login": "thegrugq_ebooks", + "account_type": "twitter", + "display_name": "the grugq" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "user-account", + "user_id": "1001", + "account_login": "jdoe", + "account_type": "unix", + "display_name": "John Doe", + "is_service_account": False, + "is_privileged": False, + "can_escalate_privs": True, + "extensions": { + "unix-account-ext": { + "gid": 1001, + "groups": ["wheel"], + "home_dir": "/home/jdoe", + "shell": "/bin/bash" + } + } + } + } + } +] _VULNERABILITY_OBJECTS = [ { "type": "vulnerability", @@ -1091,6 +1156,10 @@ def get_bundle_with_software_objects(cls): def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) + @classmethod + def get_bundle_with_user_account_objects(cls): + return cls.__assemble_bundle(*_USER_ACCOUNT_OBJECTS) + @classmethod def get_bundle_with_x509_objects(cls): return cls.__assemble_bundle(*_X509_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 6770bd1b..a6e68e4c 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -538,6 +538,58 @@ def test_stix20_bundle_with_url_attributes(self): observed_data2, s_url, 'url' ) + def test_stix20_bundle_with_user_account_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_user_account_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._assert_multiple_equal( + multiple1.name, multiple2.name, single.name, 'user-account' + ) + self._check_misp_object_fields(multiple1, observed_data1, '0') + self.assertEqual(len(multiple1.attributes), 11) + user_account = observed_data1.objects['0'] + object_id = f'{observed_data1.id} - 0' + self._check_user_account_fields( + multiple1.attributes[:7], user_account, object_id + ) + self._check_user_account_timeline_fields( + multiple1.attributes[7:-1], user_account, object_id + ) + password_last_changed = multiple1.attributes[-1] + self.assertEqual( + password_last_changed.object_relation, 'password_last_changed' + ) + self.assertEqual( + password_last_changed.value, user_account.password_last_changed + ) + self.assertEqual( + password_last_changed.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - password_last_changed' + f' - {password_last_changed.value}' + ) + ) + self._check_misp_object_fields(multiple2, observed_data1, '1') + self._check_user_account_twitter_fields( + multiple2, observed_data1.objects['1'], f'{observed_data1.id} - 1' + ) + self._check_misp_object_fields(single, observed_data2) + self.assertEqual(len(single.attributes), 11) + user_account = observed_data2.objects['0'] + self._check_user_account_fields( + single.attributes[:7], user_account, observed_data2.id + ) + self._check_user_account_extension_fields( + single.attributes[7:], user_account.extensions['unix-account-ext'], + observed_data2.id + ) + def test_stix20_bundle_with_x509_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_x509_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 198908a5..fab78056 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -990,6 +990,82 @@ ] } ] +_USER_ACCOUNT_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c", + "user-account--9bd3afcf-deee-54f9-83e2-520653cb6bba" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "user-account--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" + ] + }, + { + "type": "user-account", + "spec_version": "2.1", + "id": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c", + "user_id": "1001", + "account_login": "jdoe", + "account_type": "unix", + "display_name": "John Doe", + "is_service_account": False, + "is_privileged": False, + "can_escalate_privs": True, + "account_created": "2016-01-20T12:31:12Z", + "credential_last_changed": "2016-01-20T14:27:43Z", + "account_first_login": "2016-01-20T14:26:07Z", + "account_last_login": "2016-07-22T16:08:28Z" + }, + { + "type": "user-account", + "spec_version": "2.1", + "id": "user-account--9bd3afcf-deee-54f9-83e2-520653cb6bba", + "user_id": "thegrugq_ebooks", + "account_login": "thegrugq_ebooks", + "account_type": "twitter", + "display_name": "the grugq" + }, + { + "type": "user-account", + "spec_version": "2.1", + "id": "user-account--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "user_id": "1001", + "account_login": "jdoe", + "account_type": "unix", + "display_name": "John Doe", + "is_service_account": False, + "is_privileged": False, + "can_escalate_privs": True, + "extensions": { + "unix-account-ext": { + "gid": 1001, + "groups": ["wheel"], + "home_dir": "/home/jdoe", + "shell": "/bin/bash" + } + } + } +] _VULNERABILITY_OBJECTS = [ { "type": "vulnerability", @@ -1292,6 +1368,10 @@ def get_bundle_with_software_objects(cls): def get_bundle_with_url_attributes(cls): return cls.__assemble_bundle(*_URL_ATTRIBUTES) + @classmethod + def get_bundle_with_user_account_objects(cls): + return cls.__assemble_bundle(*_USER_ACCOUNT_OBJECTS) + @classmethod def get_bundle_with_x509_objects(cls): return cls.__assemble_bundle(*_X509_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index d6f38d75..7a65ee86 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -495,6 +495,51 @@ def test_stix21_bundle_with_url_attributes(self): self._check_generic_attribute(od1, url_2, m_url2, 'url') self._check_generic_attribute(od2, url_3, s_url, 'url') + def test_stix21_bundle_with_user_account_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_user_account_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, user1, user2, user3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + multiple1, multiple2, single = misp_objects + self._assert_multiple_equal( + multiple1.name, multiple2.name, single.name, 'user-account' + ) + self._check_misp_object_fields(multiple1, od1, user1) + self.assertEqual(len(multiple1.attributes), 11) + self._check_user_account_fields( + multiple1.attributes[:7], user1, user1.id + ) + self._check_user_account_timeline_fields( + multiple1.attributes[7:-1], user1, user1.id + ) + password_last_changed = multiple1.attributes[-1] + self.assertEqual( + password_last_changed.object_relation, 'password_last_changed' + ) + self.assertEqual( + password_last_changed.value, user1.credential_last_changed + ) + self.assertEqual( + password_last_changed.uuid, + uuid5( + self._UUIDv4, + f'{user1.id} - password_last_changed' + f' - {password_last_changed.value}' + ) + ) + self._check_misp_object_fields(multiple2, od1, user2) + self._check_user_account_twitter_fields(multiple2, user2, user2.id) + self._check_misp_object_fields(single, od2, user3) + self.assertEqual(len(single.attributes), 11) + self._check_user_account_fields(single.attributes[:7], user3, user3.id) + self._check_user_account_extension_fields( + single.attributes[7:], user3.extensions['unix-account-ext'], + user3.id + ) + def test_stix21_bundle_with_x509_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_x509_objects() self.parser.load_stix_bundle(bundle) From 92ceba1ee62b2dcd89ef3f92e24b32f4f90c3f61 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 5 Feb 2024 11:31:38 +0100 Subject: [PATCH 048/137] fix: [stix2 import] Yield syntax --- misp_stix_converter/stix2misp/stix2_to_misp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index a79e8eb1..a686dc05 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1173,7 +1173,7 @@ def _parse_markings(self, marking_refs: list): except ObjectRefLoadingError as error: self._object_ref_loading_error(error) continue - yield(marking_definition) + yield marking_definition def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: misp_object = { From 242f713b94069e8dfac94382732b9b8ab1fa709c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 5 Feb 2024 12:07:57 +0100 Subject: [PATCH 049/137] fix: [stix2 import] Some methods deduplication between main parser & converters --- .../converters/stix2_indicator_converter.py | 4 ++- .../stix2misp/converters/stix2converter.py | 36 ++----------------- .../stix2misp/stix2_to_misp.py | 27 -------------- 3 files changed, 6 insertions(+), 61 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py index 8a8dc3cc..cf8e593d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py @@ -933,7 +933,9 @@ def _parse_sigma_pattern(self, indicator: Indicator): misp_object = self._create_misp_object('sigma', indicator) if hasattr(indicator, 'object_marking_refs'): tags = tuple( - self._parse_markings(indicator.object_marking_refs) + self.main_parser._parse_markings( + indicator.object_marking_refs + ) ) for attribute in attributes: misp_attribute = misp_object.add_attribute(**attribute) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index 3fb44540..12041bd8 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -160,44 +160,12 @@ def _handle_labels(meta: dict, labels: list): if meta_labels: meta['labels'] = meta_labels - def _handle_tags_from_stix_fields(self, stix_object: _SDO_TYPING): - if hasattr(stix_object, 'confidence'): - yield self._parse_confidence_level(stix_object.confidence) - if hasattr(stix_object, 'object_marking_refs'): - yield from self._parse_markings(stix_object.object_marking_refs) - @staticmethod def _parse_AS_value(number: Union[int, str]) -> str: if isinstance(number, int) or not number.startswith('AS'): return f'AS{number}' return number - @staticmethod - def _parse_confidence_level(confidence_level: int) -> str: - if confidence_level == 100: - return 'misp:confidence-level="completely-confident"' - if confidence_level >= 75: - return 'misp:confidence-level="usually-confident"' - if confidence_level >= 50: - return 'misp:confidence-level="fairly-confident"' - if confidence_level >= 25: - return 'misp:confidence-level="rarely-confident"' - return 'misp:confidence-level="unconfident"' - - def _parse_markings(self, marking_refs: list): - for marking_ref in marking_refs: - try: - marking_definition = self.main_parser._get_stix_object( - marking_ref - ) - except ObjectTypeLoadingError as error: - self.main_parser._object_type_loading_error(error) - continue - except ObjectRefLoadingError as error: - self.main_parser._object_ref_loading_error(error) - continue - yield(marking_definition) - def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: misp_object = { 'timestamp': self._timestamp_from_date(stix_object.modified) @@ -386,7 +354,9 @@ def _handle_object_forcing(attributes: list, force_object: tuple) -> bool: def _handle_object_case(self, stix_object: _SDO_TYPING, attributes: list, name: str) -> MISPObject: misp_object = self._create_misp_object(name, stix_object) - tags = tuple(self._handle_tags_from_stix_fields(stix_object)) + tags = tuple( + self.main_parser._handle_tags_from_stix_fields(stix_object) + ) if tags: for attribute in attributes: misp_attribute = misp_object.add_attribute(**attribute) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index a686dc05..a8d9fa5d 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1100,19 +1100,6 @@ def _create_misp_galaxy_cluster(cluster_args: dict) -> MISPGalaxyCluster: cluster.from_dict(**cluster_args) return cluster - def _create_misp_object( - self, name: str, stix_object: Optional[_SDO_TYPING] = None - ) -> MISPObject: - misp_object = MISPObject( - name, - misp_objects_path_custom=_MISP_OBJECTS_PATH, - force_timestamps=True - ) - if stix_object is not None: - self._sanitise_object_uuid(misp_object, stix_object['id']) - misp_object.from_dict(**self._parse_timeline(stix_object)) - return misp_object - def _handle_tags_from_stix_fields(self, stix_object: _SDO_TYPING): if hasattr(stix_object, 'confidence'): yield self._parse_confidence_level(stix_object.confidence) @@ -1175,20 +1162,6 @@ def _parse_markings(self, marking_refs: list): continue yield marking_definition - def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: - misp_object = { - 'timestamp': self._timestamp_from_date(stix_object.modified) - } - object_type = stix_object.type - if self._mapping.timeline_mapping(object_type) is not None: - first, last = self._mapping.timeline_mapping(object_type) - if not self._skip_first_seen_last_seen(stix_object): - if hasattr(stix_object, first) and getattr(stix_object, first): - misp_object['first_seen'] = getattr(stix_object, first) - if hasattr(stix_object, last) and getattr(stix_object, last): - misp_object['last_seen'] = getattr(stix_object, last) - return misp_object - @staticmethod def _populate_object_attributes( misp_object: MISPObject, mapping: dict, values: Union[list, str]): From 6f90287ea1cd826ea5a529f8956acbab4f82a916 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 5 Feb 2024 12:15:16 +0100 Subject: [PATCH 050/137] fix: [stix2 import] Better tags from indicators parsing & simplified the tags handling method --- .../converters/stix2_indicator_converter.py | 4 ++-- .../stix2misp/stix2_to_misp.py | 23 ++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py index cf8e593d..1c7dfe19 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py @@ -933,8 +933,8 @@ def _parse_sigma_pattern(self, indicator: Indicator): misp_object = self._create_misp_object('sigma', indicator) if hasattr(indicator, 'object_marking_refs'): tags = tuple( - self.main_parser._parse_markings( - indicator.object_marking_refs + self.main_parser._handle_tags_from_stix_fields( + indicator ) ) for attribute in attributes: diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index a8d9fa5d..5449cd1f 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1104,7 +1104,16 @@ def _handle_tags_from_stix_fields(self, stix_object: _SDO_TYPING): if hasattr(stix_object, 'confidence'): yield self._parse_confidence_level(stix_object.confidence) if hasattr(stix_object, 'object_marking_refs'): - yield from self._parse_markings(stix_object.object_marking_refs) + for marking_ref in stix_object.object_marking_refs: + try: + marking_definition = self._get_stix_object(marking_ref) + except ObjectTypeLoadingError as error: + self._object_type_loading_error(error) + continue + except ObjectRefLoadingError as error: + self._object_ref_loading_error(error) + continue + yield marking_definition ############################################################################ # UTILITY METHODS. # @@ -1150,18 +1159,6 @@ def _parse_marking_definition( return marking_definition.name.lower() raise MarkingDefinitionLoadingError(marking_definition.id) - def _parse_markings(self, marking_refs: list): - for marking_ref in marking_refs: - try: - marking_definition = self._get_stix_object(marking_ref) - except ObjectTypeLoadingError as error: - self._object_type_loading_error(error) - continue - except ObjectRefLoadingError as error: - self._object_ref_loading_error(error) - continue - yield marking_definition - @staticmethod def _populate_object_attributes( misp_object: MISPObject, mapping: dict, values: Union[list, str]): From b01d4b3e38026bce5e2e0cd49b0dbc7a4c02946c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 5 Feb 2024 22:55:13 +0100 Subject: [PATCH 051/137] fix: [stix2 import] Avoiding issues with marking definitions referenced but not present in a file - Checking TLP Markings --- .../stix2misp/stix2_to_misp.py | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 5449cd1f..8e9325b9 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -33,16 +33,18 @@ from pymisp import ( AbstractMISP, MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, MISPObject, MISPSighting) +from stix2 import TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE from stix2.parsing import parse as stix2_parser from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v20.common import MarkingDefinition as MarkingDefinition_v20 from stix2.v20.observables import NetworkTraffic as NetworkTraffic_v20 from stix2.v20.sdo import ( AttackPattern as AttackPattern_v20, Campaign as Campaign_v20, - CourseOfAction as CourseOfAction_v20, Identity as Identity_v20, - Indicator as Indicator_v20, IntrusionSet as IntrusionSet_v20, - Malware as Malware_v20, ObservedData as ObservedData_v20, - Report as Report_v20, ThreatActor as ThreatActor_v20, Tool as Tool_v20, + CourseOfAction as CourseOfAction_v20, CustomObject as CustomObject_v20, + Identity as Identity_v20, Indicator as Indicator_v20, + IntrusionSet as IntrusionSet_v20, Malware as Malware_v20, + ObservedData as ObservedData_v20, Report as Report_v20, + ThreatActor as ThreatActor_v20, Tool as Tool_v20, Vulnerability as Vulnerability_v20) from stix2.v20.sro import ( Relationship as Relationship_v20, Sighting as Sighting_v20) @@ -53,13 +55,14 @@ EmailMessage, File, IPv4Address, IPv6Address, MACAddress, Mutex, NetworkTraffic as NetworkTraffic_v21, Process, Software, URL, UserAccount, WindowsRegistryKey, X509Certificate) -from stix2.v21.sdo import Grouping, MalwareAnalysis, Note, Opinion +from stix2.v21.sdo import Grouping, Location, MalwareAnalysis, Note, Opinion from stix2.v21.sdo import ( AttackPattern as AttackPattern_v21, Campaign as Campaign_v21, - CourseOfAction as CourseOfAction_v21, Identity as Identity_v21, - Indicator as Indicator_v21, IntrusionSet as IntrusionSet_v21, Location, - Malware as Malware_v21, ObservedData as ObservedData_v21, - Report as Report_v21, ThreatActor as ThreatActor_v21, Tool as Tool_v21, + CourseOfAction as CourseOfAction_v21, CustomObject as CustomObject_v21, + Identity as Identity_v21, Indicator as Indicator_v21, + IntrusionSet as IntrusionSet_v21, Malware as Malware_v21, + ObservedData as ObservedData_v21, Report as Report_v21, + ThreatActor as ThreatActor_v21, Tool as Tool_v21, Vulnerability as Vulnerability_v21) from stix2.v21.sro import ( Relationship as Relationship_v21, Sighting as Sighting_v21) @@ -171,8 +174,12 @@ Report_v20, Report_v21 ] _SDO_TYPING = Union[ + Campaign_v20, Campaign_v21, + CustomObject_v20, CustomObject_v21, + Grouping, Indicator_v20, Indicator_v21, ObservedData_v20, ObservedData_v21, + Report_v20, Report_v21, Vulnerability_v20, Vulnerability_v21 ] _SIGHTING_TYPING = Union[ @@ -598,14 +605,8 @@ def _handle_object(self, object_type: str, object_ref: str): def _handle_misp_event_tags( self, misp_event: MISPEvent, stix_object: _GROUPING_REPORT_TYPING): - if hasattr(stix_object, 'object_marking_refs'): - for marking_ref in stix_object.object_marking_refs: - try: - misp_event.add_tag(self._marking_definition[marking_ref]) - except KeyError: - self._unknown_marking_ref_warning(marking_ref) - except AttributeError: - self._unknown_marking_object_warning(marking_ref) + for tag in self._handle_tags_from_stix_fields(stix_object): + misp_event.add_tag(tag) if hasattr(stix_object, 'labels'): self._fetch_tags_from_labels(misp_event, stix_object.labels) @@ -1046,19 +1047,15 @@ def _add_misp_attribute(self, attribute: dict, stix_object: _SDO_TYPING) -> MISPAttribute: misp_attribute = MISPAttribute() misp_attribute.from_dict(**attribute) - tags = tuple(self._handle_tags_from_stix_fields(stix_object)) - if tags: - for tag in tags: - misp_attribute.add_tag(tag) + for tag in self._handle_tags_from_stix_fields(stix_object): + misp_attribute.add_tag(tag) return self.misp_event.add_attribute(**misp_attribute) def _add_misp_object(self, misp_object: MISPObject, stix_object: _SDO_TYPING) -> MISPObject: - tags = tuple(self._handle_tags_from_stix_fields(stix_object)) - if tags: + for tag in self._handle_tags_from_stix_fields(stix_object): for attribute in misp_object.attributes: - for tag in tags: - attribute.add_tag(tag) + attribute.add_tag(tag) return self.misp_event.add_object(misp_object) def _create_generic_event(self) -> MISPEvent: @@ -1108,10 +1105,17 @@ def _handle_tags_from_stix_fields(self, stix_object: _SDO_TYPING): try: marking_definition = self._get_stix_object(marking_ref) except ObjectTypeLoadingError as error: - self._object_type_loading_error(error) + if self._is_tlp_marking(marking_ref): + marking = self._get_stix_object(marking_ref) + yield self._get_stix_object(marking_ref) + else: + self._object_type_loading_error(error) continue except ObjectRefLoadingError as error: - self._object_ref_loading_error(error) + if self._is_tlp_marking(marking_ref): + yield self._get_stix_object(marking_ref) + else: + self._object_ref_loading_error(error) continue yield marking_definition @@ -1130,6 +1134,13 @@ def _fetch_tags_from_labels( if label.lower() != 'threat-report'): misp_feature.add_tag(label) + def _is_tlp_marking(self, marking_ref: str) -> bool: + for marking in (TLP_WHITE, TLP_GREEN, TLP_AMBER, TLP_RED): + if marking_ref == marking.id: + self._load_marking_definition(marking) + return True + return False + @staticmethod def _parse_AS_value(number: Union[int, str]) -> str: if isinstance(number, int) or not number.startswith('AS'): From 302b427044c03a817245b8197c067b8678337ab2 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 12 Feb 2024 23:48:49 +0100 Subject: [PATCH 052/137] fix: [stix2 import] Quick reordering to allow more reusability --- .../stix2misp/converters/stix2converter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index 12041bd8..27d79c9b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -92,8 +92,9 @@ def _generic_parser( attribute, getattr(stix_object, field), stix_object.id ) - def _populate_object_attribute(self, mapping: dict, reference: str, + def _populate_object_attribute(self, mapping: dict, object_id: str, value: Union[dict, str]) -> dict: + reference = f"{object_id} - {mapping['object_relation']}" if isinstance(value, dict): attribute_value = value['value'] return { @@ -132,12 +133,11 @@ def _populate_object_attributes( def _populate_object_attributes_with_data( self, mapping: dict, values: Union[dict, list, str], object_id: str) -> Iterator[dict]: - reference = f"{object_id} - {mapping['object_relation']}" if isinstance(values, list): for value in values: - yield self._populate_object_attribute(mapping, reference, value) + yield self._populate_object_attribute(mapping, object_id, value) else: - yield self._populate_object_attribute(mapping, reference, values) + yield self._populate_object_attribute(mapping, object_id, values) ############################################################################ # UTILITY METHODS. # From 7b22f2bd538997352340c5193d3e545d3581e528 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 00:06:03 +0100 Subject: [PATCH 053/137] fix: [stix2 import] Some typings fixed --- .../stix2_observed_data_converter.py | 49 +++++-------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 38b7b8ff..399e909c 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -7,59 +7,36 @@ from .stix2_observable_converter import ( ExternalSTIX2ObservableConverter, ExternalSTIX2ObservableMapping, InternalSTIX2ObservableConverter, InternalSTIX2ObservableMapping, - STIX2ObservableConverter, _ARTIFACT_TYPING, _AUTONOMOUS_SYSTEM_TYPING, - _DIRECTORY_TYPING, _EMAIL_ADDRESS_TYPING, _EXTENSION_TYPING, - _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) + STIX2ObservableConverter, _AUTONOMOUS_SYSTEM_TYPING, _EMAIL_ADDRESS_TYPING, + _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject from stix2.v20.observables import ( - Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, - Directory as Directory_v20, DomainName as DomainName_v20, - IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, - MACAddress as MACAddress_v20, Mutex as Mutex_v20, Process as Process_v20, - Software as Software_v20, URL as URL_v20, UserAccount as UserAccount_v20, - X509Certificate as X509Certificate_v20) + WindowsRegistryValueType as WindowsRegistryValueType_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - Artifact as Artifact_v21, AutonomousSystem as AutonomousSystem_v21, - Directory as Directory_v21, DomainName as DomainName_v21, - IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, - MACAddress as MACAddress_v21, Mutex as Mutex_v21, Process as Process_v21, - Software as Software_v21, URL as URL_v21, UserAccount as UserAccount_v21, - X509Certificate as X509Certificate_v21) + Artifact, AutonomousSystem, Directory, DomainName, File, IPv4Address, + IPv6Address, MACAddress, Mutex, Process, Software, URL, UserAccount, + X509Certificate) from stix2.v21.sdo import ObservedData as ObservedData_v21 -from typing import Iterator, Optional, TYPE_CHECKING, Union +from typing import Optional, TYPE_CHECKING, Union if TYPE_CHECKING: from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser _GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ - Artifact_v20, Artifact_v21, - Directory_v20, Directory_v21, - Software_v20, Software_v21, - Process_v20, Process_v21, - UserAccount_v20, UserAccount_v21, - X509Certificate_v20, X509Certificate_v21 + Artifact, Directory, File, Process, Software, UserAccount, + X509Certificate ] _GENERIC_OBSERVABLE_TYPING = Union[ - DomainName_v20, DomainName_v21, - IPv4Address_v20, IPv4Address_v21, - IPv6Address_v20, IPv6Address_v21, - MACAddress_v20, MACAddress_v21, - Mutex_v20, Mutex_v21, - URL_v20, URL_v21 + DomainName, IPv4Address, IPv6Address, MACAddress, Mutex, URL ] _OBSERVABLE_OBJECTS_TYPING = Union[ - Artifact_v20, Artifact_v21, - AutonomousSystem_v20, AutonomousSystem_v21, - Directory_v20, Directory_v21, - Process_v20, Process_v21, - Software_v20, Software_v21, - UserAccount_v20, UserAccount_v21, - X509Certificate_v20, X509Certificate_v21 + Artifact, AutonomousSystem, Directory, File, Process, Software, + UserAccount, X509Certificate ] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 @@ -281,7 +258,7 @@ def _parse_as_observable_objects( def _parse_autonomous_system_observable_object_ref( self, autonomous_system: _AUTONOMOUS_SYSTEM_TYPING, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: + observed_data: ObservedData_v21) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( 'asn', autonomous_system, observed_data ) From 46f764b568207dd463e13b02c9bc55a6b5e3ab96 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 00:08:55 +0100 Subject: [PATCH 054/137] fix: [stix2 import] Added observed data id as comment for misp objects converted from STIX 2.0 when it has a v5 uuid --- .../stix2misp/converters/stix2_observed_data_converter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 399e909c..efe0a4f7 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -964,6 +964,7 @@ def _create_misp_object_from_observable_object( misp_object = self._create_misp_object(name) misp_object.from_dict( uuid=self.main_parser._create_v5_uuid(object_id), + comment=f'Observed Data ID: {observed_data.id}', **self._parse_timeline(observed_data) ) return misp_object From 3dbb2713041e09308dcbeeb75b8979bdd0d20820 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 00:13:34 +0100 Subject: [PATCH 055/137] fix: [tests] Testing MISP Object comment when its uuid is v5 --- tests/test_external_stix20_import.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index a6e68e4c..1300554e 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -278,6 +278,9 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier=None) misp_object.uuid, uuid5(self._UUIDv4, f'{observed_data.id} - {identifier}') ) + self.assertEqual( + misp_object.comment, f'Observed Data ID: {observed_data.id}' + ) if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): self.assertEqual(misp_object.first_seen, observed_data.first_observed) self.assertEqual(misp_object.last_seen, observed_data.last_observed) From 40a2111b170af38ea9007cd58972e3b6c5d62473 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 00:15:54 +0100 Subject: [PATCH 056/137] fix: [tests] Passing observable ids instead of objects themselves for some tests that only need to know about ids --- tests/test_external_stix21_import.py | 55 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 7a65ee86..f28df391 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -227,22 +227,22 @@ def test_stix21_bundle_with_vulnerability_galaxy(self): def _check_artifact_object(self, misp_object, observed_data, artifact): self.assertEqual(misp_object.name, 'artifact') - self._check_misp_object_fields(misp_object, observed_data, artifact) + self._check_misp_object_fields(misp_object, observed_data, artifact.id) self._check_artifact_fields(misp_object, artifact, artifact.id) def _check_as_attribute(self, attribute, observed_data, autonomous_system): - self._check_misp_object_fields(attribute, observed_data, autonomous_system) + self._check_misp_object_fields(attribute, observed_data, autonomous_system.id) self.assertEqual(attribute.type, 'AS') self.assertEqual(attribute.value, f'AS{autonomous_system.number}') def _check_as_object(self, misp_object, observed_data, autonomous_system): self.assertEqual(misp_object.name, 'asn') - self._check_misp_object_fields(misp_object, observed_data, autonomous_system) + self._check_misp_object_fields(misp_object, observed_data, autonomous_system.id) self._check_as_fields(misp_object, autonomous_system, autonomous_system.id) def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(misp_object.name, 'directory') - self._check_misp_object_fields(misp_object, observed_data, directory) + self._check_misp_object_fields(misp_object, observed_data, directory.id) atime, ctime, mtime = self._check_directory_fields( misp_object, directory, directory.id ) @@ -251,21 +251,22 @@ def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(mtime, directory.mtime) def _check_email_address_attribute(self, observed_data, address, email_address): - self._check_misp_object_fields(address, observed_data, email_address) + self._check_misp_object_fields(address, observed_data, email_address.id) self.assertEqual(address.type, 'email-dst') self.assertEqual(address.value, email_address.value) def _check_email_address_attribute_with_display_name( self, observed_data, address, display_name, email_address): self._check_misp_object_fields( - address, observed_data, email_address, - f'{email_address.id} - email-dst - {email_address.value}' + address, observed_data, + f'{email_address.id} - email-dst - {email_address.value}', True ) self.assertEqual(address.type, 'email-dst') self.assertEqual(address.value, email_address.value) self._check_misp_object_fields( - display_name, observed_data, email_address, - f'{email_address.id} - email-dst-display-name - {email_address.display_name}' + display_name, observed_data, + f'{email_address.id} - email-dst-display-name - {email_address.display_name}', + True ) self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) @@ -273,16 +274,16 @@ def _check_email_address_attribute_with_display_name( def _check_generic_attribute( self, observed_data, observable_object, attribute, attribute_type, feature='value'): - self._check_misp_object_fields(attribute, observed_data, observable_object) + self._check_misp_object_fields(attribute, observed_data, observable_object.id) self.assertEqual(attribute.type, attribute_type) self.assertEqual(attribute.value, getattr(observable_object, feature)) def _check_misp_object_fields( - self, misp_object, observed_data, observable_object, identifier=None): - if identifier is None: - self.assertEqual(misp_object.uuid, observable_object.id.split('--')[1]) + self, misp_object, observed_data, object_id, multiple=False): + if multiple: + self.assertEqual(misp_object.uuid, uuid5(self._UUIDv4, object_id)) else: - self.assertEqual(misp_object.uuid, uuid5(self._UUIDv4, identifier)) + self.assertEqual(misp_object.uuid, object_id.split('--')[1]) self.assertEqual(misp_object.comment, f'Observed Data ID: {observed_data.id}') if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): self.assertEqual(misp_object.first_seen, observed_data.first_observed) @@ -291,12 +292,12 @@ def _check_misp_object_fields( def _check_software_object(self, misp_object, observed_data, software): self.assertEqual(misp_object.name, 'software') - self._check_misp_object_fields(misp_object, observed_data, software) + self._check_misp_object_fields(misp_object, observed_data, software.id) self._check_software_fields(misp_object, software, software.id) def _check_x509_object(self, misp_object, observed_data, x509): self.assertEqual(misp_object.name, 'x509') - self._check_misp_object_fields(misp_object, observed_data, x509) + self._check_misp_object_fields(misp_object, observed_data, x509.id) self._check_x509_fields(misp_object, x509, x509.id) def test_stix21_bundle_with_artifact_objects(self): @@ -309,7 +310,7 @@ def test_stix21_bundle_with_artifact_objects(self): self.assertEqual(len(misp_objects), 3) multiple1, multiple2, single = misp_objects self._check_artifact_object(multiple1, od1, artifact1) - self._check_misp_object_fields(multiple2, od1, artifact2) + self._check_misp_object_fields(multiple2, od1, artifact2.id) self._check_artifact_with_url_fields(multiple2, artifact2, artifact2.id) self._check_artifact_object(single, od2, artifact3) @@ -438,7 +439,7 @@ def test_stix21_bundle_with_process_objects(self): ) self._assert_multiple_equal(image1.name, image2.name, 'file') - self._check_misp_object_fields(multiple, od1, process1) + self._check_misp_object_fields(multiple, od1, process1.id) self._check_process_multiple_fields(multiple, process1, process1.id) self.assertEqual(len(multiple.references), 3) child_ref, parent_ref, binary_ref = multiple.references @@ -449,23 +450,23 @@ def test_stix21_bundle_with_process_objects(self): self.assertEqual(binary_ref.referenced_uuid, image1.uuid) self.assertEqual(binary_ref.relationship_type, 'executes') - self._check_misp_object_fields(parent, od1, process2) + self._check_misp_object_fields(parent, od1, process2.id) self._check_process_parent_fields(parent, process2, process2.id) self.assertEqual(len(parent.references), 1) reference = parent.references[0] self.assertEqual(reference.referenced_uuid, image2.uuid) self.assertEqual(reference.relationship_type, 'executes') - self._check_misp_object_fields(child, od1, process3) + self._check_misp_object_fields(child, od1, process3.id) self._check_process_child_fields(child, process3, process3.id) - self._check_misp_object_fields(image1, od1, file2) + self._check_misp_object_fields(image1, od1, file2.id) self._check_process_image_reference_fields(image1, file2, file2.id) - self._check_misp_object_fields(image2, od1, file1) + self._check_misp_object_fields(image2, od1, file1.id) self._check_process_image_reference_fields(image2, file1, file1.id) - self._check_misp_object_fields(single, od2, process4) + self._check_misp_object_fields(single, od2, process4.id) self._check_process_single_fields(single, process4, process4.id) def test_stix21_bundle_with_software_objects(self): @@ -479,7 +480,7 @@ def test_stix21_bundle_with_software_objects(self): multiple1, multiple2, single = misp_objects self._check_software_object(multiple1, od1, software1) self._check_software_object(multiple2, od1, software2) - self._check_misp_object_fields(single, od2, software3) + self._check_misp_object_fields(single, od2, software3.id) self._check_software_with_swid_fields(single, software3, software3.id) def test_stix21_bundle_with_url_attributes(self): @@ -507,7 +508,7 @@ def test_stix21_bundle_with_user_account_objects(self): self._assert_multiple_equal( multiple1.name, multiple2.name, single.name, 'user-account' ) - self._check_misp_object_fields(multiple1, od1, user1) + self._check_misp_object_fields(multiple1, od1, user1.id) self.assertEqual(len(multiple1.attributes), 11) self._check_user_account_fields( multiple1.attributes[:7], user1, user1.id @@ -530,9 +531,9 @@ def test_stix21_bundle_with_user_account_objects(self): f' - {password_last_changed.value}' ) ) - self._check_misp_object_fields(multiple2, od1, user2) + self._check_misp_object_fields(multiple2, od1, user2.id) self._check_user_account_twitter_fields(multiple2, user2, user2.id) - self._check_misp_object_fields(single, od2, user3) + self._check_misp_object_fields(single, od2, user3.id) self.assertEqual(len(single.attributes), 11) self._check_user_account_fields(single.attributes[:7], user3, user3.id) self._check_user_account_extension_fields( From 0410a1a6fdf983727c19c4583fd6716f7e9ad2ca Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 09:12:25 +0100 Subject: [PATCH 057/137] wip: [stix2 import] Converting STIX 2.x Windows Registry Key objects --- .../stix2_observed_data_converter.py | 110 +++++++++++++++++- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index efe0a4f7..39e58704 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -19,7 +19,8 @@ from stix2.v21.observables import ( Artifact, AutonomousSystem, Directory, DomainName, File, IPv4Address, IPv6Address, MACAddress, Mutex, Process, Software, URL, UserAccount, - X509Certificate) + WindowsRegistryKey, X509Certificate, + WindowsRegistryValueType as WindowsRegistryValueType_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Optional, TYPE_CHECKING, Union @@ -29,18 +30,21 @@ _GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ Artifact, Directory, File, Process, Software, UserAccount, - X509Certificate + WindowsRegistryKey, X509Certificate ] _GENERIC_OBSERVABLE_TYPING = Union[ DomainName, IPv4Address, IPv6Address, MACAddress, Mutex, URL ] _OBSERVABLE_OBJECTS_TYPING = Union[ Artifact, AutonomousSystem, Directory, File, Process, Software, - UserAccount, X509Certificate + UserAccount, WindowsRegistryKey, X509Certificate ] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 ] +_WINDOWS_REGISTRY_VALUE_TYPING = Union[ + WindowsRegistryValueType_v20, WindowsRegistryValueType_v21 +] class STIX2ObservedDataConverter(STIX2ObservableConverter, metaclass=ABCMeta): @@ -856,6 +860,106 @@ def _parse_process_reference_observable_object_ref( ) ) + def _parse_registry_key_observable_object_ref( + self, registry_key: WindowsRegistryKey, + observed_data: ObservedData_v21): + regkey_object = self._create_misp_object_from_observable_object_ref( + 'registry-key', registry_key, observed_data, + ) + for attribute in self._parse_registry_key_observable(registry_key): + regkey_object.add_attribute(**attribute) + misp_object = self.main_parser._add_misp_object( + regkey_object, observed_data + ) + if len(registry_key.get('values', [])) > 1: + for index, registry_value in enumerate(registry_key['values']): + value_uuid = self._parse_registry_key_value_observable( + registry_value, observed_data, + f'{registry_key.id} - values - {index}' + ) + misp_object.add_reference(value_uuid, 'contains') + + def _parse_registry_key_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + registry_key = observable['observable'] + self._parse_registry_key_observable_object_ref( + registry_key, observed_data + ) + observable['used'][self.event_uuid] = True + + def _parse_registry_key_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + registry_key = next(iter(observed_data.objects.values())) + if hasattr(registry_key, 'id'): + return self._parse_registry_key_observable_object_ref( + registry_key, observed_data + ) + regkey_object = self._create_misp_object_from_observable_object( + 'registry-key', observed_data + ) + attributes = self._parse_registry_key_observable( + registry_key, observed_data.id + ) + for attribute in attributes: + regkey_object.add_attribute(**attribute) + misp_object = self.main_parser._add_misp_object( + regkey_object, observed_data + ) + if len(registry_key.get('values', [])) > 1: + for index, registry_value in enumerate(registry_key['values']): + value_uuid = self._parse_registry_key_value_observable( + registry_value, observed_data, + f'{observed_data.id} - values - {index}' + ) + misp_object.add_reference(value_uuid, 'contains') + return misp_object + for identifier, registry_key in observed_data.objects.items(): + if hasattr(registry_key, 'id'): + return self._parse_registry_key_observable_object_ref( + registry_key, observed_data + ) + object_id = f'{observed_data.id} - {identifier}' + regkey_object = self._create_misp_object_from_observable_object( + 'registry-key', observed_data, object_id + ) + attributes = self._parse_registry_key_observable( + registry_key, object_id + ) + for attribute in attributes: + regkey_object.add_attribute(**attribute) + misp_object = self.main_parser._add_misp_object( + regkey_object, observed_data + ) + if len(registry_key.get('values', [])) > 1: + for index, registry_value in enumerate(registry_key['values']): + value_uuid = self._parse_registry_key_value_observable( + registry_value, observed_data, + f'{object_id} - values - {index}' + ) + misp_object.add_reference(value_uuid, 'contains') + + def _parse_registry_key_value_observable( + self, registry_value: _WINDOWS_REGISTRY_VALUE_TYPING, + observed_data: _OBSERVED_DATA_TYPING, object_id) -> str: + misp_object = self._create_misp_object_from_observable_object( + 'registry-key-value', observed_data, object_id + ) + mapping = self._mapping.registry_key_values_mapping + for field, attribute in mapping().items(): + if hasattr(registry_value, field): + misp_object.add_attribute( + **self._populate_object_attribute( + attribute, object_id, getattr(registry_value, field) + ) + ) + misp_object = self.main_parser._add_misp_object( + misp_object, observed_data + ) + return misp_object.uuid + def _parse_software_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: From a116dc71bde148a9b37757b70ec9488b5ee4a20b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 09:13:58 +0100 Subject: [PATCH 058/137] wip: [tests] Tests for STIX 2.x Windows Registry Key objects conversion --- tests/_test_stix_import.py | 81 ++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 72 +++++++++++++++++++++++ tests/test_external_stix20_import.py | 45 +++++++++++++++ tests/test_external_stix21_bundles.py | 83 +++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 36 ++++++++++++ 5 files changed, 317 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index cad3eacb..0b08abc4 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -651,6 +651,87 @@ def _check_process_single_fields(self, misp_object, process, object_id): uuid5(self._UUIDv4, f'{object_id} - pid - {pid.value}') ) + def _check_registry_key_fields(self, misp_object, registry_key, object_id): + self.assertEqual(len(misp_object.attributes), 2) + key, modified = misp_object.attributes + self.assertEqual(key.type, 'regkey') + self.assertEqual(key.object_relation, 'key') + self.assertEqual(key.value, registry_key.key) + self.assertEqual( + key.uuid, uuid5(self._UUIDv4, f'{object_id} - key - {key.value}') + ) + self.assertEqual(modified.type, 'datetime') + self.assertEqual(modified.object_relation, 'last-modified') + self.assertEqual( + modified.uuid, + uuid5( + self._UUIDv4, f'{object_id} - last-modified - {modified.value}' + ) + ) + return modified + + def _check_registry_key_value_fields(self, misp_object, registry_value, object_id): + self.assertEqual(len(misp_object.attributes), 3) + data, data_type, name = misp_object.attributes + self.assertEqual(data.type, 'text') + self.assertEqual(data.object_relation, 'data') + self.assertEqual(data.value, registry_value.data) + self.assertEqual( + data.uuid, uuid5(self._UUIDv4, f'{object_id} - data - {data.value}') + ) + self.assertEqual(data_type.type, 'text') + self.assertEqual(data_type.object_relation, 'data-type') + self.assertEqual(data_type.value, registry_value.data_type) + self.assertEqual( + data_type.uuid, + uuid5(self._UUIDv4, f'{object_id} - data-type - {data_type.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, registry_value.name) + self.assertEqual( + name.uuid, uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + + def _check_registry_key_with_values_fields(self, misp_object, registry_key, object_id): + self.assertEqual(len(misp_object.attributes), 5) + key, modified, data, data_type, name = misp_object.attributes + self.assertEqual(key.type, 'regkey') + self.assertEqual(key.object_relation, 'key') + self.assertEqual(key.value, registry_key.key) + self.assertEqual( + key.uuid, uuid5(self._UUIDv4, f'{object_id} - key - {key.value}') + ) + self.assertEqual(modified.type, 'datetime') + self.assertEqual(modified.object_relation, 'last-modified') + self.assertEqual( + modified.uuid, + uuid5( + self._UUIDv4, f'{object_id} - last-modified - {modified.value}' + ) + ) + registry_value = registry_key['values'][0] + self.assertEqual(data.type, 'text') + self.assertEqual(data.object_relation, 'data') + self.assertEqual(data.value, registry_value.data) + self.assertEqual( + data.uuid, uuid5(self._UUIDv4, f'{object_id} - data - {data.value}') + ) + self.assertEqual(data_type.type, 'text') + self.assertEqual(data_type.object_relation, 'data-type') + self.assertEqual(data_type.value, registry_value.data_type) + self.assertEqual( + data_type.uuid, + uuid5(self._UUIDv4, f'{object_id} - data-type - {data_type.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, registry_value.name) + self.assertEqual( + name.uuid, uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + return modified + def _check_software_fields(self, misp_object, software, object_id): self.assertEqual(len(misp_object.attributes), 4) language, name, vendor, version = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 1f9e2a5d..463d01b8 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -622,6 +622,74 @@ } } ] +_REGISTRY_KEY_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "windows-registry-key", + "key": "hkey_local_machine\\system\\bar\\baz", + "modified": "2020-10-25T16:22:00Z", + "values": [ + { + "name": "RegistryName", + "data": "%DATA%\\baz", + "data_type": "REG_SZ" + } + ] + }, + "1": { + "type": "windows-registry-key", + "key": "hkey_local_machine\\system\\bar\\foo", + "modified": "2020-10-25T16:22:00Z", + "number_of_subkeys": 2, + "values": [ + { + "name": "Foo", + "data": "qwerty", + "data_type": "REG_SZ" + }, + { + "name": "Bar", + "data": "42", + "data_type": "REG_DWORD" + } + ] + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "windows-registry-key", + "key": "hkey_local_machine\\system\\foo\\fortytwo", + "modified": "2020-10-25T16:22:00Z", + "values": [ + { + "name": "FortyTwoFoo", + "data": "%DATA%\\42", + "data_type": "REG_QWORD" + } + ] + } + } + } +] _SOFTWARE_OBJECTS = [ { "type": "observed-data", @@ -1148,6 +1216,10 @@ def get_bundle_with_mutex_attributes(cls): def get_bundle_with_process_objects(cls): return cls.__assemble_bundle(*_PROCESS_OBJECTS) + @classmethod + def get_bundle_with_registry_key_objects(cls): + return cls.__assemble_bundle(*_REGISTRY_KEY_OBJECTS) + @classmethod def get_bundle_with_software_objects(cls): return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 1300554e..127feedf 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -286,6 +286,36 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier=None) self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_registry_key_object( + self, misp_object, observed_data, *values, identifier=None): + self.assertEqual(misp_object.name, 'registry-key') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{observed_data.id} - {identifier}' + registry_key = observed_data.objects[identifier] + if values: + modified = self._check_registry_key_fields( + misp_object, registry_key, object_id + ) + for index, value_object in enumerate(values): + self.assertEqual(value_object.name, 'registry-key-value') + self._check_misp_object_fields( + value_object, observed_data, + f'{identifier} - values - {index}' + ) + self._check_registry_key_value_fields( + value_object, registry_key['values'][index], + f'{object_id} - values - {index}' + ) + else: + modified = self._check_registry_key_with_values_fields( + misp_object, registry_key, object_id + ) + self.assertEqual(modified.value, registry_key.modified) + def _check_software_object(self, misp_object, observed_data, identifier): self.assertEqual(misp_object.name, 'software') self._check_misp_object_fields(misp_object, observed_data, identifier) @@ -506,6 +536,21 @@ def test_stix20_bundle_with_process_objects(self): single, observed_data2.objects['0'], observed_data2.id ) + def test_stix20_bundle_with_registry_key_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_registry_key_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 5) + multiple1, multiple2, value1, value2, single = misp_objects + self._check_registry_key_object(multiple1, observed_data1, identifier='0') + self._check_registry_key_object( + multiple2, observed_data1, value1, value2, identifier='1' + ) + self._check_registry_key_object(single, observed_data2) + def test_stix20_bundle_with_software_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_software_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index fab78056..a971959e 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -775,6 +775,85 @@ "command_line": "rm -rf *" } ] +_REGISTRY_KEY_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "windows-registry-key--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "windows-registry-key--2ba37ae7-2745-5082-9dfd-9486dad41016" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "windows-registry-key--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0" + ] + }, + { + "type": "windows-registry-key", + "spec_version": "2.1", + "id": "windows-registry-key--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "key": "hkey_local_machine\\system\\bar\\baz", + "modified_time": "2020-10-25T16:22:00Z", + "values": [ + { + "name": "RegistryName", + "data": "%DATA%\\baz", + "data_type": "REG_SZ" + } + ] + }, + { + "type": "windows-registry-key", + "spec_version": "2.1", + "id": "windows-registry-key--2ba37ae7-2745-5082-9dfd-9486dad41016", + "key": "hkey_local_machine\\system\\bar\\foo", + "modified_time": "2020-10-25T16:22:00Z", + "number_of_subkeys": 2, + "values": [ + { + "name": "Foo", + "data": "qwerty", + "data_type": "REG_SZ" + }, + { + "name": "Bar", + "data": "42", + "data_type": "REG_DWORD" + } + ] + }, + { + "type": "windows-registry-key", + "spec_version": "2.1", + "id": "windows-registry-key--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0", + "key": "hkey_local_machine\\system\\foo\\fortytwo", + "modified_time": "2020-10-25T16:22:00Z", + "values": [ + { + "name": "FortyTwoFoo", + "data": "%DATA%\\42", + "data_type": "REG_QWORD" + } + ] + } +] _SOFTWARE_OBJECTS = [ { "type": "observed-data", @@ -1360,6 +1439,10 @@ def get_bundle_with_mutex_attributes(cls): def get_bundle_with_process_objects(cls): return cls.__assemble_bundle(*_PROCESS_OBJECTS) + @classmethod + def get_bundle_with_registry_key_objects(cls): + return cls.__assemble_bundle(*_REGISTRY_KEY_OBJECTS) + @classmethod def get_bundle_with_software_objects(cls): return cls.__assemble_bundle(*_SOFTWARE_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index f28df391..114e7f6e 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -290,6 +290,29 @@ def _check_misp_object_fields( self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_registry_key_object( + self, misp_object, observed_data, registry_key, *values): + self.assertEqual(misp_object.name, 'registry-key') + self._check_misp_object_fields(misp_object, observed_data, registry_key.id) + if values: + modified = self._check_registry_key_fields( + misp_object, registry_key, registry_key.id + ) + for index, value_object in enumerate(values): + object_id = f'{registry_key.id} - values - {index}' + self.assertEqual(value_object.name, 'registry-key-value') + self._check_misp_object_fields( + value_object, observed_data, object_id, True + ) + self._check_registry_key_value_fields( + value_object, registry_key['values'][index], object_id + ) + else: + modified = self._check_registry_key_with_values_fields( + misp_object, registry_key, registry_key.id + ) + self.assertEqual(modified.value, registry_key.modified_time) + def _check_software_object(self, misp_object, observed_data, software): self.assertEqual(misp_object.name, 'software') self._check_misp_object_fields(misp_object, observed_data, software.id) @@ -469,6 +492,19 @@ def test_stix21_bundle_with_process_objects(self): self._check_misp_object_fields(single, od2, process4.id) self._check_process_single_fields(single, process4, process4.id) + def test_stix21_bundle_with_registry_key_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_registry_key_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, key1, key2, key3 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 5) + multiple1, multiple2, value1, value2, single = misp_objects + self._check_registry_key_object(multiple1, od1, key1) + self._check_registry_key_object(multiple2, od1, key2, value1, value2) + self._check_registry_key_object(single, od2, key3) + def test_stix21_bundle_with_software_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_software_objects() self.parser.load_stix_bundle(bundle) From 0cdc0a66a2f8219d298f1443dd96a4e6729c3cf0 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 16:05:32 +0100 Subject: [PATCH 059/137] fix: [stix2 import] Fixed AttributeError with method from parent conversion class --- .../stix2misp/converters/stix2_observable_objects_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 7139bd90..1509ccf2 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -550,7 +550,7 @@ def _parse_registry_key_observable_object(self, registry_key_ref: str): value_object = self._create_misp_object('registry-key-value') reference = f'{object_id} - values - {index}' value_object.from_dict( - uuid=self._create_v5_uuid(reference), + uuid=self.main_parser._create_v5_uuid(reference), comment=f'Original Windows Registry Key ID: {object_id}' ) attributes = self._parse_generic_observable( From e8f28fbcc346db65067f3965f4f8ea8d5585e7ac Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 16:18:28 +0100 Subject: [PATCH 060/137] fix: [stix2 import] Setting MISP objects timestamp with the datetime value instead of an int --- misp_stix_converter/stix2misp/converters/stix2converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index 27d79c9b..d0f193a7 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -168,7 +168,7 @@ def _parse_AS_value(number: Union[int, str]) -> str: def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: misp_object = { - 'timestamp': self._timestamp_from_date(stix_object.modified) + 'timestamp': stix_object.modified } object_type = stix_object.type if self._mapping.timeline_mapping(object_type) is not None: From 23ce7f39b4dbac9301a9598b124229ae9b52ddfb Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 19:07:22 +0100 Subject: [PATCH 061/137] wip: [stix2 import] Parsing User Account observables referenced by registry keys to be the creator reference --- .../converters/stix2_observable_converter.py | 8 +- .../stix2_observed_data_converter.py | 120 ++++++++++++++---- 2 files changed, 101 insertions(+), 27 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 7d51d9ef..290dd6ca 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -430,7 +430,6 @@ class ExternalSTIX2ObservableMapping( 'software': 'software', 'url': 'url', 'user-account': 'user_account', - 'windows-registry-key': 'registry_key', 'x509-certificate': 'x509', **dict.fromkeys( ( @@ -504,6 +503,13 @@ class ExternalSTIX2ObservableMapping( 'process' ), 'process' + ), + **dict.fromkeys( + ( + 'user-account_windows-registry-key', + 'windows-registry-key' + ), + 'registry_key' ) } ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 39e58704..5e4373e4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -860,9 +860,37 @@ def _parse_process_reference_observable_object_ref( ) ) + def _parse_registry_key_observable_object( + self, observed_data: _OBSERVED_DATA_TYPING, + identifier: str) -> MISPObject: + registry_key = observed_data.objects[identifier] + if hasattr(registry_key, 'id'): + return self._parse_registry_key_observable_object_ref( + registry_key, observed_data + ) + object_id = f'{observed_data.id} - {identifier}' + regkey_object = self._create_misp_object_from_observable_object( + 'registry-key', observed_data, object_id + ) + attributes = self._parse_registry_key_observable( + registry_key, object_id + ) + for attribute in attributes: + regkey_object.add_attribute(**attribute) + misp_object = self.main_parser._add_misp_object( + regkey_object, observed_data + ) + if len(registry_key.get('values', [])) > 1: + for index, value in enumerate(registry_key['values']): + value_uuid = self._parse_registry_key_value_observable( + value, observed_data, f'{object_id} - values - {index}' + ) + misp_object.add_reference(value_uuid, 'contains') + return misp_object + def _parse_registry_key_observable_object_ref( self, registry_key: WindowsRegistryKey, - observed_data: ObservedData_v21): + observed_data: ObservedData_v21) -> MISPObject: regkey_object = self._create_misp_object_from_observable_object_ref( 'registry-key', registry_key, observed_data, ) @@ -878,16 +906,41 @@ def _parse_registry_key_observable_object_ref( f'{registry_key.id} - values - {index}' ) misp_object.add_reference(value_uuid, 'contains') + return misp_object def _parse_registry_key_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - registry_key = observable['observable'] - self._parse_registry_key_observable_object_ref( - registry_key, observed_data + observable_object = observable['observable'] + if observable_object.type == 'user-account': + if observable['used'].get(self.event_uuid, False): + continue + misp_object = self._parse_generic_observable_object_ref( + observable_object, observed_data, 'user-account', False + ) + observable['misp_object'] = misp_object + observable['used'][self.event_uuid] = True + continue + misp_object = self._parse_registry_key_observable_object_ref( + observable_object, observed_data ) observable['used'][self.event_uuid] = True + if hasattr(observable_object, 'creator_user_ref'): + creator_observable = self._fetch_observables( + observable_object.creator_user_ref + ) + if creator_observable['used'].get(self.event_uuid, False): + creator_object = creator_observable['misp_object'] + creator_object.add_reference(misp_object.uuid, 'creates') + continue + creator_object = self._parse_generic_observable_object_ref( + creator_observable['observable'], observed_data, + 'user-account', False + ) + creator_object.add_reference(misp_object.uuid, 'creates') + creator_observable['misp_object'] = creator_object + creator_observable['used'][self.event_uuid] = True def _parse_registry_key_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING): @@ -916,30 +969,45 @@ def _parse_registry_key_observable_objects( ) misp_object.add_reference(value_uuid, 'contains') return misp_object - for identifier, registry_key in observed_data.objects.items(): - if hasattr(registry_key, 'id'): - return self._parse_registry_key_observable_object_ref( - registry_key, observed_data + observable_objects = { + object_id: {'used': False, 'observable': observable} + for object_id, observable in observed_data.objects.items() + } + for identifier, observable in observable_objects.items(): + observable_object = observable['observable'] + if observable_object.type == 'user-account': + if observable['used']: + continue + misp_object = ( + self._parse_generic_observable_object_ref( + observable_object, observed_data, 'user-account', False + ) if observable['used'] else + self._parse_generic_observable_object( + observed_data, identifier, 'user-account', False + ) ) - object_id = f'{observed_data.id} - {identifier}' - regkey_object = self._create_misp_object_from_observable_object( - 'registry-key', observed_data, object_id - ) - attributes = self._parse_registry_key_observable( - registry_key, object_id - ) - for attribute in attributes: - regkey_object.add_attribute(**attribute) - misp_object = self.main_parser._add_misp_object( - regkey_object, observed_data - ) - if len(registry_key.get('values', [])) > 1: - for index, registry_value in enumerate(registry_key['values']): - value_uuid = self._parse_registry_key_value_observable( - registry_value, observed_data, - f'{object_id} - values - {index}' + observable['misp_object'] = misp_object + observable['used'] = True + continue + misp_object = self._parse_registry_key_observable_object( + observed_data, identifier + ) + if hasattr(observable_object, 'creator_user_ref'): + creator_observable = observable_objects[ + observable_object.creator_user_ref + ] + if creator_observable['used']: + creator_observable['misp_object'].add_reference( + misp_object.uuid, 'creates' ) - misp_object.add_reference(value_uuid, 'contains') + continue + creator_object = self._parse_generic_observable_object( + observed_data, observable_object.creator_user_ref, + 'user-account', False + ) + creator_object.add_reference(misp_object.uuid, 'creates') + creator_observable['misp_object'] = creator_object + creator_observable['used'] = True def _parse_registry_key_value_observable( self, registry_value: _WINDOWS_REGISTRY_VALUE_TYPING, From 0440ceda0f29e20defd7c45e3133fb2abd38abf2 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 19:21:44 +0100 Subject: [PATCH 062/137] wip: [stix2 import] Handling cases where some STIX 2.1 observable objects are referenced by multiple observed data objects --- .../stix2_observed_data_converter.py | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 5e4373e4..8f844f0e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -12,6 +12,7 @@ from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict +from datetime import datetime from pymisp import MISPObject from stix2.v20.observables import ( WindowsRegistryValueType as WindowsRegistryValueType_v20) @@ -136,6 +137,17 @@ def _parse_observable_refs(self, observed_data: ObservedData_v21): # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ + def _handle_observable_object_refs_parsing( + self, observable: dict, observed_data: ObservedData_v21, + *args: tuple) -> MISPObject: + if observable['used'].get(self.event_uuid, False): + misp_object = observable['misp_object'] + self._handle_misp_object_fields(misp_object, observed_data) + return misp_object + return self._parse_generic_observable_object_ref( + observable['observable'], observed_data, *args + ) + def _parse_artifact_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: @@ -286,14 +298,10 @@ def _parse_directory_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - directory = observable['observable'] - misp_object = ( - observable['misp_object'] if - observable['used'].get(self.event_uuid, False) else - self._parse_generic_observable_object_ref( - directory, observed_data, 'directory' - ) + misp_object = self._handle_observable_object_refs_parsing( + observable, observed_data, 'directory' ) + directory = observable['observable'] observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True if not hasattr(directory, 'contains_refs'): @@ -301,6 +309,7 @@ def _parse_directory_observable_object_refs( for contained_ref in directory.contains_refs: contained = self._fetch_observables(contained_ref) if contained['used'].get(self.event_uuid, False): + self._handle_misp_object_fields(misp_object, observed_data) misp_object.add_reference( contained['misp_object'].uuid, 'contains' ) @@ -746,15 +755,11 @@ def _parse_process_observable_object_refs( observable['used'][self.event_uuid] = True observable['misp_object'] = misp_object continue - process = observable['observable'] - misp_object = ( - observable['misp_object'] if - observable['used'].get(self.event_uuid, False) else - self._parse_generic_observable_object_ref( - process, observed_data, 'process', False - ) + misp_object = self._handle_observable_object_refs_parsing( + observable, observed_data, 'process', False ) observable['used'][self.event_uuid] = True + process = observable['observable'] if hasattr(process, 'parent_ref'): self._parse_process_reference_observable_object_ref( observed_data, misp_object, process.parent_ref, 'child-of' @@ -841,6 +846,7 @@ def _parse_process_reference_observable_object_ref( name: Optional[str] = 'process'): observable = self._fetch_observables(reference) if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields(misp_object, observed_data) misp_object.add_reference( observable['misp_object'].uuid, relationship_type ) @@ -915,6 +921,9 @@ def _parse_registry_key_observable_object_refs( observable_object = observable['observable'] if observable_object.type == 'user-account': if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) continue misp_object = self._parse_generic_observable_object_ref( observable_object, observed_data, 'user-account', False @@ -932,6 +941,9 @@ def _parse_registry_key_observable_object_refs( ) if creator_observable['used'].get(self.event_uuid, False): creator_object = creator_observable['misp_object'] + self._handle_misp_object_fields( + creator_object, observed_data + ) creator_object.add_reference(misp_object.uuid, 'creates') continue creator_object = self._parse_generic_observable_object_ref( @@ -1152,6 +1164,24 @@ def _create_misp_object_from_observable_object_ref( self.main_parser._sanitise_object_uuid(misp_object, observable.id) return misp_object + def _handle_misp_object_fields( + self, misp_object: MISPObject, observed_data: ObservedData_v21): + time_fields = self._parse_timeline(observed_data) + for field in ('timestamp', 'last_seen'): + if time_fields.get(field) is None: + continue + if time_fields[field] > misp_object.get(field, datetime.max): + setattr(misp_object, field, time_fields[field]) + if time_fields.get('first_seen') is not None: + field = 'first_seen' + if time_fields[field] < misp_object.get(field, datetime.min): + misp_object.first_seen = time_fields[field] + comment = f'Observed Data ID: {observed_data.id}' + if misp_object.get('comment') is None: + misp_object.comment = comment + elif comment not in misp_object.comment: + misp_object.comment = f'{misp_object.comment} - {comment}' + class InternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, InternalSTIX2ObservableConverter): @@ -1377,7 +1407,7 @@ def _attribute_from_first_observable_v20( def _attribute_from_first_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs[0]) + observable = self._fetch_observables(observed_data.object_refs) attribute['value'] = observable.value self.main_parser._add_misp_attribute(attribute, observed_data) From cf262674e8996d1d9e8f16bafe23571e512791d9 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 13 Feb 2024 20:02:00 +0100 Subject: [PATCH 063/137] wip: [tests] Tests for user account observable objects referenced by registry keys as creators --- tests/_test_stix_import.py | 37 +++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 8 ++++++ tests/test_external_stix20_import.py | 13 ++++++++-- tests/test_external_stix21_bundles.py | 14 +++++++++- tests/test_external_stix21_import.py | 25 +++++++++++++++--- 5 files changed, 91 insertions(+), 6 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 0b08abc4..b1bd8a22 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -453,6 +453,43 @@ def _check_as_fields(self, misp_object, autonomous_system, object_id): self.assertEqual(name.value, autonomous_system.name) self.assertEqual(name.uuid, uuid5(self._UUIDv4, f'{object_id} - description - {name.value}')) + def _check_creator_user_fields(self, misp_object, user_account, object_id): + self.assertEqual(len(misp_object.attributes), 4) + username, account_type, privileged, user_id = misp_object.attributes + self.assertEqual(username.type, 'text') + self.assertEqual(username.object_relation, 'username') + self.assertEqual(username.value, user_account.account_login) + self.assertEqual( + username.uuid, + uuid5(self._UUIDv4, f'{object_id} - username - {username.value}') + ) + self.assertEqual(account_type.type, 'text') + self.assertEqual(account_type.object_relation, 'account-type') + self.assertEqual(account_type.value, user_account.account_type) + self.assertEqual( + account_type.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - account-type - {account_type.value}' + ) + ) + self.assertEqual(privileged.type, 'boolean') + self.assertEqual(privileged.object_relation, 'privileged') + self.assertEqual(privileged.value, user_account.is_privileged) + self.assertEqual( + privileged.uuid, + uuid5( + self._UUIDv4, f'{object_id} - privileged - {privileged.value}' + ) + ) + self.assertEqual(user_id.type, 'text') + self.assertEqual(user_id.object_relation, 'user-id') + self.assertEqual(user_id.value, user_account.user_id) + self.assertEqual( + user_id.uuid, + uuid5(self._UUIDv4, f'{object_id} - user-id - {user_id.value}') + ) + def _check_directory_fields(self, misp_object, directory, object_id): self.assertEqual(len(misp_object.attributes), 5) accessed, created, modified, path, path_enc = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 463d01b8..584e7c5f 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -650,6 +650,7 @@ "key": "hkey_local_machine\\system\\bar\\foo", "modified": "2020-10-25T16:22:00Z", "number_of_subkeys": 2, + "creator_user_ref": "2", "values": [ { "name": "Foo", @@ -662,6 +663,13 @@ "data_type": "REG_DWORD" } ] + }, + "2": { + "type": "user-account", + "user_id": "john.doe", + "account_login": "JohnDoe", + "account_type": "windows-local", + "is_privileged": True } } }, diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 127feedf..e5545cdd 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -543,12 +543,21 @@ def test_stix20_bundle_with_registry_key_objects(self): event = self.parser.misp_event _, report, observed_data1, observed_data2 = bundle.objects misp_objects = self._check_misp_event_features(event, report) - self.assertEqual(len(misp_objects), 5) - multiple1, multiple2, value1, value2, single = misp_objects + self.assertEqual(len(misp_objects), 6) + multiple1, multiple2, value1, value2, creator_user, single = misp_objects self._check_registry_key_object(multiple1, observed_data1, identifier='0') self._check_registry_key_object( multiple2, observed_data1, value1, value2, identifier='1' ) + self._check_misp_object_fields(creator_user, observed_data1, '2') + self._check_creator_user_fields( + creator_user, observed_data1.objects['2'], + f'{observed_data1.id} - 2' + ) + self.assertEqual(len(creator_user.references), 1) + reference = creator_user.references[0] + self.assertEqual(reference.referenced_uuid, multiple2.uuid) + self.assertEqual(reference.relationship_type, 'creates') self._check_registry_key_object(single, observed_data2) def test_stix20_bundle_with_software_objects(self): diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index a971959e..2c6b3404 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -802,7 +802,8 @@ "last_observed": "2020-10-25T16:22:00Z", "number_observed": 1, "object_refs": [ - "windows-registry-key--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0" + "windows-registry-key--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0", + "user-account--5e384ae7-672c-4250-9cda-3b4da964451a" ] }, { @@ -826,6 +827,7 @@ "key": "hkey_local_machine\\system\\bar\\foo", "modified_time": "2020-10-25T16:22:00Z", "number_of_subkeys": 2, + "creator_user_ref": "user-account--5e384ae7-672c-4250-9cda-3b4da964451a", "values": [ { "name": "Foo", @@ -845,6 +847,7 @@ "id": "windows-registry-key--28b2fff7-ca78-483b-9c4f-6f684ee7cdd0", "key": "hkey_local_machine\\system\\foo\\fortytwo", "modified_time": "2020-10-25T16:22:00Z", + "creator_user_ref": "user-account--5e384ae7-672c-4250-9cda-3b4da964451a", "values": [ { "name": "FortyTwoFoo", @@ -852,6 +855,15 @@ "data_type": "REG_QWORD" } ] + }, + { + "type": "user-account", + "spec_version": "2.1", + "id": "user-account--5e384ae7-672c-4250-9cda-3b4da964451a", + "user_id": "john.doe", + "account_login": "JohnDoe", + "account_type": "windows-local", + "is_privileged": True } ] _SOFTWARE_OBJECTS = [ diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 114e7f6e..5e850378 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -497,13 +497,32 @@ def test_stix21_bundle_with_registry_key_objects(self): self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle() event = self.parser.misp_event - _, grouping, od1, od2, key1, key2, key3 = bundle.objects + _, grouping, od1, od2, key1, key2, key3, user = bundle.objects misp_objects = self._check_misp_event_features_from_grouping(event, grouping) - self.assertEqual(len(misp_objects), 5) - multiple1, multiple2, value1, value2, single = misp_objects + self.assertEqual(len(misp_objects), 6) + multiple1, multiple2, value1, value2, creator_user, single = misp_objects self._check_registry_key_object(multiple1, od1, key1) self._check_registry_key_object(multiple2, od1, key2, value1, value2) self._check_registry_key_object(single, od2, key3) + self.assertEqual(creator_user.uuid, user.id.split('--')[1]) + self.assertEqual(creator_user.name, 'user-account') + self.assertEqual( + creator_user.comment, + f'Observed Data ID: {od1.id} - Observed Data ID: {od2.id}' + ) + self.assertEqual(creator_user.timestamp, od1.modified) + self._assert_multiple_equal( + creator_user.first_seen, od1.first_observed, od2.first_observed + ) + self._check_creator_user_fields(creator_user, user, user.id) + self.assertEqual(len(creator_user.references), 2) + reference1, reference2 = creator_user.references + self.assertEqual(reference1.referenced_uuid, multiple2.uuid) + self._assert_multiple_equal( + reference1.relationship_type, + reference2.relationship_type, + 'creates' + ) def test_stix21_bundle_with_software_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_software_objects() From 09316e19f00219d5d9995ebbb68c6d0a10d953d9 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 14 Feb 2024 14:25:54 +0100 Subject: [PATCH 064/137] fix: [stix2 import] Reusing the STIX 2.1 observable objects fetching method --- .../converters/stix2_observable_converter.py | 10 ++++ .../stix2_observable_objects_converter.py | 52 +++++++++---------- .../stix2_observed_data_converter.py | 10 ---- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 290dd6ca..59b73f94 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -174,6 +174,16 @@ def x509_hashes_mapping(cls, field: str) -> Union[dict, None]: class STIX2ObservableConverter(STIX2Converter): + def _fetch_observables(self, object_refs: Union[list, str]): + if isinstance(object_refs, str): + return self.main_parser._observable[object_refs] + if len(object_refs) == 1: + return self.main_parser._observable[object_refs[0]] + return tuple( + self.main_parser._observable[object_ref] + for object_ref in object_refs + ) + def _parse_email_additional_header( self, observable: _EMAIL_MESSAGE_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 1509ccf2..7fab5647 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -63,7 +63,7 @@ def _create_misp_object_from_observable_object( def _parse_artifact_observable_object( self, artifact_ref: str) -> MISPObject: - observable = self.main_parser._observable[artifact_ref] + observable = self._fetch_observables(artifact_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] artifact = observable['observable'] @@ -80,7 +80,7 @@ def _parse_artifact_observable_object( return misp_object def _parse_as_observable_object(self, as_ref: str) -> _MISP_CONTENT_TYPING: - observable = self.main_parser._observable[as_ref] + observable = self._fetch_observables(as_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_object', observable.get('misp_attribute') @@ -121,7 +121,7 @@ def _parse_as_observable_object(self, as_ref: str) -> _MISP_CONTENT_TYPING: def _parse_directory_observable_object( self, directory_ref: str, child: Optional[str] = None ) -> _MISP_CONTENT_TYPING: - observable = self.main_parser._observable[directory_ref] + observable = self._fetch_observables(directory_ref) if observable['used'].get(self.event_uuid, False): return observable.get('misp_object', observable['misp_attribute']) directory = observable['observable'] @@ -148,7 +148,7 @@ def _parse_directory_observable_object( for ref in directory.contains_refs: feature = f"_parse_{ref.split('--')[0]}_observable_object" referenced_object = ( - self.main_parser._observable[child]['observable'] + self._fetch_observables(child)['misp_object'] if child == ref else getattr(self, feature)(ref) ) misp_object.add_reference( @@ -168,14 +168,14 @@ def _parse_directory_observable_object( ) observable['misp_attribute'] = misp_attribute return misp_attribute - file_object = self.main_parser._observable[child]['observable'] + file_object = self._fetch_observables(child)['misp_object'] file_object.add_attribute(**attributes[0]) observable['misp_object'] = file_object return misp_object def _parse_domain_observable_object( self, domain_ref: str) -> _MISP_CONTENT_TYPING: - observable = self.main_parser._observable[domain_ref] + observable = self._fetch_observables(domain_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_object', observable.get('misp_attribute') @@ -201,7 +201,7 @@ def _parse_domain_observable_object( referenced_domain.uuid, 'resolves-to' ) continue - resolved_ip = self.main_parser._observable[reference] + resolved_ip = self._fetch_observables(reference) ip_address = resolved_ip['observable'] misp_object.add_attribute( **self._parse_ip_observable(ip_address) @@ -210,9 +210,7 @@ def _parse_domain_observable_object( resolved_ip['misp_object'] = misp_object if hasattr(ip_address, 'resolves_to_refs'): for referenced_mac in ip_address.resolves_to_refs: - resolved_mac = self.main_parser._observable[ - referenced_mac - ] + resolved_mac = self._fetch_observables(referenced_mac) mac_address = resolved_mac['observable'] attribute = self._create_misp_attribute( 'mac-address', mac_address, @@ -236,7 +234,7 @@ def _parse_domain_observable_object( def _parse_email_address_observable_object( self, email_address_ref: str) -> _MISP_CONTENT_TYPING: - observable = self.main_parser._observable[email_address_ref] + observable = self._fetch_observables(email_address_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_attribute', observable.get('misp_object') @@ -265,7 +263,7 @@ def _parse_email_address_observable_object( def _parse_email_message_observable_object( self, email_message_ref: str) -> MISPObject: - observable = self.main_parser._observable[email_message_ref] + observable = self._fetch_observables(email_message_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] email_message = observable['observable'] @@ -280,7 +278,7 @@ def _parse_email_message_observable_object( ) observable['misp_object'] = misp_object if hasattr(email_message, 'from_ref'): - observable = self.main_parser._observable[email_message.from_ref] + observable = self._fetch_observables(email_message.from_ref) attributes = self._parse_email_reference_observable( observable['observable'], 'from' ) @@ -292,7 +290,7 @@ def _parse_email_message_observable_object( field = f'{feature}_refs' if hasattr(email_message, field): for reference in getattr(email_message, field): - observable = self.main_parser._observable[reference] + observable = self._fetch_observables(reference) attributes = self._parse_email_reference_observable( observable['observable'], feature ) @@ -309,7 +307,7 @@ def _parse_email_message_observable_object( return misp_object def _parse_file_observable_object(self, file_ref: str) -> MISPObject: - observable = self.main_parser._observable[file_ref] + observable = self._fetch_observables(file_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] _file = observable['observable'] @@ -378,7 +376,7 @@ def _parse_ip_addresses_belonging_to_AS(self, AS_id: str): def _parse_ip_address_observable_object( self, ip_address_ref: str) -> _MISP_CONTENT_TYPING: - observable = self.main_parser._observable[ip_address_ref] + observable = self._fetch_observables(ip_address_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_attribute', observable.get('misp_object') @@ -393,7 +391,7 @@ def _parse_ip_address_observable_object( def _parse_mac_address_observable_object( self, mac_address_ref: str) -> MISPAttribute: - observable = self.main_parser._observable[mac_address_ref] + observable = self._fetch_observables(mac_address_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] mac_address = observable['observable'] @@ -405,7 +403,7 @@ def _parse_mac_address_observable_object( return misp_attribute def _parse_mutex_observable_object(self, mutex_ref: str) -> MISPAttribute: - observable = self.main_parser._observable[mutex_ref] + observable = self._fetch_observables(mutex_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] mutex = observable['observable'] @@ -446,7 +444,7 @@ def _parse_network_socket_observable_object( def _parse_network_traffic_observable_object( self, network_traffic_ref: str) -> MISPObject: - observable = self.main_parser._observable[network_traffic_ref] + observable = self._fetch_observables(network_traffic_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] network_traffic = observable['observable'] @@ -459,9 +457,9 @@ def _parse_network_traffic_observable_object( observable['misp_object'] = misp_object for asset in ('src', 'dst'): if hasattr(network_traffic, f'{asset}_ref'): - referenced_object = self.main_parser._observable[ + referenced_object = self._fetch_observables( getattr(network_traffic, f'{asset}_ref') - ] + ) attributes = self._parse_network_traffic_reference_observable( asset, referenced_object['observable'] ) @@ -490,7 +488,7 @@ def _parse_network_traffic_observable_fields( return '_parse_network_connection_observable_object' def _parse_process_observable_object(self, process_ref: str) -> MISPObject: - observable = self.main_parser._observable[process_ref] + observable = self._fetch_observables(process_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] process = observable['observable'] @@ -530,7 +528,7 @@ def _parse_process_observable_object(self, process_ref: str) -> MISPObject: return misp_object def _parse_registry_key_observable_object(self, registry_key_ref: str): - observable = self.main_parser._observable[registry_key_ref] + observable = self._fetch_observables(registry_key_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] registry_key = observable['observable'] @@ -564,7 +562,7 @@ def _parse_registry_key_observable_object(self, registry_key_ref: str): def _parse_software_observable_object( self, software_ref: str) -> MISPObject: - observable = self.main_parser._observable[software_ref] + observable = self._fetch_observables(software_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] software = observable['observable'] @@ -581,7 +579,7 @@ def _parse_software_observable_object( return misp_object def _parse_url_observable_object(self, url_ref: str) -> MISPAttribute: - observable = self.main_parser._observable[url_ref] + observable = self._fetch_observables(url_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] url = observable['observable'] @@ -594,7 +592,7 @@ def _parse_url_observable_object(self, url_ref: str) -> MISPAttribute: def _parse_user_account_observable_object( self, user_account_ref: str) -> MISPObject: - observable = self.main_parser._observable[user_account_ref] + observable = self._fetch_observables(user_account_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] user_account = observable['observable'] @@ -611,7 +609,7 @@ def _parse_user_account_observable_object( return misp_object def _parse_x509_observable_object(self, x509_ref: str) -> MISPObject: - observable = self.main_parser._observable[x509_ref] + observable = self._fetch_observables(x509_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] x509 = observable['observable'] diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 8f844f0e..ddd1844b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -52,16 +52,6 @@ class STIX2ObservedDataConverter(STIX2ObservableConverter, metaclass=ABCMeta): def __init__(self, main: _MAIN_PARSER_TYPING): self._set_main_parser(main) - def _fetch_observables(self, object_refs: Union[list, str]): - if isinstance(object_refs, str): - return self.main_parser._observable[object_refs] - if len(object_refs) == 1: - return self.main_parser._observable[object_refs[0]] - return tuple( - self.main_parser._observable[object_ref] - for object_ref in object_refs - ) - class ExternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, ExternalSTIX2ObservableConverter): From c9c1c66232c55efaab1b90396f3ad247763549c8 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 15 Feb 2024 23:33:03 +0100 Subject: [PATCH 065/137] fix: [stix2 import] Clearer observable objects mapping handling in the observed data conversion methods --- .../stix2_observed_data_converter.py | 49 ++++++++++--------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index ddd1844b..783e8083 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -70,7 +70,7 @@ def parse(self, observed_data_ref: str): observed_data = self.main_parser._get_stix_object(observed_data_ref) try: if hasattr(observed_data, 'object_refs'): - self._parse_observable_refs(observed_data) + self._parse_observable_object_refs(observed_data) else: self._parse_observable_objects(observed_data) except UnknownObservableMappingError as observable_types: @@ -92,36 +92,37 @@ def _set_observable_relationships(self): # GENERIC OBSERVED DATA HANDLING METHODS # ############################################################################ - def _handle_observables_mapping(self, observable_types: set) -> str: - to_call = '_'.join(sorted(observable_types)) - mapping = self._mapping.observable_mapping(to_call) + def _parse_observable_object_refs(self, observed_data: ObservedData_v21): + observable_types = set( + reference.split('--')[0] for reference in observed_data.object_refs + ) + fields = '_'.join(observable_types) + mapping = self._mapping.observable_mapping(fields) if mapping is None: raise UnknownObservableMappingError(to_call) - return mapping + else: + feature = f'_parse_{mapping}_observable_object_refs' + try: + parser = getattr(self, feature) + except AttributeError: + raise UnknownParsingFunctionError(feature) + parser(observed_data) def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): observable_types = set( observable['type'] for observable in observed_data.objects.values() ) - mapping = self._handle_observables_mapping(observable_types) - feature = f'_parse_{mapping}_observable_objects' - try: - parser = getattr(self, feature) - except AttributeError: - raise UnknownParsingFunctionError(feature) - parser(observed_data) - - def _parse_observable_refs(self, observed_data: ObservedData_v21): - observable_types = set( - reference.split('--')[0] for reference in observed_data.object_refs - ) - mapping = self._handle_observables_mapping(observable_types) - feature = f'_parse_{mapping}_observable_object_refs' - try: - parser = getattr(self, feature) - except AttributeError: - raise UnknownParsingFunctionError(feature) - parser(observed_data) + fields = '_'.join(observable_types) + mapping = self._mapping.observable_mapping(fields) + if mapping is None: + raise UnknownObservableMappingError(fields) + else: + feature = f'_parse_{mapping}_observable_objects' + try: + parser = getattr(self, feature) + except AttributeError: + raise UnknownParsingFunctionError(feature) + parser(observed_data) ############################################################################ # OBSERVABLE OBJECTS PARSING METHODS # From 84a790be54776c85d15eb2c566448ec0ef876f49 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 19 Feb 2024 12:07:25 +0100 Subject: [PATCH 066/137] fix: [stix2 import] Added the missing sorting statement for observable objects types passed to match mapping --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 783e8083..99a0548e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -96,7 +96,7 @@ def _parse_observable_object_refs(self, observed_data: ObservedData_v21): observable_types = set( reference.split('--')[0] for reference in observed_data.object_refs ) - fields = '_'.join(observable_types) + fields = '_'.join(sorted(observable_types)) mapping = self._mapping.observable_mapping(fields) if mapping is None: raise UnknownObservableMappingError(to_call) @@ -112,7 +112,7 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): observable_types = set( observable['type'] for observable in observed_data.objects.values() ) - fields = '_'.join(observable_types) + fields = '_'.join(sorted(observable_types)) mapping = self._mapping.observable_mapping(fields) if mapping is None: raise UnknownObservableMappingError(fields) From be0c7fc46e7e718c80b4b70b02a61c4c4792499c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Feb 2024 11:44:36 +0100 Subject: [PATCH 067/137] fix: [stix2 import] Removed unsued import & added missing blank lines to make pep8 happy --- misp_stix_converter/misp_stix_converter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index f3e09a1e..f08df8b7 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -3,7 +3,6 @@ import json import os -import re import sys from .misp2stix.framing import ( _stix1_attributes_framing, _stix1_framing, _handle_namespaces, @@ -714,11 +713,13 @@ def _from_misp(stix_objects): return True return False + def _get_stix2_parser(from_misp: bool, *args: tuple) -> tuple: if from_misp: return InternalSTIX2toMISPParser, args[:-2] return ExternalSTIX2toMISPParser, args + def _load_stix_event(filename, tries=0): try: return STIXPackage.from_xml(filename) @@ -1079,4 +1080,4 @@ def _generate_traceback(debug: bool, parser, *output_names: List[Path]) -> dict: if brol: traceback[feature] = brol traceback['results'] = list(output_names) - return traceback \ No newline at end of file + return traceback From 7ff0b7f400228f4b28f5f0e2eb9c2d94677d5d2f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Feb 2024 11:47:23 +0100 Subject: [PATCH 068/137] fix: [stix2 import] Avoiding issues with STIX 2.x content coming from a TAXII collection or embedded into a single list instead of a Bundle --- misp_stix_converter/misp_stix_converter.py | 10 ++--- misp_stix_converter/stix2misp/__init__.py | 1 + misp_stix_converter/stix2misp/importparser.py | 41 +++++++++++++++++++ .../stix2misp/stix2_to_misp.py | 30 +++++++------- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index f08df8b7..4fc50608 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -14,6 +14,7 @@ from .misp2stix.stix1_mapping import NS_DICT, SCHEMALOC_DICT from .stix2misp.external_stix1_to_misp import ExternalSTIX1toMISPParser from .stix2misp.external_stix2_to_misp import ExternalSTIX2toMISPParser +from .stix2misp.importparser import _load_stix2_content from .stix2misp.internal_stix1_to_misp import InternalSTIX1toMISPParser from .stix2misp.internal_stix2_to_misp import InternalSTIX2toMISPParser from collections import defaultdict @@ -26,8 +27,6 @@ Campaigns, CoursesOfAction, Indicators, ThreatActors, STIXPackage) from stix.core.ttps import TTPs from stix2.base import STIXJSONEncoder -from stix2.exceptions import InvalidValueError -from stix2.parsing import parse as stix2_parser, ParseError from stix2.v20 import Bundle as Bundle_v20 from stix2.v21 import Bundle as Bundle_v21 from typing import List, Optional, Union @@ -672,11 +671,8 @@ def stix_2_to_misp(filename: _files_type, if isinstance(filename, str): filename = Path(filename).resolve() try: - with open(filename, 'rt', encoding='utf-8') as f: - bundle = stix2_parser( - f.read(), allow_custom=True, interoperability=True - ) - except (ParseError, InvalidValueError) as error: + bundle = _load_stix2_content(filename) + except Exception as error: return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( _from_misp(bundle.objects), distribution, sharing_group_id, diff --git a/misp_stix_converter/stix2misp/__init__.py b/misp_stix_converter/stix2misp/__init__.py index 0c3754d7..cc1ff470 100644 --- a/misp_stix_converter/stix2misp/__init__.py +++ b/misp_stix_converter/stix2misp/__init__.py @@ -1,6 +1,7 @@ from .external_stix1_to_misp import ExternalSTIX1toMISPParser # noqa from .external_stix2_mapping import ExternalSTIX2toMISPMapping # noqa from .external_stix2_to_misp import ExternalSTIX2toMISPParser # noqa +from .importparser import _load_stix2_content # noqa from .internal_stix1_to_misp import InternalSTIX1toMISPParser # noqa from .internal_stix2_mapping import InternalSTIX2toMISPMapping # noqa from .internal_stix2_to_misp import InternalSTIX2toMISPParser # noqa diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 796a6179..070bbd0a 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -9,7 +9,11 @@ from collections import defaultdict from pathlib import Path from pymisp import AbstractMISP, MISPEvent, MISPObject +from stix2.exceptions import InvalidValueError +from stix2.parsing import dict_to_stix2, parse as stix2_parser, ParseError +from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v20.sdo import Indicator as Indicator_v20 +from stix2.v21.bundle import Bundle as Bundle_v21 from stix2.v21.sdo import Indicator as Indicator_v21 from types import GeneratorType from typing import Optional, Union @@ -26,6 +30,43 @@ _UUIDv4 = UUID('76beed5f-7251-457e-8c2a-b45f7b589d3d') +def _get_stix2_content_version(stix2_content: dict): + for stix_object in stix2_content['objects']: + if stix_object.get('spec_version'): + return '2.1' + return '2.0' + + +def _handle_stix2_loading_error(stix2_content: dict): + version = _get_stix2_content_version(stix2_content) + if isinstance(stix2_content, dict): + if version == '2.1' and stix2_content.get('spec_version') == '2.0': + del stix2_content['spec_version'] + return dict_to_stix2( + stix2_content, allow_custom=True, interoperability=True + ) + if version == '2.0' and stix2_content.get('spec_version') == '2.1': + stix2_content['spec_version'] = '2.0' + return dict_to_stix2( + stix2_content, allow_custom=True, interoperability=True + ) + bundle = Bundle_v21 if version == '2.1' else Bundle_v20 + if 'objects' in stix2_content: + stix2_content = stix2_content['objects'] + return bundle(*stix2_content, allow_custom=True, interoperability=True) + + +def _load_stix2_content(filename): + with open(filename, 'rt', encoding='utf-8') as f: + stix2_content = f.read() + try: + return stix2_parser( + stix2_content, allow_custom=True, interoperability=True + ) + except (InvalidValueError, ParseError): + return _handle_stix2_loading_error(json.loads(stix2_content)) + + class STIXtoMISPParser(metaclass=ABCMeta): def __init__(self, distribution: int, sharing_group_id: Union[int, None], galaxies_as_tags: bool): diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 8e9325b9..66f7d9c6 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1,19 +1,9 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +from os import walk import sys import time -from .exceptions import ( - MarkingDefinitionLoadingError, ObjectRefLoadingError, - ObjectTypeLoadingError, SynonymsResourceJSONError, - UnavailableGalaxyResourcesError, UnavailableSynonymsResourceError, - UndefinedIndicatorError, UndefinedSTIXObjectError, UndefinedObservableError, - UnknownAttributeTypeError, UnknownObjectNameError, - UnknownParsingFunctionError, UnknownPatternTypeError, - UnknownStixObjectTypeError) -from .external_stix2_mapping import ExternalSTIX2toMISPMapping -from .importparser import STIXtoMISPParser, _INDICATOR_TYPING -from .internal_stix2_mapping import InternalSTIX2toMISPMapping from .converters import ( ExternalSTIX2AttackPatternConverter, ExternalSTIX2MalwareAnalysisConverter, ExternalSTIX2CampaignConverter, InternalSTIX2CampaignConverter, @@ -27,6 +17,18 @@ ExternalSTIX2ThreatActorConverter, InternalSTIX2ThreatActorConverter, ExternalSTIX2ToolConverter, InternalSTIX2ToolConverter, ExternalSTIX2VulnerabilityConverter, InternalSTIX2VulnerabilityConverter) +from .exceptions import ( + MarkingDefinitionLoadingError, ObjectRefLoadingError, + ObjectTypeLoadingError, SynonymsResourceJSONError, + UnavailableGalaxyResourcesError, UnavailableSynonymsResourceError, + UndefinedIndicatorError, UndefinedSTIXObjectError, UndefinedObservableError, + UnknownAttributeTypeError, UnknownObjectNameError, + UnknownParsingFunctionError, UnknownPatternTypeError, + UnknownStixObjectTypeError) +from .external_stix2_mapping import ExternalSTIX2toMISPMapping +from .importparser import ( + STIXtoMISPParser, _INDICATOR_TYPING, _load_stix2_content) +from .internal_stix2_mapping import InternalSTIX2toMISPMapping from abc import ABCMeta from collections import defaultdict from datetime import datetime @@ -34,7 +36,6 @@ AbstractMISP, MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, MISPObject, MISPSighting) from stix2 import TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE -from stix2.parsing import parse as stix2_parser from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v20.common import MarkingDefinition as MarkingDefinition_v20 from stix2.v20.observables import NetworkTraffic as NetworkTraffic_v20 @@ -283,10 +284,7 @@ def parse_stix_bundle(self, single_event: Optional[bool] = False): def parse_stix_content( self, filename: str, single_event: Optional[bool] = False): try: - with open(filename, 'rt', encoding='utf-8') as f: - bundle = stix2_parser( - f.read(), allow_custom=True, interoperability=True - ) + bundle = _load_stix2_content(filename) except Exception as exception: sys.exit(exception) self.load_stix_bundle(bundle) From c4aa78fa98de8bb74c04f1420ee056d36ee6b516 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 20 Feb 2024 23:34:24 +0100 Subject: [PATCH 069/137] fix: [stix2 import] Removed unused import - I guess this was an auto completion typo --- misp_stix_converter/stix2misp/stix2_to_misp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 66f7d9c6..2d905e53 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from os import walk import sys import time from .converters import ( From 74e3236ed5e7abd55c1c04cbb3b7cf14cc6b5328 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 21 Feb 2024 11:20:12 +0100 Subject: [PATCH 070/137] fix: [stix2 import] Fixed MISP Sightings handling --- .../stix2misp/stix2_to_misp.py | 131 ++++++++---------- 1 file changed, 54 insertions(+), 77 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 2d905e53..0127ed4b 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -486,12 +486,27 @@ def _load_observed_data(self, observed_data: _OBSERVED_DATA_TYPING): self._observed_data = {observed_data.id: observed_data} def _load_opinion(self, opinion: Opinion): + misp_sighting = MISPSighting() + sighting_args = { + 'date_sighting': self._timestamp_from_date(opinion.modified), + 'type': '1', + **self._sanitise_attribute_uuid(opinion.id) + } + if hasattr(opinion, 'x_misp_source'): + sighting_args['source'] = opinion.x_misp_source + if hasattr(opinion, 'x_misp_author_ref'): + identity = self._identity[opinion.x_misp_author_ref] + sighting_args['Organisation'] = { + 'uuid': self._sanitise_uuid(identity.id), + 'name': identity.name + } + misp_sighting.from_dict(**sighting_args) opinion_ref = self._sanitise_uuid(opinion.id) try: - self._sighting['opinion'][opinion_ref] = opinion + self._sighting['opinion'][opinion_ref] = misp_sighting except AttributeError: self._sighting = defaultdict(lambda: defaultdict(list)) - self._sighting['opinion'][opinion_ref] = opinion + self._sighting['opinion'][opinion_ref] = misp_sighting for object_ref in opinion.object_refs: sanitised_ref = self._sanitise_uuid(object_ref) self._sighting['opinion_refs'][sanitised_ref].append(opinion_ref) @@ -513,12 +528,27 @@ def _load_report(self, report: _REPORT_TYPING): self._report = {report.id: report} def _load_sighting(self, sighting: _SIGHTING_TYPING): + misp_sighting = MISPSighting() + sighting_args = { + 'date_sighting': self._timestamp_from_date(sighting.modified), + 'type': '0', + **self._sanitise_attribute_uuid(sighting.id) + } + if hasattr(sighting, 'description'): + sighting_args['source'] = sighting.description + if hasattr(sighting, 'where_sighted_refs'): + identity = self._identity[sighting.where_sighted_refs[0]] + sighting_args['Organisation'] = { + 'uuid': self._sanitise_uuid(identity.id), + 'name': identity.name + } + misp_sighting.from_dict(**sighting_args) sighting_of_ref = self._sanitise_uuid(sighting.sighting_of_ref) try: - self._sighting['sighting'][sighting_of_ref].append(sighting) + self._sighting['sighting'][sighting_of_ref].append(misp_sighting) except AttributeError: self._sighting = defaultdict(lambda: defaultdict(list)) - self._sighting['sighting'][sighting_of_ref].append(sighting) + self._sighting['sighting'][sighting_of_ref].append(misp_sighting) def _load_threat_actor(self, threat_actor: _THREAT_ACTOR_TYPING): self._check_uuid(threat_actor.id) @@ -819,21 +849,35 @@ def _handle_meta_fields(self, stix_object: _GALAXY_OBJECTS_TYPING) -> dict: def _handle_attribute_sightings(self, attribute: MISPAttribute): attribute_uuid = attribute.uuid + if attribute_uuid in self.replacement_uuids: + attribute_uuid = self.replacement_uuids[attribute_uuid] if attribute_uuid in self._sighting.get('sighting', {}): - self._parse_attribute_sightings(attribute) + for sighting in self._sighting['sighting'][attribute_uuid]: + attribute.add_sighting(sighting) if attribute_uuid in self._sighting.get('opinion_refs', {}): - self._parse_attribute_opinions(attribute) + for opinion_ref in self._sighting['opinion_refs'][attribute_uuid]: + attribute.add_sighting(self._sighting['opinion'][opinion_ref]) elif attribute_uuid in self._sighting.get('custom_opinion', {}): - self._parse_attribute_custom_opinions(attribute) + for sighting in self._sighting['custom_opinion'][attribute_uuid]: + attribute.add_sighting(sighting) def _handle_object_sightings(self, misp_object: MISPObject): object_uuid = misp_object.uuid + if object_uuid in self.replacement_uuids: + object_uuid = self.replacement_uuids[object_uuid] if object_uuid in self._sighting.get('sighting', {}): - self._parse_object_sightings(misp_object) + for sighting in self._sighting['sighting'][object_uuid]: + for attribute in misp_object.attributes: + attribute.add_sighting(sighting) if object_uuid in self._sighting.get('opinion_refs', {}): - self._parse_object_opinions(misp_object) + for opinion_ref in self._sighting['opinion_refs'][object_uuid]: + sighting = self._sighting['opinion'][opinion_ref] + for attribute in misp_object.attributes: + attribute.add_sighting(sighting) elif misp_object.uuid in self._sighting.get('custom_opinion', {}): - self._parse_object_custom_opinion(misp_object) + for sighting in self._sighting['custom_opinion'][object_uuid]: + for attribute in misp_object.attributes: + attribute.add_sighting(sighting) def _handle_opposite_reference( self, relationship_type: str, source_uuid: str, target_uuid: str): @@ -841,14 +885,6 @@ def _handle_opposite_reference( reference = (source_uuid, self.relationship_types[relationship_type]) self._relationship[sanitised_uuid].add(reference) - def _parse_attribute_custom_opinions(self, attribute: MISPAttribute): - for sighting in self._sighting['custom_opinion'][attribute.uuid]: - attribute.add_sighting(sighting) - - def _parse_attribute_opinions(self, attribute: MISPAttribute): - for opinion_ref in self._sighting['opinion_refs'][attribute.uuid]: - attribute.add_sighting(self._sighting['opinion'][opinion_ref]) - def _parse_attribute_relationships_as_container( self, attribute: MISPAttribute): clusters = defaultdict(list) @@ -882,10 +918,6 @@ def _parse_attribute_relationships_as_tag_names( relationship_type, attribute.uuid, referenced_uuid ) - def _parse_attribute_sightings(self, attribute: MISPAttribute): - for sighting in self._sighting['sighting'][attribute.uuid]: - attribute.add_sighting(self._parse_sighting(sighting)) - def _parse_cluster_relationships(self, cluster: MISPGalaxyCluster): for relationship in self._relationship[cluster.uuid]: referenced_uuid, relationship_type = relationship @@ -901,17 +933,6 @@ def _parse_galaxy_relationships(self): if cluster.uuid in self._relationship: self._parse_cluster_relationships(cluster) - def _parse_object_custom_opinion(self, misp_object: MISPObject): - for sighting in self._sighting['custom_opinion'][misp_object.uuid]: - for attribute in misp_object.attributes: - attribute.add_sighting(sighting) - - def _parse_object_opinions(self, misp_object: MISPObject): - for opinion_ref in self._sighting['opinion_refs'][misp_object.uuid]: - sighting = self._sighting['opinion'][opinion_ref] - for attribute in misp_object.attributes: - attribute.add_sighting(sighting) - def _parse_object_relationships_as_container(self, misp_object: MISPObject): clusters = defaultdict(list) for relationship in self._relationship[misp_object.uuid]: @@ -943,29 +964,6 @@ def _parse_object_relationships_as_tag_names(self, misp_object: MISPObject): self._sanitise_uuid(referenced_uuid), relationship_type ) - def _parse_object_sightings(self, misp_object: MISPObject): - for sighting in self._sighting['sighting'][misp_object.uuid]: - misp_sighting = self._parse_sighting(sighting) - for attribute in misp_object.attributes: - attribute.add_sighting(misp_sighting) - - def _parse_opinion(self, opinion: Opinion) -> MISPSighting: - sighting = MISPSighting() - sighting_args = { - 'date_sighting': self._timestamp_from_date(opinion.modified), - 'type': '1' - } - if hasattr(opinion, 'x_misp_source'): - sighting_args['source'] = opinion.x_misp_source - if hasattr(opinion, 'x_misp_author_ref'): - identity = self._identity[opinion.x_misp_author_ref] - sighting_args['Organisation'] = { - 'uuid': self._sanitise_uuid(identity.id), - 'name': identity.name - } - sighting.from_dict(**sighting_args) - return sighting - def _parse_relationships(self): for attribute in self.misp_event.attributes: if attribute.uuid in self._relationship: @@ -987,8 +985,6 @@ def _parse_relationships(self): self._parse_galaxy_relationships() def _parse_relationships_and_sightings(self): - for opinion_id, opinion in self._sighting['opinion'].items(): - self._sighting['opinion'][opinion_id] = self._parse_opinion(opinion) for attribute in self.misp_event.attributes: if attribute.uuid in self._relationship: getattr( @@ -1010,26 +1006,7 @@ def _parse_relationships_and_sightings(self): if not self.galaxies_as_tags: self._parse_galaxy_relationships() - def _parse_sighting(self, sighting: _SIGHTING_TYPING) -> MISPSighting: - misp_sighting = MISPSighting() - sighting_args = { - 'date_sighting': self._timestamp_from_date(sighting.modified), - 'type': '0' - } - if hasattr(sighting, 'description'): - sighting_args['source'] = sighting.description - if hasattr(sighting, 'where_sighted_refs'): - identity = self._identity[sighting.where_sighted_refs[0]] - sighting_args['Organisation'] = { - 'uuid': self._sanitise_uuid(identity.id), - 'name': identity.name - } - misp_sighting.from_dict(**sighting_args) - return misp_sighting - def _parse_sightings(self): - for opinion_id, opinion in self._sighting['opinion'].items(): - self._sighting['opinion'][opinion_id] = self._parse_opinion(opinion) for attribute in self.misp_event.attributes: self._handle_attribute_sightings(attribute) for misp_object in self.misp_event.objects: From 011fc43459ef55be36c39ee6e929c020cb24dcae Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Feb 2024 13:54:44 +0100 Subject: [PATCH 071/137] fix: [stix2 import] Fixed relationships handling between sighting & opinion objects, and their references --- .../stix2_observed_data_converter.py | 3 + .../stix2misp/stix2_to_misp.py | 57 +++++++++++-------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 99a0548e..7f72c588 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1153,6 +1153,9 @@ def _create_misp_object_from_observable_object_ref( **self._parse_timeline(observed_data) ) self.main_parser._sanitise_object_uuid(misp_object, observable.id) + self.main_parser._check_sighting_replacements( + self.main_parser._sanitise_uuid(observed_data.id), misp_object.uuid + ) return misp_object def _handle_misp_object_fields( diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 0127ed4b..beaf91c2 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -486,30 +486,32 @@ def _load_observed_data(self, observed_data: _OBSERVED_DATA_TYPING): self._observed_data = {observed_data.id: observed_data} def _load_opinion(self, opinion: Opinion): - misp_sighting = MISPSighting() - sighting_args = { - 'date_sighting': self._timestamp_from_date(opinion.modified), - 'type': '1', - **self._sanitise_attribute_uuid(opinion.id) - } - if hasattr(opinion, 'x_misp_source'): - sighting_args['source'] = opinion.x_misp_source - if hasattr(opinion, 'x_misp_author_ref'): - identity = self._identity[opinion.x_misp_author_ref] - sighting_args['Organisation'] = { - 'uuid': self._sanitise_uuid(identity.id), - 'name': identity.name + if opinion.opinion != 'neutral': + misp_sighting = MISPSighting() + sighting_args = { + 'date_sighting': self._timestamp_from_date(opinion.modified), + 'type': '1' if 'disagree' in opinion.opinion else '0' } - misp_sighting.from_dict(**sighting_args) - opinion_ref = self._sanitise_uuid(opinion.id) - try: - self._sighting['opinion'][opinion_ref] = misp_sighting - except AttributeError: - self._sighting = defaultdict(lambda: defaultdict(list)) - self._sighting['opinion'][opinion_ref] = misp_sighting - for object_ref in opinion.object_refs: - sanitised_ref = self._sanitise_uuid(object_ref) - self._sighting['opinion_refs'][sanitised_ref].append(opinion_ref) + if hasattr(opinion, 'x_misp_source'): + sighting_args['source'] = opinion.x_misp_source + if hasattr(opinion, 'x_misp_author_ref'): + identity = self._identity[opinion.x_misp_author_ref] + sighting_args['Organisation'] = { + 'uuid': self._sanitise_uuid(identity.id), + 'name': identity.name + } + misp_sighting.from_dict(**sighting_args) + opinion_ref = self._sanitise_uuid(opinion.id) + try: + self._sighting['opinion'][opinion_ref] = misp_sighting + except AttributeError: + self._sighting = defaultdict(lambda: defaultdict(list)) + self._sighting['opinion'][opinion_ref] = misp_sighting + for object_ref in opinion.object_refs: + sanitised_ref = self._sanitise_uuid(object_ref) + self._sighting['opinion_refs'][sanitised_ref].append( + opinion_ref + ) def _load_relationship(self, relationship: _RELATIONSHIP_TYPING): reference = (relationship.target_ref, relationship.relationship_type) @@ -531,8 +533,7 @@ def _load_sighting(self, sighting: _SIGHTING_TYPING): misp_sighting = MISPSighting() sighting_args = { 'date_sighting': self._timestamp_from_date(sighting.modified), - 'type': '0', - **self._sanitise_attribute_uuid(sighting.id) + 'type': '0' } if hasattr(sighting, 'description'): sighting_args['source'] = sighting.description @@ -847,6 +848,12 @@ def _handle_meta_fields(self, stix_object: _GALAXY_OBJECTS_TYPING) -> dict: # RELATIONSHIPS & SIGHTINGS PARSING METHODS. # ############################################################################ + def _check_sighting_replacements( + self, parent_uuid: str, replaced_uuid: str): + for field in ('opinion_refs', 'sighting'): + if parent_uuid in getattr(self, '_sighting', {}).get(field, {}): + self.replacement_uuids[replaced_uuid] = parent_uuid + def _handle_attribute_sightings(self, attribute: MISPAttribute): attribute_uuid = attribute.uuid if attribute_uuid in self.replacement_uuids: From e56fa9f4c6915d70084b1db1f67f4d82cdbc31ba Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Feb 2024 14:46:35 +0100 Subject: [PATCH 072/137] fix: [stix2 import] In the end we have to parse the Sighting & Opinion objects and convert them as MISP Sighting when they are used - Parsing them when the loading methods are called can raise issues with some referenced identity objects are not loaded already --- .../stix2misp/stix2_to_misp.py | 102 ++++++++++-------- 1 file changed, 55 insertions(+), 47 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index beaf91c2..6eb93c3a 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -486,32 +486,15 @@ def _load_observed_data(self, observed_data: _OBSERVED_DATA_TYPING): self._observed_data = {observed_data.id: observed_data} def _load_opinion(self, opinion: Opinion): - if opinion.opinion != 'neutral': - misp_sighting = MISPSighting() - sighting_args = { - 'date_sighting': self._timestamp_from_date(opinion.modified), - 'type': '1' if 'disagree' in opinion.opinion else '0' - } - if hasattr(opinion, 'x_misp_source'): - sighting_args['source'] = opinion.x_misp_source - if hasattr(opinion, 'x_misp_author_ref'): - identity = self._identity[opinion.x_misp_author_ref] - sighting_args['Organisation'] = { - 'uuid': self._sanitise_uuid(identity.id), - 'name': identity.name - } - misp_sighting.from_dict(**sighting_args) - opinion_ref = self._sanitise_uuid(opinion.id) - try: - self._sighting['opinion'][opinion_ref] = misp_sighting - except AttributeError: - self._sighting = defaultdict(lambda: defaultdict(list)) - self._sighting['opinion'][opinion_ref] = misp_sighting - for object_ref in opinion.object_refs: - sanitised_ref = self._sanitise_uuid(object_ref) - self._sighting['opinion_refs'][sanitised_ref].append( - opinion_ref - ) + opinion_ref = self._sanitise_uuid(opinion.id) + try: + self._sighting['opinion'][opinion_ref] = opinion + except AttributeError: + self._sighting = defaultdict(lambda: defaultdict(list)) + self._sighting['opinion'][opinion_ref] = opinion + for object_ref in opinion.object_refs: + sanitised_ref = self._sanitise_uuid(object_ref) + self._sighting['opinion_refs'][sanitised_ref].append(opinion_ref) def _load_relationship(self, relationship: _RELATIONSHIP_TYPING): reference = (relationship.target_ref, relationship.relationship_type) @@ -530,26 +513,12 @@ def _load_report(self, report: _REPORT_TYPING): self._report = {report.id: report} def _load_sighting(self, sighting: _SIGHTING_TYPING): - misp_sighting = MISPSighting() - sighting_args = { - 'date_sighting': self._timestamp_from_date(sighting.modified), - 'type': '0' - } - if hasattr(sighting, 'description'): - sighting_args['source'] = sighting.description - if hasattr(sighting, 'where_sighted_refs'): - identity = self._identity[sighting.where_sighted_refs[0]] - sighting_args['Organisation'] = { - 'uuid': self._sanitise_uuid(identity.id), - 'name': identity.name - } - misp_sighting.from_dict(**sighting_args) sighting_of_ref = self._sanitise_uuid(sighting.sighting_of_ref) try: - self._sighting['sighting'][sighting_of_ref].append(misp_sighting) + self._sighting['sighting'][sighting_of_ref].append(sighting) except AttributeError: self._sighting = defaultdict(lambda: defaultdict(list)) - self._sighting['sighting'][sighting_of_ref].append(misp_sighting) + self._sighting['sighting'][sighting_of_ref].append(sighting) def _load_threat_actor(self, threat_actor: _THREAT_ACTOR_TYPING): self._check_uuid(threat_actor.id) @@ -860,10 +829,12 @@ def _handle_attribute_sightings(self, attribute: MISPAttribute): attribute_uuid = self.replacement_uuids[attribute_uuid] if attribute_uuid in self._sighting.get('sighting', {}): for sighting in self._sighting['sighting'][attribute_uuid]: - attribute.add_sighting(sighting) + attribute.add_sighting(self._parse_sighting(sighting)) if attribute_uuid in self._sighting.get('opinion_refs', {}): for opinion_ref in self._sighting['opinion_refs'][attribute_uuid]: - attribute.add_sighting(self._sighting['opinion'][opinion_ref]) + attribute.add_sighting( + self._parse_opinion(self._sighting['opinion'][opinion_ref]) + ) elif attribute_uuid in self._sighting.get('custom_opinion', {}): for sighting in self._sighting['custom_opinion'][attribute_uuid]: attribute.add_sighting(sighting) @@ -874,13 +845,16 @@ def _handle_object_sightings(self, misp_object: MISPObject): object_uuid = self.replacement_uuids[object_uuid] if object_uuid in self._sighting.get('sighting', {}): for sighting in self._sighting['sighting'][object_uuid]: + misp_sighting = self._parse_sighting(sighting) for attribute in misp_object.attributes: - attribute.add_sighting(sighting) + attribute.add_sighting(misp_sighting) if object_uuid in self._sighting.get('opinion_refs', {}): for opinion_ref in self._sighting['opinion_refs'][object_uuid]: - sighting = self._sighting['opinion'][opinion_ref] + misp_sighting = self._parse_opinion( + self._sighting['opinion'][opinion_ref] + ) for attribute in misp_object.attributes: - attribute.add_sighting(sighting) + attribute.add_sighting(misp_sighting) elif misp_object.uuid in self._sighting.get('custom_opinion', {}): for sighting in self._sighting['custom_opinion'][object_uuid]: for attribute in misp_object.attributes: @@ -971,6 +945,23 @@ def _parse_object_relationships_as_tag_names(self, misp_object: MISPObject): self._sanitise_uuid(referenced_uuid), relationship_type ) + def _parse_opinion(self, opinion: Opinion) -> MISPSighting: + misp_sighting = MISPSighting() + sighting_args = { + 'date_sighting': self._timestamp_from_date(opinion.modified), + 'type': '1' if 'disagree' in opinion.opinion else '0' + } + if hasattr(opinion, 'x_misp_source'): + sighting_args['source'] = opinion.x_misp_source + if hasattr(opinion, 'x_misp_author_ref'): + identity = self._identity[opinion.x_misp_author_ref] + sighting_args['Organisation'] = { + 'uuid': self._sanitise_uuid(identity.id), + 'name': identity.name + } + misp_sighting.from_dict(**sighting_args) + return misp_sighting + def _parse_relationships(self): for attribute in self.misp_event.attributes: if attribute.uuid in self._relationship: @@ -1013,6 +1004,23 @@ def _parse_relationships_and_sightings(self): if not self.galaxies_as_tags: self._parse_galaxy_relationships() + def _parse_sighting(self, sighting: _SIGHTING_TYPING) -> MISPSighting: + misp_sighting = MISPSighting() + sighting_args = { + 'date_sighting': self._timestamp_from_date(sighting.modified), + 'type': '0' + } + if hasattr(sighting, 'description'): + sighting_args['source'] = sighting.description + if hasattr(sighting, 'where_sighted_refs'): + identity = self._identity[sighting.where_sighted_refs[0]] + sighting_args['Organisation'] = { + 'uuid': self._sanitise_uuid(identity.id), + 'name': identity.name + } + misp_sighting.from_dict(**sighting_args) + return misp_sighting + def _parse_sightings(self): for attribute in self.misp_event.attributes: self._handle_attribute_sightings(attribute) From f8bd81c579598e3573e9190517b9cdffbb043428 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 22 Feb 2024 15:54:41 +0100 Subject: [PATCH 073/137] fix: [stix2 import] Fixed the case with multiple events as result - As `single_event` was set again for each report or grouping, there was no possibility the multiple events were saved accordingly on different result files --- misp_stix_converter/stix2misp/stix2_to_misp.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 6eb93c3a..bc3ff959 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -608,13 +608,11 @@ def _handle_misp_event_tags( self._fetch_tags_from_labels(misp_event, stix_object.labels) def _misp_event_from_grouping(self, grouping: Grouping) -> MISPEvent: - self.__single_event = True misp_event = self._create_misp_event(grouping) misp_event.published = False return misp_event def _misp_event_from_report(self, report: _REPORT_TYPING) -> MISPEvent: - self.__single_event = True misp_event = self._create_misp_event(report) if report.published != report.modified: misp_event.published = True @@ -639,20 +637,19 @@ def _parse_bundle_with_multiple_reports(self): self._handle_object_refs(grouping.object_refs) self._handle_unparsed_content() else: - events = [] + self.__misp_events = [] if hasattr(self, '_report') and self._report is not None: for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) self._handle_object_refs(report.object_refs) self._handle_unparsed_content() - events.append(self.misp_event) + self.__misp_events.append(self.misp_event) if hasattr(self, '_grouping') and self._grouping is not None: for grouping in self._grouping.values(): self.__misp_event = self._misp_event_from_grouping(grouping) self._handle_object_refs(grouping.object_refs) self._handle_unparsed_content() - events.append(self.misp_event) - self.__misp_events = events + self.__misp_events.append(self.misp_event) def _parse_bundle_with_no_report(self): self.__single_event = True From 2aba763d973fae07df72e5f1f6ee2fcf8ab6c721 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 23 Feb 2024 14:34:16 +0100 Subject: [PATCH 074/137] fix: [stix2 import] Setting `single_event` when parsing a bundle with a single report/grouping, to avoid issues raised with multiple reports/groupings handling methods --- misp_stix_converter/stix2misp/stix2_to_misp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index bc3ff959..292bc827 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -658,6 +658,7 @@ def _parse_bundle_with_no_report(self): self._handle_unparsed_content() def _parse_bundle_with_single_report(self): + self.__single_event = True if hasattr(self, '_report') and self._report is not None: for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) From b6dfa0483965d0da14e9be62f998957894f632ae Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 23 Feb 2024 21:27:00 +0100 Subject: [PATCH 075/137] add: [stix2 import] Added `organisation_uuid` argument to use to generate the custom clusters UUID --- misp_stix_converter/__init__.py | 5 +++ misp_stix_converter/misp_stix_converter.py | 10 ++++-- misp_stix_converter/stix2misp/__init__.py | 2 +- .../stix2misp/converters/stix2converter.py | 7 ++-- .../stix2misp/external_stix2_to_misp.py | 34 ++++++++++++------- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 782f3afc..10bf865a 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -30,6 +30,7 @@ from .stix2misp import ExternalSTIX2toMISPParser, InternalSTIX2toMISPParser # noqa from .stix2misp import ExternalSTIX2toMISPMapping, InternalSTIX2toMISPMapping # noqa from .stix2misp import STIX2PatternParser # noqa +from .stix2misp import MISP_org_uuid # noqa from pathlib import Path @@ -147,6 +148,10 @@ def main(): '--galaxies_as_tags', action='store_true', help='Import MISP Galaxies as tag names instead of the standard Galaxy format.' ) + import_parser.add_argument( + '--org_uuid', default=MISP_org_uuid, + help='Organisation UUID to use when creating custom Galaxy clusters.' + ) import_parser.add_argument( '-cd', '--cluster_distribution', type=int, default=0, help='Galaxy Clusters distribution level in case of External STIX 2 content.' diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 4fc50608..4afe0cde 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -13,7 +13,8 @@ from .misp2stix.misp_to_stix21 import MISPtoSTIX21Parser from .misp2stix.stix1_mapping import NS_DICT, SCHEMALOC_DICT from .stix2misp.external_stix1_to_misp import ExternalSTIX1toMISPParser -from .stix2misp.external_stix2_to_misp import ExternalSTIX2toMISPParser +from .stix2misp.external_stix2_to_misp import ( + ExternalSTIX2toMISPParser, MISP_org_uuid) from .stix2misp.importparser import _load_stix2_content from .stix2misp.internal_stix1_to_misp import InternalSTIX1toMISPParser from .stix2misp.internal_stix2_to_misp import InternalSTIX2toMISPParser @@ -664,6 +665,7 @@ def stix_2_to_misp(filename: _files_type, debug: Optional[bool] = False, distribution: Optional[int] = 0, galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, output_dir: Optional[_files_type]=None, output_name: Optional[_files_type]=None, sharing_group_id: Optional[int] = None, @@ -676,7 +678,8 @@ def stix_2_to_misp(filename: _files_type, return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( _from_misp(bundle.objects), distribution, sharing_group_id, - galaxies_as_tags, cluster_distribution, cluster_sharing_group_id + galaxies_as_tags, organisation_uuid, + cluster_distribution, cluster_sharing_group_id ) stix_parser = parser(*args) stix_parser.load_stix_bundle(bundle) @@ -712,7 +715,7 @@ def _from_misp(stix_objects): def _get_stix2_parser(from_misp: bool, *args: tuple) -> tuple: if from_misp: - return InternalSTIX2toMISPParser, args[:-2] + return InternalSTIX2toMISPParser, args[:-3] return ExternalSTIX2toMISPParser, args @@ -1017,6 +1020,7 @@ def _stix_to_misp(stix_args): cluster_sharing_group_id=stix_args.cluster_sharing_group, debug=stix_args.debug, distribution=stix_args.distribution, galaxies_as_tags=stix_args.galaxies_as_tags, + organisation_uuid=stix_args.org_uuid, output_dir=stix_args.output_dir, output_name=stix_args.output_name, sharing_group_id=stix_args.sharing_group, diff --git a/misp_stix_converter/stix2misp/__init__.py b/misp_stix_converter/stix2misp/__init__.py index cc1ff470..b4f7633a 100644 --- a/misp_stix_converter/stix2misp/__init__.py +++ b/misp_stix_converter/stix2misp/__init__.py @@ -1,6 +1,6 @@ from .external_stix1_to_misp import ExternalSTIX1toMISPParser # noqa from .external_stix2_mapping import ExternalSTIX2toMISPMapping # noqa -from .external_stix2_to_misp import ExternalSTIX2toMISPParser # noqa +from .external_stix2_to_misp import ExternalSTIX2toMISPParser, MISP_org_uuid # noqa from .importparser import _load_stix2_content # noqa from .internal_stix1_to_misp import InternalSTIX1toMISPParser # noqa from .internal_stix2_mapping import InternalSTIX2toMISPMapping # noqa diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index d0f193a7..29b74e08 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -220,8 +220,11 @@ def _create_cluster_args( cluster_value: Optional[str] = None) -> dict: value = cluster_value or stix_object.name cluster_args = { - 'uuid': self.main_parser._sanitise_uuid(stix_object.id), - 'value': value, **self.main_parser.cluster_distribution + 'value': value, **self.main_parser.cluster_distribution, + 'uuid': self.main_parser._create_v5_uuid( + f'{self.main_parser._extract_uuid(stix_object.id)} -' + f' {self.main_parser.organisation_uuid}' + ) } if galaxy_type is None: version = getattr(stix_object, 'spec_version', '2.0') diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index edcb8421..bc50e61f 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -50,6 +50,8 @@ ObservedData as ObservedData_v21, Vulnerability as Vulnerability_v21) from typing import Optional, Tuple, Union +MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' + # Useful lists _observable_skip_properties = ( 'content_ref', 'content_type', 'decryption_key', 'defanged', @@ -130,6 +132,7 @@ class ExternalSTIX2toMISPParser(STIX2toMISPParser): def __init__(self, distribution: Optional[int] = 0, sharing_group_id: Optional[int] = None, galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, cluster_distribution: Optional[int] = 0, cluster_sharing_group_id: Optional[int] = None): super().__init__(distribution, sharing_group_id, galaxies_as_tags) @@ -137,6 +140,7 @@ def __init__(self, distribution: Optional[int] = 0, self._sanitise_distribution(cluster_distribution), self._sanitise_sharing_group_id(cluster_sharing_group_id) ) + self.__organisation_uuid = organisation_uuid self._mapping = ExternalSTIX2toMISPMapping # parsers self._attack_pattern_parser: ExternalSTIX2AttackPatternConverter @@ -163,10 +167,14 @@ def observable_object_parser(self) -> STIX2ObservableObjectConverter: self._set_observable_object_parser() return self._observable_object_parser - def _set_attack_pattern_parser(self) -> ExternalSTIX2AttackPatternConverter: + @property + def organisation_uuid(self) -> str: + return self.__organisation_uuid + + def _set_attack_pattern_parser(self): self._attack_pattern_parser = ExternalSTIX2AttackPatternConverter(self) - def _set_campaign_parser(self) -> ExternalSTIX2CampaignConverter: + def _set_campaign_parser(self): self._campaign_parser = ExternalSTIX2CampaignConverter(self) def _set_cluster_distribution( @@ -176,37 +184,37 @@ def _set_cluster_distribution( cluster_distribution['sharing_group_id'] = sharing_group_id self.__cluster_distribution = cluster_distribution - def _set_course_of_action_parser(self) -> ExternalSTIX2CourseOfActionConverter: + def _set_course_of_action_parser(self): self._course_of_action_parser = ExternalSTIX2CourseOfActionConverter(self) - def _set_identity_parser(self) -> ExternalSTIX2IdentityConverter: + def _set_identity_parser(self): self._identity_parser = ExternalSTIX2IdentityConverter(self) - def _set_indicator_parser(self) -> ExternalSTIX2IndicatorConverter: + def _set_indicator_parser(self): self._indicator_parser = ExternalSTIX2IndicatorConverter(self) - def _set_intrusion_set_parser(self) -> ExternalSTIX2IntrusionSetConverter: + def _set_intrusion_set_parser(self): self._intrusion_set_parser = ExternalSTIX2IntrusionSetConverter(self) - def _set_location_parser(self) -> ExternalSTIX2LocationConverter: + def _set_location_parser(self): self._location_parser = ExternalSTIX2LocationConverter(self) - def _set_malware_analysis_parser(self) -> ExternalSTIX2MalwareAnalysisConverter: + def _set_malware_analysis_parser(self): self._malware_analysis_parser = ExternalSTIX2MalwareAnalysisConverter(self) - def _set_malware_parser(self) -> ExternalSTIX2MalwareConverter: + def _set_malware_parser(self): self._malware_parser = ExternalSTIX2MalwareConverter(self) - def _set_observable_object_parser(self) -> STIX2ObservableObjectConverter: + def _set_observable_object_parser(self): self._observable_object_parser = STIX2ObservableObjectConverter(self) - def _set_threat_actor_parser(self) -> ExternalSTIX2ThreatActorConverter: + def _set_threat_actor_parser(self): self._threat_actor_parser = ExternalSTIX2ThreatActorConverter(self) - def _set_tool_parser(self) -> ExternalSTIX2ToolConverter: + def _set_tool_parser(self): self._tool_parser = ExternalSTIX2ToolConverter(self) - def _set_vulnerability_parser(self) -> ExternalSTIX2VulnerabilityConverter: + def _set_vulnerability_parser(self): self._vulnerability_parser = ExternalSTIX2VulnerabilityConverter(self) ############################################################################ From 7b4b895ddf973a89c13b80104c52476bcde80709 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 23 Feb 2024 21:28:49 +0100 Subject: [PATCH 076/137] fix: [tests] Fixed tests for external STIX 2.x SDOs imported as Galaxy Clusters following the recent add of the `organisation_uuid` argument --- tests/_test_stix_import.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index b1bd8a22..89b4169d 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -6,7 +6,7 @@ from collections import defaultdict from misp_stix_converter import ( ExternalSTIX2toMISPMapping, ExternalSTIX2toMISPParser, - InternalSTIX2toMISPParser) + InternalSTIX2toMISPParser, MISP_org_uuid) from uuid import UUID, uuid5 from ._test_stix import TestSTIX from .update_documentation import AttributesDocumentationUpdater, ObjectsDocumentationUpdater @@ -334,7 +334,13 @@ def _check_galaxy_features(self, galaxies, stix_object): galaxy = galaxies[0] self.assertEqual(len(galaxy.clusters), 1) cluster = galaxy.clusters[0] - self.assertEqual(cluster.uuid, stix_object.id.split('--')[1]) + self.assertEqual( + cluster.uuid, + uuid5( + self._UUIDv4, + f"{stix_object.id.split('--')[1]} - {MISP_org_uuid}" + ) + ) version = getattr(stix_object, 'spec_version', '2.0') self._assert_multiple_equal( galaxy.type, cluster.type, f'stix-{version}-{stix_object.type}' From 289b6f8783cfeca67e1ecba6c5a811f1475b37e5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 26 Feb 2024 10:37:29 +0100 Subject: [PATCH 077/137] wip: [tests] Tests for some objects referenced by Opinions --- tests/test_external_stix21_bundles.py | 34 +++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 28 ++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 2c6b3404..4cc7443d 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -1447,6 +1447,40 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_opinion_objects(cls): + agree_opinion = { + "type": "opinion", + "spec_version": "2.1", + "id": "opinion--3b7f3754-a31c-4bf8-a97f-a8ff10aab5a3", + "created_by_ref": "identity--b3bca3c2-1f3d-4b54-b44f-dac42c3a8f01", + "created": "2024-02-17T00:47:42.000Z", + "modified": "2024-02-17T00:47:42.000Z", + "opinion": "agree", + "explanation": "Not confirmed; possibly malicious (if marked or otherwise evaluated as malicious-activity) or benign (if marked as benign); no other information on the subject known to the opinion author. Please see AIS Scoring Framework used for Indicator Enrichment at https://www.cisa.gov/ais.", + "object_refs": [ + "observed-data--e812789e-e49d-47e2-b334-8ee0e8a766ce" + ] + } + strongly_disagree_opinion = { + "type": "opinion", + "spec_version": "2.1", + "id": "opinion--b01efc25-77b4-4003-b18b-f6e24b5cd9f7", + "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", + "created": "2016-05-12T08:17:27.000Z", + "modified": "2016-05-12T08:17:27.000Z", + "opinion": "strongly-disagree", + "explanation": "This doesn't seem like it is feasible.", + "object_refs": [ + "software--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" + ] + } + return cls.__assemble_bundle( + agree_opinion, strongly_disagree_opinion, + deepcopy(_DIRECTORY_OBJECTS[1]), deepcopy(_SOFTWARE_OBJECTS[0]), + deepcopy(_DIRECTORY_OBJECTS[-1]), *deepcopy(_SOFTWARE_OBJECTS[2:-1]) + ) + @classmethod def get_bundle_with_process_objects(cls): return cls.__assemble_bundle(*_PROCESS_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 5e850378..2ef133fd 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -4,6 +4,7 @@ from .test_external_stix21_bundles import TestExternalSTIX21Bundles from ._test_stix import TestSTIX21 from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import +from datetime import datetime from uuid import uuid5 @@ -448,6 +449,33 @@ def test_stix21_bundle_with_mutex_attributes(self): self._check_generic_attribute(od1, mutex_2, m_mutex2, 'mutex', 'name') self._check_generic_attribute(od2, mutex_3, s_mutex, 'mutex', 'name') + def test_stix21_bundle_with_opinion_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_opinion_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, opinion1, opinion2, od1, od2, directory, software1, software2 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + sighted_directory, sighted_software, software = misp_objects + self._check_directory_object(sighted_directory, od1, directory) + self._check_software_object(sighted_software, od2, software1) + self._check_software_object(software, od2, software2) + for directory_attribute in sighted_directory.attributes: + self.assertEqual(len(directory_attribute.sightings), 1) + sighting = directory_attribute.sightings[0] + self.assertEqual( + sighting.date_sighting, datetime.timestamp(opinion1.modified) + ) + self.assertEqual(sighting.type, '0') + for software_attribute in sighted_software.attributes: + self.assertEqual(len(software_attribute.sightings), 1) + sighting = software_attribute.sightings[0] + self.assertEqual( + sighting.date_sighting, datetime.timestamp(opinion2.modified) + ) + self.assertEqual(sighting.type, '1') + def test_stix21_bundle_with_process_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_process_objects() self.parser.load_stix_bundle(bundle) From 3f43d502fc6b2f6ea379bce3827df6a897e5bb22 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 26 Feb 2024 15:16:24 +0100 Subject: [PATCH 078/137] fix: [stix2 import] Fixed wrong variable name for a MISP object meta fields check --- .../stix2misp/converters/stix2_observed_data_converter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 7f72c588..168f5fe2 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -300,10 +300,11 @@ def _parse_directory_observable_object_refs( for contained_ref in directory.contains_refs: contained = self._fetch_observables(contained_ref) if contained['used'].get(self.event_uuid, False): - self._handle_misp_object_fields(misp_object, observed_data) - misp_object.add_reference( - contained['misp_object'].uuid, 'contains' + contained_object = contained['misp_object'] + self._handle_misp_object_fields( + contained_object, observed_data ) + misp_object.add_reference(contained_object.uuid, 'contains') continue if contained_ref not in observed_data.object_refs: self.observable_relationships[misp_object.uuid].add( From 66413dd0a2686c01ea908868b79eed559a07f60c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 26 Feb 2024 21:15:13 +0100 Subject: [PATCH 079/137] fix: [stix2 import] Returning MISPAttributes in some generic observable objects conversion methods --- .../converters/stix2_observed_data_converter.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 168f5fe2..9601b314 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -13,7 +13,7 @@ from abc import ABCMeta from collections import defaultdict from datetime import datetime -from pymisp import MISPObject +from pymisp import MISPAttribute, MISPObject from stix2.v20.observables import ( WindowsRegistryValueType as WindowsRegistryValueType_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 @@ -559,13 +559,14 @@ def _parse_generic_observable_object( def _parse_generic_observable_object_as_attribute( self, observed_data: _OBSERVED_DATA_TYPING, identifier: str, - attribute_type: str, feature: Optional[str] = 'value'): + attribute_type: str, feature: Optional[str] = 'value' + ) -> MISPAttribute: observable_object = observed_data.objects[identifier] if hasattr(observable_object, 'id'): return self._parse_generic_observable_object_ref_as_attribute( observable_object, observed_data, attribute_type, feature ) - self.main_parser._add_misp_attribute( + return self.main_parser._add_misp_attribute( { 'type': attribute_type, 'value': getattr(observable_object, feature), @@ -599,8 +600,8 @@ def _parse_generic_observable_object_ref( def _parse_generic_observable_object_ref_as_attribute( self, observable_object: _GENERIC_OBSERVABLE_TYPING, observed_data: _OBSERVED_DATA_TYPING, attribute_type: str, - feature: Optional[str] = 'value'): - self.main_parser._add_misp_attribute( + feature: Optional[str] = 'value') -> MISPAttribute: + return self.main_parser._add_misp_attribute( { 'type': attribute_type, 'value': getattr(observable_object, feature), @@ -1160,7 +1161,8 @@ def _create_misp_object_from_observable_object_ref( return misp_object def _handle_misp_object_fields( - self, misp_object: MISPObject, observed_data: ObservedData_v21): + self, misp_object: MISPAttribute | MISPObject, + observed_data: ObservedData_v21): time_fields = self._parse_timeline(observed_data) for field in ('timestamp', 'last_seen'): if time_fields.get(field) is None: From 2adb8c07d7a09cda773b7ff454eef2eb4361594d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 26 Feb 2024 21:36:03 +0100 Subject: [PATCH 080/137] wip: [stix2 import] Parsing the observable objects referenced with `contains_refs` references in a generic method that will be reused later --- .../stix2_observed_data_converter.py | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9601b314..54e647af 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -12,6 +12,7 @@ from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta from collections import defaultdict +from collections.abc import Generator from datetime import datetime from pymisp import MISPAttribute, MISPObject from stix2.v20.observables import ( @@ -285,41 +286,54 @@ def _parse_autonomous_system_observable_object_ref( ) return self.main_parser._add_misp_object(misp_object, observed_data) + def _parse_contained_object_refs( + self, observed_data: ObservedData_v21, misp_object_uuid: str, + *contains_refs: tuple) -> Generator: + for contained_ref in contains_refs: + contained = self._fetch_observables(contained_ref) + if contained['used'].get(self.event_uuid, False): + contained_object = contained['misp_object'] + self._handle_misp_object_fields(contained_object, observed_data) + yield contained_object.uuid + continue + if contained_ref not in observed_data.object_refs: + self.observable_relationships[misp_object_uuid].add( + ( + self.main_parser._sanitise_uuid(contained_ref), + 'contains' + ) + ) + continue + observable_object = contained['observable'] + contained_object = self._parse_generic_observable_object_ref( + observable_object, observed_data, observable_object.type, + (observable_object.type == 'directory') + ) + contained['misp_object'] = contained_object + contained['used'][self.event_uuid] = True + yield contained_object.uuid + def _parse_directory_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue misp_object = self._handle_observable_object_refs_parsing( observable, observed_data, 'directory' ) directory = observable['observable'] observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True - if not hasattr(directory, 'contains_refs'): - continue - for contained_ref in directory.contains_refs: - contained = self._fetch_observables(contained_ref) - if contained['used'].get(self.event_uuid, False): - contained_object = contained['misp_object'] - self._handle_misp_object_fields( - contained_object, observed_data - ) - misp_object.add_reference(contained_object.uuid, 'contains') - continue - if contained_ref not in observed_data.object_refs: - self.observable_relationships[misp_object.uuid].add( - ( - self.main_parser._sanitise_uuid(contained_ref), - 'contains' - ) - ) - continue - contained_object = self._parse_generic_observable_object_ref( - contained['observable'], observed_data, 'directory' + if hasattr(directory, 'contains_refs'): + contained_uuids = self._parse_contained_object_refs( + observed_data, misp_object.uuid, *directory.contains_refs ) - contained['misp_object'] = contained_object - contained['used'][self.event_uuid] = True - misp_object.add_reference(contained_object.uuid, 'contains') + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') def _parse_directory_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING): From 7172bf47b902fb7b2886c9741a5814b666f6f967 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 29 Feb 2024 00:17:48 +0100 Subject: [PATCH 081/137] wip: [stix2 import] Better embedded directory observable object references parsing --- .../stix2_observed_data_converter.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 54e647af..6d0cb508 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -313,6 +313,23 @@ def _parse_contained_object_refs( contained['used'][self.event_uuid] = True yield contained_object.uuid + def _parse_contained_objects( + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: dict, *contained_refs: tuple) -> Generator: + for contained_ref in contained_refs: + contained = observable_objects[contained_ref] + if contained['used']: + yield contained['misp_object'].uuid + continue + observable_object = observed_data.objects[contained_ref] + misp_object = self._parse_generic_observable_object( + observed_data, contained_ref, observable_object.type, + (observable_object.type == 'directory') + ) + contained['misp_object'] = misp_object + contained['used'] = True + yield misp_object.uuid + def _parse_directory_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: @@ -356,21 +373,11 @@ def _parse_directory_observable_objects( observable['misp_object'] = misp_object observable['used'] = True if hasattr(directory, 'contains_refs'): - for contained_ref in directory.contains_refs: - contained = observable_objects[contained_ref] - if contained['used']: - misp_object.add_reference( - contained['misp_object'].uuid, 'contains' - ) - continue - contained_object = self._parse_generic_observable_object( - observed_data, contained_ref, 'directory' - ) - contained['misp_object'] = contained_object - contained['used'] = True - misp_object.add_reference( - contained_object.uuid, 'contains' - ) + contained_uuids = self._parse_contained_objects( + observed_data, observable_objects, *directory.contains_refs + ) + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') def _parse_domain_observable_object_refs( self, observed_data: ObservedData_v21): From d1b42f4511572ae6bdd10d11a2e9e5a6254cc7f1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 29 Feb 2024 23:14:14 +0100 Subject: [PATCH 082/137] wip: [stix2 import] Better observable objects parsing --- .../stix2_observed_data_converter.py | 270 +++++++++++------- 1 file changed, 165 insertions(+), 105 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 6d0cb508..22808ce3 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -136,18 +136,39 @@ def _handle_observable_object_refs_parsing( misp_object = observable['misp_object'] self._handle_misp_object_fields(misp_object, observed_data) return misp_object - return self._parse_generic_observable_object_ref( + misp_object = self._parse_generic_observable_object_ref( observable['observable'], observed_data, *args ) + observable['misp_object'] = misp_object + observable['used'][self.event_uuid] = True + return misp_object + + def _handle_observable_objects_parsing( + self, observable_objects: dict, object_id: str, + observed_data: _OBSERVED_DATA_TYPING, *args: tuple) -> MISPObject: + observable = observable_objects[object_id] + if observable['used']: + return observable['misp_object'] + misp_object = self._parse_generic_observable_object( + observed_data, object_id, *args + ) + observable.update({'misp_object': misp_object, 'used': True}) + return misp_object def _parse_artifact_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue artifact = observable['observable'] - self._parse_generic_observable_object_ref( + misp_object = self._parse_generic_observable_object_ref( artifact, observed_data, 'artifact', False ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True def _parse_artifact_observable_objects( @@ -201,25 +222,36 @@ def _parse_as_observable_object( def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable[ + 'misp_object' if 'misp_object' in observable + else 'misp_attribute' + ], + observed_data + ) + continue autonomous_system = observable['observable'] - if hasattr(autonomous_system, 'name'): - self._parse_autonomous_system_observable_object_ref( - autonomous_system, observed_data + if not hasattr(autonomous_system, 'name'): + attribute = self.main_parser._add_misp_attribute( + { + 'type': 'AS', + 'value': self._parse_AS_value(autonomous_system.number), + **self._parse_timeline(observed_data), + **self.main_parser._sanitise_attribute_uuid( + autonomous_system.id, + comment=f'Observed Data ID: {observed_data.id}' + ) + }, + observed_data ) + observable['misp_attribute'] = attribute observable['used'][self.event_uuid] = True continue - self.main_parser._add_misp_attribute( - { - 'type': 'AS', - 'value': self._parse_AS_value(autonomous_system.number), - **self._parse_timeline(observed_data), - **self.main_parser._sanitise_attribute_uuid( - autonomous_system.id, - comment=f'Observed Data ID: {observed_data.id}' - ) - }, - observed_data + misp_object = self._parse_autonomous_system_observable_object_ref( + autonomous_system, observed_data ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True def _parse_as_observable_objects( @@ -326,8 +358,7 @@ def _parse_contained_objects( observed_data, contained_ref, observable_object.type, (observable_object.type == 'directory') ) - contained['misp_object'] = misp_object - contained['used'] = True + contained.update({'misp_object': misp_object, 'used': True}) yield misp_object.uuid def _parse_directory_observable_object_refs( @@ -343,8 +374,6 @@ def _parse_directory_observable_object_refs( observable, observed_data, 'directory' ) directory = observable['observable'] - observable['misp_object'] = misp_object - observable['used'][self.event_uuid] = True if hasattr(directory, 'contains_refs'): contained_uuids = self._parse_contained_object_refs( observed_data, misp_object.uuid, *directory.contains_refs @@ -359,19 +388,13 @@ def _parse_directory_observable_objects( observed_data, 'directory' ) observable_objects = { - object_id: {'used': False, 'observable': observable} - for object_id, observable in observed_data.objects.items() + object_id: {'used': False} for object_id in observed_data.objects } - for object_id, observable in observable_objects.items(): - directory = observable['observable'] - misp_object = ( - observable['misp_object'] if observable['used'] else - self._parse_generic_observable_object( - observed_data, object_id, 'directory' - ) + for object_id in observable_objects.keys(): + misp_object = self._handle_observable_objects_parsing( + observable_objects, object_id, observed_data, 'directory' ) - observable['misp_object'] = misp_object - observable['used'] = True + directory = observed_data.objects[object_id] if hasattr(directory, 'contains_refs'): contained_uuids = self._parse_contained_objects( observed_data, observable_objects, *directory.contains_refs @@ -383,10 +406,16 @@ def _parse_domain_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue domain = observable['observable'] - self._parse_generic_observable_object_ref_as_attribute( + attribute = self._parse_generic_observable_object_ref_as_attribute( domain, observed_data, 'domain' ) + observable['misp_attribute'] = attribute observable['used'][self.event_uuid] = True def _parse_domain_observable_objects( @@ -454,14 +483,14 @@ def _parse_email_address_observable_object( def _parse_email_address_observable_object_ref( self, email_address: _EMAIL_ADDRESS_TYPING, - observed_data: _OBSERVED_DATA_TYPING): + observed_data: _OBSERVED_DATA_TYPING) -> Generator: if hasattr(email_address, 'display_name'): attribute = { 'comment': f'Observed Data ID: {observed_data.id}', **self._parse_timeline(observed_data) } address = email_address.value - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': address, **attribute, 'uuid': self.main_parser._create_v5_uuid( @@ -472,7 +501,7 @@ def _parse_email_address_observable_object_ref( ) attr_type = 'email-dst-display-name' display_name = email_address.display_name - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': attr_type, 'value': display_name, **attribute, 'uuid': self.main_parser._create_v5_uuid( @@ -482,7 +511,7 @@ def _parse_email_address_observable_object_ref( observed_data ) else: - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': email_address.value, **self._parse_timeline(observed_data), @@ -498,9 +527,15 @@ def _parse_email_address_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + for attribute in observable['misp_attribute']: + self._handle_misp_object_fields(attribute, observed_data) + continue email_address = observable['observable'] - self._parse_email_address_observable_object_ref( - email_address, observed_data + observable['misp_attribute'] = tuple( + self._parse_email_address_observable_object_ref( + email_address, observed_data + ) ) observable['used'][self.event_uuid] = True @@ -662,10 +697,15 @@ def _parse_ip_address_observable_object_refs( self, observed_data: _OBSERVED_DATA_TYPING): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - ip_address = observable['observable'] - self._parse_generic_observable_object_ref_as_attribute( - ip_address, observed_data, 'ip-dst' + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue + attribute = self._parse_generic_observable_object_ref_as_attribute( + observable['observable'], observed_data, 'ip-dst' ) + observable['misp_attributes'] = attribute observable['used'][self.event_uuid] = True def _parse_ip_address_observable_objects( @@ -695,10 +735,15 @@ def _parse_mac_address_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - mac_address = observable['observable'] - self._parse_generic_observable_object_ref_as_attribute( - mac_address, observed_data, 'mac-address' + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue + attribute = self._parse_generic_observable_object_ref_as_attribute( + observable['observable'], observed_data, 'mac-address' ) + observable['misp_attribute'] = attribute observable['used'][self.event_uuid] = True def _parse_mac_address_observable_objects( @@ -728,10 +773,15 @@ def _parse_mutex_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - mutex = observable['observable'] - self._parse_generic_observable_object_ref_as_attribute( - mutex, observed_data, 'mutex', feature='name' + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue + attribute = self._parse_generic_observable_object_ref_as_attribute( + observable['observable'], observed_data, 'mutex', feature='name' ) + observable['misp_attribute'] = attribute observable['used'][self.event_uuid] = True def _parse_mutex_observable_objects( @@ -760,19 +810,13 @@ def _parse_mutex_observable_objects( def _parse_process_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: + object_type = object_ref.split('--')[0] observable = self._fetch_observables(object_ref) - if object_ref.startswith('file--'): - if not observable['used'].get(self.event_uuid, False): - misp_object = self._parse_generic_observable_object_ref( - observable['observable'], observed_data, 'file', False - ) - observable['used'][self.event_uuid] = True - observable['misp_object'] = misp_object - continue misp_object = self._handle_observable_object_refs_parsing( - observable, observed_data, 'process', False + observable, observed_data, object_type, False ) - observable['used'][self.event_uuid] = True + if object_type == 'file': + continue process = observable['observable'] if hasattr(process, 'parent_ref'): self._parse_process_reference_observable_object_ref( @@ -796,43 +840,39 @@ def _parse_process_observable_objects( observed_data, 'process' ) observable_objects = { - object_id: {'used': False, 'observable': observable} - for object_id, observable in observed_data.objects.items() + object_id: {'used': False} for object_id in observed_data.objects } for object_id, observable in observable_objects.items(): - if observable['observable'].type == 'file': + observable_object = observed_data.objects[object_id] + object_type = observable_object.type + if object_type == 'file': if not observable['used']: misp_object = self._parse_generic_observable_object( - observed_data, object_id, 'file', False + observed_data, object_id, object_type, False + ) + observable.update( + {'misp_object': misp_object, 'used': True} ) - observable['used'] = True - observable['misp_object'] = misp_object continue - process = observable['observable'] - misp_object = ( - observable['misp_object'] if observable['used'] else - self._parse_generic_observable_object( - observed_data, object_id, 'process', False - ) + misp_object = self._handle_observable_objects_parsing( + observable_objects, object_id, observed_data, 'process', False ) - observable['misp_object'] = misp_object - observable['used'] = True - if hasattr(process, 'parent_ref'): + if hasattr(observable_object, 'parent_ref'): self._parse_process_reference_observable_object( observed_data, misp_object, - observable_objects[process.parent_ref], - process.parent_ref, 'child-of' + observable_objects[observable_object.parent_ref], + observable_object.parent_ref, 'child-of' ) - if hasattr(process, 'child_refs'): - for child_ref in process.child_refs: + if hasattr(observable_object, 'child_refs'): + for child_ref in observable_object.child_refs: self._parse_process_reference_observable_object( observed_data, misp_object, observable_objects[child_ref], child_ref, 'parent-of' ) for feature in ('binary', 'image'): - if hasattr(process, f'{feature}_ref'): - reference = getattr(process, f'{feature}_ref') + if hasattr(observable_object, f'{feature}_ref'): + reference = getattr(observable_object, f'{feature}_ref') self._parse_process_reference_observable_object( observed_data, misp_object, observable_objects[reference], @@ -932,13 +972,13 @@ def _parse_registry_key_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue observable_object = observable['observable'] if observable_object.type == 'user-account': - if observable['used'].get(self.event_uuid, False): - self._handle_misp_object_fields( - observable['misp_object'], observed_data - ) - continue misp_object = self._parse_generic_observable_object_ref( observable_object, observed_data, 'user-account', False ) @@ -948,6 +988,7 @@ def _parse_registry_key_observable_object_refs( misp_object = self._parse_registry_key_observable_object_ref( observable_object, observed_data ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True if hasattr(observable_object, 'creator_user_ref'): creator_observable = self._fetch_observables( @@ -996,11 +1037,10 @@ def _parse_registry_key_observable_objects( misp_object.add_reference(value_uuid, 'contains') return misp_object observable_objects = { - object_id: {'used': False, 'observable': observable} - for object_id, observable in observed_data.objects.items() + object_id: {'used': False} for object_id in observed_data.objects } - for identifier, observable in observable_objects.items(): - observable_object = observable['observable'] + for object_id, observable in observable_objects.items(): + observable_object = observed_data.objects[object_id] if observable_object.type == 'user-account': if observable['used']: continue @@ -1009,14 +1049,13 @@ def _parse_registry_key_observable_objects( observable_object, observed_data, 'user-account', False ) if observable['used'] else self._parse_generic_observable_object( - observed_data, identifier, 'user-account', False + observed_data, object_id, 'user-account', False ) ) - observable['misp_object'] = misp_object - observable['used'] = True + observable.update({'misp_object': misp_object, 'used': True}) continue misp_object = self._parse_registry_key_observable_object( - observed_data, identifier + observed_data, object_id ) if hasattr(observable_object, 'creator_user_ref'): creator_observable = observable_objects[ @@ -1032,8 +1071,9 @@ def _parse_registry_key_observable_objects( 'user-account', False ) creator_object.add_reference(misp_object.uuid, 'creates') - creator_observable['misp_object'] = creator_object - creator_observable['used'] = True + creator_observable.update( + {'misp_object': creator_object, 'used': True} + ) def _parse_registry_key_value_observable( self, registry_value: _WINDOWS_REGISTRY_VALUE_TYPING, @@ -1058,10 +1098,15 @@ def _parse_software_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - software = observable['observable'] - self._parse_generic_observable_object_ref( - software, observed_data, 'software' + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + misp_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, 'software' ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True def _parse_software_observable_objects( @@ -1079,10 +1124,15 @@ def _parse_url_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - url = observable['observable'] - self._parse_generic_observable_object_ref_as_attribute( - url, observed_data, 'url' + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue + attribute = self._parse_generic_observable_object_ref_as_attribute( + observable['observable'], observed_data, 'url' ) + observable['misp_attribute'] = attribute observable['used'][self.event_uuid] = True def _parse_url_observable_objects( @@ -1112,10 +1162,15 @@ def _parse_user_account_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - user_account = observable['observable'] - self._parse_generic_observable_object_ref( - user_account, observed_data, 'user-account', False + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + misp_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, 'user-account', False ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True def _parse_user_account_observable_objects( @@ -1133,10 +1188,15 @@ def _parse_x509_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: observable = self._fetch_observables(object_ref) - x509 = observable['observable'] - self._parse_generic_observable_object_ref( - x509, observed_data, 'x509', False + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + misp_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, 'x509', False ) + observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True def _parse_x509_observable_objects( From f2011e44b609c55fff199f7250c2a83b0977d634 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Sat, 2 Mar 2024 00:00:06 +0100 Subject: [PATCH 083/137] wip: [stix2 import] Converting File observable objects and their Directory & Artifact references --- .../stix2_observed_data_converter.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 22808ce3..ca8a72f1 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -590,6 +590,106 @@ def _parse_email_address_observable_objects( observed_data, identifier ) + def _parse_file_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: + object_type = object_ref.split('--')[0] + observable = self._fetch_observables(object_ref) + misp_object = self._handle_observable_object_refs_parsing( + observable, observed_data, object_type, + (object_type == 'directory') + ) + if object_type == 'artifact': + continue + observable_object = observable['observable'] + if hasattr(observable_object, 'contains_refs'): + contained_uuids = self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *observable_object.contains_refs + ) + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') + if object_type == 'directory': + continue + if hasattr(observable_object, 'parent_directory_ref'): + parent_ref = observable_object.parent_directory_ref + if parent_ref not in observed_data.object_refs: + self.observable_relationships[misp_object.uuid].add( + ( + self.main_parser._sanitise_uuid(parent_ref), + 'contained-in' + ) + ) + else: + parent = self._fetch_observables(parent_ref) + parent_object = self._handle_observable_object_refs_parsing( + parent, observed_data, 'directory' + ) + misp_object.add_reference( + parent_object.uuid, 'contained-in' + ) + if hasattr(observable_object, 'content_ref'): + content_ref = observable_object.content_ref + if content_ref not in observed_data.object_refs: + content_uuid = self.main_parser._sanitise_uuid(content_ref) + self.observable_relationships[content_uuid].add( + (misp_object.uuid, 'content-of') + ) + else: + content = self._fetch_observables(content_ref) + artifact = self._handle_observable_object_refs_parsing( + content, observed_data, 'artifact', False + ) + artifact.add_reference(misp_object.uuid, 'content-of') + + def _parse_file_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + return self._parse_generic_single_observable_object( + observed_data, 'file', False + ) + observable_objects = { + object_id: {'used': False} for object_id in observed_data.objects + } + for object_id, observable in observable_objects.items(): + observable_object = observed_data.objects[object_id] + object_type = observable_object.type + if object_type == 'artifact': + if not observable['used']: + misp_object = self._parse_generic_observable_object( + observed_data, object_id, object_type, False + ) + observable.update( + {'misp_object': misp_object, 'used': True} + ) + continue + misp_object = self._handle_observable_objects_parsing( + observable_objects, object_id, observed_data, + object_type, (object_type == 'directory') + ) + if hasattr(observable_object, 'contains_refs'): + contained_uuids = self._parse_contained_objects( + observed_data, observable_objects, + *observable_object.contains_refs + ) + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') + if object_type == 'directory': + continue + if hasattr(observable_object, 'parent_directory_ref'): + parent_ref = observable_object.parent_directory_ref + parent_object = self._handle_observable_objects_parsing( + observable_objects, parent_ref, observed_data, 'directory' + ) + misp_object.add_reference(parent_object.uuid, 'contained-in') + if hasattr(observable_object, 'content_ref'): + content_ref = observable_object.content_ref + artifact = self._handle_observable_objects_parsing( + observable_objects, content_ref, observed_data, + 'artifact', False + ) + artifact.add_reference(misp_object.uuid, 'content-of') + def _parse_generic_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, object_id: str, name: str, generic: Optional[bool] = True) -> MISPObject: From da8e4666b9d84d07c7f9dcc082aaf880bec26319 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Sat, 2 Mar 2024 12:31:10 +0100 Subject: [PATCH 084/137] wip: [tests] Tests for File objects and their Directory & Artifact references import from STIX 2.x --- tests/_test_stix_import.py | 91 +++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 50 +++++++++++++++ tests/test_external_stix20_import.py | 59 +++++++++++++++++ tests/test_external_stix21_bundles.py | 60 ++++++++++++++++++ tests/test_external_stix21_import.py | 49 ++++++++++++++- 5 files changed, 308 insertions(+), 1 deletion(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 89b4169d..c4f53cfc 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -459,6 +459,45 @@ def _check_as_fields(self, misp_object, autonomous_system, object_id): self.assertEqual(name.value, autonomous_system.name) self.assertEqual(name.uuid, uuid5(self._UUIDv4, f'{object_id} - description - {name.value}')) + def _check_content_ref_fields(self, misp_object, artifact, object_id): + self.assertEqual(len(misp_object.attributes), 4) + payload_bin, md5, decryption_key, mime_type = misp_object.attributes + self.assertEqual(payload_bin.type, 'attachment') + self.assertEqual(payload_bin.object_relation, 'payload_bin') + self.assertEqual( + payload_bin.value, + getattr(artifact, 'id', object_id.split(' - ')[0]).split('--')[1] + ) + self.assertEqual(self._get_data_value(payload_bin.data), artifact.payload_bin) + self.assertEqual( + payload_bin.uuid, + uuid5( + self._UUIDv4, f'{object_id} - payload_bin - {payload_bin.value}' + ) + ) + self._assert_multiple_equal(md5.type, md5.object_relation, 'md5') + self.assertEqual(md5.value, artifact.hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + self.assertEqual(decryption_key.type, 'text') + self.assertEqual(decryption_key.object_relation, 'decryption_key') + self.assertEqual(decryption_key.value, artifact.decryption_key) + self.assertEqual( + decryption_key.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - decryption_key - {decryption_key.value}' + ) + ) + self.assertEqual(mime_type.type, 'mime-type') + self.assertEqual(mime_type.object_relation, 'mime_type') + self.assertEqual(mime_type.value, artifact.mime_type) + self.assertEqual( + mime_type.uuid, + uuid5(self._UUIDv4, f'{object_id} - mime_type - {mime_type.value}') + ) + def _check_creator_user_fields(self, misp_object, user_account, object_id): self.assertEqual(len(misp_object.attributes), 4) username, account_type, privileged, user_id = misp_object.attributes @@ -528,6 +567,58 @@ def _check_directory_fields(self, misp_object, directory, object_id): ) return accessed.value, created.value, modified.value + def _check_file_fields(self, misp_object, observable_object, object_id): + self.assertEqual(len(misp_object.attributes), 6) + md5, sha1, sha256, filename, encoding, size = misp_object.attributes + hashes = observable_object.hashes + self._assert_multiple_equal(md5.type, md5.object_relation, 'md5') + self.assertEqual(md5.value, hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + self._assert_multiple_equal(sha1.type, sha1.object_relation, 'sha1') + self.assertEqual(sha1.value, hashes['SHA-1']) + self.assertEqual( + sha1.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha1 - {sha1.value}') + ) + self._assert_multiple_equal( + sha256.type, sha256.object_relation, 'sha256' + ) + self.assertEqual(sha256.value, hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha256 - {sha256.value}') + ) + self._assert_multiple_equal( + filename.type, filename.object_relation, 'filename' + ) + self.assertEqual(filename.value, observable_object.name) + self.assertEqual( + filename.uuid, + uuid5(self._UUIDv4, f'{object_id} - filename - {filename.value}') + ) + self.assertEqual(encoding.type, 'text') + self.assertEqual(encoding.object_relation, 'file-encoding') + self.assertEqual(encoding.value, observable_object.name_enc) + self.assertEqual( + encoding.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - file-encoding - {encoding.value}' + ) + ) + self._assert_multiple_equal( + size.type, size.object_relation, 'size-in-bytes' + ) + self.assertEqual(size.value, observable_object.size) + self.assertEqual( + size.uuid, + uuid5( + self._UUIDv4, f'{object_id} - size-in-bytes - {size.value}' + ) + ) + def _check_process_child_fields(self, misp_object, process, object_id): self.assertEqual(len(misp_object.attributes), 2) name, pid = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 584e7c5f..5e22e325 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -1,9 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from base64 import b64encode from copy import deepcopy +from pathlib import Path from stix2.parsing import dict_to_stix2 +_TESTFILES_PATH = Path(__file__).parent.resolve() / 'attachment_test_files' _ARTIFACT_OBJECTS = [ { @@ -367,6 +370,46 @@ } } ] +_FILE_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--5e384ae7-672c-4250-9cda-3b4da964451a", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "file", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802", + "SHA-1": "46aba99aa7158e4609aaa72b50990842fd22ae86", + "SHA-256": "ec5aedf5ecc6bdadd4120932170d1b10f6cfa175cfda22951dfd882928ab279b" + }, + "size": 35, + "name": "oui", + "name_enc": "UTF-8", + "parent_directory_ref": "1", + "content_ref": "2" + }, + "1": { + "type": "directory", + "path": "/var/www/MISP/app/files/scripts/tmp" + }, + "2": { + "type": "artifact", + "mime_type": "application/zip", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802" + }, + "encryption_algorithm": "mime-type-indicated", + "decryption_key": "infected" + } + } + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -1208,6 +1251,13 @@ def get_bundle_with_domain_attributes(cls): def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + @classmethod + def get_bundle_with_file_objects(cls): + observed_data = deepcopy(_FILE_OBJECTS) + with open(_TESTFILES_PATH / 'malware_sample.zip', 'rb') as f: + observed_data[0]['objects']['2']['payload_bin'] = b64encode(f.read()).decode() + return cls.__assemble_bundle(*observed_data) + @classmethod def get_bundle_with_ip_address_attributes(cls): return cls.__assemble_bundle(*_IP_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index e5545cdd..ffe880a0 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -213,6 +213,18 @@ def _check_as_object(self, misp_object, observed_data, identifier=None): autonomous_system = observed_data.objects[identifier] self._check_as_fields(misp_object, autonomous_system, object_id) + def _check_content_ref_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + self._check_content_ref_fields( + misp_object, observed_data.objects[identifier], object_id + ) + def _check_directory_object(self, misp_object, observed_data, identifier=None): self.assertEqual(misp_object.name, 'directory') self._check_misp_object_fields(misp_object, observed_data, identifier) @@ -262,6 +274,18 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_file_object(self, misp_object, observed_data, identifier=None): + self.assertEqual(misp_object.name, 'file') + self._check_misp_object_fields(misp_object, observed_data, identifier) + object_id = observed_data.id + if identifier is None: + identifier = '0' + else: + object_id = f'{object_id} - {identifier}' + self._check_file_fields( + misp_object, observed_data.objects[identifier], object_id + ) + def _check_generic_attribute( self, observed_data, attribute, attribute_type, identifier=None, feature='value'): @@ -423,6 +447,41 @@ def test_stix20_bundle_with_email_address_objects(self): ) self._check_email_address_attribute(observed_data3, ss_address) + def test_stix20_bundle_with_file_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_file_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + file_object, directory_object, artifact_object = misp_objects + self._check_file_object(file_object, observed_data, '0') + self.assertEqual(directory_object.name, 'directory') + self._check_misp_object_fields(directory_object, observed_data, '1') + artifact = observed_data.objects['1'] + self.assertEqual(len(directory_object.attributes), 1) + path_attribute = directory_object.attributes[0] + self.assertEqual(path_attribute.type, 'text') + self.assertEqual(path_attribute.object_relation, 'path') + self.assertEqual(path_attribute.value, artifact.path) + self.assertEqual( + path_attribute.uuid, + uuid5( + self._UUIDv4, + f'{observed_data.id} - 1 - path - {path_attribute.value}' + ) + ) + self._check_content_ref_object(artifact_object, observed_data, '2') + self.assertEqual(len(file_object.references), 1) + file_reference = file_object.references[0] + self.assertEqual(file_reference.referenced_uuid, directory_object.uuid) + self.assertEqual(file_reference.relationship_type, 'contained-in') + self.assertEqual(len(artifact_object.references), 1) + artifact_reference = artifact_object.references[0] + self.assertEqual(artifact_reference.referenced_uuid, file_object.uuid) + self.assertEqual(artifact_reference.relationship_type, 'content-of') + def test_stix20_bundle_with_ip_address_attributes(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_ip_address_attributes() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 4cc7443d..0f77df83 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -1,9 +1,12 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from base64 import b64encode from copy import deepcopy +from pathlib import Path from stix2.parsing import dict_to_stix2 +_TESTFILES_PATH = Path(__file__).parent.resolve() / 'attachment_test_files' _ARTIFACT_OBJECTS = [ { @@ -439,6 +442,56 @@ "value": "donald.duck@gmail.com" } ] +_FILE_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--5e384ae7-672c-4250-9cda-3b4da964451a", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "file--5e384ae7-672c-4250-9cda-3b4da964451a", + "directory--34cb1a7c-55ec-412a-8684-ba4a88d83a45", + "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" + ], + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--5e384ae7-672c-4250-9cda-3b4da964451a", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802", + "SHA-1": "46aba99aa7158e4609aaa72b50990842fd22ae86", + "SHA-256": "ec5aedf5ecc6bdadd4120932170d1b10f6cfa175cfda22951dfd882928ab279b" + }, + "size": 35, + "name": "oui", + "name_enc": "UTF-8", + "parent_directory_ref": "directory--34cb1a7c-55ec-412a-8684-ba4a88d83a45", + "content_ref": "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" + }, + { + "type": "directory", + "spec_version": "2.1", + "id": "directory--34cb1a7c-55ec-412a-8684-ba4a88d83a45", + "path": "/var/www/MISP/app/files/scripts/tmp" + }, + { + "type": "artifact", + "spec_version": "2.1", + "id": "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f", + "mime_type": "application/zip", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802" + }, + "encryption_algorithm": "mime-type-indicated", + "decryption_key": "infected" + } +] _INTRUSION_SET_OBJECTS = [ { "type": "intrusion-set", @@ -1435,6 +1488,13 @@ def get_bundle_with_domain_attributes(cls): def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + @classmethod + def get_bundle_with_file_objects(cls): + observables = deepcopy(_FILE_OBJECTS) + with open(_TESTFILES_PATH / 'malware_sample.zip', 'rb') as f: + observables[-1]['payload_bin'] = b64encode(f.read()).decode() + return cls.__assemble_bundle(*observables) + @classmethod def get_bundle_with_ip_address_attributes(cls): return cls.__assemble_bundle(*_IP_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 2ef133fd..c82730a1 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -241,6 +241,11 @@ def _check_as_object(self, misp_object, observed_data, autonomous_system): self._check_misp_object_fields(misp_object, observed_data, autonomous_system.id) self._check_as_fields(misp_object, autonomous_system, autonomous_system.id) + def _check_content_ref_object(self, misp_object, observed_data, artifact): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, artifact.id) + self._check_content_ref_fields(misp_object, artifact, artifact.id) + def _check_directory_object(self, misp_object, observed_data, directory): self.assertEqual(misp_object.name, 'directory') self._check_misp_object_fields(misp_object, observed_data, directory.id) @@ -272,6 +277,15 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_file_object(self, misp_object, observed_data, observable_object): + self.assertEqual(misp_object.name, 'file') + self._check_misp_object_fields( + misp_object, observed_data, observable_object.id + ) + self._check_file_fields( + misp_object, observable_object, observable_object.id + ) + def _check_generic_attribute( self, observed_data, observable_object, attribute, attribute_type, feature='value'): @@ -410,6 +424,39 @@ def test_stix21_bundle_with_email_address_objects(self): ) self._check_email_address_attribute(od3, ss_address, ea4) + def test_stix21_bundle_with_file_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_file_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, file1, directory, artifact = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + file_object, directory_object, artifact_object = misp_objects + self._check_file_object(file_object, od1, file1) + self.assertEqual(directory_object.name, 'directory') + self._check_misp_object_fields(directory_object, od1, directory.id) + self.assertEqual(len(directory_object.attributes), 1) + path_attribute = directory_object.attributes[0] + self.assertEqual(path_attribute.type, 'text') + self.assertEqual(path_attribute.object_relation, 'path') + self.assertEqual(path_attribute.value, directory.path) + self.assertEqual( + path_attribute.uuid, + uuid5( + self._UUIDv4, f'{directory.id} - path - {path_attribute.value}' + ) + ) + self._check_content_ref_object(artifact_object, od1, artifact) + self.assertEqual(len(file_object.references), 1) + file_reference = file_object.references[0] + self.assertEqual(file_reference.referenced_uuid, directory_object.uuid) + self.assertEqual(file_reference.relationship_type, 'contained-in') + self.assertEqual(len(artifact_object.references), 1) + artifact_reference = artifact_object.references[0] + self.assertEqual(artifact_reference.referenced_uuid, file_object.uuid) + self.assertEqual(artifact_reference.relationship_type, 'content-of') + def test_stix21_bundle_with_ip_address_attributes(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_ip_address_attributes() self.parser.load_stix_bundle(bundle) @@ -635,4 +682,4 @@ def test_stix21_bundle_with_x509_objects(self): multiple1, multiple2, single = misp_objects self._check_x509_object(multiple1, od1, cert1) self._check_x509_object(multiple2, od1, cert2) - self._check_x509_object(single, od2, cert3) \ No newline at end of file + self._check_x509_object(single, od2, cert3) From e6067925ecaa27d5e0f13d9672b9ee4f9de97079 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 4 Mar 2024 21:17:46 +0100 Subject: [PATCH 085/137] wip: [stix2 import] Parsing Observable objects referenced together by a single Observed Data object with no specific mapping --- .../stix2_observed_data_converter.py | 84 ++++++++++++------- 1 file changed, 56 insertions(+), 28 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index ca8a72f1..70b6496d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -100,7 +100,9 @@ def _parse_observable_object_refs(self, observed_data: ObservedData_v21): fields = '_'.join(sorted(observable_types)) mapping = self._mapping.observable_mapping(fields) if mapping is None: - raise UnknownObservableMappingError(to_call) + if len(observable_types) == 1: + raise UnknownObservableMappingError(fields) + self._parse_multiple_observable_object_refs(observed_data) else: feature = f'_parse_{mapping}_observable_object_refs' try: @@ -156,8 +158,8 @@ def _handle_observable_objects_parsing( return misp_object def _parse_artifact_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -219,8 +221,9 @@ def _parse_as_observable_object( observed_data ) - def _parse_as_observable_object_refs(self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + def _parse_as_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -362,8 +365,8 @@ def _parse_contained_objects( yield misp_object.uuid def _parse_directory_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -403,8 +406,8 @@ def _parse_directory_observable_objects( misp_object.add_reference(contained_uuid, 'contains') def _parse_domain_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -524,8 +527,8 @@ def _parse_email_address_observable_object_ref( ) def _parse_email_address_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): for attribute in observable['misp_attribute']: @@ -794,8 +797,8 @@ def _parse_generic_single_observable_object( return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_ip_address_observable_object_refs( - self, observed_data: _OBSERVED_DATA_TYPING): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -832,8 +835,8 @@ def _parse_ip_address_observable_objects( ) def _parse_mac_address_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -869,9 +872,34 @@ def _parse_mac_address_observable_objects( observed_data, identifier, 'mac-address' ) - def _parse_mutex_observable_object_refs( + def _parse_multiple_observable_object_refs( self, observed_data: ObservedData_v21): for object_ref in observed_data.object_refs: + observable = self._fetch_observables(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + object_type = object_ref.split('--')[0] + mapping = self._mapping.observable_mapping(object_type) + if mapping is None: + self.main_parser._observable_mapping_error( + observed_data.id, object_type + ) + continue + feature = f'_parse_{mapping}_observable_object_refs' + try: + parser = getattr(self, feature) + except AttributeError: + self.main_parser._unknown_parsing_function_error(feature) + continue + parser(observed_data, object_ref) + + def _parse_mutex_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: + print(f'Mutex object ref: {object_ref}') observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -908,8 +936,8 @@ def _parse_mutex_observable_objects( ) def _parse_process_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: object_type = object_ref.split('--')[0] observable = self._fetch_observables(object_ref) misp_object = self._handle_observable_object_refs_parsing( @@ -1069,8 +1097,8 @@ def _parse_registry_key_observable_object_ref( return misp_object def _parse_registry_key_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -1195,8 +1223,8 @@ def _parse_registry_key_value_observable( return misp_object.uuid def _parse_software_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -1221,8 +1249,8 @@ def _parse_software_observable_objects( ) def _parse_url_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -1259,8 +1287,8 @@ def _parse_url_observable_objects( ) def _parse_user_account_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -1285,8 +1313,8 @@ def _parse_user_account_observable_objects( ) def _parse_x509_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: observable = self._fetch_observables(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( From 742c91468415763af853b80e0315afe38ec58d51 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 5 Mar 2024 22:59:41 +0100 Subject: [PATCH 086/137] wip: [stix2 import] Better observable objects fetching methods --- .../converters/stix2_observable_converter.py | 11 +-- .../stix2_observable_objects_converter.py | 48 +++++----- .../stix2_observed_data_converter.py | 89 ++++++++++--------- 3 files changed, 72 insertions(+), 76 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 59b73f94..43a1568c 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -174,15 +174,8 @@ def x509_hashes_mapping(cls, field: str) -> Union[dict, None]: class STIX2ObservableConverter(STIX2Converter): - def _fetch_observables(self, object_refs: Union[list, str]): - if isinstance(object_refs, str): - return self.main_parser._observable[object_refs] - if len(object_refs) == 1: - return self.main_parser._observable[object_refs[0]] - return tuple( - self.main_parser._observable[object_ref] - for object_ref in object_refs - ) + def _fetch_observable(self, object_ref: str) -> dict: + return self.main_parser._observable[object_ref] def _parse_email_additional_header( self, observable: _EMAIL_MESSAGE_TYPING, diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 7fab5647..c7a6e10f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -63,7 +63,7 @@ def _create_misp_object_from_observable_object( def _parse_artifact_observable_object( self, artifact_ref: str) -> MISPObject: - observable = self._fetch_observables(artifact_ref) + observable = self._fetch_observable(artifact_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] artifact = observable['observable'] @@ -80,7 +80,7 @@ def _parse_artifact_observable_object( return misp_object def _parse_as_observable_object(self, as_ref: str) -> _MISP_CONTENT_TYPING: - observable = self._fetch_observables(as_ref) + observable = self._fetch_observable(as_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_object', observable.get('misp_attribute') @@ -121,7 +121,7 @@ def _parse_as_observable_object(self, as_ref: str) -> _MISP_CONTENT_TYPING: def _parse_directory_observable_object( self, directory_ref: str, child: Optional[str] = None ) -> _MISP_CONTENT_TYPING: - observable = self._fetch_observables(directory_ref) + observable = self._fetch_observable(directory_ref) if observable['used'].get(self.event_uuid, False): return observable.get('misp_object', observable['misp_attribute']) directory = observable['observable'] @@ -148,7 +148,7 @@ def _parse_directory_observable_object( for ref in directory.contains_refs: feature = f"_parse_{ref.split('--')[0]}_observable_object" referenced_object = ( - self._fetch_observables(child)['misp_object'] + self._fetch_observable(child)['misp_object'] if child == ref else getattr(self, feature)(ref) ) misp_object.add_reference( @@ -168,14 +168,14 @@ def _parse_directory_observable_object( ) observable['misp_attribute'] = misp_attribute return misp_attribute - file_object = self._fetch_observables(child)['misp_object'] + file_object = self._fetch_observable(child)['misp_object'] file_object.add_attribute(**attributes[0]) observable['misp_object'] = file_object return misp_object def _parse_domain_observable_object( self, domain_ref: str) -> _MISP_CONTENT_TYPING: - observable = self._fetch_observables(domain_ref) + observable = self._fetch_observable(domain_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_object', observable.get('misp_attribute') @@ -201,7 +201,7 @@ def _parse_domain_observable_object( referenced_domain.uuid, 'resolves-to' ) continue - resolved_ip = self._fetch_observables(reference) + resolved_ip = self._fetch_observable(reference) ip_address = resolved_ip['observable'] misp_object.add_attribute( **self._parse_ip_observable(ip_address) @@ -210,7 +210,7 @@ def _parse_domain_observable_object( resolved_ip['misp_object'] = misp_object if hasattr(ip_address, 'resolves_to_refs'): for referenced_mac in ip_address.resolves_to_refs: - resolved_mac = self._fetch_observables(referenced_mac) + resolved_mac = self._fetch_observable(referenced_mac) mac_address = resolved_mac['observable'] attribute = self._create_misp_attribute( 'mac-address', mac_address, @@ -234,7 +234,7 @@ def _parse_domain_observable_object( def _parse_email_address_observable_object( self, email_address_ref: str) -> _MISP_CONTENT_TYPING: - observable = self._fetch_observables(email_address_ref) + observable = self._fetch_observable(email_address_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_attribute', observable.get('misp_object') @@ -263,7 +263,7 @@ def _parse_email_address_observable_object( def _parse_email_message_observable_object( self, email_message_ref: str) -> MISPObject: - observable = self._fetch_observables(email_message_ref) + observable = self._fetch_observable(email_message_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] email_message = observable['observable'] @@ -278,7 +278,7 @@ def _parse_email_message_observable_object( ) observable['misp_object'] = misp_object if hasattr(email_message, 'from_ref'): - observable = self._fetch_observables(email_message.from_ref) + observable = self._fetch_observable(email_message.from_ref) attributes = self._parse_email_reference_observable( observable['observable'], 'from' ) @@ -290,7 +290,7 @@ def _parse_email_message_observable_object( field = f'{feature}_refs' if hasattr(email_message, field): for reference in getattr(email_message, field): - observable = self._fetch_observables(reference) + observable = self._fetch_observable(reference) attributes = self._parse_email_reference_observable( observable['observable'], feature ) @@ -307,7 +307,7 @@ def _parse_email_message_observable_object( return misp_object def _parse_file_observable_object(self, file_ref: str) -> MISPObject: - observable = self._fetch_observables(file_ref) + observable = self._fetch_observable(file_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] _file = observable['observable'] @@ -376,7 +376,7 @@ def _parse_ip_addresses_belonging_to_AS(self, AS_id: str): def _parse_ip_address_observable_object( self, ip_address_ref: str) -> _MISP_CONTENT_TYPING: - observable = self._fetch_observables(ip_address_ref) + observable = self._fetch_observable(ip_address_ref) if observable['used'].get(self.event_uuid, False): return observable.get( 'misp_attribute', observable.get('misp_object') @@ -391,7 +391,7 @@ def _parse_ip_address_observable_object( def _parse_mac_address_observable_object( self, mac_address_ref: str) -> MISPAttribute: - observable = self._fetch_observables(mac_address_ref) + observable = self._fetch_observable(mac_address_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] mac_address = observable['observable'] @@ -403,7 +403,7 @@ def _parse_mac_address_observable_object( return misp_attribute def _parse_mutex_observable_object(self, mutex_ref: str) -> MISPAttribute: - observable = self._fetch_observables(mutex_ref) + observable = self._fetch_observable(mutex_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] mutex = observable['observable'] @@ -444,7 +444,7 @@ def _parse_network_socket_observable_object( def _parse_network_traffic_observable_object( self, network_traffic_ref: str) -> MISPObject: - observable = self._fetch_observables(network_traffic_ref) + observable = self._fetch_observable(network_traffic_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] network_traffic = observable['observable'] @@ -457,7 +457,7 @@ def _parse_network_traffic_observable_object( observable['misp_object'] = misp_object for asset in ('src', 'dst'): if hasattr(network_traffic, f'{asset}_ref'): - referenced_object = self._fetch_observables( + referenced_object = self._fetch_observable( getattr(network_traffic, f'{asset}_ref') ) attributes = self._parse_network_traffic_reference_observable( @@ -488,7 +488,7 @@ def _parse_network_traffic_observable_fields( return '_parse_network_connection_observable_object' def _parse_process_observable_object(self, process_ref: str) -> MISPObject: - observable = self._fetch_observables(process_ref) + observable = self._fetch_observable(process_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] process = observable['observable'] @@ -528,7 +528,7 @@ def _parse_process_observable_object(self, process_ref: str) -> MISPObject: return misp_object def _parse_registry_key_observable_object(self, registry_key_ref: str): - observable = self._fetch_observables(registry_key_ref) + observable = self._fetch_observable(registry_key_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] registry_key = observable['observable'] @@ -562,7 +562,7 @@ def _parse_registry_key_observable_object(self, registry_key_ref: str): def _parse_software_observable_object( self, software_ref: str) -> MISPObject: - observable = self._fetch_observables(software_ref) + observable = self._fetch_observable(software_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] software = observable['observable'] @@ -579,7 +579,7 @@ def _parse_software_observable_object( return misp_object def _parse_url_observable_object(self, url_ref: str) -> MISPAttribute: - observable = self._fetch_observables(url_ref) + observable = self._fetch_observable(url_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_attribute'] url = observable['observable'] @@ -592,7 +592,7 @@ def _parse_url_observable_object(self, url_ref: str) -> MISPAttribute: def _parse_user_account_observable_object( self, user_account_ref: str) -> MISPObject: - observable = self._fetch_observables(user_account_ref) + observable = self._fetch_observable(user_account_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] user_account = observable['observable'] @@ -609,7 +609,7 @@ def _parse_user_account_observable_object( return misp_object def _parse_x509_observable_object(self, x509_ref: str) -> MISPObject: - observable = self._fetch_observables(x509_ref) + observable = self._fetch_observable(x509_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] x509 = observable['observable'] diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 70b6496d..481357d9 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -160,7 +160,7 @@ def _handle_observable_objects_parsing( def _parse_artifact_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -224,7 +224,7 @@ def _parse_as_observable_object( def _parse_as_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable[ @@ -325,7 +325,7 @@ def _parse_contained_object_refs( self, observed_data: ObservedData_v21, misp_object_uuid: str, *contains_refs: tuple) -> Generator: for contained_ref in contains_refs: - contained = self._fetch_observables(contained_ref) + contained = self._fetch_observable(contained_ref) if contained['used'].get(self.event_uuid, False): contained_object = contained['misp_object'] self._handle_misp_object_fields(contained_object, observed_data) @@ -367,7 +367,7 @@ def _parse_contained_objects( def _parse_directory_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -408,7 +408,7 @@ def _parse_directory_observable_objects( def _parse_domain_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_attribute'], observed_data @@ -529,7 +529,7 @@ def _parse_email_address_observable_object_ref( def _parse_email_address_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): for attribute in observable['misp_attribute']: self._handle_misp_object_fields(attribute, observed_data) @@ -597,7 +597,7 @@ def _parse_file_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: object_type = object_ref.split('--')[0] - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) misp_object = self._handle_observable_object_refs_parsing( observable, observed_data, object_type, (object_type == 'directory') @@ -624,7 +624,7 @@ def _parse_file_observable_object_refs( ) ) else: - parent = self._fetch_observables(parent_ref) + parent = self._fetch_observable(parent_ref) parent_object = self._handle_observable_object_refs_parsing( parent, observed_data, 'directory' ) @@ -639,7 +639,7 @@ def _parse_file_observable_object_refs( (misp_object.uuid, 'content-of') ) else: - content = self._fetch_observables(content_ref) + content = self._fetch_observable(content_ref) artifact = self._handle_observable_object_refs_parsing( content, observed_data, 'artifact', False ) @@ -799,7 +799,7 @@ def _parse_generic_single_observable_object( def _parse_ip_address_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_attribute'], observed_data @@ -837,7 +837,7 @@ def _parse_ip_address_observable_objects( def _parse_mac_address_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_attribute'], observed_data @@ -899,8 +899,7 @@ def _parse_multiple_observable_object_refs( def _parse_mutex_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - print(f'Mutex object ref: {object_ref}') - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_attribute'], observed_data @@ -939,7 +938,7 @@ def _parse_process_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: object_type = object_ref.split('--')[0] - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) misp_object = self._handle_observable_object_refs_parsing( observable, observed_data, object_type, False ) @@ -1026,7 +1025,7 @@ def _parse_process_reference_observable_object_ref( self, observed_data: _OBSERVED_DATA_TYPING, misp_object: MISPObject, reference: str, relationship_type: str, name: Optional[str] = 'process'): - observable = self._fetch_observables(reference) + observable = self._fetch_observable(reference) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields(misp_object, observed_data) misp_object.add_reference( @@ -1099,7 +1098,7 @@ def _parse_registry_key_observable_object_ref( def _parse_registry_key_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -1119,7 +1118,7 @@ def _parse_registry_key_observable_object_refs( observable['misp_object'] = misp_object observable['used'][self.event_uuid] = True if hasattr(observable_object, 'creator_user_ref'): - creator_observable = self._fetch_observables( + creator_observable = self._fetch_observable( observable_object.creator_user_ref ) if creator_observable['used'].get(self.event_uuid, False): @@ -1225,7 +1224,7 @@ def _parse_registry_key_value_observable( def _parse_software_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -1251,7 +1250,7 @@ def _parse_software_observable_objects( def _parse_url_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_attribute'], observed_data @@ -1289,7 +1288,7 @@ def _parse_url_observable_objects( def _parse_user_account_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -1315,7 +1314,7 @@ def _parse_user_account_observable_objects( def _parse_x509_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( observable['misp_object'], observed_data @@ -1369,6 +1368,10 @@ def _create_misp_object_from_observable_object_ref( ) return misp_object + def _fetch_observables(self, object_refs: Union[tuple, str]) -> Generator: + for object_ref in object_refs: + yield self.main_parser._observable[object_ref] + def _handle_misp_object_fields( self, misp_object: MISPAttribute | MISPObject, observed_data: ObservedData_v21): @@ -1434,7 +1437,7 @@ def _attribute_from_address_observable_v21( attribute = self._create_attribute_dict(observed_data) for reference in observed_data.object_refs: if '-addr' in reference: - observable = self._fetch_observables(reference) + observable = self._fetch_observable(reference) attribute['value'] = observable.value break self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1449,7 +1452,7 @@ def _attribute_from_AS_observable_v20( def _attribute_from_AS_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs) attribute['value'] = self._parse_AS_value(observable.number) self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1478,8 +1481,8 @@ def _attribute_from_attachment_observable_v20( def _attribute_from_attachment_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observables = self._fetch_observables(observed_data.object_refs) - if isinstance(observables, tuple): + observables = tuple(self._fetch_observables(observed_data.object_refs)) + if len(observables) > 1: attribute.update( self._attribute_from_attachment_observable(observables) ) @@ -1510,7 +1513,7 @@ def _attribute_from_email_attachment_observable_v20( def _attribute_from_email_attachment_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs[1]) + observable = self._fetch_observable(observed_data.object_refs[1]) attribute['value'] = observable.name self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1523,7 +1526,7 @@ def _attribute_from_email_body_observable_v20( def _attribute_from_email_body_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.body self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1536,14 +1539,14 @@ def _attribute_from_email_header_observable_v20( def _attribute_from_email_header_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.received_lines[0] self.main_parser._add_misp_attribute(attribute, observed_data) def _attribute_from_email_message_id_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.message_id self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1557,7 +1560,7 @@ def _attribute_from_email_reply_to_observable_v20( def _attribute_from_email_reply_to_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.additional_header_fields['Reply-To'] self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1570,7 +1573,7 @@ def _attribute_from_email_subject_observable_v20( def _attribute_from_email_subject_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.subject self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1584,7 +1587,7 @@ def _attribute_from_email_x_mailer_observable_v20( def _attribute_from_email_x_mailer_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.additional_header_fields['X-Mailer'] self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1599,7 +1602,7 @@ def _attribute_from_filename_hash_observable_v20( def _attribute_from_filename_hash_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) hash_value = list(observable.hashes.values())[0] attribute['value'] = f'{observable.name}|{hash_value}' self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1613,14 +1616,14 @@ def _attribute_from_first_observable_v20( def _attribute_from_first_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.value self.main_parser._add_misp_attribute(attribute, observed_data) def _attribute_from_github_username_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.account_login self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1633,7 +1636,7 @@ def _attribute_from_hash_observable_v20( def _attribute_from_hash_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = list(observable.hashes.values())[0] self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1705,8 +1708,8 @@ def _attribute_from_malware_sample_observable_v20( def _attribute_from_malware_sample_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observables = self._fetch_observables(observed_data.object_refs) - if isinstance(observables, tuple): + observables = tuple(self._fetch_observables(observed_data.object_refs)) + if len(observables) > 1: attribute.update( self._attribute_from_malware_sample_observable(observables) ) @@ -1725,7 +1728,7 @@ def _attribute_from_name_observable_v20( def _attribute_from_name_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.name self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1738,7 +1741,7 @@ def _attribute_from_regkey_observable_v20( def _attribute_from_regkey_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = observable.key self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1752,7 +1755,7 @@ def _attribute_from_regkey_value_observable_v20( def _attribute_from_regkey_value_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observables(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = f"{observable.key}|{observable['values'][0].data}" self.main_parser._add_misp_attribute(attribute, observed_data) @@ -1862,14 +1865,14 @@ def _object_from_domain_ip_observable_v21( ip_parsed = set() for object_ref in observed_data.object_refs: if object_ref.startswith('domain-name--'): - observable = self._fetch_observables(object_ref) + observable = self._fetch_observable(object_ref) for attribute in self._parse_domain_ip_observable(observable): misp_object.add_attribute(**attribute) if hasattr(observable, 'resolves_to_refs'): for reference in observable.resolves_to_refs: if reference in ip_parsed: continue - address = self._fetch_observables(reference) + address = self._fetch_observable(reference) misp_object.add_attribute( **{ 'value': address.value, From e1840e9b7d140c14fc81c3871de7a7bc21656184 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 6 Mar 2024 16:25:51 +0100 Subject: [PATCH 087/137] wip: [stix2 import] Parsing STIX 2.0 Observed Data objects with multiple embedded observable objects with no specific mapping --- .../stix2_observed_data_converter.py | 332 ++++++++++++++---- 1 file changed, 273 insertions(+), 59 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 481357d9..9fe22895 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -11,7 +11,7 @@ _EXTENSION_TYPING, _NETWORK_TRAFFIC_TYPING, _PROCESS_TYPING) from .stix2converter import _MAIN_PARSER_TYPING from abc import ABCMeta -from collections import defaultdict +from collections import defaultdict, deque from collections.abc import Generator from datetime import datetime from pymisp import MISPAttribute, MISPObject @@ -118,7 +118,9 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): fields = '_'.join(sorted(observable_types)) mapping = self._mapping.observable_mapping(fields) if mapping is None: - raise UnknownObservableMappingError(fields) + if len(observable_types) == 1: + raise UnknownObservableMappingError(fields) + self._parse_multiple_observable_objects(observed_data) else: feature = f'_parse_{mapping}_observable_objects' try: @@ -127,6 +129,102 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): raise UnknownParsingFunctionError(feature) parser(observed_data) + ############################################################################ + # MULTIPLE OBSERVABLE OBJECTS PARSING METHODS. # + ############################################################################ + + def _fetch_multiple_observable_ids( + self, observed_data: _OBSERVED_DATA_TYPING, + object_id: str) -> Generator: + yield object_id + for key, value in observed_data.objects[object_id].items(): + if key.endswith('_ref'): + yield from self._fetch_multiple_observable_ids( + observed_data, value + ) + if key.endswith('_refs'): + for reference in value: + yield from self._fetch_multiple_observable_ids( + observed_data, reference + ) + + def _parse_multiple_observable_object_refs( + self, observed_data: ObservedData_v21): + for object_ref in observed_data.object_refs: + observable = self._fetch_observable(object_ref) + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + object_type = object_ref.split('--')[0] + mapping = self._mapping.observable_mapping(object_type) + if mapping is None: + self.main_parser._observable_mapping_error( + observed_data.id, object_type + ) + continue + feature = f'_parse_{mapping}_observable_object_refs' + try: + parser = getattr(self, feature) + except AttributeError: + self.main_parser._unknown_parsing_function_error(feature) + continue + parser(observed_data, object_ref) + + def _parse_multiple_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + observable_objects = { + object_id: {'used': False} for object_id in observed_data.objects + } + for object_id in observable_objects.keys(): + if observable_objects[object_id]['used']: + continue + observables = { + identifier: observable_objects[identifier] + for identifier in set( + self._fetch_multiple_observable_ids( + observed_data, object_id + ) + ) + } + observable_types = set( + observed_data.objects[identifier].type + for identifier in observables + ) + object_type = '_'.join(sorted(observable_types)) + mapping = self._mapping.observable_mapping(object_type) + if mapping is not None: + feature = f'_parse_{mapping}_observable_objects' + try: + parser = getattr(self, feature) + except AttributeError: + self.main_parser._unknown_parsing_function_error(feature) + continue + parser(observed_data, observables) + observable_objects.update(observables) + continue + if len(observable_types) == 1: + self.main_parser._observable_mapping_error( + observed_data.id, object_type + ) + continue + mapping = self._mapping.observable_mapping( + observed_data.objects[object_id]['type'] + ) + if mapping is None: + self.main_parser._observable_mapping_error( + observed_data.id, object_type + ) + continue + feature = f'_parse_{mapping}_observable_objects' + try: + parser = getattr(self, feature) + except AttributeError: + self.main_parser._unknown_parsing_function_error(feature) + continue + parser(observed_data, observable_objects) + ############################################################################ # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ @@ -174,7 +272,17 @@ def _parse_artifact_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_artifact_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + misp_object = self._parse_generic_observable_object( + observed_data, object_id, 'artifact', False + ) + observable.update({'misp_object': misp_object, 'used': True}) + return if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'artifact', False @@ -258,7 +366,21 @@ def _parse_as_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_as_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + misp_content = self._parse_as_observable_object( + observed_data, object_id + ) + feature = ( + 'misp_object' if isinstance(misp_content, MISPObject) + else 'misp_attribute' + ) + observable.update({feature: misp_content, 'used': True}) + return if len(observed_data.objects) == 1: autonomous_system = next(iter(observed_data.objects.values())) if autonomous_system.get('id') is not None: @@ -385,14 +507,17 @@ def _parse_directory_observable_object_refs( misp_object.add_reference(contained_uuid, 'contains') def _parse_directory_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'directory' ) - observable_objects = { - object_id: {'used': False} for object_id in observed_data.objects - } + if observable_objects is None: + observable_objects = { + object_id: {'used': False} + for object_id in observed_data.objects + } for object_id in observable_objects.keys(): misp_object = self._handle_observable_objects_parsing( observable_objects, object_id, observed_data, 'directory' @@ -422,7 +547,17 @@ def _parse_domain_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_domain_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attribute = self._parse_generic_observable_object_as_attribute( + observed_data, object_id, 'domain' + ) + observable.update({'misp_attribute': attribute, 'used': True}) + return if len(observed_data.objects) == 1: domain = next(iter(observed_data.objects.values())) if domain.get('id') is not None: @@ -445,7 +580,8 @@ def _parse_domain_observable_objects( ) def _parse_email_address_observable_object( - self, observed_data: ObservedData_v20, identifier: str): + self, observed_data: ObservedData_v20, + identifier: str) -> Generator: attribute = { 'comment': f'Observed Data ID: {observed_data.id}', **self._parse_timeline(observed_data) @@ -453,7 +589,7 @@ def _parse_email_address_observable_object( email_address = observed_data.objects[identifier] object_id = f'{observed_data.id} - {identifier}' if hasattr(email_address, 'display_name'): - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': email_address.value, 'uuid': self.main_parser._create_v5_uuid( @@ -465,7 +601,7 @@ def _parse_email_address_observable_object( ) attr_type = 'email-dst-display-name' display_name = email_address.display_name - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': attr_type, 'value': display_name, **attribute, 'uuid': self.main_parser._create_v5_uuid( @@ -475,7 +611,7 @@ def _parse_email_address_observable_object( observed_data ) else: - self.main_parser._add_misp_attribute( + yield self.main_parser._add_misp_attribute( { 'type': 'email-dst', 'value': email_address.value, 'uuid': self.main_parser._create_v5_uuid(object_id), @@ -543,7 +679,27 @@ def _parse_email_address_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_email_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attributes = tuple( + self._parse_email_address_observable_object( + observed_data, object_id + ) + ) + observable.update( + { + 'used': True, + 'misp_attribute': ( + attributes[0] if len(attributes) == 1 + else attributes + ) + } + ) + return if len(observed_data.objects) == 1: email_address = next(iter(observed_data.objects.values())) if email_address.get('id') is not None: @@ -589,8 +745,11 @@ def _parse_email_address_observable_objects( ) else: for identifier in observed_data.objects: - self._parse_email_address_observable_object( - observed_data, identifier + deque( + self._parse_email_address_observable_object( + observed_data, identifier + ), + maxlen=0 ) def _parse_file_observable_object_refs( @@ -646,14 +805,17 @@ def _parse_file_observable_object_refs( artifact.add_reference(misp_object.uuid, 'content-of') def _parse_file_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'file', False ) - observable_objects = { - object_id: {'used': False} for object_id in observed_data.objects - } + if observable_objects is None: + observable_objects = { + object_id: {'used': False} + for object_id in observed_data.objects + } for object_id, observable in observable_objects.items(): observable_object = observed_data.objects[object_id] object_type = observable_object.type @@ -812,7 +974,17 @@ def _parse_ip_address_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_ip_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attribute = self._parse_generic_observable_object_as_attribute( + observed_data, object_id, 'ip-dst' + ) + observable.update({'misp_attribute': attribute, 'used': True}) + return if len(observed_data.objects) == 1: ip_address = next(iter(observed_data.objects.values())) if ip_address.get('id') is not None: @@ -850,7 +1022,17 @@ def _parse_mac_address_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_mac_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attribute = self._parse_generic_observable_object_as_attribute( + observed_data, object_id, 'mac-address' + ) + observable.update({'misp_attribute': attribute, 'used': True}) + return if len(observed_data.objects) == 1: mac_address = next(iter(observed_data.objects.values())) if mac_address.get('id') is not None: @@ -872,30 +1054,6 @@ def _parse_mac_address_observable_objects( observed_data, identifier, 'mac-address' ) - def _parse_multiple_observable_object_refs( - self, observed_data: ObservedData_v21): - for object_ref in observed_data.object_refs: - observable = self._fetch_observables(object_ref) - if observable['used'].get(self.event_uuid, False): - self._handle_misp_object_fields( - observable['misp_object'], observed_data - ) - continue - object_type = object_ref.split('--')[0] - mapping = self._mapping.observable_mapping(object_type) - if mapping is None: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) - continue - feature = f'_parse_{mapping}_observable_object_refs' - try: - parser = getattr(self, feature) - except AttributeError: - self.main_parser._unknown_parsing_function_error(feature) - continue - parser(observed_data, object_ref) - def _parse_mutex_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: @@ -912,7 +1070,17 @@ def _parse_mutex_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_mutex_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attribute = self._parse_generic_observable_object_as_attribute( + observed_data, object_id, 'mutex', feature='name' + ) + observable.update({'misp_attribute': attribute, 'used': True}) + return if len(observed_data.objects) == 1: mutex = next(iter(observed_data.objects.values())) if mutex.get('id') is not None: @@ -961,14 +1129,17 @@ def _parse_process_observable_object_refs( ) def _parse_process_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'process' ) - observable_objects = { - object_id: {'used': False} for object_id in observed_data.objects - } + if observable_objects is None: + observable_objects = { + object_id: {'used': False} + for object_id in observed_data.objects + } for object_id, observable in observable_objects.items(): observable_object = observed_data.objects[object_id] object_type = observable_object.type @@ -1137,7 +1308,8 @@ def _parse_registry_key_observable_object_refs( creator_observable['used'][self.event_uuid] = True def _parse_registry_key_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): if len(observed_data.objects) == 1: registry_key = next(iter(observed_data.objects.values())) if hasattr(registry_key, 'id'): @@ -1163,9 +1335,11 @@ def _parse_registry_key_observable_objects( ) misp_object.add_reference(value_uuid, 'contains') return misp_object - observable_objects = { - object_id: {'used': False} for object_id in observed_data.objects - } + if observable_objects is None: + observable_objects = { + object_id: {'used': False} + for object_id in observed_data.objects + } for object_id, observable in observable_objects.items(): observable_object = observed_data.objects[object_id] if observable_object.type == 'user-account': @@ -1237,7 +1411,17 @@ def _parse_software_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_software_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + misp_object = self._parse_generic_observable_object( + observed_data, object_id, 'software' + ) + observable.update({'misp_object': misp_object, 'used': True}) + return if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'software' @@ -1263,7 +1447,17 @@ def _parse_url_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_url_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + attribute = self._parse_generic_observable_object_as_attribute( + observed_data, object_id, 'url' + ) + observable.update({'misp_attribute': attribute, 'used': True}) + return if len(observed_data.objects) == 1: url = next(iter(observed_data.objects.values())) if url.get('id') is not None: @@ -1301,7 +1495,17 @@ def _parse_user_account_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_user_account_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + misp_object = self._parse_generic_observable_object( + observed_data, object_id, 'user-account', False + ) + observable.update({'misp_object': misp_object, 'used': True}) + return if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'user-account', False @@ -1327,7 +1531,17 @@ def _parse_x509_observable_object_refs( observable['used'][self.event_uuid] = True def _parse_x509_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING): + self, observed_data: _OBSERVED_DATA_TYPING, + observable_objects: Optional[dict] = None): + if observable_objects is not None: + for object_id, observable in observable_objects.items(): + if observable['used']: + continue + misp_object = self._parse_generic_observable_object( + observed_data, object_id, 'x509', False + ) + observable.update({'misp_object': misp_object, 'used': True}) + return if len(observed_data.objects) == 1: return self._parse_generic_single_observable_object( observed_data, 'x509', False From 4753e38c46d28cf856e5cc4ad810a0c4130c169d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 6 Mar 2024 16:43:01 +0100 Subject: [PATCH 088/137] fix: [stix2 import] Avoiding issues with observables references, by keeping track of each reference within a single STIX 2.0 observed data objects list --- .../stix2_observed_data_converter.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9fe22895..19e8e3cc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -133,6 +133,19 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): # MULTIPLE OBSERVABLE OBJECTS PARSING METHODS. # ############################################################################ + @staticmethod + def _extract_referenced_ids_from_observable_objects( + **observable_objects: dict) -> dict: + referenced_ids = defaultdict(set) + for identifier, observable_object in observable_objects.items(): + for key, value in observable_object.items(): + if key.endswith('_ref'): + referenced_ids[value].add(identifier) + if key.endswith('_refs'): + for reference in value: + referenced_ids[reference].add(identifier) + return referenced_ids + def _fetch_multiple_observable_ids( self, observed_data: _OBSERVED_DATA_TYPING, object_id: str) -> Generator: @@ -177,6 +190,9 @@ def _parse_multiple_observable_objects( observable_objects = { object_id: {'used': False} for object_id in observed_data.objects } + referenced_ids = self._extract_referenced_ids_from_observable_objects( + **observed_data.objects + ) for object_id in observable_objects.keys(): if observable_objects[object_id]['used']: continue @@ -188,6 +204,9 @@ def _parse_multiple_observable_objects( ) ) } + if object_id in referenced_ids: + for reference in referenced_ids[object_id]: + observables[reference] = observable_objects[reference] observable_types = set( observed_data.objects[identifier].type for identifier in observables From 3d662c247677805de41cba58221fd936c09658a6 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 7 Mar 2024 21:30:47 +0100 Subject: [PATCH 089/137] wip: [stix2 import] Parsing File objects extensions --- .../stix2_observed_data_converter.py | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 19e8e3cc..7b0d334b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -16,12 +16,14 @@ from datetime import datetime from pymisp import MISPAttribute, MISPObject from stix2.v20.observables import ( + WindowsPEBinaryExt as WindowsPEBinaryExt_v20, WindowsRegistryValueType as WindowsRegistryValueType_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( Artifact, AutonomousSystem, Directory, DomainName, File, IPv4Address, IPv6Address, MACAddress, Mutex, Process, Software, URL, UserAccount, WindowsRegistryKey, X509Certificate, + WindowsPEBinaryExt as WindowsPEBinaryExt_v21, WindowsRegistryValueType as WindowsRegistryValueType_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 from typing import Optional, TYPE_CHECKING, Union @@ -44,6 +46,9 @@ _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 ] +_WINDOWS_PE_BINARY_EXT_TYPING = Union[ + WindowsPEBinaryExt_v20, WindowsPEBinaryExt_v21 +] _WINDOWS_REGISTRY_VALUE_TYPING = Union[ WindowsRegistryValueType_v20, WindowsRegistryValueType_v21 ] @@ -792,6 +797,29 @@ def _parse_file_observable_object_refs( misp_object.add_reference(contained_uuid, 'contains') if object_type == 'directory': continue + if hasattr(observable_object, 'extensions'): + extensions = observable_object.extensions + if extensions.get('archive-ext'): + archive_ext = extensions['archive-ext'] + if hasattr(archive_ext, 'comment'): + misp_object.from_dict( + comment=' - '.join( + (archive_ext.comment, misp_object.comment) + ) + ) + contained_uuids = self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *archive_ext.contains_refs + ) + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') + if extensions.get('windows-pebinary-ext'): + windows_pe_ext = extensions['windows-pebinary-ext'] + pe_object_uuid = self._parse_pe_extension_observable( + windows_pe_ext, observed_data, + f'{observable_object.id} - windows-pebinary-ext' + ) + misp_object.add_reference(pe_object_uuid, 'includes') if hasattr(observable_object, 'parent_directory_ref'): parent_ref = observable_object.parent_directory_ref if parent_ref not in observed_data.object_refs: @@ -860,6 +888,30 @@ def _parse_file_observable_objects( misp_object.add_reference(contained_uuid, 'contains') if object_type == 'directory': continue + if hasattr(observable_object, 'extensions'): + extensions = observable_object.extensions + if extensions.get('archive-ext'): + archive_ext = extensions['archive-ext'] + if hasattr(archive_ext, 'comment'): + misp_object.from_dict( + comment=' - '.join( + (archive_ext.comment, misp_object.comment) + ) + ) + contained_uuids = self._parse_contained_objects( + observed_data, observable_objects, + *archive_ext.contains_refs + ) + for contained_uuid in contained_uuids: + misp_object.add_reference(contained_uuid, 'contains') + if extensions.get('windows-pebinary-ext'): + windows_pe_ext = extensions['windows-pebinary-ext'] + pe_object_uuid = self._parse_pe_extension_observable( + windows_pe_ext, observed_data, + f'{observable_object.id} - windows-pebinary-ext' + f' - {object_id}' + ) + misp_object.add_reference(pe_object_uuid, 'includes') if hasattr(observable_object, 'parent_directory_ref'): parent_ref = observable_object.parent_directory_ref parent_object = self._handle_observable_objects_parsing( @@ -874,6 +926,35 @@ def _parse_file_observable_objects( ) artifact.add_reference(misp_object.uuid, 'content-of') + def _parse_file_pe_extension_observable( + self, pe_extension: _WINDOWS_PE_BINARY_EXT_TYPING, + observed_data: _OBSERVED_DATA_TYPING, object_id: str) -> str: + pe_object = self._create_misp_object_from_observable_object( + 'pe', observed_data, object_id + ) + attributes = self._parse_pe_extension_observable( + pe_extension, object_id + ) + for attribute in attributes: + pe_object.add_attribute(**attribute) + misp_object = self.main_parser._add_misp_object( + pe_object, observed_data + ) + if hasattr(pe_extension, 'sections'): + for section_id, section in enumerate(pe_extension.sections): + section_reference = f'{object_id} - section - {section_id}' + section_object = self._create_misp_object_from_observable_object( + 'pe-section', observed_data, section_reference + ) + attributes = self._parse_pe_section_observable( + section, section_reference + ) + for attribute in attributes: + section_object.add_attribute(**attribute) + self.main_parser._add_misp_object(section_object, observed_data) + misp_object.add_reference(section_object.uuid, 'includes') + return misp_object.uuid + def _parse_generic_observable_object( self, observed_data: _OBSERVED_DATA_TYPING, object_id: str, name: str, generic: Optional[bool] = True) -> MISPObject: @@ -1397,7 +1478,7 @@ def _parse_registry_key_observable_objects( def _parse_registry_key_value_observable( self, registry_value: _WINDOWS_REGISTRY_VALUE_TYPING, - observed_data: _OBSERVED_DATA_TYPING, object_id) -> str: + observed_data: _OBSERVED_DATA_TYPING, object_id: str) -> str: misp_object = self._create_misp_object_from_observable_object( 'registry-key-value', observed_data, object_id ) From d92ea92dc3adeebfafbc2af05676871768f49b14 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 8 Mar 2024 09:00:30 +0100 Subject: [PATCH 090/137] fix: [stix2 import] Fixed File PE extension parsing method name to avoid confusion with the generic method used then from the observable objects converter class --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 7b0d334b..f87b380d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -815,7 +815,7 @@ def _parse_file_observable_object_refs( misp_object.add_reference(contained_uuid, 'contains') if extensions.get('windows-pebinary-ext'): windows_pe_ext = extensions['windows-pebinary-ext'] - pe_object_uuid = self._parse_pe_extension_observable( + pe_object_uuid = self._parse_file_pe_extension_observable( windows_pe_ext, observed_data, f'{observable_object.id} - windows-pebinary-ext' ) @@ -906,7 +906,7 @@ def _parse_file_observable_objects( misp_object.add_reference(contained_uuid, 'contains') if extensions.get('windows-pebinary-ext'): windows_pe_ext = extensions['windows-pebinary-ext'] - pe_object_uuid = self._parse_pe_extension_observable( + pe_object_uuid = self._parse_file_pe_extension_observable( windows_pe_ext, observed_data, f'{observable_object.id} - windows-pebinary-ext' f' - {object_id}' From 68911dd839f0e2a5d60bc0003f3521ce3b7566db Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 8 Mar 2024 15:58:33 +0100 Subject: [PATCH 091/137] fix: [stix2 import] Deduplication of MISP object references - Checking the presence of references with the same referenced uuid AND relationship type before adding a reference to a MISP object --- .../stix2_observed_data_converter.py | 152 ++++++++++++------ 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index f87b380d..ee1b123d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -89,7 +89,11 @@ def parse_relationships(self): object_uuid = misp_object.uuid if object_uuid in self.observable_relationships: for relationship in self.observable_relationships[object_uuid]: - misp_object.add_reference(*relationship) + referenced_uuid, relationship_type = relationship + self._handle_misp_object_references( + misp_object, referenced_uuid, + relationship_type=relationship_type + ) def _set_observable_relationships(self): self._observable_relationships = defaultdict(set) @@ -524,11 +528,13 @@ def _parse_directory_observable_object_refs( ) directory = observable['observable'] if hasattr(directory, 'contains_refs'): - contained_uuids = self._parse_contained_object_refs( - observed_data, misp_object.uuid, *directory.contains_refs + self._handle_misp_object_references( + misp_object, + *self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *directory.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') def _parse_directory_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING, @@ -548,11 +554,13 @@ def _parse_directory_observable_objects( ) directory = observed_data.objects[object_id] if hasattr(directory, 'contains_refs'): - contained_uuids = self._parse_contained_objects( - observed_data, observable_objects, *directory.contains_refs + self._handle_object_reference( + misp_object, + *self._parse_contained_objects( + observed_data, observable_objects, + *directory.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') def _parse_domain_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): @@ -789,12 +797,13 @@ def _parse_file_observable_object_refs( continue observable_object = observable['observable'] if hasattr(observable_object, 'contains_refs'): - contained_uuids = self._parse_contained_object_refs( - observed_data, misp_object.uuid, - *observable_object.contains_refs + self._handle_misp_object_references( + misp_object, + *self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *observable_object.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') if object_type == 'directory': continue if hasattr(observable_object, 'extensions'): @@ -807,12 +816,13 @@ def _parse_file_observable_object_refs( (archive_ext.comment, misp_object.comment) ) ) - contained_uuids = self._parse_contained_object_refs( - observed_data, misp_object.uuid, - *archive_ext.contains_refs + self._handle_misp_object_references( + misp_object, + *self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *archive_ext.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') if extensions.get('windows-pebinary-ext'): windows_pe_ext = extensions['windows-pebinary-ext'] pe_object_uuid = self._parse_file_pe_extension_observable( @@ -834,8 +844,9 @@ def _parse_file_observable_object_refs( parent_object = self._handle_observable_object_refs_parsing( parent, observed_data, 'directory' ) - misp_object.add_reference( - parent_object.uuid, 'contained-in' + self._handle_misp_object_references( + misp_object, parent_object.uuid, + relationship_type='contained-in' ) if hasattr(observable_object, 'content_ref'): content_ref = observable_object.content_ref @@ -849,7 +860,10 @@ def _parse_file_observable_object_refs( artifact = self._handle_observable_object_refs_parsing( content, observed_data, 'artifact', False ) - artifact.add_reference(misp_object.uuid, 'content-of') + self._handle_misp_object_references( + artifact, misp_object.uuid, + relationship_type='content-of' + ) def _parse_file_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING, @@ -880,12 +894,13 @@ def _parse_file_observable_objects( object_type, (object_type == 'directory') ) if hasattr(observable_object, 'contains_refs'): - contained_uuids = self._parse_contained_objects( - observed_data, observable_objects, - *observable_object.contains_refs + self._handle_misp_object_references( + misp_object, + *self._parse_contained_objects( + observed_data, observable_objects, + *observable_object.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') if object_type == 'directory': continue if hasattr(observable_object, 'extensions'): @@ -898,12 +913,13 @@ def _parse_file_observable_objects( (archive_ext.comment, misp_object.comment) ) ) - contained_uuids = self._parse_contained_objects( - observed_data, observable_objects, - *archive_ext.contains_refs + self._handle_misp_object_references( + misp_object, + *self._parse_contained_objects( + observed_data, observable_objects, + *archive_ext.contains_refs + ) ) - for contained_uuid in contained_uuids: - misp_object.add_reference(contained_uuid, 'contains') if extensions.get('windows-pebinary-ext'): windows_pe_ext = extensions['windows-pebinary-ext'] pe_object_uuid = self._parse_file_pe_extension_observable( @@ -917,14 +933,19 @@ def _parse_file_observable_objects( parent_object = self._handle_observable_objects_parsing( observable_objects, parent_ref, observed_data, 'directory' ) - misp_object.add_reference(parent_object.uuid, 'contained-in') + self._handle_misp_object_references( + misp_object, parent_object.uuid, + relationship_type='contained-in' + ) if hasattr(observable_object, 'content_ref'): content_ref = observable_object.content_ref artifact = self._handle_observable_objects_parsing( observable_objects, content_ref, observed_data, 'artifact', False ) - artifact.add_reference(misp_object.uuid, 'content-of') + self._handle_misp_object_references( + artifact, misp_object.uuid, relationship_type='content-of' + ) def _parse_file_pe_extension_observable( self, pe_extension: _WINDOWS_PE_BINARY_EXT_TYPING, @@ -1282,14 +1303,18 @@ def _parse_process_reference_observable_object( observable: dict, reference: str, relationship_type: str, name: Optional[str] = 'process'): if observable['used']: - misp_object.add_reference( - observable['misp_object'].uuid, relationship_type + self._handle_misp_object_references( + misp_object, observable['misp_object'].uuid, + relationship_type=relationship_type ) return referenced_object = self._parse_generic_observable_object( observed_data, reference, name, False ) - misp_object.add_reference(referenced_object.uuid, relationship_type) + self._handle_misp_object_references( + misp_object, referenced_object.uuid, + relationship_type=relationship_type + ) observable.update({'used': True, 'misp_object': referenced_object}) def _parse_process_reference_observable_object_ref( @@ -1299,8 +1324,9 @@ def _parse_process_reference_observable_object_ref( observable = self._fetch_observable(reference) if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields(misp_object, observed_data) - misp_object.add_reference( - observable['misp_object'].uuid, relationship_type + self._handle_misp_object_references( + misp_object, observable['misp_object'].uuid, + relationship_type=relationship_type ) return if reference in observed_data.object_refs: @@ -1309,7 +1335,10 @@ def _parse_process_reference_observable_object_ref( ) observable['misp_object'] = referenced_object observable['used'][self.event_uuid] = True - misp_object.add_reference(referenced_object.uuid, relationship_type) + self._handle_misp_object_references( + misp_object, referenced_object.uuid, + relationship_type=relationship_type + ) else: self.observable_relationships[misp_object.uuid].add( ( @@ -1343,7 +1372,9 @@ def _parse_registry_key_observable_object( value_uuid = self._parse_registry_key_value_observable( value, observed_data, f'{object_id} - values - {index}' ) - misp_object.add_reference(value_uuid, 'contains') + self._handle_misp_object_references( + misp_object, value_uuid + ) return misp_object def _parse_registry_key_observable_object_ref( @@ -1363,7 +1394,9 @@ def _parse_registry_key_observable_object_ref( registry_value, observed_data, f'{registry_key.id} - values - {index}' ) - misp_object.add_reference(value_uuid, 'contains') + self._handle_misp_object_references( + misp_object, value_uuid + ) return misp_object def _parse_registry_key_observable_object_refs( @@ -1397,13 +1430,19 @@ def _parse_registry_key_observable_object_refs( self._handle_misp_object_fields( creator_object, observed_data ) - creator_object.add_reference(misp_object.uuid, 'creates') + self._handle_misp_object_references( + creator_object, misp_object.uuid, + relationship_type='creates' + ) continue creator_object = self._parse_generic_observable_object_ref( creator_observable['observable'], observed_data, 'user-account', False ) - creator_object.add_reference(misp_object.uuid, 'creates') + self._handle_misp_object_references( + creator_object, misp_object.uuid, + relationship_type='creates' + ) creator_observable['misp_object'] = creator_object creator_observable['used'][self.event_uuid] = True @@ -1433,7 +1472,9 @@ def _parse_registry_key_observable_objects( registry_value, observed_data, f'{observed_data.id} - values - {index}' ) - misp_object.add_reference(value_uuid, 'contains') + self._handle_misp_object_references( + misp_object, value_uuid + ) return misp_object if observable_objects is None: observable_objects = { @@ -1463,15 +1504,19 @@ def _parse_registry_key_observable_objects( observable_object.creator_user_ref ] if creator_observable['used']: - creator_observable['misp_object'].add_reference( - misp_object.uuid, 'creates' + self._handle_misp_object_references( + creator_observable['misp_object'], misp_object.uuid, + relationship_type='creates' ) continue creator_object = self._parse_generic_observable_object( observed_data, observable_object.creator_user_ref, 'user-account', False ) - creator_object.add_reference(misp_object.uuid, 'creates') + self._handle_misp_object_references( + creator_object, misp_object.uuid, + relationship_type='creates' + ) creator_observable.update( {'misp_object': creator_object, 'used': True} ) @@ -1705,6 +1750,17 @@ def _handle_misp_object_fields( elif comment not in misp_object.comment: misp_object.comment = f'{misp_object.comment} - {comment}' + @staticmethod + def _handle_misp_object_references( + misp_object: MISPObject, *object_ids: tuple, + relationship_type: str = 'contains'): + for object_id in object_ids: + for reference in misp_object.references: + if (reference.referenced_uuid == object_id and + reference.relationship_type == relationship_type): + break + misp_object.add_reference(object_id, relationship_type) + class InternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, InternalSTIX2ObservableConverter): From 63a8e05c388358d30a682d0a7094dc86cf0c5429 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 8 Mar 2024 16:34:46 +0100 Subject: [PATCH 092/137] fix: [stix2 import] Fixed the MISP object reference duplicates checking --- .../converters/stix2_observed_data_converter.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index ee1b123d..9427d178 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1755,11 +1755,10 @@ def _handle_misp_object_references( misp_object: MISPObject, *object_ids: tuple, relationship_type: str = 'contains'): for object_id in object_ids: - for reference in misp_object.references: - if (reference.referenced_uuid == object_id and - reference.relationship_type == relationship_type): - break - misp_object.add_reference(object_id, relationship_type) + if not any(reference.referenced_uuid == object_id and + reference.relationship_type == relationship_type + for reference in misp_object.references): + misp_object.add_reference(object_id, relationship_type) class InternalSTIX2ObservedDataConverter( From 2207e452b7e721b05a7686758b27c29281a172e4 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 11 Mar 2024 23:40:39 +0100 Subject: [PATCH 093/137] fix: [stix2 import] Error exceptions handling method name --- .../stix2misp/converters/stix2_observable_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 43a1568c..2447be2c 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -237,7 +237,7 @@ def _parse_file_observable( for hash_type, value in observable.hashes.items(): attribute = self._mapping.file_hashes_mapping(hash_type) if attribute is None: - self.main_parser.hash_type_error(hash_type) + self.main_parser._hash_type_error(hash_type) continue yield from self._populate_object_attributes( attribute, value, object_id From 858c41f0193192d047647bc476f57d95aba24f1a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 11 Mar 2024 23:41:44 +0100 Subject: [PATCH 094/137] fix: [stix2 import] MISP object references handling method name --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9427d178..666ec2b2 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -554,7 +554,7 @@ def _parse_directory_observable_objects( ) directory = observed_data.objects[object_id] if hasattr(directory, 'contains_refs'): - self._handle_object_reference( + self._handle_misp_object_references( misp_object, *self._parse_contained_objects( observed_data, observable_objects, From 64aa90bdecfa091bbdd79fd74acee3d730c5e6c3 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 11 Mar 2024 23:42:43 +0100 Subject: [PATCH 095/137] fix: [stix2 import] Better STIX 2.0 `windows-pebinary-ext` within File observable object handling --- .../stix2_observed_data_converter.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 666ec2b2..bd978dbc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -869,9 +869,19 @@ def _parse_file_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING, observable_objects: Optional[dict] = None): if len(observed_data.objects) == 1: - return self._parse_generic_single_observable_object( + misp_object = self._parse_generic_single_observable_object( observed_data, 'file', False ) + observable = observed_data.objects['0'] + if getattr(observable, 'extensions', {}).get( + 'windows-pebinary-ext'): + object_id = getattr(observable, 'id', observed_data.id) + pe_object_uuid = self._parse_file_pe_extension_observable( + observable.extensions['windows-pebinary-ext'], + observed_data, f'{object_id} - windows-pebinary-ext' + ) + misp_object.add_reference(pe_object_uuid, 'includes') + return misp_object if observable_objects is None: observable_objects = { object_id: {'used': False} @@ -921,11 +931,10 @@ def _parse_file_observable_objects( ) ) if extensions.get('windows-pebinary-ext'): - windows_pe_ext = extensions['windows-pebinary-ext'] pe_object_uuid = self._parse_file_pe_extension_observable( - windows_pe_ext, observed_data, - f'{observable_object.id} - windows-pebinary-ext' - f' - {object_id}' + extensions['windows-pebinary-ext'], observed_data, + f'{observed_data.id} - ' + f'{object_id} - windows-pebinary-ext' ) misp_object.add_reference(pe_object_uuid, 'includes') if hasattr(observable_object, 'parent_directory_ref'): From 11269086e4cdd7e363587b1974fe7ee94a358567 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 11 Mar 2024 23:48:18 +0100 Subject: [PATCH 096/137] fix: [stix2 import] Updated pe object mapping with the `compilation-timestamp` attribute --- .../stix2misp/converters/stix2mapping.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index f3e7b80b..60fad927 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -47,6 +47,9 @@ class STIX2Mapping: __comment_attribute = Mapping( **{'type': 'comment', 'object_relation': 'comment'} ) + __compilation_timestamp_attribute = Mapping( + **{'type': 'datetime', 'object_relation': 'compilation-timestamp'} + ) __content_type_attribute = Mapping( **{'type': 'other', 'object_relation': 'content-type'} ) @@ -344,6 +347,7 @@ class STIX2Mapping: socket_type={'type': 'text', 'object_relation': 'socket-type'} ) __pe_object_mapping = Mapping( + time_date_stamp=__compilation_timestamp_attribute, imphash=__imphash_attribute, number_of_sections={'type': 'counter', 'object_relation': 'number-sections'}, pe_type=__type_attribute @@ -451,6 +455,10 @@ def command_line_attribute(cls) -> dict: def comment_attribute(cls) -> dict: return cls.__comment_attribute + @classmethod + def compilation_timestamp_attribute(cls) -> dict: + return cls.__compilation_timestamp_attribute + @classmethod def connection_protocols(cls, field: str) -> Union[str, None]: return cls.__connection_protocols.get(field) @@ -1049,7 +1057,6 @@ class InternalSTIX2Mapping(STIX2Mapping): __account_name_attribute = {'type': 'text', 'object_relation': 'account-name'} __bio_attribute = {'type': 'text', 'object_relation': 'bio'} __community_id_attribute = {'type': 'community-id', 'object_relation': 'community-id'} - __compilation_timestamp_attribute = {'type': 'datetime', 'object_relation': 'compilation-timestamp'} __followers_attribute = {'type': 'text', 'object_relation': 'followers'} __following_attribute = {'type': 'text', 'object_relation': 'following'} __fullpath_attribute = {'type': 'text', 'object_relation': 'fullpath'} @@ -1224,7 +1231,7 @@ class InternalSTIX2Mapping(STIX2Mapping): **STIX2Mapping.file_object_mapping(), x_misp_attachment=__attachment_attribute, x_misp_certificate={'type': 'x509-fingerprint-sha1', 'object_relation': 'certificate'}, - x_misp_compilation_timestamp=__compilation_timestamp_attribute, + x_misp_compilation_timestamp=STIX2Mapping.compilation_timestamp_attribute(), x_misp_entropy=STIX2Mapping.entropy_attribute(), x_misp_fullpath=__fullpath_attribute, x_misp_path=STIX2Mapping.path_attribute(), @@ -1385,7 +1392,7 @@ class InternalSTIX2Mapping(STIX2Mapping): **STIX2Mapping.pe_object_mapping(), x_misp_authentihash=__authentihash_attribute, x_misp_company_name={'type': 'text', 'object_relation': 'company-name'}, - x_misp_compilation_timestamp=__compilation_timestamp_attribute, + x_misp_compilation_timestamp=STIX2Mapping.compilation_timestamp_attribute(), x_misp_entrypoint_section_at_position={ 'type': 'text', 'object_relation': 'entrypoint-section-at-position' From 8854631fb9a6b7571ca8d33010da56c91a04bac8 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 12 Mar 2024 12:14:19 +0100 Subject: [PATCH 097/137] fix: [stix2 import] Avoiding issues with `ssdeep` hash type in STIX 2.0 external content --- misp_stix_converter/stix2misp/converters/stix2mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 60fad927..8a44466b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -326,6 +326,7 @@ class STIX2Mapping: 'SHA3-256': __sha3_256_attribute, 'SHA3-512': __sha3_512_attribute, 'SSDEEP': __ssdeep_attribute, + 'ssdeep': __ssdeep_attribute, 'TLSH': __tlsh_attribute } ) @@ -1223,7 +1224,6 @@ class InternalSTIX2Mapping(STIX2Mapping): SHA3384=__sha3_384_attribute, SHA384=__sha384_attribute, SHA512=STIX2Mapping.sha512_attribute(), - ssdeep=STIX2Mapping.ssdeep_attribute(), TELFHASH=__telfhash_attribute, VHASH=__vhash_attribute ) From 266b01550dd2479ca1abd0773d2f52b29318c509 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 12 Mar 2024 12:20:11 +0100 Subject: [PATCH 098/137] wip: [tests] Added tests for file objects with extensions --- tests/_test_stix_import.py | 147 +++++++++++++++++++ tests/test_external_stix20_bundles.py | 125 +++++++++++++++- tests/test_external_stix20_import.py | 108 ++++++++++---- tests/test_external_stix21_bundles.py | 199 ++++++++++++++++++++------ tests/test_external_stix21_import.py | 82 ++++++++--- 5 files changed, 570 insertions(+), 91 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index c4f53cfc..96a0a183 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -364,6 +364,48 @@ def _check_galaxy_features(self, galaxies, stix_object): # OBSERVED DATA OBJECTS CHECKING METHODS # ############################################################################ + def _check_archive_file_object(self, misp_object, observed_data, + observable_object, object_id = None): + self.assertEqual(misp_object.name, 'file') + if object_id is None: + self.assertEqual( + misp_object.uuid, observable_object.id.split('--')[1] + ) + object_id = observable_object.id + else: + self.assertEqual(misp_object.uuid, uuid5(self._UUIDv4, object_id)) + self.assertEqual( + misp_object.comment, + f"{observable_object.extensions['archive-ext'].comment} - " + f'Observed Data ID: {observed_data.id}' + ) + self.assertEqual(misp_object.timestamp, observed_data.modified) + self.assertEqual(len(misp_object.attributes), 3) + sha256, mime_type, filename = misp_object.attributes + self._assert_multiple_equal( + sha256.type, sha256.object_relation, 'sha256' + ) + self.assertEqual(sha256.value, observable_object.hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha256 - {sha256.value}') + ) + self.assertEqual(mime_type.type, 'mime-type') + self.assertEqual(mime_type.object_relation, 'mimetype') + self.assertEqual(mime_type.value, observable_object.mime_type) + self.assertEqual( + mime_type.uuid, + uuid5(self._UUIDv4, f'{object_id} - mimetype - {mime_type.value}') + ) + self._assert_multiple_equal( + filename.type, filename.object_relation, 'filename' + ) + self.assertEqual(filename.value, observable_object.name) + self.assertEqual( + filename.uuid, + uuid5(self._UUIDv4, f'{object_id} - filename - {filename.value}') + ) + def _check_artifact_fields(self, misp_object, artifact, object_id): self.assertEqual(len(misp_object.attributes), 6) payload_bin, md5, sha1, sha256, decryption, mime_type = misp_object.attributes @@ -619,6 +661,111 @@ def _check_file_fields(self, misp_object, observable_object, object_id): ) ) + def _check_file_with_pe_fields(self, misp_object, file_object, object_id): + self.assertEqual(len(misp_object.attributes), 7) + md5, sha1, sha256, sha512, ssdeep, filename, size = misp_object.attributes + hashes = file_object.hashes + self._assert_multiple_equal(md5.type, md5.object_relation, 'md5') + self.assertEqual(md5.value, hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + self._assert_multiple_equal(sha1.type, sha1.object_relation, 'sha1') + self.assertEqual(sha1.value, hashes['SHA-1']) + self.assertEqual( + sha1.uuid, uuid5(self._UUIDv4, f'{object_id} - sha1 - {sha1.value}') + ) + self._assert_multiple_equal(sha256.type, sha256.object_relation, 'sha256') + self.assertEqual(sha256.value, hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha256 - {sha256.value}') + ) + self._assert_multiple_equal(sha512.type, sha512.object_relation, 'sha512') + self.assertEqual(sha512.value, hashes['SHA-512']) + self.assertEqual( + sha512.uuid, + uuid5(self._UUIDv4, f'{object_id} - sha512 - {sha512.value}') + ) + self._assert_multiple_equal(ssdeep.type, ssdeep.object_relation, 'ssdeep') + self.assertEqual(ssdeep.value, hashes.get('SSDEEP') or hashes.get('ssdeep')) + self.assertEqual( + ssdeep.uuid, + uuid5(self._UUIDv4, f'{object_id} - ssdeep - {ssdeep.value}') + ) + self._assert_multiple_equal(filename.type, filename.object_relation, 'filename') + self.assertEqual(filename.value, file_object.name) + self.assertEqual( + filename.uuid, + uuid5(self._UUIDv4, f'{object_id} - filename - {filename.value}') + ) + self._assert_multiple_equal(size.type, size.object_relation, 'size-in-bytes') + self.assertEqual(size.value, file_object.size) + self.assertEqual( + size.uuid, + uuid5(self._UUIDv4, f'{object_id} - size-in-bytes - {size.value}') + ) + + def _check_pe_fields(self, misp_object, pe_extension, object_id): + self.assertEqual(len(misp_object.attributes), 3) + compilation_timestamp, number_of_sections, pe_type = misp_object.attributes + self.assertEqual(compilation_timestamp.type, 'datetime') + self.assertEqual(compilation_timestamp.object_relation, 'compilation-timestamp') + self.assertEqual(compilation_timestamp.value, pe_extension.time_date_stamp) + self.assertEqual( + compilation_timestamp.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - compilation-timestamp' + f' - {compilation_timestamp.value}' + ) + ) + self.assertEqual(number_of_sections.type, 'counter') + self.assertEqual(number_of_sections.object_relation, 'number-sections') + self.assertEqual(number_of_sections.value, pe_extension.number_of_sections) + self.assertEqual( + number_of_sections.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - number-sections - {number_of_sections.value}' + ) + ) + self.assertEqual(pe_type.type, 'text') + self.assertEqual(pe_type.object_relation, 'type') + self.assertEqual(pe_type.value, pe_extension.pe_type) + self.assertEqual( + pe_type.uuid, + uuid5(self._UUIDv4, f'{object_id} - type - {pe_type.value}') + ) + + def _check_pe_section_fields(self, misp_object, pe_section, object_id): + self.assertEqual(len(misp_object.attributes), 4) + entropy, name, size, md5 = misp_object.attributes + self.assertEqual(entropy.type, 'float') + self.assertEqual(entropy.object_relation, 'entropy') + self.assertEqual(entropy.value, pe_section.entropy) + self.assertEqual( + entropy.uuid, + uuid5(self._UUIDv4, f'{object_id} - entropy - {entropy.value}') + ) + self.assertEqual(name.type, 'text') + self.assertEqual(name.object_relation, 'name') + self.assertEqual(name.value, pe_section.name) + self.assertEqual( + name.uuid, uuid5(self._UUIDv4, f'{object_id} - name - {name.value}') + ) + self._assert_multiple_equal(size.type, size.object_relation, 'size-in-bytes') + self.assertEqual(size.value, pe_section.size) + self.assertEqual( + size.uuid, + uuid5(self._UUIDv4, f'{object_id} - size-in-bytes - {size.value}') + ) + self._assert_multiple_equal(md5.type, md5.object_relation, 'md5') + self.assertEqual(md5.value, pe_section.hashes['MD5']) + self.assertEqual( + md5.uuid, uuid5(self._UUIDv4, f'{object_id} - md5 - {md5.value}') + ) + def _check_process_child_fields(self, misp_object, process, object_id): self.assertEqual(len(misp_object.attributes), 2) name, pid = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 5e22e325..5dfeba80 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -408,6 +408,127 @@ "decryption_key": "infected" } } + }, + { + "type": "observed-data", + "id": "observed-data--1a165e68-ea72-44e6-b821-3b88f2cc46d8", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "file", + "name": "oui.zip", + "hashes": { + "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + }, + "mime_type": "application/zip", + "extensions": { + "archive-ext": { + "comment": "Zip file containing `oui` in the tmp directory", + "contains_refs": [ + "1", + "2" + ] + } + } + }, + "1": { + "type": "file", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802", + "SHA-1": "46aba99aa7158e4609aaa72b50990842fd22ae86", + "SHA-256": "ec5aedf5ecc6bdadd4120932170d1b10f6cfa175cfda22951dfd882928ab279b" + }, + "size": 35, + "name": "oui", + "name_enc": "UTF-8", + "parent_directory_ref": "2", + "content_ref": "3" + }, + "2": { + "type": "directory", + "path": "/var/www/MISP/app/files/scripts/tmp" + }, + "3": { + "type": "artifact", + "mime_type": "application/zip", + "hashes": { + "MD5": "8764605c6f388c89096b534d33565802" + }, + "encryption_algorithm": "mime-type-indicated", + "decryption_key": "infected" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "file", + "hashes": { + "MD5": "b1de37bf229890ac181bdef1ad8ee0c2", + "SHA-1": "ffdb3cc7ab5b01d276d23ac930eb21ffe3202d11", + "SHA-256": "99b80c5ac352081a64129772ed5e1543d94cad708ba2adc46dc4ab7a0bd563f1", + "SHA-512": "e41df636a36ac0cce38e7db5c2ce4d04a1a7f9bc274bdf808912d14067dc1ef478268035521d0d4b7bcf96facce7f515560b38a7ebe47995d861b9c482e07e25", + "SSDEEP": "98304:z2eyMq4PuR5d7wgdo0OFfnFJkEUCGdaQLhpYYEfRTl6sysy:ryxzbdo0ifnoEOdz9pY7j5" + }, + "size": 3712512, + "name": "SMSvcService.exe", + "extensions": { + "windows-pebinary-ext": { + "pe_type": "exe", + "number_of_sections": 4, + "time_date_stamp": "1970-01-01T00:00:00Z", + "size_of_optional_header": 512, + "sections": [ + { + "name": "header", + "size": 512, + "entropy": 2.499747, + "hashes": { + "MD5": "7f8e8722da728b6e834260b5a314cbac" + } + }, + { + "name": "UPX0", + "size": 0, + "entropy": 0.0, + "hashes": { + "MD5": "d41d8cd98f00b204e9800998ecf8427e" + } + }, + { + "name": "UPX1", + "size": 3711488, + "entropy": 7.890727, + "hashes": { + "MD5": "f9943591918adeeeee7da80e4d985a49" + } + }, + { + "name": "UPX2", + "size": 512, + "entropy": 1.371914, + "hashes": { + "MD5": "5c0061445ac2f8e6cadf694e54146914" + } + } + ] + } + } + } + } } ] _INTRUSION_SET_OBJECTS = [ @@ -1255,7 +1376,9 @@ def get_bundle_with_email_address_attributes(cls): def get_bundle_with_file_objects(cls): observed_data = deepcopy(_FILE_OBJECTS) with open(_TESTFILES_PATH / 'malware_sample.zip', 'rb') as f: - observed_data[0]['objects']['2']['payload_bin'] = b64encode(f.read()).decode() + payload_bin = b64encode(f.read()).decode() + observed_data[0]['objects']['2']['payload_bin'] = payload_bin + observed_data[1]['objects']['3']['payload_bin'] = payload_bin return cls.__assemble_bundle(*observed_data) @classmethod diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index ffe880a0..c35cfef4 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -274,6 +274,54 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_file_and_pe_objects(self, observed_data, file_object, + pe_object, *sections): + self.assertEqual(file_object.name, 'file') + self._check_misp_object_fields(file_object, observed_data) + observable_object = observed_data.objects['0'] + self._check_file_with_pe_fields( + file_object, observable_object, observed_data.id + ) + file_reference = file_object.references[0] + self.assertEqual(file_reference.referenced_uuid, pe_object.uuid) + self.assertEqual(file_reference.relationship_type, 'includes') + self.assertEqual(pe_object.name, 'pe') + self._check_misp_object_fields( + pe_object, observed_data, 'windows-pebinary-ext' + ) + object_id = f'{observed_data.id} - windows-pebinary-ext' + extension = observable_object.extensions['windows-pebinary-ext'] + self._check_pe_fields(pe_object, extension, object_id) + self.assertEqual(len(pe_object.references), len(sections)) + for reference, section in zip(pe_object.references, sections): + self.assertEqual(reference.referenced_uuid, section.uuid) + self.assertEqual(reference.relationship_type, 'includes') + for index, section in enumerate(sections): + self.assertEqual(section.name, 'pe-section') + section_id = f'windows-pebinary-ext - section - {index}' + self._check_misp_object_fields(section, observed_data, section_id) + self._check_pe_section_fields( + section, extension.sections[index], + f'{observed_data.id} - {section_id}' + ) + + def _check_file_directory_object(self, misp_object, observed_data, object_id): + self.assertEqual(misp_object.name, 'directory') + self._check_misp_object_fields(misp_object, observed_data, object_id) + directory = observed_data.objects[object_id] + self.assertEqual(len(misp_object.attributes), 1) + path_attribute = misp_object.attributes[0] + self.assertEqual(path_attribute.type, 'text') + self.assertEqual(path_attribute.object_relation, 'path') + self.assertEqual(path_attribute.value, directory.path) + self.assertEqual( + path_attribute.uuid, + uuid5( + self._UUIDv4, + f'{observed_data.id} - {object_id} - path - {path_attribute.value}' + ) + ) + def _check_file_object(self, misp_object, observed_data, identifier=None): self.assertEqual(misp_object.name, 'file') self._check_misp_object_fields(misp_object, observed_data, identifier) @@ -452,35 +500,45 @@ def test_stix20_bundle_with_file_objects(self): self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle() event = self.parser.misp_event - _, report, observed_data = bundle.objects + _, report, observed_data1, observed_data2, observed_data3 = bundle.objects misp_objects = self._check_misp_event_features(event, report) - self.assertEqual(len(misp_objects), 3) - file_object, directory_object, artifact_object = misp_objects - self._check_file_object(file_object, observed_data, '0') - self.assertEqual(directory_object.name, 'directory') - self._check_misp_object_fields(directory_object, observed_data, '1') - artifact = observed_data.objects['1'] - self.assertEqual(len(directory_object.attributes), 1) - path_attribute = directory_object.attributes[0] - self.assertEqual(path_attribute.type, 'text') - self.assertEqual(path_attribute.object_relation, 'path') - self.assertEqual(path_attribute.value, artifact.path) - self.assertEqual( - path_attribute.uuid, - uuid5( - self._UUIDv4, - f'{observed_data.id} - 1 - path - {path_attribute.value}' - ) + self.assertEqual(len(misp_objects), 13) + file1, directory1, artifact1, zip_file, file2, directory2, artifact2, file3, pe, *sections = misp_objects + self._check_file_object(file1, observed_data1, '0') + self._check_file_directory_object(directory1, observed_data1, '1') + self._check_content_ref_object(artifact1, observed_data1, '2') + self.assertEqual(len(file1.references), 1) + file_reference = file1.references[0] + self.assertEqual(file_reference.referenced_uuid, directory1.uuid) + self.assertEqual(file_reference.relationship_type, 'contained-in') + self.assertEqual(len(artifact1.references), 1) + artifact_reference = artifact1.references[0] + self.assertEqual(artifact_reference.referenced_uuid, file1.uuid) + self.assertEqual(artifact_reference.relationship_type, 'content-of') + self._check_archive_file_object( + zip_file, observed_data2, observed_data2.objects['0'], + f'{observed_data2.id} - 0' ) - self._check_content_ref_object(artifact_object, observed_data, '2') - self.assertEqual(len(file_object.references), 1) - file_reference = file_object.references[0] - self.assertEqual(file_reference.referenced_uuid, directory_object.uuid) + self.assertEqual(len(zip_file.references), 2) + directory_reference, file_reference = zip_file.references + self.assertEqual( + directory_reference.referenced_uuid, file2.uuid + ) + self.assertEqual(directory_reference.relationship_type, 'contains') + self.assertEqual(file_reference.referenced_uuid, directory2.uuid) + self.assertEqual(file_reference.relationship_type, 'contains') + self._check_file_object(file2, observed_data2, '1') + self._check_file_directory_object(directory2, observed_data2, '2') + self._check_content_ref_object(artifact2, observed_data2, '3') + self.assertEqual(len(file2.references), 1) + file_reference = file2.references[0] + self.assertEqual(file_reference.referenced_uuid, directory2.uuid) self.assertEqual(file_reference.relationship_type, 'contained-in') - self.assertEqual(len(artifact_object.references), 1) - artifact_reference = artifact_object.references[0] - self.assertEqual(artifact_reference.referenced_uuid, file_object.uuid) + self.assertEqual(len(artifact2.references), 1) + artifact_reference = artifact2.references[0] + self.assertEqual(artifact_reference.referenced_uuid, file2.uuid) self.assertEqual(artifact_reference.relationship_type, 'content-of') + self._check_file_and_pe_objects(observed_data3, file3, pe, *sections) def test_stix20_bundle_with_ip_address_attributes(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_ip_address_attributes() diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 0f77df83..81614448 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -459,6 +459,36 @@ "artifact--91ae0a21-c7ae-4c7f-b84b-b84a7ce53d1f" ], }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--1a165e68-ea72-44e6-b821-3b88f2cc46d8", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "file--b52bc0df-84e3-44bf-8a55-2889e26723fa", + "file--5e384ae7-672c-4250-9cda-3b4da964451a", + "directory--34cb1a7c-55ec-412a-8684-ba4a88d83a45" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "file--6ee289fa-fe03-5ca5-bbdf-451603a31436" + ] + }, { "type": "file", "spec_version": "2.1", @@ -490,6 +520,81 @@ }, "encryption_algorithm": "mime-type-indicated", "decryption_key": "infected" + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--b52bc0df-84e3-44bf-8a55-2889e26723fa", + "name": "oui.zip", + "hashes": { + "SHA-256": "35a01331e9ad96f751278b891b6ea09699806faedfa237d40513d92ad1b7100f" + }, + "mime_type": "application/zip", + "extensions": { + "archive-ext": { + "comment": "Zip file containing `oui` in the tmp directory", + "contains_refs": [ + "file--5e384ae7-672c-4250-9cda-3b4da964451a", + "directory--34cb1a7c-55ec-412a-8684-ba4a88d83a45" + ] + } + } + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--6ee289fa-fe03-5ca5-bbdf-451603a31436", + "hashes": { + "MD5": "b1de37bf229890ac181bdef1ad8ee0c2", + "SHA-1": "ffdb3cc7ab5b01d276d23ac930eb21ffe3202d11", + "SHA-256": "99b80c5ac352081a64129772ed5e1543d94cad708ba2adc46dc4ab7a0bd563f1", + "SHA-512": "e41df636a36ac0cce38e7db5c2ce4d04a1a7f9bc274bdf808912d14067dc1ef478268035521d0d4b7bcf96facce7f515560b38a7ebe47995d861b9c482e07e25", + "SSDEEP": "98304:z2eyMq4PuR5d7wgdo0OFfnFJkEUCGdaQLhpYYEfRTl6sysy:ryxzbdo0ifnoEOdz9pY7j5" + }, + "size": 3712512, + "name": "SMSvcService.exe", + "extensions": { + "windows-pebinary-ext": { + "pe_type": "exe", + "number_of_sections": 4, + "time_date_stamp": "1970-01-01T00:00:00Z", + "size_of_optional_header": 512, + "sections": [ + { + "name": "header", + "size": 512, + "entropy": 2.499747, + "hashes": { + "MD5": "7f8e8722da728b6e834260b5a314cbac" + } + }, + { + "name": "UPX0", + "size": 0, + "entropy": 0.0, + "hashes": { + "MD5": "d41d8cd98f00b204e9800998ecf8427e" + } + }, + { + "name": "UPX1", + "size": 3711488, + "entropy": 7.890727, + "hashes": { + "MD5": "f9943591918adeeeee7da80e4d985a49" + } + }, + { + "name": "UPX2", + "size": 512, + "entropy": 1.371914, + "hashes": { + "MD5": "5c0061445ac2f8e6cadf694e54146914" + } + } + ] + } + } } ] _INTRUSION_SET_OBJECTS = [ @@ -1039,6 +1144,52 @@ "primary_motivation": "organizational-gain" } ] +_TOOL_OBJECTS = [ + { + "type": "tool", + "spec_version": "2.1", + "id": "tool--ce45f721-af14-4fc0-938c-000c16186418", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "cachedump", + "tool_types": [ + "credential-exploitation" + ], + "description": "This program extracts cached password hashes from a system’s registry.", + "kill_chain_phases": [ + { + "kill_chain_name": "mandiant-attack-lifecycle-model", + "phase_name": "escalate-privileges" + } + ] + }, + { + "type": "tool", + "spec_version": "2.1", + "id": "tool--e9778c42-bc2f-4eda-9fb4-6a931834f68c", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "fgdump", + "tool_types": [ + "credential-exploitation" + ], + "description": "Windows password hash dumper", + "kill_chain_phases": [ + { + "kill_chain_name": "mandiant-attack-lifecycle-model", + "phase_name": "escalate-privileges" + } + ], + "external_references": [ + { + "source_name": "fgdump", + "url": "http://www.foofus.net/fizzgig/fgdump/" + } + ] + } +] _URL_ATTRIBUTES = [ { "type": "observed-data", @@ -1088,52 +1239,6 @@ "value": "https://misp-project.org/blog/" } ] -_TOOL_OBJECTS = [ - { - "type": "tool", - "spec_version": "2.1", - "id": "tool--ce45f721-af14-4fc0-938c-000c16186418", - "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", - "created": "2020-10-25T16:22:00.000Z", - "modified": "2020-10-25T16:22:00.000Z", - "name": "cachedump", - "tool_types": [ - "credential-exploitation" - ], - "description": "This program extracts cached password hashes from a system’s registry.", - "kill_chain_phases": [ - { - "kill_chain_name": "mandiant-attack-lifecycle-model", - "phase_name": "escalate-privileges" - } - ] - }, - { - "type": "tool", - "spec_version": "2.1", - "id": "tool--e9778c42-bc2f-4eda-9fb4-6a931834f68c", - "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", - "created": "2020-10-25T16:22:00.000Z", - "modified": "2020-10-25T16:22:00.000Z", - "name": "fgdump", - "tool_types": [ - "credential-exploitation" - ], - "description": "Windows password hash dumper", - "kill_chain_phases": [ - { - "kill_chain_name": "mandiant-attack-lifecycle-model", - "phase_name": "escalate-privileges" - } - ], - "external_references": [ - { - "source_name": "fgdump", - "url": "http://www.foofus.net/fizzgig/fgdump/" - } - ] - } -] _USER_ACCOUNT_OBJECTS = [ { "type": "observed-data", @@ -1492,7 +1597,7 @@ def get_bundle_with_email_address_attributes(cls): def get_bundle_with_file_objects(cls): observables = deepcopy(_FILE_OBJECTS) with open(_TESTFILES_PATH / 'malware_sample.zip', 'rb') as f: - observables[-1]['payload_bin'] = b64encode(f.read()).decode() + observables[-3]['payload_bin'] = b64encode(f.read()).decode() return cls.__assemble_bundle(*observables) @classmethod diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index c82730a1..7c577f8b 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -265,22 +265,55 @@ def _check_email_address_attribute_with_display_name( self, observed_data, address, display_name, email_address): self._check_misp_object_fields( address, observed_data, - f'{email_address.id} - email-dst - {email_address.value}', True + f'{email_address.id} - email-dst - {email_address.value}', + multiple=True ) self.assertEqual(address.type, 'email-dst') self.assertEqual(address.value, email_address.value) self._check_misp_object_fields( display_name, observed_data, f'{email_address.id} - email-dst-display-name - {email_address.display_name}', - True + multiple=True ) self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) - def _check_file_object(self, misp_object, observed_data, observable_object): + def _check_file_and_pe_objects(self, observed_data, observable_object, + file_object, pe_object, *sections): + self.assertEqual(file_object.name, 'file') + self._check_misp_object_fields(file_object, observed_data, observable_object.id) + self._check_file_with_pe_fields( + file_object, observable_object, observable_object.id + ) + self.assertEqual(len(file_object.references), 1) + file_reference = file_object.references[0] + self.assertEqual(file_reference.referenced_uuid, pe_object.uuid) + self.assertEqual(file_reference.relationship_type, 'includes') + self.assertEqual(pe_object.name, 'pe') + object_id = f'{observable_object.id} - windows-pebinary-ext' + self._check_misp_object_fields( + pe_object, observed_data, object_id, multiple=True + ) + extension = observable_object.extensions['windows-pebinary-ext'] + self._check_pe_fields(pe_object, extension, object_id) + self.assertEqual(len(pe_object.references), len(sections)) + for reference, section in zip(pe_object.references, sections): + self.assertEqual(reference.referenced_uuid, section.uuid) + self.assertEqual(reference.relationship_type, 'includes') + for index, section in enumerate(sections): + self.assertEqual(section.name, 'pe-section') + section_id = f'{object_id} - section - {index}' + self._check_misp_object_fields( + section, observed_data, section_id, multiple=True + ) + self._check_pe_section_fields( + section, extension.sections[index], section_id + ) + + def _check_file_object(self, misp_object, observed_data, observable_object, *object_ids): self.assertEqual(misp_object.name, 'file') self._check_misp_object_fields( - misp_object, observed_data, observable_object.id + misp_object, observed_data, observable_object.id, *object_ids ) self._check_file_fields( misp_object, observable_object, observable_object.id @@ -293,13 +326,17 @@ def _check_generic_attribute( self.assertEqual(attribute.type, attribute_type) self.assertEqual(attribute.value, getattr(observable_object, feature)) - def _check_misp_object_fields( - self, misp_object, observed_data, object_id, multiple=False): + def _check_misp_object_fields(self, misp_object, observed_data, object_id, + *additional_ids, multiple=False): if multiple: self.assertEqual(misp_object.uuid, uuid5(self._UUIDv4, object_id)) else: self.assertEqual(misp_object.uuid, object_id.split('--')[1]) - self.assertEqual(misp_object.comment, f'Observed Data ID: {observed_data.id}') + pattern = 'Observed Data ID: ' + self.assertEqual( + misp_object.comment, + f"{pattern}{f' - {pattern}'.join((observed_data.id, *additional_ids))}" + ) if not (observed_data.modified == observed_data.first_observed == observed_data.last_observed): self.assertEqual(misp_object.first_seen, observed_data.first_observed) self.assertEqual(misp_object.last_seen, observed_data.last_observed) @@ -317,7 +354,7 @@ def _check_registry_key_object( object_id = f'{registry_key.id} - values - {index}' self.assertEqual(value_object.name, 'registry-key-value') self._check_misp_object_fields( - value_object, observed_data, object_id, True + value_object, observed_data, object_id, multiple=True ) self._check_registry_key_value_fields( value_object, registry_key['values'][index], object_id @@ -406,7 +443,7 @@ def test_stix21_bundle_with_domain_attributes(self): self._check_generic_attribute(od1, domain_2, m_domain2, 'domain') self._check_generic_attribute(od2, domain_3, s_domain, 'domain') - def test_stix21_bundle_with_email_address_objects(self): + def test_stix21_bundle_with_email_address_attributes(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_email_address_attributes() self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle() @@ -429,14 +466,13 @@ def test_stix21_bundle_with_file_objects(self): self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle() event = self.parser.misp_event - _, grouping, od1, file1, directory, artifact = bundle.objects + _, grouping, od1, od2, od3, file1, directory, artifact, file2, file3 = bundle.objects misp_objects = self._check_misp_event_features_from_grouping(event, grouping) - self.assertEqual(len(misp_objects), 3) - file_object, directory_object, artifact_object = misp_objects - self._check_file_object(file_object, od1, file1) + self.assertEqual(len(misp_objects), 10) + file_object1, directory_object, artifact_object, archive, file_object2, pe_object, *sections = misp_objects + self._check_file_object(file_object1, od1, file1, od2.id) self.assertEqual(directory_object.name, 'directory') - self._check_misp_object_fields(directory_object, od1, directory.id) - self.assertEqual(len(directory_object.attributes), 1) + self._check_misp_object_fields(directory_object, od1, directory.id, od2.id) path_attribute = directory_object.attributes[0] self.assertEqual(path_attribute.type, 'text') self.assertEqual(path_attribute.object_relation, 'path') @@ -448,14 +484,24 @@ def test_stix21_bundle_with_file_objects(self): ) ) self._check_content_ref_object(artifact_object, od1, artifact) - self.assertEqual(len(file_object.references), 1) - file_reference = file_object.references[0] + self.assertEqual(len(file_object1.references), 1) + file_reference = file_object1.references[0] self.assertEqual(file_reference.referenced_uuid, directory_object.uuid) self.assertEqual(file_reference.relationship_type, 'contained-in') self.assertEqual(len(artifact_object.references), 1) artifact_reference = artifact_object.references[0] - self.assertEqual(artifact_reference.referenced_uuid, file_object.uuid) + self.assertEqual(artifact_reference.referenced_uuid, file_object1.uuid) self.assertEqual(artifact_reference.relationship_type, 'content-of') + self._check_archive_file_object(archive, od2, file2) + self.assertEqual(len(archive.references), 2) + directory_reference, file_reference = archive.references + self.assertEqual( + directory_reference.referenced_uuid, file_object1.uuid + ) + self.assertEqual(directory_reference.relationship_type, 'contains') + self.assertEqual(file_reference.referenced_uuid, directory_object.uuid) + self.assertEqual(file_reference.relationship_type, 'contains') + self._check_file_and_pe_objects(od3, file3, file_object2, pe_object, *sections) def test_stix21_bundle_with_ip_address_attributes(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_ip_address_attributes() From 6e389e3fb99a7af893bc18e68c26e73318d82c1a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 13 Mar 2024 23:56:52 +0100 Subject: [PATCH 099/137] fix: [stix2 import] Making the multiple observables fetching method available to both internal and external STIX 2 Observed Data object converters --- .../stix2misp/converters/stix2_observed_data_converter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index bd978dbc..3938aa32 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -58,6 +58,10 @@ class STIX2ObservedDataConverter(STIX2ObservableConverter, metaclass=ABCMeta): def __init__(self, main: _MAIN_PARSER_TYPING): self._set_main_parser(main) + def _fetch_observables(self, object_refs: Union[tuple, str]) -> Generator: + for object_ref in object_refs: + yield self.main_parser._observable[object_ref] + class ExternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, ExternalSTIX2ObservableConverter): @@ -1736,9 +1740,6 @@ def _create_misp_object_from_observable_object_ref( ) return misp_object - def _fetch_observables(self, object_refs: Union[tuple, str]) -> Generator: - for object_ref in object_refs: - yield self.main_parser._observable[object_ref] def _handle_misp_object_fields( self, misp_object: MISPAttribute | MISPObject, From 181545fe6bcef7c858e6a6271b5d3cf9c38b0d4a Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 13 Mar 2024 23:58:00 +0100 Subject: [PATCH 100/137] fix: [stix2 import] Avoiding issues with the internal STIX 2.1 Autonomous System observable objects fetching method --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 3938aa32..2c44d8f3 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1831,7 +1831,7 @@ def _attribute_from_AS_observable_v20( def _attribute_from_AS_observable_v21( self, observed_data: ObservedData_v21): attribute = self._create_attribute_dict(observed_data) - observable = self._fetch_observable(observed_data.object_refs) + observable = self._fetch_observable(observed_data.object_refs[0]) attribute['value'] = self._parse_AS_value(observable.number) self.main_parser._add_misp_attribute(attribute, observed_data) From 8207d3d68e0215f73ed85f376958ff55de609a25 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 14 Mar 2024 14:41:23 +0100 Subject: [PATCH 101/137] fix: [stix2 import] Avoiding issues introduced since we updated the observables fetching method - As observables are fetched in a generator, we have to handle it before returning a single or multiple observable(s) to avoid breaking the automation on conversion of the internal STIX 2.x content without modifying the different methods --- .../stix2misp/converters/stix2_observed_data_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 2c44d8f3..fd19213f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -3055,7 +3055,8 @@ def _fetch_observables_v20(observed_data: ObservedData_v20): return observables[0] if len(observables) == 1 else observables def _fetch_observables_v21(self, observed_data: ObservedData_v21): - return self._fetch_observables(observed_data.object_refs) + observables = tuple(self._fetch_observables(observed_data.object_refs)) + return observables[0] if len(observables) == 1 else observables @staticmethod def _fetch_observables_with_id_v20(observed_data: ObservedData_v20) -> dict: From 88489c1a35c9de88ea291695ebb8eb71c347a444 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 25 Mar 2024 11:36:36 +0100 Subject: [PATCH 102/137] wip: [stix2 import] Parsing `archive-ext` from standalone file observable objects --- .../stix2_observable_objects_converter.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index c7a6e10f..16631a4d 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -328,9 +328,29 @@ def _parse_file_observable_object(self, file_ref: str) -> MISPObject: self._parse_directory_observable_object( _file.parent_directory_ref, child=_file.id ) - if getattr(_file, 'extensions', {}).get('windows-pebinary-ext'): - pe_object = self._parse_file_pe_extension_observable_object(_file) - misp_object.add_reference(pe_object.uuid, 'includes') + if hasattr(_file, 'extensions'): + extensions = _file.extensions + if extensions.get('archive-ext'): + archive_ext = extensions['archive-ext'] + if hasattr(archive_ext, 'comment'): + comment = archive_ext.comment + if hasattr(misp_object, 'comment'): + comment = f'{comment} - {misp_object.comment}' + misp_object.comment = comment + for contains_ref in archive_ext.contains_refs: + object_type = contains_ref.split('--')[0] + contained_object = getattr( + self, + f'_parse_{object_type}_observable_object' + )( + contains_ref + ) + misp_object.add_reference(contained_object.uuid, 'contains') + if extensions.get('windows-pebinary-ext'): + pe_object = self._parse_file_pe_extension_observable_object( + _file + ) + misp_object.add_reference(pe_object.uuid, 'includes') return misp_object def _parse_file_pe_extension_observable_object( From ca6b9b071a17e708cbb7760efc55cecdd221e7e5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 2 Apr 2024 21:26:31 +0200 Subject: [PATCH 103/137] wip: [stix2 import] Parsing `DomainName` and IP observable objects resolving each others --- .../stix2_observed_data_converter.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index fd19213f..f82792a6 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -76,6 +76,14 @@ def observable_relationships(self): self._set_observable_relationships() return self._observable_relationships + @property + def referenced_ids(self): + try: + return self.__referenced_ids + except AttributeError: + self._extract_referenced_ids_from_observable_object_refs() + return self.__referenced_ids + def parse(self, observed_data_ref: str): observed_data = self.main_parser._get_stix_object(observed_data_ref) try: @@ -146,6 +154,16 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): # MULTIPLE OBSERVABLE OBJECTS PARSING METHODS. # ############################################################################ + def _extract_referenced_ids_from_observable_object_refs(self): + self.__referenced_ids = defaultdict(set) + for object_id, observable_object in self._observable.items(): + for key, value in observable_object['observable'].items(): + if key.endswith('_ref'): + self.referenced_ids[value].add(object_id) + if key.endswith('_refs'): + for reference in value: + self.referenced_ids[reference].add(object_id) + @staticmethod def _extract_referenced_ids_from_observable_objects( **observable_objects: dict) -> dict: @@ -566,6 +584,104 @@ def _parse_directory_observable_objects( ) ) + def _parse_domain_ip_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: + if object_ref in self.referenced_ids: + continue + observable = self._fetch_observable(object_ref) + domain = observable['observable'] + if hasattr(domain, 'resolves_to_refs'): + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_object'], observed_data + ) + continue + domain_object = self._create_misp_object('domain-ip') + domain_object.from_dict( + comment=f'Observed Data ID: {observed_data.id}', + **self._parse_timeline(observed_data) + ) + self.main_parser._check_sighting_replacements( + self.main_parser._sanitise_uuid(observed_data.id), + domain_object.uuid + ) + domain_object.uuid = self.main_parser._create_v5_uuid( + ' - '.join((domain.id, *domain.resolves_to_refs)) + ) + domain_object.add_attribute( + 'domain', domain.value, + uuid=self.main_parser._sanitise_attribute_uuid(domain.id) + ) + misp_object = self.main_parser._add_misp_object( + domain_object, observed_data + ) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + for resolved_ref in domain.resolves_to_refs: + resolved_observable = self._fetch_observable(resolved_ref) + resolved_object= resolved_observable['observable'] + misp_object.add_attribute( + ( + 'domain' if resolved_object.type == 'domain-name' + else 'ip' + ), + resolved_object.value, + uuid=self.main_parser._sanitise_attribute_uuid( + resolved_ref + ) + ) + resolved_observable['used'][self.event_uuid] = True + resolved_observable['misp_object'] = misp_object + continue + if observable['used'].get(self.event_uuid, False): + self._handle_misp_object_fields( + observable['misp_attribute'], observed_data + ) + continue + attribute = self._parse_generic_observable_object_ref_as_attribute( + domain, observed_data, 'domain' + ) + observable['misp_attribute'] = attribute + observable['used'][self.event_uuid] = True + + def _parse_domain_ip_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + referenced_ids = self._extract_referenced_ids_from_observable_objects( + **observed_data.objects + ) + for identifier, observable_object in observed_data.objects: + if identifier in referenced_ids: + continue + if hasattr(observable_object, 'resolves_to_refs'): + object_id = f'{observed_data.id} - {identifier}' + misp_object = self._create_misp_object_from_observable_object( + 'domain-ip', observed_data, ' - '.join( + (object_id, *observable_object.resolves_to_refs) + ) + ) + misp_object.add_attribute( + 'domain', observable_object.value, + uuid=self.main_parser._create_v5_uuid(object_id) + ) + for resolved_ref in observable_object.resolves_to_refs: + resolved_object = observed_data.objects[resolved_ref] + misp_object.add_attribute( + ( + 'domain' if resolved_object.type == 'domain-name' + else 'ip' + ), + resolved_object.value, + uuid=self.main_parser._create_v5_uuid( + f'{observed_data.id} - {resolved_ref}' + ) + ) + self.main_parser._add_misp_object(misp_object, observed_data) + continue + self._parse_generic_observable_object_as_attribute( + observed_data, identifier, 'domain' + ) + def _parse_domain_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: From 252d272e8e3e0f838ac7192e278ff27c142914a1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 2 Apr 2024 21:31:40 +0200 Subject: [PATCH 104/137] wip: [stix2 import] Reusing `EmailMessage` observable parsing method --- .../converters/stix2_observable_converter.py | 18 ++++++++++++------ .../stix2_observable_objects_converter.py | 8 +------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 2447be2c..c5a90b60 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -177,18 +177,24 @@ class STIX2ObservableConverter(STIX2Converter): def _fetch_observable(self, object_ref: str) -> dict: return self.main_parser._observable[object_ref] - def _parse_email_additional_header( + def _parse_email_observable( self, observable: _EMAIL_MESSAGE_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: if object_id is None: object_id = observable.id - mapping = self._mapping.email_additional_header_fields_mapping - email_header = observable.additional_header_fields - for field, attribute in mapping().items(): - if email_header.get(field): + for field, attribute in self._mapping.email_object_mapping.items(): + if hasattr(observable, field): yield from self._populate_object_attributes( - attribute, email_header[field], object_id + attribute, getattr(observable, field), object_id ) + if hasattr(observable, 'additional_header_fields'): + mapping = self._mapping.email_additional_header_fields_mapping + email_header = observable.additional_header_fields + for field, attribute in mapping().items(): + if email_header.get(field): + yield from self._populate_object_attributes( + attribute, email_header[field], object_id + ) def _parse_email_reference_observable( self, observable: _EMAIL_ADDRESS_TYPING, feature: str, diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 16631a4d..59a7dcc6 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -270,7 +270,7 @@ def _parse_email_message_observable_object( email_object = self._create_misp_object_from_observable_object( 'email', email_message ) - for attribute in self._parse_generic_observable(email_message, 'email'): + for attribute in self._parse_email_observable(email_message): email_object.add_attribute(**attribute) observable['used'][self.event_uuid] = True misp_object = self.main_parser._add_misp_object( @@ -298,12 +298,6 @@ def _parse_email_message_observable_object( misp_object.add_attribute(**attribute) observable['used'][self.event_uuid] = True observable['misp_object'] = misp_object - if hasattr(email_message, 'additional_header_fields'): - attributes = self._parse_email_additional_header( - email_message.additional_header_fields - ) - for attribute in attributes: - misp_object.add_attribute(**attribute) return misp_object def _parse_file_observable_object(self, file_ref: str) -> MISPObject: From 1a4259a98af5b6792107b49879020cc61123b0be Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 4 Apr 2024 17:19:11 +0200 Subject: [PATCH 105/137] wip: [stix2 import] Parsing EmailMessage observable objects from Observed Data converter --- .../converters/stix2_observable_converter.py | 2 +- .../stix2_observed_data_converter.py | 338 ++++++++++++++---- 2 files changed, 276 insertions(+), 64 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index c5a90b60..9b3c7587 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -466,7 +466,7 @@ class ExternalSTIX2ObservableMapping( 'email-message', 'email-message_file' ), - 'email' + 'email_message' ), **dict.fromkeys( ( diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index f82792a6..b35e71d4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -16,13 +16,13 @@ from datetime import datetime from pymisp import MISPAttribute, MISPObject from stix2.v20.observables import ( - WindowsPEBinaryExt as WindowsPEBinaryExt_v20, + File as File_v20, WindowsPEBinaryExt as WindowsPEBinaryExt_v20, WindowsRegistryValueType as WindowsRegistryValueType_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - Artifact, AutonomousSystem, Directory, DomainName, File, IPv4Address, + Artifact, AutonomousSystem, Directory, DomainName, IPv4Address, IPv6Address, MACAddress, Mutex, Process, Software, URL, UserAccount, - WindowsRegistryKey, X509Certificate, + File as File_v21, WindowsRegistryKey, X509Certificate, WindowsPEBinaryExt as WindowsPEBinaryExt_v21, WindowsRegistryValueType as WindowsRegistryValueType_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 @@ -32,15 +32,18 @@ from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser +_FILE_TYPING = Union[ + File_v20, File_v21 +] _GENERIC_OBSERVABLE_OBJECT_TYPING = Union[ - Artifact, Directory, File, Process, Software, UserAccount, + Artifact, Directory, File_v21, Process, Software, UserAccount, WindowsRegistryKey, X509Certificate ] _GENERIC_OBSERVABLE_TYPING = Union[ DomainName, IPv4Address, IPv6Address, MACAddress, Mutex, URL ] _OBSERVABLE_OBJECTS_TYPING = Union[ - Artifact, AutonomousSystem, Directory, File, Process, Software, + Artifact, AutonomousSystem, Directory, File_v21, Process, Software, UserAccount, WindowsRegistryKey, X509Certificate ] _OBSERVED_DATA_TYPING = Union[ @@ -904,6 +907,270 @@ def _parse_email_address_observable_objects( maxlen=0 ) + def _parse_email_message_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: + if object_ref.split('--')[0] != 'email-message': + continue + observable = self._fetch_observable(object_ref) + email_message = observable['observable'] + misp_object = self._parse_generic_observable_object_ref( + email_message, observed_data, 'email', False + ) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + if hasattr(email_message, 'from_ref'): + observable = self._fetch_observable(email_message.from_ref) + attributes = self._parse_email_reference_observable( + observable['observable'], 'from' + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + for feature in ('to', 'cc', 'bcc'): + field = f'{feature}_refs' + if hasattr(email_message, field): + for reference in getattr(email_message, field): + observable = self._fetch_observable(reference) + attributes = self._parse_email_reference_observable( + observable['observable'], feature + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + if hasattr(email_message, 'body_multipart'): + for index, multipart in enumerate(email_message.body_multipart): + if hasattr(multipart, 'body'): + misp_object.add_attribute( + 'email-body', multipart.body, + uuid=self.main_parser._create_v5_uuid( + f'{email_message.id} - body_multipart - {index}' + f' - email-body - {multipart.body}' + ) + ) + continue + observable = self._fetch_observable( + multipart.body_raw_ref + ) + if observable['used'].get(self.event_uuid, False): + referenced_object = observable['misp_object'] + self._handle_misp_object_fields( + referenced_object, observed_data + ) + misp_object.add_reference( + referenced_object.uuid, 'contains' + ) + continue + observable_object = observable['observable'] + if observable_object.type == 'artifact': + artifact = self._parse_generic_observable_object_ref( + observable_object, observed_data, 'artifact', False + ) + misp_object.add_reference(artifact.uuid, 'contains') + observable['misp_object'] = artifact + observable['used'][self.event_uuid] = True + continue + file_object = self._parse_generic_observable_object_ref( + observable_object, observed_data, 'file', False + ) + misp_object.add_reference(file_object.uuid, 'contains') + observable['misp_object'] = file_object + observable['used'][self.event_uuid] = True + self._parse_file_observable_object_ref_references( + file_object, observable_object, observed_data + ) + + def _parse_email_message_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + if len(observed_data.objects) == 1: + misp_object = self._parse_generic_single_observable_object( + observed_data, 'email', False + ) + email_message = observed_data.objects['0'] + if hasattr(email_message, 'body_multipart'): + for index, multipart in enumerate(email_message.body_multipart): + if hasattr(multipart, 'body'): + misp_object.add_attribute( + 'email-body', multipart.body, + uuid=self.main_parser._create_v5_uuid( + f'{observed_data.id} - body_multipart - {index}' + f' - email-body - {multipart.body}' + ) + ) + continue + observable_objects = { + object_id: {'used': False} + for object_id, observable in observed_data.objects.items() + if observable.type in ('file', 'artifact') + } + for identifier, observable in observed_data.objects.items(): + if observable.type != 'email-message': + continue + misp_object = self._parse_generic_observable_object( + observed_data, identifier, 'email', False + ) + object_id = f'{observed_data.id} - {identifier}' + if hasattr(observable, 'from_ref'): + attributes = self._parse_email_reference_observable( + observed_data.objects[observable.from_ref], + 'from', object_id + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + for feature in ('to', 'cc', 'bcc'): + field = f'{feature}_refs' + if hasattr(observable, field): + for reference in getattr(observable, field): + attributes = self._parse_email_reference_observable( + observed_data.objects[reference], + feature, object_id + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + if hasattr(observable, 'body_multipart'): + for index, multipart in enumerate(observable.body_multipart): + if hasattr(multipart, 'body'): + misp_object.add_attribute( + 'email-body', multipart.body, + uuid=self.main_parser._create_v5_uuid( + f'{object_id} - body_multipart - {index}' + f' - email-body - {multipart.body}' + ) + ) + continue + body_ref = multipart.body_raw_ref + if observable_objects[body_ref]['used']: + misp_object.add_reference( + observable_objects[body_ref]['misp_object'].uuid, + 'contains' + ) + continue + if observed_data.objects[body_ref].type == 'artifact': + artifact = self._parse_generic_observable_object( + observed_data, body_ref, 'artifact', False + ) + misp_object.add_reference(artifact.uuid, 'contains') + continue + file_object = self._parse_generic_observable_object( + observed_data, body_ref, 'file', False + ) + misp_object.add_reference(file_object.uuid, 'contains') + self._parse_file_observable_object_references( + file_object, observable, observed_data, + observable_objects, body_ref + ) + + def _parse_file_observable_object_ref_references( + self, misp_object: MISPObject, observable_object: File_v21, + observed_data: ObservedData_v21): + if hasattr(observable_object, 'extensions'): + extensions = observable_object.extensions + if extensions.get('archive-ext'): + archive_ext = extensions['archive-ext'] + if hasattr(archive_ext, 'comment'): + misp_object.from_dict( + comment=' - '.join( + (archive_ext.comment, misp_object.comment) + ) + ) + self._handle_misp_object_references( + misp_object, + *self._parse_contained_object_refs( + observed_data, misp_object.uuid, + *archive_ext.contains_refs + ) + ) + if extensions.get('windows-pebinary-ext'): + windows_pe_ext = extensions['windows-pebinary-ext'] + pe_object_uuid = self._parse_file_pe_extension_observable( + windows_pe_ext, observed_data, + f'{observable_object.id} - windows-pebinary-ext' + ) + misp_object.add_reference(pe_object_uuid, 'includes') + if hasattr(observable_object, 'parent_directory_ref'): + parent_ref = observable_object.parent_directory_ref + if parent_ref not in observed_data.object_refs: + self.observable_relationships[misp_object.uuid].add( + ( + self.main_parser._sanitise_uuid(parent_ref), + 'contained-in' + ) + ) + else: + parent = self._fetch_observable(parent_ref) + parent_object = self._handle_observable_object_refs_parsing( + parent, observed_data, 'directory' + ) + self._handle_misp_object_references( + misp_object, parent_object.uuid, + relationship_type='contained-in' + ) + if hasattr(observable_object, 'content_ref'): + content_ref = observable_object.content_ref + if content_ref not in observed_data.object_refs: + content_uuid = self.main_parser._sanitise_uuid(content_ref) + self.observable_relationships[content_uuid].add( + (misp_object.uuid, 'content-of') + ) + else: + content = self._fetch_observable(content_ref) + artifact = self._handle_observable_object_refs_parsing( + content, observed_data, 'artifact', False + ) + self._handle_misp_object_references( + artifact, misp_object.uuid, + relationship_type='content-of' + ) + + def _parse_file_observable_object_references( + self, misp_object: MISPObject, file_object: _FILE_TYPING, + observed_data: _OBSERVED_DATA_TYPING, + observable_objects: dict, object_id: str): + if hasattr(file_object, 'extensions'): + extensions = file_object.extensions + if extensions.get('archive-ext'): + archive_ext = extensions['archive-ext'] + if hasattr(archive_ext, 'comment'): + misp_object.from_dict( + comment=' - '.join( + (archive_ext.comment, misp_object.comment) + ) + ) + self._handle_misp_object_references( + misp_object, + *self._parse_contained_objects( + observed_data, observable_objects, + *archive_ext.contains_refs + ) + ) + if extensions.get('windows-pebinary-ext'): + pe_object_uuid = self._parse_file_pe_extension_observable( + extensions['windows-pebinary-ext'], observed_data, + f'{observed_data.id} - ' + f'{object_id} - windows-pebinary-ext' + ) + misp_object.add_reference(pe_object_uuid, 'includes') + if hasattr(file_object, 'parent_directory_ref'): + parent_ref = file_object.parent_directory_ref + parent_object = self._handle_observable_objects_parsing( + observable_objects, parent_ref, observed_data, 'directory' + ) + self._handle_misp_object_references( + misp_object, parent_object.uuid, + relationship_type='contained-in' + ) + if hasattr(file_object, 'content_ref'): + content_ref = file_object.content_ref + artifact = self._handle_observable_objects_parsing( + observable_objects, content_ref, observed_data, + 'artifact', False + ) + self._handle_misp_object_references( + artifact, misp_object.uuid, relationship_type='content-of' + ) + def _parse_file_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: @@ -926,64 +1193,9 @@ def _parse_file_observable_object_refs( ) if object_type == 'directory': continue - if hasattr(observable_object, 'extensions'): - extensions = observable_object.extensions - if extensions.get('archive-ext'): - archive_ext = extensions['archive-ext'] - if hasattr(archive_ext, 'comment'): - misp_object.from_dict( - comment=' - '.join( - (archive_ext.comment, misp_object.comment) - ) - ) - self._handle_misp_object_references( - misp_object, - *self._parse_contained_object_refs( - observed_data, misp_object.uuid, - *archive_ext.contains_refs - ) - ) - if extensions.get('windows-pebinary-ext'): - windows_pe_ext = extensions['windows-pebinary-ext'] - pe_object_uuid = self._parse_file_pe_extension_observable( - windows_pe_ext, observed_data, - f'{observable_object.id} - windows-pebinary-ext' - ) - misp_object.add_reference(pe_object_uuid, 'includes') - if hasattr(observable_object, 'parent_directory_ref'): - parent_ref = observable_object.parent_directory_ref - if parent_ref not in observed_data.object_refs: - self.observable_relationships[misp_object.uuid].add( - ( - self.main_parser._sanitise_uuid(parent_ref), - 'contained-in' - ) - ) - else: - parent = self._fetch_observable(parent_ref) - parent_object = self._handle_observable_object_refs_parsing( - parent, observed_data, 'directory' - ) - self._handle_misp_object_references( - misp_object, parent_object.uuid, - relationship_type='contained-in' - ) - if hasattr(observable_object, 'content_ref'): - content_ref = observable_object.content_ref - if content_ref not in observed_data.object_refs: - content_uuid = self.main_parser._sanitise_uuid(content_ref) - self.observable_relationships[content_uuid].add( - (misp_object.uuid, 'content-of') - ) - else: - content = self._fetch_observable(content_ref) - artifact = self._handle_observable_object_refs_parsing( - content, observed_data, 'artifact', False - ) - self._handle_misp_object_references( - artifact, misp_object.uuid, - relationship_type='content-of' - ) + self._parse_file_observable_object_ref_references( + misp_object, observable_object, observed_data + ) def _parse_file_observable_objects( self, observed_data: _OBSERVED_DATA_TYPING, From ee1e09b5726d11e34808cb445be9072b8f4ed332 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 9 Apr 2024 09:20:38 +0200 Subject: [PATCH 106/137] fix: [stix2 import] Fixed email message objects parsing --- .../stix2misp/converters/stix2_observable_converter.py | 2 +- .../converters/stix2_observed_data_converter.py | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 9b3c7587..72a6268f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -182,7 +182,7 @@ def _parse_email_observable( object_id: Optional[str] = None) -> Iterator[dict]: if object_id is None: object_id = observable.id - for field, attribute in self._mapping.email_object_mapping.items(): + for field, attribute in self._mapping.email_object_mapping().items(): if hasattr(observable, field): yield from self._populate_object_attributes( attribute, getattr(observable, field), object_id diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index b35e71d4..d66af902 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -2619,17 +2619,9 @@ def _object_from_email_observable( for attribute in attributes: misp_object.add_attribute(**attribute) object_id = getattr(observable, 'id', observed_data.id) - attributes = self._parse_generic_observable( - observable, 'email', object_id - ) + attributes = self._parse_email_observable(observable, object_id) for attribute in attributes: misp_object.add_attribute(**attribute) - if hasattr(observable, 'additional_header_fields'): - attributes = self._parse_email_additional_header( - observable, object_id - ) - for attribute in attributes: - misp_object.add_attribute(**attribute) if hasattr(observable, 'body_multipart'): for body_part in observable.body_multipart: relation, value = body_part.content_disposition.split(';') From 5a46d9728130e93d4ff4a4d0cbf2d7e2de49d4c5 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 9 Apr 2024 15:06:53 +0200 Subject: [PATCH 107/137] chg: [stix2 import] Making the network-traffic objects parsing more generic - Some parts will be more easily reused for network traffic objects associated to some observed data objects --- .../converters/stix2_observable_converter.py | 7 +++ .../stix2_observable_objects_converter.py | 49 +++++-------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 72a6268f..7c754a9a 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -618,6 +618,13 @@ def _parse_ip_observable(self, observable: _IP_OBSERVABLE_TYPING, ) return attribute + @staticmethod + def _parse_network_traffic_observable_fields( + observable: NetworkTraffic_v21) -> str: + if getattr(observable, 'extensions', {}).get('socket-ext'): + return 'network-socket' + return 'network-connection' + def _parse_url_observable(self, observable: _URL_TYPING, observed_data_id: str) -> Iterator[dict]: object_id = getattr(observable, 'id', observed_data_id) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 59a7dcc6..5c57c705 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -428,42 +428,24 @@ def _parse_mutex_observable_object(self, mutex_ref: str) -> MISPAttribute: observable['misp_attribute'] = misp_attribute return misp_attribute - def _parse_network_connection_observable_object( - self, observable: NetworkTraffic) -> MISPObject: - connection_object = self._create_misp_object_from_observable_object( - 'network-connection', observable - ) - attributes = self._parse_generic_observable( - observable, 'network_traffic' - ) - for attribute in attributes: - connection_object.add_attribute(**attribute) - for attribute in self._parse_network_connection_observable(observable): - connection_object.add_attribute(**attribute) - return connection_object - - def _parse_network_socket_observable_object( - self, observable: NetworkTraffic) -> MISPObject: - socket_object = self._create_misp_object_from_observable_object( - 'network-socket', observable - ) - attributes = self._parse_generic_observable( - observable, 'network_traffic' - ) - for attribute in attributes: - socket_object.add_attribute(**attribute) - for attribute in self._parse_network_socket_observable(observable): - socket_object.add_attribute(**attribute) - return socket_object - def _parse_network_traffic_observable_object( self, network_traffic_ref: str) -> MISPObject: observable = self._fetch_observable(network_traffic_ref) if observable['used'].get(self.event_uuid, False): return observable['misp_object'] network_traffic = observable['observable'] - feature = self._parse_network_traffic_observable_fields(network_traffic) - network_object = getattr(self, feature)(network_traffic) + name = self._parse_network_traffic_observable_fields(network_traffic) + network_object = self._create_misp_object_from_observable_object( + name, network_traffic + ) + attributes = self._parse_generic_observable( + network_traffic, 'network_traffic' + ) + for attribute in attributes: + network_object.add_attribute(**attribute) + feature = f"_parse_{name.replace('-', '_')}_observable" + for attribute in getattr(self, feature)(network_traffic): + network_object.add_attribute(**attribute) observable['used'][self.event_uuid] = True misp_object = self.main_parser._add_misp_object( network_object, network_traffic @@ -494,13 +476,6 @@ def _parse_network_traffic_observable_object( misp_object.add_reference(referenced.uuid, 'encapsulated-by') return misp_object - @staticmethod - def _parse_network_traffic_observable_fields( - observable: NetworkTraffic) -> str: - if getattr(observable, 'extensions', {}).get('socket-ext'): - return '_parse_network_socket_observable_object' - return '_parse_network_connection_observable_object' - def _parse_process_observable_object(self, process_ref: str) -> MISPObject: observable = self._fetch_observable(process_ref) if observable['used'].get(self.event_uuid, False): From c9e5d4f9dfa2e4200d1e83e73c322f23774a1b03 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 9 Apr 2024 18:09:26 +0200 Subject: [PATCH 108/137] wip: [stix2 import] Parsing Network Traffic Observable objects referenced in Observed Data from the Observed Data Converter --- .../stix2_observed_data_converter.py | 163 ++++++++++++++++-- 1 file changed, 152 insertions(+), 11 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index d66af902..7daf7b55 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -20,9 +20,9 @@ WindowsRegistryValueType as WindowsRegistryValueType_v20) from stix2.v20.sdo import ObservedData as ObservedData_v20 from stix2.v21.observables import ( - Artifact, AutonomousSystem, Directory, DomainName, IPv4Address, - IPv6Address, MACAddress, Mutex, Process, Software, URL, UserAccount, - File as File_v21, WindowsRegistryKey, X509Certificate, + Artifact, AutonomousSystem, Directory, DomainName, File as File_v21, + IPv4Address, IPv6Address, MACAddress, Mutex, Process, Software, URL, + UserAccount, WindowsRegistryKey, X509Certificate, WindowsPEBinaryExt as WindowsPEBinaryExt_v21, WindowsRegistryValueType as WindowsRegistryValueType_v21) from stix2.v21.sdo import ObservedData as ObservedData_v21 @@ -282,6 +282,16 @@ def _parse_multiple_observable_objects( # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ + def _handle_misp_object_storage( + self, observable: dict, misp_object: MISPObject): + observable['used'][self.event_uuid] = True + if observable.get('misp_object') is None: + observable['misp_object'] = misp_object + elif isinstance(observable['misp_object'], list): + observable['misp_object'].append(misp_object) + else: + observable['misp_object'] = [observable['misp_object'], misp_object] + def _handle_observable_object_refs_parsing( self, observable: dict, observed_data: ObservedData_v21, *args: tuple) -> MISPObject: @@ -926,8 +936,7 @@ def _parse_email_message_observable_object_refs( ) for attribute in attributes: misp_object.add_attribute(**attribute) - observable['used'][self.event_uuid] = True - observable['misp_object'] = misp_object + self._handle_misp_object_storage(observable, misp_object) for feature in ('to', 'cc', 'bcc'): field = f'{feature}_refs' if hasattr(email_message, field): @@ -938,8 +947,9 @@ def _parse_email_message_observable_object_refs( ) for attribute in attributes: misp_object.add_attribute(**attribute) - observable['used'][self.event_uuid] = True - observable['misp_object'] = misp_object + self._handle_misp_object_storage( + observable, misp_object + ) if hasattr(email_message, 'body_multipart'): for index, multipart in enumerate(email_message.body_multipart): if hasattr(multipart, 'body'): @@ -1365,14 +1375,16 @@ def _parse_generic_observable_object_as_attribute( def _parse_generic_observable_object_ref( self, observable_object: _GENERIC_OBSERVABLE_OBJECT_TYPING, observed_data: ObservedData_v21, name: str, - generic: Optional[bool] = True) -> MISPObject: + generic: Optional[bool] = True, + mapping_name: Optional[str] = None) -> MISPObject: misp_object = self._create_misp_object_from_observable_object_ref( name, observable_object, observed_data ) - _name = name.replace('-', '_') + if mapping_name is None: + mapping_name = name.replace('-', '_') attributes = ( - self._parse_generic_observable(observable_object, _name) - if generic else getattr(self, f'_parse_{_name}_observable')( + self._parse_generic_observable(observable_object, mapping_name) + if generic else getattr(self, f'_parse_{mapping_name}_observable')( observable_object ) ) @@ -1564,6 +1576,135 @@ def _parse_mutex_observable_objects( observed_data, identifier, 'mutex', feature='name' ) + def _parse_network_traffic_observable_object( + self, observable_objects: dict, object_id: str, + observed_data: _OBSERVED_DATA_TYPING, name: str) -> MISPObject: + observable = observable_objects[object_id] + if observable['used']: + return observable['misp_object'] + misp_object = self._parse_generic_observable_object( + observed_data, object_id, name, True, 'network_traffic' + ) + feature = f"_parse_{name.replace('-', '_')}_observable" + attributes = getattr(self, feature)(observed_data.objects[object_id]) + for attribute in attributes: + misp_object.add_attribute(**attribute) + observable.update({'misp_object': misp_object, 'used': True}) + return misp_object + + def _parse_network_traffic_observable_object_ref( + self, observable: dict, observed_data: ObservedData_v21, + name: str) -> MISPObject: + if observable['used'].get(self.event_uuid, False): + misp_object = observable['misp_object'] + self._handle_misp_object_fields(misp_object, observed_data) + return misp_object + misp_object = self._parse_generic_observable_object_ref( + observable['observable'], observed_data, name, + True, 'network_traffic' + ) + feature = f"_parse_{name.replace('-', '_')}_observable" + attributes = getattr(self, feature)(observable['observable']) + for attribute in attributes: + misp_object.add_attribute(**attribute) + observable['used'][self.event_uuid] = True + observable['misp_object'] = misp_object + return misp_object + + def _parse_network_traffic_observable_object_refs( + self, observed_data: ObservedData_v21, *object_refs: tuple): + for object_ref in object_refs or observed_data.object_refs: + if object_ref.split('--')[0] != 'network-traffic': + continue + observable = self._fetch_observable(object_ref) + network_traffic = observable['observable'] + name = self._parse_network_traffic_observable_fields( + network_traffic + ) + misp_object = self._parse_network_traffic_observable_object_ref( + observable, observed_data, name + ) + for asset in ('src', 'dst'): + if hasattr(network_traffic, f'{asset}_ref'): + referenced = self._fetch_observable( + getattr(network_traffic, f'{asset}_ref') + ) + attributes = self._parse_network_traffic_reference_observable( + asset, referenced['observable'] + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + self._handle_misp_object_storage(referenced, misp_object) + if hasattr(network_traffic, 'encapsulates_refs'): + for reference in network_traffic.encapsulates_refs: + encapsulated_observable = self._fetch_observable(reference) + name = self._parse_network_traffic_observable_fields( + encapsulated_observable['observable'] + ) + encapsulated = self._parse_network_traffic_observable_object_ref( + encapsulated_observable, observed_data, name + ) + misp_object.add_reference(encapsulated.uuid, 'encapsulates') + if hasattr(network_traffic, 'encapsulated_by_ref'): + referenced_observable = self._fetch_observable( + network_traffic.encapsulated_by_ref + ) + name = self._parse_network_traffic_observable_fields( + referenced_observable['observable'] + ) + referenced = self._parse_network_traffic_observable_object_ref( + referenced_observable, observed_data, name + ) + misp_object.add_reference(referenced.uuid, 'encapsulated-by') + + def _parse_network_traffic_observable_objects( + self, observed_data: _OBSERVED_DATA_TYPING): + observable_objects = { + object_id: {'used': False} + for object_id, observable in observed_data.objects.items() + if observable.type == 'network-traffic' + } + for object_id in observed_data.objects.values(): + network_traffic = observed_data.objects[object_id] + name = self._parse_network_traffic_observable_fields( + network_traffic + ) + misp_object = self._parse_network_traffic_observable_object( + observable_objects, object_id, observed_data, name + ) + for asset in ('src', 'dst'): + if hasattr(network_traffic, f'{asset}_ref'): + referenced = observed_data.objects[ + getattr(network_traffic, f'{asset}_ref') + ] + attributes = self._parse_network_traffic_reference_observable( + asset, referenced, f'{observed_data.id} - {object_id}' + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) + if hasattr(network_traffic, 'encapsulates_refs'): + for reference in network_traffic.encapsulates_refs: + observable = observed_data.objects[reference] + name = self._parse_network_traffic_observable_fields( + observable + ) + encapsulated = self._parse_network_traffic_observable_object( + observable_objects, reference, observed_data, name + ) + misp_object.add_reference(encapsulated.uuid, 'encapsulates') + if hasattr(network_traffic, 'encapsulated_by_ref'): + referenced = observed_data.objects[ + network_traffic.encapsulated_by_ref + ] + name = self._parse_network_traffic_observable_fields(referenced) + referenced_object = self._parse_network_traffic_observable_object( + observable_objects, network_traffic.encapsulated_by_ref, + observed_data, name + ) + misp_object.add_reference( + referenced_object.uuid, 'encapsulated-by' + ) + def _parse_process_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: From b788db12fa379803b5a89976e135e9b89bcb17ac Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 9 Apr 2024 18:11:33 +0200 Subject: [PATCH 109/137] chg: [stix2 import] Using the file observable references parsing method to convert v2.0 observable objects --- .../stix2_observed_data_converter.py | 46 ++----------------- 1 file changed, 4 insertions(+), 42 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 7daf7b55..678b9b25 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1255,48 +1255,10 @@ def _parse_file_observable_objects( ) if object_type == 'directory': continue - if hasattr(observable_object, 'extensions'): - extensions = observable_object.extensions - if extensions.get('archive-ext'): - archive_ext = extensions['archive-ext'] - if hasattr(archive_ext, 'comment'): - misp_object.from_dict( - comment=' - '.join( - (archive_ext.comment, misp_object.comment) - ) - ) - self._handle_misp_object_references( - misp_object, - *self._parse_contained_objects( - observed_data, observable_objects, - *archive_ext.contains_refs - ) - ) - if extensions.get('windows-pebinary-ext'): - pe_object_uuid = self._parse_file_pe_extension_observable( - extensions['windows-pebinary-ext'], observed_data, - f'{observed_data.id} - ' - f'{object_id} - windows-pebinary-ext' - ) - misp_object.add_reference(pe_object_uuid, 'includes') - if hasattr(observable_object, 'parent_directory_ref'): - parent_ref = observable_object.parent_directory_ref - parent_object = self._handle_observable_objects_parsing( - observable_objects, parent_ref, observed_data, 'directory' - ) - self._handle_misp_object_references( - misp_object, parent_object.uuid, - relationship_type='contained-in' - ) - if hasattr(observable_object, 'content_ref'): - content_ref = observable_object.content_ref - artifact = self._handle_observable_objects_parsing( - observable_objects, content_ref, observed_data, - 'artifact', False - ) - self._handle_misp_object_references( - artifact, misp_object.uuid, relationship_type='content-of' - ) + self._parse_file_observable_object_references( + misp_object, observable_object, observed_data, + observable_objects, object_id + ) def _parse_file_pe_extension_observable( self, pe_extension: _WINDOWS_PE_BINARY_EXT_TYPING, From 5b8933f74863d10793620a53fb0fa7d45d5af5ed Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 16 Apr 2024 16:19:16 +0200 Subject: [PATCH 110/137] fix: [stix2 import] Importing Network Traffic observable objects referenced by external Observed Data objects with the `network-traffic` generic MISP object template --- .../converters/stix2_observable_converter.py | 62 ++++++++++++++----- .../stix2_observable_objects_converter.py | 5 -- .../stix2_observed_data_converter.py | 13 ++-- 3 files changed, 51 insertions(+), 29 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 7c754a9a..bac12abc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -266,22 +266,6 @@ def _parse_generic_observable( mapping, getattr(observable, field), object_id ) - def _parse_network_connection_observable( - self, observable: _NETWORK_TRAFFIC_TYPING, - object_id: Optional[str] = None) -> Iterator[dict]: - if object_id is None: - object_id = observable.id - for protocol in observable.protocols: - layer = self._mapping.connection_protocols(protocol) - if layer is not None: - yield { - 'object_relation': f'layer{layer}-protocol', - 'type': 'text', 'value': protocol.upper(), - 'uuid': self.main_parser._create_v5_uuid( - f'{object_id} - layer{layer}-protocol - {protocol}' - ) - } - def _parse_network_socket_observable( self, observable: _NETWORK_TRAFFIC_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: @@ -310,6 +294,25 @@ def _parse_network_socket_observable( ) } + def _parse_network_traffic_observable( + self, observable: _NETWORK_TRAFFIC_TYPING, + object_id: Optional[str] = None) -> Iterator[dict]: + if object_id is None: + object_id = observable.id + mapping = self._mapping.network_traffic_object_mapping() + for field, attribute in mapping.items(): + if hasattr(observable, field): + yield from self._populate_object_attributes( + attribute, getattr(observable, field), object_id + ) + for protocol in observable.protocols: + yield { + 'value': protocol.upper(), **self._mapping.protocol_attribute(), + 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - protocol - {protocol}' + ) + } + def _parse_network_traffic_reference_observable( self, asset: str, observable: _NETWORK_TRAFFIC_REFERENCE_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: @@ -623,7 +626,7 @@ def _parse_network_traffic_observable_fields( observable: NetworkTraffic_v21) -> str: if getattr(observable, 'extensions', {}).get('socket-ext'): return 'network-socket' - return 'network-connection' + return 'network-traffic' def _parse_url_observable(self, observable: _URL_TYPING, observed_data_id: str) -> Iterator[dict]: @@ -1017,3 +1020,28 @@ def _parse_netflow_references( f'{observed_data_id} - {feature}-as - {value}' ) } + + def _parse_network_connection_observable( + self, observable: _NETWORK_TRAFFIC_TYPING, + object_id: Optional[str] = None) -> Iterator[dict]: + if object_id is None: + object_id = observable.id + for protocol in observable.protocols: + layer = self._mapping.connection_protocols(protocol) + if layer is None: + args = ( + (object_id.split(' - ')[0].split('--')[1], 'Observed Data') + if ' - ' in object_id else + (object_id.split('--')[1], 'Network Traffic observable') + ) + self.main_parser._unknown_network_protocol_warning( + protocol, *args + ) + continue + yield { + 'object_relation': f'layer{layer}-protocol', + 'type': 'text', 'value': protocol.upper(), + 'uuid': self.main_parser._create_v5_uuid( + f'{object_id} - layer{layer}-protocol - {protocol}' + ) + } diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 5c57c705..41df8623 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -438,11 +438,6 @@ def _parse_network_traffic_observable_object( network_object = self._create_misp_object_from_observable_object( name, network_traffic ) - attributes = self._parse_generic_observable( - network_traffic, 'network_traffic' - ) - for attribute in attributes: - network_object.add_attribute(**attribute) feature = f"_parse_{name.replace('-', '_')}_observable" for attribute in getattr(self, feature)(network_traffic): network_object.add_attribute(**attribute) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 678b9b25..df85025f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1544,15 +1544,15 @@ def _parse_network_traffic_observable_object( observable = observable_objects[object_id] if observable['used']: return observable['misp_object'] - misp_object = self._parse_generic_observable_object( - observed_data, object_id, name, True, 'network_traffic' + misp_object = self._create_misp_object_from_observable_object( + name, observed_data, object_id ) feature = f"_parse_{name.replace('-', '_')}_observable" attributes = getattr(self, feature)(observed_data.objects[object_id]) for attribute in attributes: misp_object.add_attribute(**attribute) observable.update({'misp_object': misp_object, 'used': True}) - return misp_object + return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_network_traffic_observable_object_ref( self, observable: dict, observed_data: ObservedData_v21, @@ -1561,9 +1561,8 @@ def _parse_network_traffic_observable_object_ref( misp_object = observable['misp_object'] self._handle_misp_object_fields(misp_object, observed_data) return misp_object - misp_object = self._parse_generic_observable_object_ref( - observable['observable'], observed_data, name, - True, 'network_traffic' + misp_object = self._create_misp_object_from_observable_object_ref( + name, observable['observable'], observed_data ) feature = f"_parse_{name.replace('-', '_')}_observable" attributes = getattr(self, feature)(observable['observable']) @@ -1571,7 +1570,7 @@ def _parse_network_traffic_observable_object_ref( misp_object.add_attribute(**attribute) observable['used'][self.event_uuid] = True observable['misp_object'] = misp_object - return misp_object + return self.main_parser._add_misp_object(misp_object, observed_data) def _parse_network_traffic_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): From 5649197b32211f2202b86a59b742a5253d83052b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 16 Apr 2024 17:00:09 +0200 Subject: [PATCH 111/137] fix: [stix2 import] Updated Network Traffic observables objects mapping to MISP objects --- .../stix2misp/converters/stix2mapping.py | 82 +++++++++---------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 8a44466b..bd807b5f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -173,6 +173,9 @@ class STIX2Mapping: __pid_attribute = Mapping( **{'type': 'text', 'object_relation': 'pid'} ) + __protocol_attribute = Mapping( + **{'type': 'text', 'object_relation': 'protocol'} + ) __reference_attribute = Mapping( **{'type': 'link', 'object_relation': 'reference'} ) @@ -296,21 +299,6 @@ class STIX2Mapping: __password_last_changed_attribute = {'type': 'datetime', 'object_relation': 'password_last_changed'} # MISP OBJECTS MAPPING - __connection_protocols = Mapping( - **{ - **dict.fromkeys(('tcp', 'TCP', 'udp', 'UDP'), '4'), - **dict.fromkeys( - ( - 'arp', 'icmp', 'ip', 'ipv4', 'ipv6', - 'ARP', 'ICMP', 'IP', 'IPV4', 'IPV6' - ), - '3' - ), - **dict.fromkeys( - ('http', 'HTTP', 'https', 'HTTPS', 'ftp', 'FTP'), '7' - ) - } - ) __email_object_mapping = Mapping( body=__email_body_attribute, date=__send_date_attribute, @@ -347,6 +335,27 @@ class STIX2Mapping: protocol_family=__domain_family_attribute, socket_type={'type': 'text', 'object_relation': 'socket-type'} ) + __network_socket_object_mapping = Mapping( + src_port=__src_port_attribute, + dst_port=__dst_port_attribute, + start=__first_packet_seen_attribute, + end=__last_packet_seen_attribute, + src_byte_count={'type': 'size-in-bytes', 'object_relation': 'src-byte-count'}, + dst_byte_count={'type': 'size-in-bytes', 'object_relation': 'dst-byte-count'}, + src_packets={'type': 'counter', 'object_relation': 'src-packets'}, + dst_packets={'type': 'counter', 'object_relation': 'dst-packets'} + ) + __network_traffic_object_mapping = Mapping( + src_port={'type': 'port', 'object_relation': 'src_port'}, + dst_port={'type': 'port', 'object_relation': 'dst_port'}, + start={'type': 'datetime', 'object_relation': 'start_time'}, + end={'type': 'datetime', 'object_relation': 'end_time'}, + is_active={'type': 'boolean', 'object_relation': 'is_active'}, + src_byte_count={'type': 'size-in-bytes', 'object_relation': 'src_byte_count'}, + dst_byte_count={'type': 'size-in-bytes', 'object_relation': 'dst_byte_count'}, + src_packets={'type': 'counter', 'object_relation': 'src_packets'}, + dst_packets={'type': 'counter', 'object_relation': 'dst_packets'} + ) __pe_object_mapping = Mapping( time_date_stamp=__compilation_timestamp_attribute, imphash=__imphash_attribute, @@ -460,10 +469,6 @@ def comment_attribute(cls) -> dict: def compilation_timestamp_attribute(cls) -> dict: return cls.__compilation_timestamp_attribute - @classmethod - def connection_protocols(cls, field: str) -> Union[str, None]: - return cls.__connection_protocols.get(field) - @classmethod def content_type_attribute(cls) -> dict: return cls.__content_type_attribute @@ -616,6 +621,14 @@ def name_attribute(cls) -> dict: def network_socket_extension_mapping(cls) -> dict: return cls.__network_socket_extension_mapping + @classmethod + def network_socket_object_mapping(cls) -> dict: + return cls.__network_socket_object_mapping + + @classmethod + def network_traffic_object_mapping(cls) -> dict: + return cls.__network_traffic_object_mapping + @classmethod def password_attribute(cls) -> dict: return cls.__password_attribute @@ -879,16 +892,6 @@ class ExternalSTIX2Mapping(STIX2Mapping): path=STIX2Mapping.path_attribute(), path_enc={'type': 'text', 'object_relation': 'path-encoding'} ) - __network_traffic_object_mapping = Mapping( - src_port=STIX2Mapping.src_port_attribute(), - dst_port=STIX2Mapping.dst_port_attribute(), - start=STIX2Mapping.first_packet_seen_attribute(), - end=STIX2Mapping.last_packet_seen_attribute(), - src_byte_count={'type': 'counter', 'object_relation': 'src-bytes-count'}, - dst_byte_count={'type': 'counter', 'object_relation': 'dst-bytes-count'}, - src_packets={'type': 'counter', 'object_relation': 'src-packets-count'}, - dst_packets={'type': 'counter', 'object_relation': 'dst-packets-count'} - ) __process_object_mapping = Mapping( command_line=STIX2Mapping.command_line_attribute(), created=STIX2Mapping.creation_time_attribute(), @@ -918,10 +921,6 @@ def file_hashes_mapping(cls, field: str) -> Union[dict, None]: def galaxy_name_mapping(cls, field) -> Union[dict, None]: return cls.__galaxy_name_mapping.get(field) - @classmethod - def network_traffic_object_mapping(cls) -> dict: - return cls.__network_traffic_object_mapping - @classmethod def process_object_mapping(cls) -> dict: return cls.__process_object_mapping @@ -990,9 +989,6 @@ class InternalSTIX2Mapping(STIX2Mapping): __parent_process_path_attribute = Mapping( **{'type': 'text', 'object_relation': 'parent-process-path'} ) - __protocol_attribute = Mapping( - **{'type': 'text', 'object_relation': 'protocol'} - ) __script_attribute = Mapping( **{'type': 'text', 'object_relation': 'script'} ) @@ -1351,16 +1347,20 @@ class InternalSTIX2Mapping(STIX2Mapping): x_misp_ip_version={'type': 'counter', 'object_relation': 'ip_version'} ) __network_connection_object_mapping = Mapping( - dst_port=STIX2Mapping.dst_port_attribute(), src_port=STIX2Mapping.src_port_attribute(), + dst_port=STIX2Mapping.dst_port_attribute(), start=STIX2Mapping.first_packet_seen_attribute(), + end=STIX2Mapping.last_packet_seen_attribute(), + src_byte_count={'type': 'size-in-bytes', 'object_relation': 'src-bytes-count'}, + dst_byte_count={'type': 'size-in-bytes', 'object_relation': 'dst-bytes-count'}, + src_packets={'type': 'counter', 'object_relation': 'src-packets-count'}, + dst_packets={'type': 'counter', 'object_relation': 'dst-packets-count'}, x_misp_community_id=__community_id_attribute, x_misp_hostname_dst=STIX2Mapping.hostname_dst_attribute(), x_misp_hostname_src=STIX2Mapping.hostname_src_attribute() ) __network_socket_object_mapping = Mapping( - dst_port=STIX2Mapping.dst_port_attribute(), - src_port=STIX2Mapping.src_port_attribute(), + **STIX2Mapping.network_socket_object_mapping(), x_misp_address_family=STIX2Mapping.address_family_attribute(), x_misp_domain_family=STIX2Mapping.domain_family_attribute(), x_misp_filename=STIX2Mapping.filename_attribute(), @@ -1718,10 +1718,6 @@ def pe_section_object_mapping(cls) -> dict: def process_object_mapping(cls) -> dict: return cls.__process_object_mapping - @classmethod - def protocol_attribute(cls) -> dict: - return cls.__protocol_attribute - @classmethod def reddit_account_object_mapping(cls) -> dict: return cls.__reddit_account_object_mapping From 497ab48f01e591eb2399c83a49eef2ad2bd5a33f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 17 Apr 2024 16:09:53 +0200 Subject: [PATCH 112/137] wip: [stix2 import] Better conversion of Network Traffic references observable objects - Such as IP addresses, Domain names and Mac addresses referenced with the `src_ref` and `dst_ref` fields --- .../converters/stix2_observable_converter.py | 70 ++++++++++++++----- .../stix2_observable_objects_converter.py | 9 +-- .../stix2_observed_data_converter.py | 17 ++--- 3 files changed, 64 insertions(+), 32 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index bac12abc..0998d96f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -7,6 +7,7 @@ from .stix2mapping import ( ExternalSTIX2Mapping, InternalSTIX2Mapping, STIX2Mapping) from abc import ABCMeta +from pymisp import MISPObject from stix2.v20.observables import ( Artifact as Artifact_v20, AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, DomainName as DomainName_v20, @@ -122,8 +123,20 @@ class STIX2ObservableMapping(STIX2Mapping, metaclass=ABCMeta): 'X-Mailer': STIX2Mapping.x_mailer_attribute() } ) - __mac_dst_attribute = {'type': 'mac-address', 'object_relation': 'mac-dst'} - __mac_src_attribute = {'type': 'mac-address', 'object_relation': 'mac-src'} + __dst_ip_attribute = {'type': 'ip-dst', 'object_relation': 'dst_ip'} + __src_ip_attribute = {'type': 'ip-src', 'object_relation': 'src_ip'} + __generic_network_traffic_reference_mapping = Mapping( + **{ + 'domain-name_dst': {'type': 'hostname', 'object_relation': 'dst_hostname'}, + 'domain-name_src': {'type': 'hostname', 'object_relation': 'src_hostname'}, + 'ipv4-addr_dst': __dst_ip_attribute, + 'ipv4-addr_src': __src_ip_attribute, + 'ipv6-addr_dst': __dst_ip_attribute, + 'ipv6-addr_src': __src_ip_attribute, + 'mac-address_dst': {'type': 'mac-address', 'object_relation': 'dst_mac'}, + 'mac-address_src': {'type': 'mac-address', 'object_relation': 'src_mac'} + } + ) __network_traffic_reference_mapping = Mapping( **{ 'domain-name_dst': STIX2Mapping.hostname_dst_attribute(), @@ -132,8 +145,8 @@ class STIX2ObservableMapping(STIX2Mapping, metaclass=ABCMeta): 'ipv4-addr_src': STIX2Mapping.ip_src_attribute(), 'ipv6-addr_dst': STIX2Mapping.ip_dst_attribute(), 'ipv6-addr_src': STIX2Mapping.ip_src_attribute(), - 'mac-address_dst': __mac_dst_attribute, - 'mac-address_src': __mac_src_attribute + 'mac-address_dst': {'type': 'mac-address', 'object_relation': 'mac-dst'}, + 'mac-address_src': {'type': 'mac-address', 'object_relation': 'mac-src'} } ) __software_object_mapping = Mapping( @@ -161,9 +174,17 @@ def email_additional_header_fields_mapping(cls) -> dict: return cls.__email_additional_header_fields_mapping @classmethod - def network_traffic_reference_mapping(cls, field) -> dict: + def network_traffic_reference_mapping(cls, field: str) -> dict: + return cls.__generic_network_traffic_reference_mapping.get(field) + + @classmethod + def network_socket_reference_mapping(cls, field: str) -> dict: return cls.__network_traffic_reference_mapping.get(field) + @classmethod + def network_traffic_references(cls) -> dict: + return cls.__network_traffic_reference_mapping + @classmethod def software_object_mapping(cls) -> dict: return cls.__software_object_mapping @@ -177,6 +198,16 @@ class STIX2ObservableConverter(STIX2Converter): def _fetch_observable(self, object_ref: str) -> dict: return self.main_parser._observable[object_ref] + def _handle_misp_object_storage( + self, observable: dict, misp_object: MISPObject): + observable['used'][self.event_uuid] = True + if observable.get('misp_object') is None: + observable['misp_object'] = misp_object + elif isinstance(observable['misp_object'], list): + observable['misp_object'].append(misp_object) + else: + observable['misp_object'] = [observable['misp_object'], misp_object] + def _parse_email_observable( self, observable: _EMAIL_MESSAGE_TYPING, object_id: Optional[str] = None) -> Iterator[dict]: @@ -272,10 +303,11 @@ def _parse_network_socket_observable( if object_id is None: object_id = observable.id for protocol in observable.protocols: + protocol_value = protocol.upper() yield { - 'value': protocol.upper(), **self._mapping.protocol_attribute(), + 'value': protocol_value, **self._mapping.protocol_attribute(), 'uuid': self.main_parser._create_v5_uuid( - f'{object_id} - protocol - {protocol}' + f'{object_id} - protocol - {protocol_value}' ) } socket_extension = observable.extensions['socket-ext'] @@ -306,25 +338,27 @@ def _parse_network_traffic_observable( attribute, getattr(observable, field), object_id ) for protocol in observable.protocols: + protocol_value = protocol.upper() yield { - 'value': protocol.upper(), **self._mapping.protocol_attribute(), + 'value': protocol_value, **self._mapping.protocol_attribute(), 'uuid': self.main_parser._create_v5_uuid( - f'{object_id} - protocol - {protocol}' + f'{object_id} - protocol - {protocol_value}' ) } def _parse_network_traffic_reference_observable( self, asset: str, observable: _NETWORK_TRAFFIC_REFERENCE_TYPING, - object_id: Optional[str] = None) -> Iterator[dict]: - mapping = self._mapping.network_traffic_reference_mapping( + object_id: str) -> Iterator[dict]: + attribute = self._mapping.network_traffic_reference_mapping( f'{observable.type}_{asset}' ) - if mapping is not None: - if object_id is None: - object_id = observable.id + if attribute is not None: yield { - 'value': observable.value, **mapping, - **self.main_parser._sanitise_attribute_uuid(object_id) + 'value': observable.value, **attribute, + 'uuid': self.main_parser._create_v5_uuid( + f"{object_id} - {attribute['object_relation']}" + f' - {observable.value}' + ) } def _parse_pe_extension_observable(self, extension: _EXTENSION_TYPING, @@ -770,6 +804,10 @@ def http_request_header_mapping(cls) -> dict: def malware_sample_attribute(cls) -> dict: return cls.__malware_sample_attribute + @classmethod + def network_connection_reference_mapping(cls, field: str) -> dict: + return cls.network_traffic_references.get(field) + @classmethod def parent_process_object_mapping(cls) -> dict: return cls.__parent_process_object_mapping diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py index 41df8623..c5805505 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_objects_converter.py @@ -448,16 +448,17 @@ def _parse_network_traffic_observable_object( observable['misp_object'] = misp_object for asset in ('src', 'dst'): if hasattr(network_traffic, f'{asset}_ref'): - referenced_object = self._fetch_observable( + referenced = self._fetch_observable( getattr(network_traffic, f'{asset}_ref') ) + referenced_observable = referenced['observable'] attributes = self._parse_network_traffic_reference_observable( - asset, referenced_object['observable'] + asset, referenced_observable, + f'{network_traffic.id} - {referenced_observable.id}' ) for attribute in attributes: misp_object.add_attribute(**attribute) - referenced_object['used'][self.event_uuid] = True - referenced_object['misp_object'] = misp_object + self._handle_misp_object_storage(referenced, misp_object) if hasattr(network_traffic, 'encapsulates_refs'): for reference in network_traffic.encapsulates_refs: referenced = self._parse_network_traffic_observable_object( diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index df85025f..00c79f41 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -282,16 +282,6 @@ def _parse_multiple_observable_objects( # OBSERVABLE OBJECTS PARSING METHODS # ############################################################################ - def _handle_misp_object_storage( - self, observable: dict, misp_object: MISPObject): - observable['used'][self.event_uuid] = True - if observable.get('misp_object') is None: - observable['misp_object'] = misp_object - elif isinstance(observable['misp_object'], list): - observable['misp_object'].append(misp_object) - else: - observable['misp_object'] = [observable['misp_object'], misp_object] - def _handle_observable_object_refs_parsing( self, observable: dict, observed_data: ObservedData_v21, *args: tuple) -> MISPObject: @@ -1585,13 +1575,16 @@ def _parse_network_traffic_observable_object_refs( misp_object = self._parse_network_traffic_observable_object_ref( observable, observed_data, name ) + feature = f"_parse_{name.replace('-', '_')}_reference_observable" for asset in ('src', 'dst'): if hasattr(network_traffic, f'{asset}_ref'): referenced = self._fetch_observable( getattr(network_traffic, f'{asset}_ref') ) - attributes = self._parse_network_traffic_reference_observable( - asset, referenced['observable'] + referenced_observable = referenced['observable'] + attributes = getattr(self, feature)( + asset, referenced_observable, + f'{network_traffic.id} - {referenced_observable.id}' ) for attribute in attributes: misp_object.add_attribute(**attribute) From 1e03762af9018825910ab019dd8970675d265ca7 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 18 Apr 2024 12:43:12 +0200 Subject: [PATCH 113/137] fix: [stix2 import] Better handling of internal Galaxy & Cluster description --- misp_stix_converter/stix2misp/converters/stix2converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index 29b74e08..96c11e25 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -495,7 +495,7 @@ def _parse_galaxy_as_tag_names( def _parse_galaxy_cluster( self, stix_object: _GALAXY_OBJECTS_TYPING, galaxy_type: str, description: Optional[str] = None) -> Tuple[MISPGalaxyCluster, str]: - if ' | ' in getattr(stix_object, 'description', ''): + if getattr(stix_object, 'description', '').count(' | ') == 1: _, description = stix_object.description.split(' | ') return self._create_cluster( stix_object, description=description, galaxy_type=galaxy_type From 9d9138be1a305defb08cd0425a423f4d67417ae0 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 22 Apr 2024 15:51:06 +0200 Subject: [PATCH 114/137] fix: [stix2 import] Added missing `protocol_attribute` property in STIX2Mapping parent class --- misp_stix_converter/stix2misp/converters/stix2mapping.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index bd807b5f..d8636f49 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -653,6 +653,10 @@ def pe_section_object_mapping(cls) -> dict: def pid_attribute(cls) -> dict: return cls.__pid_attribute + @classmethod + def protocol_attribute(cls) -> dict: + return cls.__protocol_attribute + @classmethod def references_attribute(cls) -> dict: return cls.__references_attribute From 2afb20a09594777e2c47c21fc0deb655858e62c8 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 24 Apr 2024 14:55:41 +0200 Subject: [PATCH 115/137] fix: [stix2 import] Fixed STIX 2.0 Network Traffic Observable objects parsing --- .../stix2_observed_data_converter.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 00c79f41..dbf90160 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1529,16 +1529,22 @@ def _parse_mutex_observable_objects( ) def _parse_network_traffic_observable_object( - self, observable_objects: dict, object_id: str, + self, observable_objects: dict, identifier: str, observed_data: _OBSERVED_DATA_TYPING, name: str) -> MISPObject: - observable = observable_objects[object_id] + network_traffic = observed_data.objects[identifier] + if hasattr(network_traffic, 'id'): + return self._parse_network_traffic_observable_object_ref( + network_traffic, observed_data, name + ) + observable = observable_objects[identifier] if observable['used']: return observable['misp_object'] + object_id = f'{observed_data.id} - {identifier}' misp_object = self._create_misp_object_from_observable_object( name, observed_data, object_id ) feature = f"_parse_{name.replace('-', '_')}_observable" - attributes = getattr(self, feature)(observed_data.objects[object_id]) + attributes = getattr(self, feature)(network_traffic, object_id) for attribute in attributes: misp_object.add_attribute(**attribute) observable.update({'misp_object': misp_object, 'used': True}) @@ -1618,7 +1624,7 @@ def _parse_network_traffic_observable_objects( for object_id, observable in observed_data.objects.items() if observable.type == 'network-traffic' } - for object_id in observed_data.objects.values(): + for object_id, observable in observable_objects.items(): network_traffic = observed_data.objects[object_id] name = self._parse_network_traffic_observable_fields( network_traffic @@ -1628,11 +1634,11 @@ def _parse_network_traffic_observable_objects( ) for asset in ('src', 'dst'): if hasattr(network_traffic, f'{asset}_ref'): - referenced = observed_data.objects[ - getattr(network_traffic, f'{asset}_ref') - ] + referenced_id = getattr(network_traffic, f'{asset}_ref') + referenced = observed_data.objects[referenced_id] attributes = self._parse_network_traffic_reference_observable( - asset, referenced, f'{observed_data.id} - {object_id}' + asset, referenced, + f'{observed_data.id} - {object_id} - {referenced_id}' ) for attribute in attributes: misp_object.add_attribute(**attribute) From 8f91043d5e5695af87329d8502f97ff5a838a36b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 24 Apr 2024 15:02:08 +0200 Subject: [PATCH 116/137] wip: [tests] Tests for Network Traffic Observable objects imported from external STIX 2 bundles as `network-traffic` objects --- tests/test_external_stix20_bundles.py | 118 ++++++++++++++++++ tests/test_external_stix20_import.py | 170 ++++++++++++++++++++++++++ tests/test_external_stix21_bundles.py | 144 ++++++++++++++++++++++ tests/test_external_stix21_import.py | 162 ++++++++++++++++++++++++ 4 files changed, 594 insertions(+) diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 5dfeba80..fc9a2192 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -711,6 +711,120 @@ } } ] +_NETWORK_TRAFFIC_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "ipv4-addr", + "value": "198.51.100.2" + }, + "1": { + "type": "ipv4-addr", + "value": "203.0.113.1" + }, + "2": { + "type": "ipv4-addr", + "value": "203.0.113.2" + }, + "3": { + "type": "network-traffic", + "src_ref": "0", + "dst_ref": "1", + "src_port": 2487, + "dst_port": 1723, + "protocols": [ + "ipv4", + "pptp" + ], + "src_byte_count": 35779, + "dst_byte_count": 935750, + "encapsulates_refs": [ + "4" + ] + }, + "4": { + "type": "network-traffic", + "src_ref": "0", + "dst_ref": "2", + "src_port": 24678, + "dst_port": 80, + "protocols": [ + "ipv4", + "tcp", + "http" + ], + "src_packets": 14356, + "dst_packets": 14356, + "encapsulated_by_ref": "3" + } + } + }, + { + "type": "observed-data", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "ipv4-addr", + "value": "203.0.113.1" + }, + "1": { + "type": "ipv4-addr", + "value": "198.51.100.34" + }, + "2": { + "type": "ipv4-addr", + "value": "198.51.100.54" + }, + "3": { + "type": "network-traffic", + "src_ref": "0", + "dst_ref": "1", + "src_port": 2487, + "dst_port": 53, + "protocols": [ + "ipv4", + "udp", + "dns" + ], + "src_byte_count": 35779, + "dst_byte_count": 935750, + "encapsulates_refs": [ + "4" + ] + }, + "4": { + "type": "network-traffic", + "src_ref": "1", + "dst_ref": "2", + "src_port": 24678, + "dst_port": 443, + "protocols": [ + "ipv4", + "tcp", + "ssl", + "http" + ], + "src_packets": 14356, + "dst_packets": 14356, + "encapsulated_by_ref": "3" + } + } + } +] _PROCESS_OBJECTS = [ { "type": "observed-data", @@ -1393,6 +1507,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_network_traffic_objects(cls): + return cls.__assemble_bundle(*_NETWORK_TRAFFIC_OBJECTS) + @classmethod def get_bundle_with_process_objects(cls): return cls.__assemble_bundle(*_PROCESS_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index c35cfef4..4c9d972e 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -358,6 +358,132 @@ def _check_misp_object_fields(self, misp_object, observed_data, identifier=None) self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_network_traffic_fields( + self, attributes, network_traffic, src_ip, dst_ip, + object_id, src_ip_id, dst_ip_id): + src_port, dst_port, src_return, dst_return, *protocols, ip_src, ip_dst = attributes + self.assertEqual(src_port.type, 'port') + self.assertEqual(src_port.object_relation, 'src_port') + self.assertEqual(src_port.value, network_traffic.src_port) + self.assertEqual( + src_port.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - src_port - {src_port.value}' + ) + ) + self.assertEqual(dst_port.type, 'port') + self.assertEqual(dst_port.object_relation, 'dst_port') + self.assertEqual(dst_port.value, network_traffic.dst_port) + self.assertEqual( + dst_port.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - dst_port - {dst_port.value}' + ) + ) + for index, protocol in enumerate(protocols): + protocol_value = network_traffic.protocols[index].upper() + self.assertEqual(protocol.type, 'text') + self.assertEqual(protocol.object_relation, 'protocol') + self.assertEqual(protocol.value, protocol_value) + self.assertEqual( + protocol.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - protocol - {protocol_value}' + ) + ) + self.assertEqual(ip_src.type, 'ip-src') + self.assertEqual(ip_src.object_relation, 'src_ip') + self.assertEqual(ip_src.value, src_ip.value) + self.assertEqual( + ip_src.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - {src_ip_id} - src_ip - {src_ip.value}' + ) + ) + self.assertEqual(ip_dst.type, 'ip-dst') + self.assertEqual(ip_dst.object_relation, 'dst_ip') + self.assertEqual(ip_dst.value, dst_ip.value) + self.assertEqual( + ip_dst.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - {dst_ip_id} - dst_ip - {dst_ip.value}' + ) + ) + return src_return, dst_return + + def _check_network_traffic_object_with_packet_counts( + self, misp_object, observed_data, network_traffic_id, + src_ip_id, dst_ip_id,attributes_count): + self.assertEqual(misp_object.name, 'network-traffic') + self._check_misp_object_fields(misp_object, observed_data, network_traffic_id) + network_traffic = observed_data.objects[network_traffic_id] + attributes = misp_object.attributes + self.assertEqual(len(attributes), attributes_count) + object_id = f'{observed_data.id} - {network_traffic_id}' + src_packets, dst_packets = self._check_network_traffic_fields( + attributes, network_traffic, observed_data.objects[src_ip_id], + observed_data.objects[dst_ip_id], object_id, src_ip_id, dst_ip_id + ) + self.assertEqual(src_packets.type, 'counter') + self.assertEqual(src_packets.object_relation, 'src_packets') + self.assertEqual(src_packets.value, network_traffic.src_packets) + self.assertEqual( + src_packets.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - src_packets - {src_packets.value}' + ) + ) + self.assertEqual(dst_packets.type, 'counter') + self.assertEqual(dst_packets.object_relation, 'dst_packets') + self.assertEqual(dst_packets.value, network_traffic.dst_packets) + self.assertEqual( + dst_packets.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - dst_packets - {dst_packets.value}' + ) + ) + + def _check_network_traffic_object_with_packet_sizes( + self, misp_object, observed_data, network_traffic_id, + src_ip_id, dst_ip_id, attributes_count): + self.assertEqual(misp_object.name, 'network-traffic') + self._check_misp_object_fields(misp_object, observed_data, network_traffic_id) + network_traffic = observed_data.objects[network_traffic_id] + attributes = misp_object.attributes + self.assertEqual(len(attributes), attributes_count) + object_id = f'{observed_data.id} - {network_traffic_id}' + src_bytes, dst_bytes = self._check_network_traffic_fields( + attributes, network_traffic, observed_data.objects[src_ip_id], + observed_data.objects[dst_ip_id], object_id, src_ip_id, dst_ip_id + ) + self.assertEqual(src_bytes.type, 'size-in-bytes') + self.assertEqual(src_bytes.object_relation, 'src_byte_count') + self.assertEqual(src_bytes.value, network_traffic.src_byte_count) + self.assertEqual( + src_bytes.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - src_byte_count - {src_bytes.value}' + ) + ) + self.assertEqual(dst_bytes.type, 'size-in-bytes') + self.assertEqual(dst_bytes.object_relation, 'dst_byte_count') + self.assertEqual(dst_bytes.value, network_traffic.dst_byte_count) + self.assertEqual( + dst_bytes.uuid, + uuid5( + self._UUIDv4, + f'{object_id} - dst_byte_count - {dst_bytes.value}' + ) + ) + def _check_registry_key_object( self, misp_object, observed_data, *values, identifier=None): self.assertEqual(misp_object.name, 'registry-key') @@ -597,6 +723,50 @@ def test_stix20_bundle_with_mutex_attributes(self): observed_data2, s_mutex, 'mutex', feature='name' ) + def test_stix20_bundle_with_network_traffic_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_network_traffic_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data1, observed_data2 = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 4) + nt1, nt2, nt3, nt4 = misp_objects + self._check_network_traffic_object_with_packet_sizes( + nt1, observed_data1, '3', '0', '1', 8 + ) + self.assertEqual(len(nt1.references), 1) + encapsulates1 = nt1.references[0] + self.assertEqual(encapsulates1.referenced_uuid, nt2.uuid) + self._check_network_traffic_object_with_packet_counts( + nt2, observed_data1, '4', '0', '2', 9 + ) + self.assertEqual(len(nt2.references), 1) + encapsulated1 = nt2.references[0] + self.assertEqual(encapsulated1.referenced_uuid, nt1.uuid) + self._check_network_traffic_object_with_packet_sizes( + nt3, observed_data2, '3', '0', '1', 9 + ) + self.assertEqual(len(nt3.references), 1) + encapsulates2 = nt3.references[0] + self.assertEqual(encapsulates2.referenced_uuid, nt4.uuid) + self._check_network_traffic_object_with_packet_counts( + nt4, observed_data2, '4', '1', '2', 10 + ) + self.assertEqual(len(nt4.references), 1) + encapsulated2 = nt4.references[0] + self.assertEqual(encapsulated2.referenced_uuid, nt3.uuid) + self._assert_multiple_equal( + encapsulates1.relationship_type, + encapsulates2.relationship_type, + 'encapsulates' + ) + self._assert_multiple_equal( + encapsulated1.relationship_type, + encapsulated2.relationship_type, + 'encapsulated-by' + ) + def test_stix20_bundle_with_process_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_process_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 81614448..d44900c9 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -838,6 +838,146 @@ "name": "sensitive_resource_lock" } ] +_NETWORK_TRAFFIC_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53", + "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7", + "ipv4-addr--ffe65ce3-bf2a-577c-bb7e-947d39198637", + "network-traffic--ac267abc-1a41-536d-8e8d-98458d9bf491", + "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3" + ] + }, + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3451329f-2525-4bcb-9659-7bd0e6f1eb0d", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7", + "ipv4-addr--f2d3c796-6c1a-5c4f-8516-d4db54727f89", + "ipv4-addr--bb884ffe-f2e4-56bb-a0c3-21f6711cb649", + "network-traffic--b4a8c150-e214-57a3-9017-e85dfa345f46", + "network-traffic--65a6016d-a91c-5781-baad-178cd55f01d4" + ] + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53", + "value": "198.51.100.2" + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7", + "value": "203.0.113.1" + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--ffe65ce3-bf2a-577c-bb7e-947d39198637", + "value": "203.0.113.2" + }, + { + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--ac267abc-1a41-536d-8e8d-98458d9bf491", + "src_ref": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53", + "dst_ref": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7", + "src_port": 2487, + "dst_port": 1723, + "protocols": [ + "ipv4", + "pptp" + ], + "src_byte_count": 35779, + "dst_byte_count": 935750, + "encapsulates_refs": [ + "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3" + ] + }, + { + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3", + "src_ref": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53", + "dst_ref": "ipv4-addr--ffe65ce3-bf2a-577c-bb7e-947d39198637", + "src_port": 24678, + "dst_port": 80, + "protocols": [ + "ipv4", + "tcp", + "http" + ], + "src_packets": 14356, + "dst_packets": 14356, + "encapsulated_by_ref": "network-traffic--ac267abc-1a41-536d-8e8d-98458d9bf491" + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--f2d3c796-6c1a-5c4f-8516-d4db54727f89", + "value": "198.51.100.34" + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--bb884ffe-f2e4-56bb-a0c3-21f6711cb649", + "value": "198.51.100.54" + }, + { + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--b4a8c150-e214-57a3-9017-e85dfa345f46", + "src_ref": "ipv4-addr--e42c19c8-f9fe-5ae9-9fc8-22c398f78fb7", + "dst_ref": "ipv4-addr--f2d3c796-6c1a-5c4f-8516-d4db54727f89", + "src_port": 2487, + "dst_port": 53, + "protocols": [ + "ipv4", + "udp", + "dns" + ], + "src_byte_count": 35779, + "dst_byte_count": 935750, + "encapsulates_refs": [ + "network-traffic--65a6016d-a91c-5781-baad-178cd55f01d4" + ] + }, + { + "type": "network-traffic", + "spec_version": "2.1", + "id": "network-traffic--65a6016d-a91c-5781-baad-178cd55f01d4", + "src_ref": "ipv4-addr--f2d3c796-6c1a-5c4f-8516-d4db54727f89", + "dst_ref": "ipv4-addr--bb884ffe-f2e4-56bb-a0c3-21f6711cb649", + "src_port": 24678, + "dst_port": 443, + "protocols": [ + "ipv4", + "tcp", + "ssl", + "http" + ], + "src_packets": 14356, + "dst_packets": 14356, + "encapsulated_by_ref": "network-traffic--b4a8c150-e214-57a3-9017-e85dfa345f46" + } +] _PROCESS_OBJECTS = [ { "type": "observed-data", @@ -1612,6 +1752,10 @@ def get_bundle_with_mac_address_attributes(cls): def get_bundle_with_mutex_attributes(cls): return cls.__assemble_bundle(*_MUTEX_ATTRIBUTES) + @classmethod + def get_bundle_with_network_traffic_objects(cls): + return cls.__assemble_bundle(*_NETWORK_TRAFFIC_OBJECTS) + @classmethod def get_bundle_with_opinion_objects(cls): agree_opinion = { diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 7c577f8b..61e2cfb3 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -342,6 +342,124 @@ def _check_misp_object_fields(self, misp_object, observed_data, object_id, self.assertEqual(misp_object.last_seen, observed_data.last_observed) self.assertEqual(misp_object.timestamp, observed_data.modified) + def _check_network_traffic_fields(self, attributes, network_traffic, src_ip, dst_ip): + src_port, dst_port, src_return, dst_return, *protocols, ip_src, ip_dst = attributes + self.assertEqual(src_port.type, 'port') + self.assertEqual(src_port.object_relation, 'src_port') + self.assertEqual(src_port.value, network_traffic.src_port) + self.assertEqual( + src_port.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - src_port - {src_port.value}' + ) + ) + self.assertEqual(dst_port.type, 'port') + self.assertEqual(dst_port.object_relation, 'dst_port') + self.assertEqual(dst_port.value, network_traffic.dst_port) + self.assertEqual( + dst_port.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - dst_port - {dst_port.value}' + ) + ) + for index, protocol in enumerate(protocols): + protocol_value = network_traffic.protocols[index].upper() + self.assertEqual(protocol.type, 'text') + self.assertEqual(protocol.object_relation, 'protocol') + self.assertEqual(protocol.value, protocol_value) + self.assertEqual( + protocol.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - protocol - {protocol_value}' + ) + ) + self.assertEqual(ip_src.type, 'ip-src') + self.assertEqual(ip_src.object_relation, 'src_ip') + self.assertEqual(ip_src.value, src_ip.value) + self.assertEqual( + ip_src.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - {src_ip.id} - src_ip - {src_ip.value}' + ) + ) + self.assertEqual(ip_dst.type, 'ip-dst') + self.assertEqual(ip_dst.object_relation, 'dst_ip') + self.assertEqual(ip_dst.value, dst_ip.value) + self.assertEqual( + ip_dst.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - {dst_ip.id} - dst_ip - {dst_ip.value}' + ) + ) + return src_return, dst_return + + def _check_network_traffic_object_with_packet_counts( + self, misp_object, obbserved_data, network_traffic, + src_ip, dst_ip, attributes_count): + self.assertEqual(misp_object.name, 'network-traffic') + self._check_misp_object_fields(misp_object, obbserved_data, network_traffic.id) + attributes = misp_object.attributes + self.assertEqual(len(attributes), attributes_count) + src_packets, dst_packets = self._check_network_traffic_fields( + attributes, network_traffic, src_ip, dst_ip + ) + self.assertEqual(src_packets.type, 'counter') + self.assertEqual(src_packets.object_relation, 'src_packets') + self.assertEqual(src_packets.value, network_traffic.src_packets) + self.assertEqual( + src_packets.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - src_packets - {src_packets.value}' + ) + ) + self.assertEqual(dst_packets.type, 'counter') + self.assertEqual(dst_packets.object_relation, 'dst_packets') + self.assertEqual(dst_packets.value, network_traffic.dst_packets) + self.assertEqual( + dst_packets.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - dst_packets - {dst_packets.value}' + ) + ) + + def _check_network_traffic_object_with_packet_sizes( + self, misp_object, obbserved_data, network_traffic, + src_ip, dst_ip, attributes_count): + self.assertEqual(misp_object.name, 'network-traffic') + self._check_misp_object_fields(misp_object, obbserved_data, network_traffic.id) + attributes = misp_object.attributes + self.assertEqual(len(attributes), attributes_count) + src_bytes, dst_bytes = self._check_network_traffic_fields( + attributes, network_traffic, src_ip, dst_ip + ) + self.assertEqual(src_bytes.type, 'size-in-bytes') + self.assertEqual(src_bytes.object_relation, 'src_byte_count') + self.assertEqual(src_bytes.value, network_traffic.src_byte_count) + self.assertEqual( + src_bytes.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - src_byte_count - {src_bytes.value}' + ) + ) + self.assertEqual(dst_bytes.type, 'size-in-bytes') + self.assertEqual(dst_bytes.object_relation, 'dst_byte_count') + self.assertEqual(dst_bytes.value, network_traffic.dst_byte_count) + self.assertEqual( + dst_bytes.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic.id} - dst_byte_count - {dst_bytes.value}' + ) + ) + def _check_registry_key_object( self, misp_object, observed_data, registry_key, *values): self.assertEqual(misp_object.name, 'registry-key') @@ -542,6 +660,50 @@ def test_stix21_bundle_with_mutex_attributes(self): self._check_generic_attribute(od1, mutex_2, m_mutex2, 'mutex', 'name') self._check_generic_attribute(od2, mutex_3, s_mutex, 'mutex', 'name') + def test_stix21_bundle_with_network_traffic_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_network_traffic_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, od1, od2, ip1, ip2, ip3, nt1, nt2, ip4, ip5, nt3, nt4 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 4) + nt_object1, nt_object2, nt_object3, nt_object4 = misp_objects + self._check_network_traffic_object_with_packet_sizes( + nt_object1, od1, nt1, ip1, ip2, 8 + ) + self.assertEqual(len(nt_object1.references), 1) + encapsulates1 = nt_object1.references[0] + self.assertEqual(encapsulates1.referenced_uuid, nt_object2.uuid) + self._check_network_traffic_object_with_packet_counts( + nt_object2, od1, nt2, ip1, ip3, 9 + ) + self.assertEqual(len(nt_object2.references), 1) + encapsulated1 = nt_object2.references[0] + self.assertEqual(encapsulated1.referenced_uuid, nt_object1.uuid) + self._check_network_traffic_object_with_packet_sizes( + nt_object3, od2, nt3, ip2, ip4, 9 + ) + self.assertEqual(len(nt_object3.references), 1) + encapsulates2 = nt_object3.references[0] + self.assertEqual(encapsulates2.referenced_uuid, nt_object4.uuid) + self._check_network_traffic_object_with_packet_counts( + nt_object4, od2, nt4, ip4, ip5, 10 + ) + self.assertEqual(len(nt_object4.references), 1) + encapsulated2 = nt_object4.references[0] + self.assertEqual(encapsulated2.referenced_uuid, nt_object3.uuid) + self._assert_multiple_equal( + encapsulates1.relationship_type, + encapsulates2.relationship_type, + 'encapsulates' + ) + self._assert_multiple_equal( + encapsulated1.relationship_type, + encapsulated2.relationship_type, + 'encapsulated-by' + ) + def test_stix21_bundle_with_opinion_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_opinion_objects() self.parser.load_stix_bundle(bundle) From 2e8dff827ca53355884f3338933c2aba02f4e744 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 24 Apr 2024 17:39:15 +0200 Subject: [PATCH 117/137] fix: [stix2 import] Fixed Network Traffic Observable objects from internal STIX 2.x content parsing - Added `connection_protocols` mapping to the internal mapping as it was removed from the parent mapping to avoid issues with the external mapping but was supposed to be moved and not completely removed - Added specific parsing for internal network traffic references objects --- .../converters/stix2_observable_converter.py | 10 +++++++--- .../stix2_observed_data_converter.py | 6 ++++-- .../stix2misp/converters/stix2mapping.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 0998d96f..502b746e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -348,8 +348,8 @@ def _parse_network_traffic_observable( def _parse_network_traffic_reference_observable( self, asset: str, observable: _NETWORK_TRAFFIC_REFERENCE_TYPING, - object_id: str) -> Iterator[dict]: - attribute = self._mapping.network_traffic_reference_mapping( + object_id: str, name: Optional[str] = 'network_traffic'): + attribute = getattr(self._mapping, f'{name}_reference_mapping')( f'{observable.type}_{asset}' ) if attribute is not None: @@ -800,13 +800,17 @@ def http_request_extension_mapping(cls) -> dict: def http_request_header_mapping(cls) -> dict: return cls.__http_request_header_mapping + @classmethod + def http_request_reference_mapping(cls, field: str) -> dict: + return cls.network_traffic_references().get(field) + @classmethod def malware_sample_attribute(cls) -> dict: return cls.__malware_sample_attribute @classmethod def network_connection_reference_mapping(cls, field: str) -> dict: - return cls.network_traffic_references.get(field) + return cls.network_traffic_references().get(field) @classmethod def parent_process_object_mapping(cls) -> dict: diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index dbf90160..ee0654c3 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -2917,7 +2917,8 @@ def _object_from_http_request_observable( ] content = self._parse_network_traffic_reference_observable( feature, address, - getattr(address, 'id', observed_data.id) + getattr(address, 'id', observed_data.id), + name='http_request' ) for attribute in content: misp_object.add_attribute(**attribute) @@ -3205,7 +3206,8 @@ def _object_from_network_traffic_observable( if hasattr(observable, f'{asset}_ref'): address = observables[getattr(observable, f'{asset}_ref')] attributes = self._parse_network_traffic_reference_observable( - asset, address, getattr(address, 'id', observed_data.id) + asset, address, getattr(address, 'id', observed_data.id), + name.replace('-', '_') ) for attribute in attributes: misp_object.add_attribute(**attribute) diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index d8636f49..456f8302 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -1154,6 +1154,21 @@ class InternalSTIX2Mapping(STIX2Mapping): x_misp_mp_import={'type': 'text', 'object_relation': 'mp-import'}, x_misp_subnet_announced={'type': 'ip-src', 'object_relation': 'subnet-announced'} ) + __connection_protocols = Mapping( + **{ + **dict.fromkeys(('tcp', 'TCP', 'udp', 'UDP'), '4'), + **dict.fromkeys( + ( + 'arp', 'icmp', 'ip', 'ipv4', 'ipv6', + 'ARP', 'ICMP', 'IP', 'IPV4', 'IPV6' + ), + '3' + ), + **dict.fromkeys( + ('http', 'HTTP', 'https', 'HTTPS', 'ftp', 'FTP'), '7' + ) + } + ) __cpe_asset_object_mapping = Mapping( cpe=STIX2Mapping.cpe_attribute(), languages=STIX2Mapping.language_attribute(), @@ -1574,6 +1589,10 @@ def child_pid_attribute(cls) -> dict: def comment_text_attribute(cls) -> dict: return cls.__comment_text_attribute + @classmethod + def connection_protocols(cls, field: str) -> Union[str, None]: + return cls.__connection_protocols.get(field) + @classmethod def cpe_asset_object_mapping(cls) -> dict: return cls.__cpe_asset_object_mapping From 5899dbb7bdf779609002eef6d80c377d03152ff3 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 25 Apr 2024 22:07:48 +0200 Subject: [PATCH 118/137] fix: [stix2 import] Fixing the internal STIX2 Network Traffic Observable objects and references IDs handling --- .../stix2_observed_data_converter.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index ee0654c3..784c19e1 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -3202,21 +3202,26 @@ def _object_from_network_traffic_observable( observables: dict, observable_id: str) -> MISPObject: misp_object = self._create_misp_object(name, observed_data) observable = observables[observable_id] + attributes = self._parse_generic_observable( + observable, name.replace('-', '_'), getattr( + observable, 'id', f'{observed_data.id} - {observable_id}' + ) + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) for asset in ('src', 'dst'): if hasattr(observable, f'{asset}_ref'): - address = observables[getattr(observable, f'{asset}_ref')] + address_ref = getattr(observable, f'{asset}_ref') + address = observables[address_ref] attributes = self._parse_network_traffic_reference_observable( - asset, address, getattr(address, 'id', observed_data.id), + asset, address, + getattr( + address, 'id', f'{observed_data.id} - {address_ref}' + ), name.replace('-', '_') ) for attribute in attributes: misp_object.add_attribute(**attribute) - attributes = self._parse_generic_observable( - observable, name.replace('-', '_'), - getattr(observable, 'id', observed_data.id) - ) - for attribute in attributes: - misp_object.add_attribute(**attribute) return misp_object def _object_from_parler_account_observable_v20( From 6e66bde4642eab82b95a646e25a09b06a74d0ecf Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 26 Apr 2024 23:29:33 +0200 Subject: [PATCH 119/137] fix: [stix2 import] Better handling of attributes uuid for values converted from internal Network Traffic Observable objects --- .../stix2misp/converters/stix2_observable_converter.py | 2 +- .../stix2misp/converters/stix2_observed_data_converter.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 502b746e..6aa09eed 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -1084,6 +1084,6 @@ def _parse_network_connection_observable( 'object_relation': f'layer{layer}-protocol', 'type': 'text', 'value': protocol.upper(), 'uuid': self.main_parser._create_v5_uuid( - f'{object_id} - layer{layer}-protocol - {protocol}' + f'{object_id} - layer{layer}-protocol - {protocol.upper()}' ) } diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 784c19e1..b861131b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -3157,7 +3157,9 @@ def _object_from_network_connection_observable( observable_id ) attributes = self._parse_network_connection_observable( - observable, getattr(observable, 'id', observed_data.id) + observable, getattr( + observable, 'id', f'{observed_data.id} - {observable_id}' + ) ) for attribute in attributes: misp_object.add_attribute(**attribute) @@ -3183,7 +3185,9 @@ def _object_from_network_socket_observable( 'network-socket', observed_data, observables, observable_id ) attributes = self._parse_network_socket_observable( - observable, getattr(observable, 'id', observed_data.id) + observable, getattr( + observable, 'id', f'{observed_data.id} - {observable_id}' + ) ) for attribute in attributes: misp_object.add_attribute(**attribute) From b0c71a828fcc745e378b4706f30f0b37fbfdbb2b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 29 Apr 2024 07:29:42 +0200 Subject: [PATCH 120/137] fix: [stix2 import] Better internal http-request objects import from Observable objects --- .../stix2_observed_data_converter.py | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index b861131b..18758a3e 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -2895,39 +2895,36 @@ def _object_from_http_request_observable( observables = getattr(self, f'_fetch_observables_with_id_{version}')( observed_data ) - for observable in observables.values(): + for observable_id, observable in observables.items(): + object_id = getattr( + observable, 'id', f'{observed_data.id} - {observable_id}' + ) if observable.type == 'domain-name': attribute = { 'value': observable.value, **self._mapping.host_attribute() } - if hasattr(observable, 'id'): - attribute.update( - self.main_parser._sanitise_attribute_uuid(observable.id) - ) - else: - attribute['uuid'] = self.main_parser._create_v5_uuid( - f'{observed_data.id} - host - {observable.value}' - ) + attribute['uuid'] = self.main_parser._create_v5_uuid( + f'{object_id} - host - {observable.value}' + ) misp_object.add_attribute(**attribute) continue + attributes = self._parse_generic_observable( + observable, 'http_request', object_id + ) + for attribute in attributes: + misp_object.add_attribute(**attribute) for feature in ('src', 'dst'): if hasattr(observable, f'{feature}_ref'): - address = observables[ - getattr(observable, f'{feature}_ref') - ] + address_ref = getattr(observable, f'{feature}_ref') + address = observables[address_ref] content = self._parse_network_traffic_reference_observable( - feature, address, - getattr(address, 'id', observed_data.id), + feature, address, getattr( + address, 'id', f'{observed_data.id} - {address_ref}' + ), name='http_request' ) for attribute in content: misp_object.add_attribute(**attribute) - object_id = getattr(observable, 'id', observed_data.id) - attributes = self._parse_generic_observable( - observable, 'http_request', object_id - ) - for attribute in attributes: - misp_object.add_attribute(**attribute) if getattr(observable, 'extensions', {}).get('http-request-ext'): attributes = self._parse_http_request_extension_observable( observable.extensions['http-request-ext'], object_id From 90771c7e9ee54d2d4743402d0b31b136f91567d3 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 29 Apr 2024 07:31:58 +0200 Subject: [PATCH 121/137] fix: [tests] Better UUID tests for objects imported from STIX 2.x Network Traffic Observable objects --- tests/_test_stix_import.py | 177 ++++++++++++++++++++++++++- tests/test_internal_stix20_import.py | 10 +- tests/test_internal_stix21_import.py | 60 +-------- 3 files changed, 183 insertions(+), 64 deletions(-) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 96a0a183..649ba995 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -2404,35 +2404,81 @@ def _check_http_request_indicator_object(self, attributes, pattern): self.assertEqual(url.object_relation, 'url') self.assertEqual(url.value, self._get_pattern_value(request_url)) - def _check_http_request_observable_object(self, attributes, observables): + def _check_http_request_observable_object( + self, attributes, observables, observed_data_id = None): self.assertEqual(len(attributes), 8) + network_traffic_id, address1_id, address2_id, domain_id = observables.keys() + if observed_data_id is not None: + network_traffic_id = f'{observed_data_id} - {network_traffic_id}' + address1_id = f'{observed_data_id} - {address1_id}' + address2_id = f'{observed_data_id} - {address2_id}' + domain_id = f'{observed_data_id} - {domain_id}' network_traffic, address1, address2, domain_name = observables.values() - ip_src, ip_dst, url, method, uri, content_type, user_agent, host = attributes + url, ip_src, ip_dst, method, uri, content_type, user_agent, host = attributes self.assertEqual(ip_src.type, 'ip-src') self.assertEqual(ip_src.object_relation, 'ip-src') self.assertEqual(ip_src.value, address1.value) + self.assertEqual( + ip_src.uuid, + uuid5(self._UUIDv4, f'{address1_id} - ip-src - {ip_src.value}') + ) self.assertEqual(ip_dst.type, 'ip-dst') self.assertEqual(ip_dst.object_relation, 'ip-dst') self.assertEqual(ip_dst.value, address2.value) + self.assertEqual( + ip_dst.uuid, + uuid5(self._UUIDv4, f'{address2_id} - ip-dst - {ip_dst.value}') + ) self.assertEqual(url.type, 'url') self.assertEqual(url.object_relation, 'url') self.assertEqual(url.value, network_traffic.x_misp_url) + self.assertEqual( + url.uuid, + uuid5(self._UUIDv4, f'{network_traffic_id} - url - {url.value}') + ) extension = network_traffic.extensions['http-request-ext'] self.assertEqual(method.type, 'http-method') self.assertEqual(method.object_relation, 'method') self.assertEqual(method.value, extension.request_method) + self.assertEqual( + method.uuid, + uuid5( + self._UUIDv4, f'{network_traffic_id} - method - {method.value}' + ) + ) self.assertEqual(uri.type, 'uri') self.assertEqual(uri.object_relation, 'uri') self.assertEqual(uri.value, extension.request_value) + self.assertEqual( + uri.uuid, + uuid5(self._UUIDv4, f'{network_traffic_id} - uri - {uri.value}') + ) self.assertEqual(content_type.type, 'other') self.assertEqual(content_type.object_relation, 'content-type') self.assertEqual(content_type.value, extension.request_header['Content-Type']) + self.assertEqual( + content_type.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - content-type - {content_type.value}' + ) + ) self.assertEqual(user_agent.type, 'text') self.assertEqual(user_agent.object_relation, 'user-agent') self.assertEqual(user_agent.value, extension.request_header['User-Agent']) + self.assertEqual( + user_agent.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - user-agent - {user_agent.value}' + ) + ) self.assertEqual(host.type, 'hostname') self.assertEqual(host.object_relation, 'host') self.assertEqual(host.value, domain_name.value) + self.assertEqual( + host.uuid, uuid5(self._UUIDv4, f'{domain_id} - host - {host.value}') + ) def _check_identity_object(self, misp_object, identity): self.assertEqual(misp_object.uuid, identity.id.split('--')[1]) @@ -2794,34 +2840,90 @@ def _check_network_connection_indicator_object(self, attributes, pattern): self.assertEqual(layer7.object_relation, 'layer7-protocol') self.assertEqual(layer7.value, self._get_pattern_value(protocol3).upper()) - def _check_network_connection_observable_object(self, attributes, observables): + def _check_network_connection_observable_object( + self, attributes, observables, observed_data_id = None): self.assertEqual(len(attributes), 8) - ip_src, ip_dst, dst_port, src_port, hostname, layer3, layer4, layer7 = attributes + src_port, dst_port, hostname, ip_src, ip_dst, layer3, layer4, layer7 = attributes + network_traffic_id, address1_id, address2_id = observables.keys() + if observed_data_id is not None: + network_traffic_id = f'{observed_data_id} - {network_traffic_id}' + address1_id = f'{observed_data_id} - {address1_id}' + address2_id = f'{observed_data_id} - {address2_id}' network_traffic, address1, address2 = observables.values() self.assertEqual(ip_src.type, 'ip-src') self.assertEqual(ip_src.object_relation, 'ip-src') self.assertEqual(ip_src.value, address1.value) + self.assertEqual( + ip_src.uuid, + uuid5(self._UUIDv4, f'{address1_id} - ip-src - {ip_src.value}') + ) self.assertEqual(ip_dst.type, 'ip-dst') self.assertEqual(ip_dst.object_relation, 'ip-dst') self.assertEqual(ip_dst.value, address2.value) + self.assertEqual( + ip_dst.uuid, + uuid5(self._UUIDv4, f'{address2_id} - ip-dst - {ip_dst.value}') + ) self.assertEqual(dst_port.type, 'port') self.assertEqual(dst_port.object_relation, 'dst-port') self.assertEqual(dst_port.value, network_traffic.dst_port) + self.assertEqual( + dst_port.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - dst-port - {dst_port.value}' + ) + ) self.assertEqual(src_port.type, 'port') self.assertEqual(src_port.object_relation, 'src-port') self.assertEqual(src_port.value, network_traffic.src_port) + self.assertEqual( + src_port.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - src-port - {src_port.value}' + ) + ) self.assertEqual(hostname.type, 'hostname') self.assertEqual(hostname.object_relation, 'hostname-dst') self.assertEqual(hostname.value, network_traffic.x_misp_hostname_dst) + self.assertEqual( + hostname.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - hostname-dst - {hostname.value}' + ) + ) self.assertEqual(layer3.type, 'text') self.assertEqual(layer3.object_relation, 'layer3-protocol') self.assertEqual(layer3.value, network_traffic.protocols[0].upper()) + self.assertEqual( + layer3.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - layer3-protocol - {layer3.value}' + ) + ) self.assertEqual(layer4.type, 'text') self.assertEqual(layer4.object_relation, 'layer4-protocol') self.assertEqual(layer4.value, network_traffic.protocols[1].upper()) + self.assertEqual( + layer4.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - layer4-protocol - {layer4.value}' + ) + ) self.assertEqual(layer7.type, 'text') self.assertEqual(layer7.object_relation, 'layer7-protocol') self.assertEqual(layer7.value, network_traffic.protocols[2].upper()) + self.assertEqual( + layer7.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - layer7-protocol - {layer7.value}' + ) + ) def _check_network_socket_indicator_object(self, attributes, pattern): self.assertEqual(len(attributes), 10) @@ -2858,38 +2960,101 @@ def _check_network_socket_indicator_object(self, attributes, pattern): self.assertEqual(domain_family.object_relation, 'domain-family') self.assertEqual(domain_family.value, self._get_pattern_value(protocolFamily)) - def _check_network_socket_observable_object(self, attributes, observables): + def _check_network_socket_observable_object( + self, attributes, observables, observed_data_id = None): self.assertEqual(len(attributes), 10 - 1) # 10 expected attributes minus the one tested separately - ip_src, ip_dst, port_dst, port_src, hostname, protocol, address_family, socket_type, listening = attributes + port_src, port_dst, hostname, ip_src, ip_dst, protocol, address_family, socket_type, listening = attributes + network_traffic_id, address1_id, address2_id = observables.keys() + if observed_data_id is not None: + network_traffic_id = f'{observed_data_id} - {network_traffic_id}' + address1_id = f'{observed_data_id} - {address1_id}' + address2_id = f'{observed_data_id} - {address2_id}' network_traffic, address1, address2 = observables.values() self.assertEqual(ip_src.type, 'ip-src') self.assertEqual(ip_src.object_relation, 'ip-src') self.assertEqual(ip_src.value, address1.value) + self.assertEqual( + ip_src.uuid, + uuid5(self._UUIDv4, f'{address1_id} - ip-src - {ip_src.value}') + ) self.assertEqual(ip_dst.type, 'ip-dst') self.assertEqual(ip_dst.object_relation, 'ip-dst') self.assertEqual(ip_dst.value, address2.value) + self.assertEqual( + ip_dst.uuid, + uuid5(self._UUIDv4, f'{address2_id} - ip-dst - {ip_dst.value}') + ) self.assertEqual(port_dst.type, 'port') self.assertEqual(port_dst.object_relation, 'dst-port') self.assertEqual(port_dst.value, network_traffic.dst_port) + self.assertEqual( + port_dst.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - dst-port - {port_dst.value}' + ) + ) self.assertEqual(port_src.type, 'port') self.assertEqual(port_src.object_relation, 'src-port') self.assertEqual(port_src.value, network_traffic.src_port) + self.assertEqual( + port_src.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - src-port - {port_src.value}' + ) + ) self.assertEqual(hostname.type, 'hostname') self.assertEqual(hostname.object_relation, 'hostname-dst') self.assertEqual(hostname.value, network_traffic.x_misp_hostname_dst) + self.assertEqual( + hostname.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - hostname-dst - {hostname.value}' + ) + ) self.assertEqual(protocol.type, 'text') self.assertEqual(protocol.object_relation, 'protocol') self.assertEqual(protocol.value, network_traffic.protocols[0].upper()) + self.assertEqual( + protocol.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - protocol - {protocol.value}' + ) + ) socket_ext = network_traffic.extensions['socket-ext'] self.assertEqual(address_family.type, 'text') self.assertEqual(address_family.object_relation, 'address-family') self.assertEqual(address_family.value, socket_ext.address_family) + self.assertEqual( + address_family.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - address-family - {address_family.value}' + ) + ) self.assertEqual(socket_type.type, 'text') self.assertEqual(socket_type.object_relation, 'socket-type') self.assertEqual(socket_type.value, socket_ext.socket_type) + self.assertEqual( + socket_type.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - socket-type - {socket_type.value}' + ) + ) self.assertEqual(listening.type, 'text') self.assertEqual(listening.object_relation, 'state') self.assertEqual(listening.value, 'listening') + self.assertEqual( + listening.uuid, + uuid5( + self._UUIDv4, + f'{network_traffic_id} - state - {listening.value}' + ) + ) def _check_news_agency_object(self, misp_object, identity): self.assertEqual(misp_object.uuid, identity.id.split('--')[1]) diff --git a/tests/test_internal_stix20_import.py b/tests/test_internal_stix20_import.py index 35e4cd0c..fad2799d 100644 --- a/tests/test_internal_stix20_import.py +++ b/tests/test_internal_stix20_import.py @@ -1845,7 +1845,9 @@ def test_stix20_bundle_with_http_request_observable_object(self): _, report, observed_data = bundle.objects misp_object = self._check_misp_event_features(event, report)[0] observables = self._check_observed_data_object(misp_object, observed_data) - self._check_http_request_observable_object(misp_object.attributes, observables) + self._check_http_request_observable_object( + misp_object.attributes, observables, observed_data.id + ) self._populate_documentation( misp_object = json.loads(misp_object.to_json()), observed_data = observed_data @@ -2070,7 +2072,9 @@ def test_stix20_bundle_with_network_connection_observable_object(self): _, report, observed_data = bundle.objects misp_object = self._check_misp_event_features(event, report)[0] observables = self._check_observed_data_object(misp_object, observed_data) - self._check_network_connection_observable_object(misp_object.attributes, observables) + self._check_network_connection_observable_object( + misp_object.attributes, observables, observed_data.id + ) self._populate_documentation( misp_object = json.loads(misp_object.to_json()), observed_data = observed_data @@ -2115,7 +2119,7 @@ def test_stix20_bundle_with_network_socket_observable_object(self): ip_src, ip_dst, port_dst, port_src, hostname, protocol, address_family, socket_type, listening ), - observables + observables, observed_data.id ) self.assertEqual(domain_family.type, 'text') self.assertEqual(domain_family.object_relation, 'domain-family') diff --git a/tests/test_internal_stix21_import.py b/tests/test_internal_stix21_import.py index c042a7d7..44e72dec 100644 --- a/tests/test_internal_stix21_import.py +++ b/tests/test_internal_stix21_import.py @@ -2305,27 +2305,7 @@ def test_stix21_bundle_with_http_request_observable_object(self): event = self.parser.misp_event _, grouping, observed_data, network_traffic, address1, address2, domain_name = bundle.objects misp_object = self._check_misp_event_features_from_grouping(event, grouping)[0] - network_traffic_ref, address1_ref, address2_ref, domain_name_ref = self._check_observed_data_object(misp_object, observed_data) - self._assert_multiple_equal( - misp_object.uuid, - network_traffic.id.split('--')[1], - network_traffic_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[0].uuid, - address1.id.split('--')[1], - address1_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[1].uuid, - address2.id.split('--')[1], - address2_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[-1].uuid, - domain_name.id.split('--')[1], - domain_name_ref.split('--')[1] - ) + self._check_observed_data_object(misp_object, observed_data) self._check_http_request_observable_object( misp_object.attributes, { @@ -2649,22 +2629,7 @@ def test_stix21_bundle_with_network_connection_observable_object(self): event = self.parser.misp_event _, grouping, observed_data, network_traffic, address1, address2 = bundle.objects misp_object = self._check_misp_event_features_from_grouping(event, grouping)[0] - network_ref, address1_ref, address2_ref = self._check_observed_data_object(misp_object, observed_data) - self._assert_multiple_equal( - misp_object.uuid, - network_traffic.id.split('--')[1], - network_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[0].uuid, - address1.id.split('--')[1], - address1_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[1].uuid, - address2.id.split('--')[1], - address2_ref.split('--')[1] - ) + self._check_observed_data_object(misp_object, observed_data) self._check_network_connection_observable_object( misp_object.attributes, { @@ -2703,25 +2668,10 @@ def test_stix21_bundle_with_network_socket_observable_object(self): event = self.parser.misp_event _, grouping, observed_data, network_traffic, address1, address2 = bundle.objects misp_object = self._check_misp_event_features_from_grouping(event, grouping)[0] - network_ref, address1_ref, address2_ref = self._check_observed_data_object(misp_object, observed_data) - self._assert_multiple_equal( - misp_object.uuid, - network_traffic.id.split('--')[1], - network_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[0].uuid, - address1.id.split('--')[1], - address1_ref.split('--')[1] - ) - self._assert_multiple_equal( - misp_object.attributes[1].uuid, - address2.id.split('--')[1], - address2_ref.split('--')[1] - ) - ip_src, ip_dst, port_dst, port_src, domain_family, *attributes = misp_object.attributes + self._check_observed_data_object(misp_object, observed_data) + port_src, port_dst, domain_family, *attributes = misp_object.attributes self._check_network_socket_observable_object( - (ip_src, ip_dst, port_dst, port_src, *attributes), + (port_src, port_dst, *attributes), { network_traffic.id: network_traffic, address1.id: address1, From ca77e3160d8c1333affde6610c5438068c8ed9ef Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 30 Apr 2024 23:11:20 +0200 Subject: [PATCH 122/137] fix: [stix2 import] Protocols error message made clearer --- misp_stix_converter/stix2misp/importparser.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 070bbd0a..58d9e975 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -357,10 +357,15 @@ def _unknown_marking_ref_warning(self, marking_ref: str): ) def _unknown_network_protocol_warning( - self, protocol: str, indicator_id: str): + self, protocol: str, object_id: str, + object_type: Optional[str] = 'indicator'): + message = ( + 'in patterning expression within the indicator with id' + if object_type == 'indicator' else + f'within the {object_type} object with id' + ) self.__warnings[self._identifier].add( - f'Unknown network protocol: {protocol} in the patterning ' - f'expression describing the Indicator with id {indicator_id}' + f'Unknown network protocol: {protocol}, {message} {object_id}' ) def _unknown_object_name_warning(self, name: str): From ba586923a6981da534bd5c715573f9c6cf7e0f66 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 28 May 2024 12:08:11 +0200 Subject: [PATCH 123/137] fix: [stix2 import] Fixed `_observable` variable name --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 18758a3e..c150b468 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -159,8 +159,8 @@ def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): def _extract_referenced_ids_from_observable_object_refs(self): self.__referenced_ids = defaultdict(set) - for object_id, observable_object in self._observable.items(): - for key, value in observable_object['observable'].items(): + for object_id, observable in self.main_parser._observable.items(): + for key, value in observable['observable'].items(): if key.endswith('_ref'): self.referenced_ids[value].add(object_id) if key.endswith('_refs'): From b5aa37ee0b1264aa1a90200d3d903bba53ae52db Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 28 May 2024 13:43:31 +0200 Subject: [PATCH 124/137] fix: [stix2 import] Fixed `domain-ip` object attributes handling as `_sanitise_attribute_uuid` already returns a dict with the `uuid` key included --- .../stix2misp/converters/stix2_observed_data_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index c150b468..3c2e8fdf 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -614,7 +614,7 @@ def _parse_domain_ip_observable_object_refs( ) domain_object.add_attribute( 'domain', domain.value, - uuid=self.main_parser._sanitise_attribute_uuid(domain.id) + **self.main_parser._sanitise_attribute_uuid(domain.id) ) misp_object = self.main_parser._add_misp_object( domain_object, observed_data @@ -630,7 +630,7 @@ def _parse_domain_ip_observable_object_refs( else 'ip' ), resolved_object.value, - uuid=self.main_parser._sanitise_attribute_uuid( + **self.main_parser._sanitise_attribute_uuid( resolved_ref ) ) From 16b2596604b831b5b437e63f8bf64e158d1631bf Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 29 May 2024 17:42:31 +0200 Subject: [PATCH 125/137] fix: [stix2 import] Fixed `domain-ip` attributes UUIDs handling --- .../stix2_observed_data_converter.py | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 3c2e8fdf..9be58995 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -614,7 +614,9 @@ def _parse_domain_ip_observable_object_refs( ) domain_object.add_attribute( 'domain', domain.value, - **self.main_parser._sanitise_attribute_uuid(domain.id) + uuid=self.main_parser._create_v5_uuid( + f'{domain.id} - domain - {domain.value}' + ) ) misp_object = self.main_parser._add_misp_object( domain_object, observed_data @@ -624,14 +626,16 @@ def _parse_domain_ip_observable_object_refs( for resolved_ref in domain.resolves_to_refs: resolved_observable = self._fetch_observable(resolved_ref) resolved_object= resolved_observable['observable'] + object_relation = ( + 'domain' if resolved_object.type == 'domain-name' + else 'ip' + ) + value = resolved_object.value misp_object.add_attribute( - ( - 'domain' if resolved_object.type == 'domain-name' - else 'ip' - ), - resolved_object.value, - **self.main_parser._sanitise_attribute_uuid( - resolved_ref + object_relation, value, + uuid=self.main_parser._create_v5_uuid( + f'{domain.id} - {resolved_ref} - ' + f'{object_relation} - {value}' ) ) resolved_observable['used'][self.event_uuid] = True @@ -653,7 +657,7 @@ def _parse_domain_ip_observable_objects( referenced_ids = self._extract_referenced_ids_from_observable_objects( **observed_data.objects ) - for identifier, observable_object in observed_data.objects: + for identifier, observable_object in observed_data.objects.items(): if identifier in referenced_ids: continue if hasattr(observable_object, 'resolves_to_refs'): @@ -665,18 +669,21 @@ def _parse_domain_ip_observable_objects( ) misp_object.add_attribute( 'domain', observable_object.value, - uuid=self.main_parser._create_v5_uuid(object_id) + uuid=self.main_parser._create_v5_uuid( + f'{object_id} - domain - {observable_object.value}' + ) ) for resolved_ref in observable_object.resolves_to_refs: resolved_object = observed_data.objects[resolved_ref] + object_relation = ( + 'domain' if resolved_object.type == 'domain-name' + else 'ip' + ) misp_object.add_attribute( - ( - 'domain' if resolved_object.type == 'domain-name' - else 'ip' - ), - resolved_object.value, + object_relation, resolved_object.value, uuid=self.main_parser._create_v5_uuid( - f'{observed_data.id} - {resolved_ref}' + f'{object_id} - {resolved_ref} - ' + f'{object_relation} - {resolved_object.value}' ) ) self.main_parser._add_misp_object(misp_object, observed_data) From 42eaa9f3b713ca28ed2145641e0f4646069f2864 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 29 May 2024 17:44:14 +0200 Subject: [PATCH 126/137] wip: [tests] Tests for `domain-ip` objects import from external STIX 2.x --- tests/_test_stix_import.py | 32 +++++++++++++++++++ tests/test_external_stix20_bundles.py | 34 +++++++++++++++++++++ tests/test_external_stix20_import.py | 19 ++++++++++++ tests/test_external_stix21_bundles.py | 44 +++++++++++++++++++++++++++ tests/test_external_stix21_import.py | 20 ++++++++++++ 5 files changed, 149 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 649ba995..6c325388 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -609,6 +609,38 @@ def _check_directory_fields(self, misp_object, directory, object_id): ) return accessed.value, created.value, modified.value + def _check_domain_ip_fields(self, misp_object, domain, ipv4, ipv6, *object_ids): + self.assertEqual(len(misp_object.attributes), 3) + domain_attribute, ipv4_attribute, ipv6_attribute = misp_object.attributes + domain_id, ipv4_id, ipv6_id = object_ids + self._assert_multiple_equal( + domain_attribute.type, domain_attribute.object_relation, 'domain' + ) + self.assertEqual(domain_attribute.value, domain.value) + self.assertEqual( + domain_attribute.uuid, + uuid5( + self._UUIDv4, + f'{domain_id} - domain - {domain_attribute.value}' + ) + ) + self._assert_multiple_equal( + ipv4_attribute.type, ipv6_attribute.type, 'ip-dst' + ) + self._assert_multiple_equal( + ipv4_attribute.object_relation, ipv6_attribute.object_relation, 'ip' + ) + self.assertEqual(ipv4_attribute.value, ipv4.value) + self.assertEqual( + ipv4_attribute.uuid, + uuid5(self._UUIDv4, f'{ipv4_id} - ip - {ipv4_attribute.value}') + ) + self.assertEqual(ipv6_attribute.value, ipv6.value) + self.assertEqual( + ipv6_attribute.uuid, + uuid5(self._UUIDv4, f'{ipv6_id} - ip - {ipv6_attribute.value}') + ) + def _check_file_fields(self, misp_object, observable_object, object_id): self.assertEqual(len(misp_object.attributes), 6) md5, sha1, sha256, filename, encoding, size = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index fc9a2192..3c95e5b2 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -314,6 +314,36 @@ } } ] +_DOMAIN_IP_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "domain-name", + "value": "example.com", + "resolves_to_refs": [ + "1", + "2" + ] + }, + "1": { + "type": "ipv4-addr", + "value": "198.51.100.3" + }, + "2": { + "type": "ipv6-addr", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + } + } + } +] _EMAIL_ADDRESS_ATTRIBUTES = [ { "type": "observed-data", @@ -1482,6 +1512,10 @@ def get_bundle_with_directory_objects(cls): def get_bundle_with_domain_attributes(cls): return cls.__assemble_bundle(*_DOMAIN_ATTRIBUTES) + @classmethod + def get_bundle_with_domain_ip_objects(cls): + return cls.__assemble_bundle(*_DOMAIN_IP_OBJECTS) + @classmethod def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 4c9d972e..40299d4e 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -603,6 +603,25 @@ def test_stix20_bundle_with_domain_attributes(self): observed_data2, s_domain, 'domain' ) + def test_stix20_bundle_with_domain_ip_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_domain_ip_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data = bundle.objects + misp_object = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_object), 1) + domain_ip_object = misp_object[0] + self.assertEqual(domain_ip_object.name, 'domain-ip') + self._check_misp_object_fields( + domain_ip_object, observed_data, '0 - 1 - 2' + ) + object_id = f'{observed_data.id} - 0' + self._check_domain_ip_fields( + domain_ip_object, *observed_data.objects.values(), + object_id, f'{object_id} - 1', f'{object_id} - 2' + ) + def test_stix20_bundle_with_email_address_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_email_address_attributes() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index d44900c9..efc1781a 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -371,6 +371,46 @@ "value": "misp-project.org" } ] +_DOMAIN_IP_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5", + "ipv4-addr--ff26c055-6336-5bc5-b98d-13d6226742dd", + "ipv6-addr--1e61d36c-a16c-53b7-a80f-2a00161c96b1" + ] + }, + { + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5", + "value": "example.com", + "resolves_to_refs": [ + "ipv4-addr--ff26c055-6336-5bc5-b98d-13d6226742dd", + "ipv6-addr--1e61d36c-a16c-53b7-a80f-2a00161c96b1" + ] + }, + { + "type": "ipv4-addr", + "spec_version": "2.1", + "id": "ipv4-addr--ff26c055-6336-5bc5-b98d-13d6226742dd", + "value": "198.51.100.3" + }, + { + "type": "ipv6-addr", + "spec_version": "2.1", + "id": "ipv6-addr--1e61d36c-a16c-53b7-a80f-2a00161c96b1", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + } +] _EMAIL_ADDRESS_ATTRIBUTES = [ { "type": "observed-data", @@ -1729,6 +1769,10 @@ def get_bundle_with_directory_objects(cls): def get_bundle_with_domain_attributes(cls): return cls.__assemble_bundle(*_DOMAIN_ATTRIBUTES) + @classmethod + def get_bundle_with_domain_ip_objects(cls): + return cls.__assemble_bundle(*_DOMAIN_IP_OBJECTS) + @classmethod def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 61e2cfb3..669c0042 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -561,6 +561,26 @@ def test_stix21_bundle_with_domain_attributes(self): self._check_generic_attribute(od1, domain_2, m_domain2, 'domain') self._check_generic_attribute(od2, domain_3, s_domain, 'domain') + def test_stix21_bundle_with_domain_ip_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_domain_ip_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, observed_data, domain, ipv4, ipv6 = bundle.objects + misp_object = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_object), 1) + domain_ip_object = misp_object[0] + self.assertEqual(domain_ip_object.name, 'domain-ip') + self._check_misp_object_fields( + domain_ip_object, observed_data, + f'{domain.id} - {ipv4.id} - {ipv6.id}', + multiple=True + ) + self._check_domain_ip_fields( + domain_ip_object, domain, ipv4, ipv6, + domain.id, f'{domain.id} - {ipv4.id}', f'{domain.id} - {ipv6.id}' + ) + def test_stix21_bundle_with_email_address_attributes(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_email_address_attributes() self.parser.load_stix_bundle(bundle) From d2e8da620a12d3ed2cbb6143b8f1b4f3dca9dbbb Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 30 May 2024 23:50:13 +0200 Subject: [PATCH 127/137] fix: [stix2 import] Avoiding `domain-name` observable objects to be skipped because they're referenced by another domain-name object --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 9be58995..8a303ada 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -590,7 +590,7 @@ def _parse_directory_observable_objects( def _parse_domain_ip_observable_object_refs( self, observed_data: ObservedData_v21, *object_refs: tuple): for object_ref in object_refs or observed_data.object_refs: - if object_ref in self.referenced_ids: + if not object_ref.startswith('domain-name--'): continue observable = self._fetch_observable(object_ref) domain = observable['observable'] From bf85ab8e68992b328cebf6ba599a7137fbe32049 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 31 May 2024 23:52:20 +0200 Subject: [PATCH 128/137] fix: [stix2 import] Removed unnecessary intermediary method --- .../stix2misp/converters/stix2_observed_data_converter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 8a303ada..f87348c6 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -76,7 +76,7 @@ def __init__(self, main: 'ExternalSTIX2toMISPParser'): @property def observable_relationships(self): if not hasattr(self, '_observable_relationships'): - self._set_observable_relationships() + self._observable_relationships = defaultdict(set) return self._observable_relationships @property @@ -110,9 +110,6 @@ def parse_relationships(self): relationship_type=relationship_type ) - def _set_observable_relationships(self): - self._observable_relationships = defaultdict(set) - ############################################################################ # GENERIC OBSERVED DATA HANDLING METHODS # ############################################################################ From a9fb5a23b37b2962c3e2f7d11f5d5774663dcb1e Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 31 May 2024 23:53:40 +0200 Subject: [PATCH 129/137] fix: [stix2 import] Handling domains resolving other domains with object references --- .../stix2_observed_data_converter.py | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index f87348c6..890ea457 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -621,22 +621,36 @@ def _parse_domain_ip_observable_object_refs( observable['used'][self.event_uuid] = True observable['misp_object'] = misp_object for resolved_ref in domain.resolves_to_refs: - resolved_observable = self._fetch_observable(resolved_ref) - resolved_object= resolved_observable['observable'] - object_relation = ( - 'domain' if resolved_object.type == 'domain-name' - else 'ip' - ) - value = resolved_object.value + resolved = self._fetch_observable(resolved_ref) + resolved_observable = resolved['observable'] + if resolved_observable.type == 'domain-name': + if resolved['used'].get(self.event_uuid, False): + resolved_object = ( + resolved['misp_object'] + if resolved.get('misp_object') is not None + else resolved['misp_attribute'] + ) + misp_object.add_reference( + resolved_object.uuid, 'resolves-to' + ) + continue + value = resolved_observable.value misp_object.add_attribute( - object_relation, value, - uuid=self.main_parser._create_v5_uuid( - f'{domain.id} - {resolved_ref} - ' - f'{object_relation} - {value}' + 'ip', value, uuid=self.main_parser._create_v5_uuid( + f'{domain.id} - {resolved_ref} - ip - {value}' ) ) - resolved_observable['used'][self.event_uuid] = True - resolved_observable['misp_object'] = misp_object + resolved['used'][self.event_uuid] = True + resolved['misp_object'] = misp_object + if object_ref in self.referenced_ids: + for referencing_id in self.referenced_ids[object_ref]: + referencing = self._fetch_observable(referencing_id) + if referencing['observable'].type != 'domain-name': + continue + if referencing['used'].get(self.event_uuid, False): + referencing['misp_object'].add_reference( + misp_object.uuid, 'resolves-to' + ) continue if observable['used'].get(self.event_uuid, False): self._handle_misp_object_fields( @@ -699,6 +713,11 @@ def _parse_domain_observable_object_refs( ) continue domain = observable['observable'] + if hasattr(domain, 'resolves_to_refs'): + self._parse_domain_ip_observable_object_refs( + observed_data, object_ref + ) + continue attribute = self._parse_generic_observable_object_ref_as_attribute( domain, observed_data, 'domain' ) From d96628dee53cbff4434bd0b41ca4bbcb5a6fd1d8 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Sat, 1 Jun 2024 13:19:18 +0200 Subject: [PATCH 130/137] fix: [stix2 import] Fixed `domain-ip` objects UUID handling --- .../converters/stix2_observed_data_converter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 890ea457..0f6a75fc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -607,7 +607,16 @@ def _parse_domain_ip_observable_object_refs( domain_object.uuid ) domain_object.uuid = self.main_parser._create_v5_uuid( - ' - '.join((domain.id, *domain.resolves_to_refs)) + ' - '.join( + ( + domain.id, + *( + resolved_id for resolved_id + in domain.resolves_to_refs + if not resolved_id.startswith('domain-name--') + ) + ) + ) ) domain_object.add_attribute( 'domain', domain.value, From df98f421c226a56f1ccabd00263e4b20563c0224 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 4 Jun 2024 11:35:22 +0200 Subject: [PATCH 131/137] chg: [tests] Updated tests for `domain-ip` objects import from STIX 2.1 to cover specific cases with UUIDs handling --- tests/test_external_stix21_bundles.py | 23 +++++++++++++++++ tests/test_external_stix21_import.py | 36 ++++++++++++++++++++------- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index efc1781a..c93ced72 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -372,6 +372,20 @@ } ] _DOMAIN_IP_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--1a165e68-ea72-44e6-b821-3b88f2cc46d8", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-10-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "domain-name--cd890f31-5825-4fea-85ca-0b3ab3872926" + ] + }, { "type": "observed-data", "spec_version": "2.1", @@ -409,6 +423,15 @@ "spec_version": "2.1", "id": "ipv6-addr--1e61d36c-a16c-53b7-a80f-2a00161c96b1", "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + }, + { + "type": "domain-name", + "spec_version": "2.1", + "id": "domain-name--cd890f31-5825-4fea-85ca-0b3ab3872926", + "value": "blog.example.com", + "resolves_to_refs": [ + "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5" + ] } ] _EMAIL_ADDRESS_ATTRIBUTES = [ diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 669c0042..3faa619d 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -566,19 +566,37 @@ def test_stix21_bundle_with_domain_ip_objects(self): self.parser.load_stix_bundle(bundle) self.parser.parse_stix_bundle() event = self.parser.misp_event - _, grouping, observed_data, domain, ipv4, ipv6 = bundle.objects - misp_object = self._check_misp_event_features_from_grouping(event, grouping) - self.assertEqual(len(misp_object), 1) - domain_ip_object = misp_object[0] - self.assertEqual(domain_ip_object.name, 'domain-ip') + _, grouping, od1, od2, domain1, ipv4, ipv6, domain2 = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 2) + domain_object, domain_ip_object = misp_objects + self._assert_multiple_equal( + domain_object.name, domain_ip_object.name, 'domain-ip' + ) + self._check_misp_object_fields( + domain_object, od1, domain2.id, multiple=True + ) + self.assertEqual(len(domain_object.attributes), 1) + domain_attribute = domain_object.attributes[0] + self._assert_multiple_equal( + domain_attribute.type, domain_attribute.object_relation, 'domain' + ) + self.assertEqual(domain_attribute.value, domain2.value) + self.assertEqual( + domain_attribute.uuid, + uuid5(self._UUIDv4, f'{domain2.id} - domain - {domain2.value}') + ) + self.assertEqual(len(domain_object.references), 1) + reference = domain_object.references[0] + self.assertEqual(reference.referenced_uuid, domain_ip_object.uuid) + self.assertEqual(reference.relationship_type, 'resolves-to') self._check_misp_object_fields( - domain_ip_object, observed_data, - f'{domain.id} - {ipv4.id} - {ipv6.id}', + domain_ip_object, od2, f'{domain1.id} - {ipv4.id} - {ipv6.id}', multiple=True ) self._check_domain_ip_fields( - domain_ip_object, domain, ipv4, ipv6, - domain.id, f'{domain.id} - {ipv4.id}', f'{domain.id} - {ipv6.id}' + domain_ip_object, domain1, ipv4, ipv6, + domain1.id, f'{domain1.id} - {ipv4.id}', f'{domain1.id} - {ipv6.id}' ) def test_stix21_bundle_with_email_address_attributes(self): From 34a40b7d00f8314a7f5c5620d304399d17d50026 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 5 Jun 2024 17:05:31 +0200 Subject: [PATCH 132/137] add: [stix2 import] Updated the STIX 2.x Email objects mappings --- .../converters/stix2_observable_converter.py | 3 ++- .../stix2misp/converters/stix2mapping.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 6aa09eed..2c73d588 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -120,7 +120,8 @@ class STIX2ObservableMapping(STIX2Mapping, metaclass=ABCMeta): __email_additional_header_fields_mapping = Mapping( **{ 'Reply-To': STIX2Mapping.reply_to_attribute(), - 'X-Mailer': STIX2Mapping.x_mailer_attribute() + 'X-Mailer': STIX2Mapping.x_mailer_attribute(), + 'X-Originating-IP': STIX2Mapping.received_header_ip_attribute(), } ) __dst_ip_attribute = {'type': 'ip-dst', 'object_relation': 'dst_ip'} diff --git a/misp_stix_converter/stix2misp/converters/stix2mapping.py b/misp_stix_converter/stix2misp/converters/stix2mapping.py index 456f8302..594342de 100644 --- a/misp_stix_converter/stix2misp/converters/stix2mapping.py +++ b/misp_stix_converter/stix2misp/converters/stix2mapping.py @@ -116,6 +116,9 @@ class STIX2Mapping: 'object_relation': 'from-display-name' } ) + __header_attribute = Mapping( + **{'type': 'email-header', 'object_relation': 'header'} + ) __hidden_attribute = Mapping( **{'type': 'boolean', 'object_relation': 'hidden'} ) @@ -176,6 +179,9 @@ class STIX2Mapping: __protocol_attribute = Mapping( **{'type': 'text', 'object_relation': 'protocol'} ) + __received_header_ip_attribute = Mapping( + **{'type': 'ip-src', 'object_relation': 'received-header-ip'} + ) __reference_attribute = Mapping( **{'type': 'link', 'object_relation': 'reference'} ) @@ -303,6 +309,7 @@ class STIX2Mapping: body=__email_body_attribute, date=__send_date_attribute, message_id=__message_id_attribute, + received_lines=__header_attribute, subject=__email_subject_attribute ) __file_hashes = Mapping( @@ -557,6 +564,10 @@ def from_attribute(cls) -> dict: def from_display_name_attribute(cls) -> dict: return cls.__from_display_name_attribute + @classmethod + def header_attribute(cls) -> dict: + return cls.__header_attribute + @classmethod def hidden_attribute(cls) -> dict: return cls.__hidden_attribute @@ -657,6 +668,10 @@ def pid_attribute(cls) -> dict: def protocol_attribute(cls) -> dict: return cls.__protocol_attribute + @classmethod + def received_header_ip_attribute(cls) -> dict: + return cls.__received_header_ip_attribute + @classmethod def references_attribute(cls) -> dict: return cls.__references_attribute From 7eedb76c389c501c75610d515bc8f2b9a2fb9b7c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 5 Jun 2024 23:17:11 +0200 Subject: [PATCH 133/137] fix: [stix2 import] Fixed UUID handling for `email` object attributes parsed from `email-message` references --- .../stix2misp/converters/stix2_observed_data_converter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 0f6a75fc..0e6b75e4 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -1045,9 +1045,10 @@ def _parse_email_message_observable_objects( ) object_id = f'{observed_data.id} - {identifier}' if hasattr(observable, 'from_ref'): + from_ref = observable.from_ref attributes = self._parse_email_reference_observable( - observed_data.objects[observable.from_ref], - 'from', object_id + observed_data.objects[from_ref], + 'from', f'{object_id} - {from_ref}' ) for attribute in attributes: misp_object.add_attribute(**attribute) @@ -1057,7 +1058,7 @@ def _parse_email_message_observable_objects( for reference in getattr(observable, field): attributes = self._parse_email_reference_observable( observed_data.objects[reference], - feature, object_id + feature, f'{object_id} - {reference}' ) for attribute in attributes: misp_object.add_attribute(**attribute) From 71b80a81cb63b704851b5d04ce354e46b6058315 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 6 Jun 2024 10:02:24 +0200 Subject: [PATCH 134/137] add: [tests] Tests for Email Message objects - and references - import from STIX 2.x --- tests/_test_stix_import.py | 174 ++++++++++++++++++++++++++ tests/test_external_stix20_bundles.py | 85 +++++++++++++ tests/test_external_stix20_import.py | 51 ++++++++ tests/test_external_stix21_bundles.py | 103 +++++++++++++++ tests/test_external_stix21_import.py | 39 ++++++ 5 files changed, 452 insertions(+) diff --git a/tests/_test_stix_import.py b/tests/_test_stix_import.py index 6c325388..750fab10 100644 --- a/tests/_test_stix_import.py +++ b/tests/_test_stix_import.py @@ -641,6 +641,180 @@ def _check_domain_ip_fields(self, misp_object, domain, ipv4, ipv6, *object_ids): uuid5(self._UUIDv4, f'{ipv6_id} - ip - {ipv6_attribute.value}') ) + def _check_email_artifact_object_fields(self, misp_object, artifact, artifact_id): + self.assertEqual(len(misp_object.attributes), 3) + payload_bin, sha256, mime_type = misp_object.attributes + self.assertEqual(payload_bin.type, 'attachment') + self.assertEqual(payload_bin.object_relation, 'payload_bin') + self.assertEqual( + payload_bin.value, artifact_id.split('--')[1].split(' - ')[0] + ) + self.assertEqual( + self._get_data_value(payload_bin.data), artifact.payload_bin + ) + self.assertEqual( + payload_bin.uuid, + uuid5( + self._UUIDv4, + f'{artifact_id} - payload_bin - {payload_bin.value}' + ) + ) + self._assert_multiple_equal( + sha256.type, sha256.object_relation, 'sha256' + ) + self.assertEqual(sha256.value, artifact.hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{artifact_id} - sha256 - {sha256.value}') + ) + self.assertEqual(mime_type.type, 'mime-type') + self.assertEqual(mime_type.object_relation, 'mime_type') + self.assertEqual(mime_type.value, artifact.mime_type) + self.assertEqual( + mime_type.uuid, + uuid5( + self._UUIDv4, f'{artifact_id} - mime_type - {mime_type.value}' + ) + ) + + def _check_email_file_object_fields(self, misp_object, _file, file_id): + self.assertEqual(len(misp_object.attributes), 2) + sha256, filename = misp_object.attributes + self._assert_multiple_equal( + sha256.type, sha256.object_relation, 'sha256' + ) + self.assertEqual(sha256.value, _file.hashes['SHA-256']) + self.assertEqual( + sha256.uuid, + uuid5(self._UUIDv4, f'{file_id} - sha256 - {sha256.value}') + ) + self._assert_multiple_equal( + filename.type, filename.object_relation, 'filename' + ) + self.assertEqual(filename.value, _file.name) + self.assertEqual( + filename.uuid, + uuid5(self._UUIDv4, f'{file_id} - filename - {filename.value}') + ) + + def _check_email_object_fields( + self, misp_object, email_message, from_address, to_address, cc_address, + email_message_id, from_address_id, to_address_id, cc_address_id): + self.assertEqual(len(misp_object.attributes), 12) + send_date, header, subject, x_mailer, received_ip, *addresses, body = misp_object.attributes + self.assertEqual(send_date.type, 'datetime') + self.assertEqual(send_date.object_relation, 'send-date') + self.assertEqual(send_date.value, email_message.date) + self.assertEqual( + send_date.uuid, + uuid5( + self._UUIDv4, + f'{email_message_id} - send-date - {send_date.value}' + ) + ) + self.assertEqual(header.type, 'email-header') + self.assertEqual(header.object_relation, 'header') + self.assertEqual(header.value, email_message.received_lines[0]) + self.assertEqual( + header.uuid, + uuid5(self._UUIDv4, f'{email_message_id} - header - {header.value}') + ) + self.assertEqual(subject.type, 'email-subject') + self.assertEqual(subject.object_relation, 'subject') + self.assertEqual(subject.value, email_message.subject) + self.assertEqual( + subject.uuid, + uuid5( + self._UUIDv4, f'{email_message_id} - subject - {subject.value}' + ) + ) + additional_header_fields = email_message.additional_header_fields + self.assertEqual(x_mailer.type, 'email-x-mailer') + self.assertEqual(x_mailer.object_relation, 'x-mailer') + self.assertEqual(x_mailer.value, additional_header_fields.get('X-Mailer')) + self.assertEqual( + x_mailer.uuid, + uuid5( + self._UUIDv4, + f'{email_message_id} - x-mailer - {x_mailer.value}' + ) + ) + self.assertEqual(received_ip.type, 'ip-src') + self.assertEqual(received_ip.object_relation, 'received-header-ip') + self.assertEqual( + received_ip.value, additional_header_fields['X-Originating-IP'] + ) + self.assertEqual( + received_ip.uuid, + uuid5( + self._UUIDv4, + f'{email_message_id} - received-header-ip - {received_ip.value}' + ) + ) + _from, from_name, _to, to_name, _cc, cc_name = addresses + self.assertEqual(_from.type, 'email-src') + self.assertEqual(_from.object_relation, 'from') + self.assertEqual(_from.value, from_address.value) + self.assertEqual( + _from.uuid, + uuid5(self._UUIDv4, f'{from_address_id} - from - {_from.value}') + ) + self.assertEqual(from_name.type, 'email-src-display-name') + self.assertEqual(from_name.object_relation, 'from-display-name') + self.assertEqual(from_name.value, from_address.display_name) + self.assertEqual( + from_name.uuid, + uuid5( + self._UUIDv4, + f'{from_address_id} - from-display-name - {from_name.value}' + ) + ) + self._assert_multiple_equal(_to.type, _cc.type, 'email-dst') + self.assertEqual(_to.object_relation, 'to') + self.assertEqual(_to.value, to_address.value) + self.assertEqual( + _to.uuid, + uuid5(self._UUIDv4, f'{to_address_id} - to - {_to.value}') + ) + self._assert_multiple_equal( + to_name.type, cc_name.type, 'email-dst-display-name' + ) + self.assertEqual(to_name.object_relation, 'to-display-name') + self.assertEqual(to_name.value, to_address.display_name) + self.assertEqual( + to_name.uuid, + uuid5( + self._UUIDv4, + f'{to_address_id} - to-display-name - {to_name.value}' + ) + ) + self.assertEqual(_cc.object_relation, 'cc') + self.assertEqual(_cc.value, cc_address.value) + self.assertEqual( + _cc.uuid, uuid5(self._UUIDv4, f'{cc_address_id} - cc - {_cc.value}') + ) + self.assertEqual(cc_name.object_relation, 'cc-display-name') + self.assertEqual(cc_name.value, cc_address.display_name) + self.assertEqual( + cc_name.uuid, + uuid5( + self._UUIDv4, + f'{cc_address_id} - cc-display-name - {cc_name.value}' + ) + ) + self._assert_multiple_equal( + body.type, body.object_relation, 'email-body' + ) + self.assertEqual(body.value, email_message.body_multipart[0].body) + self.assertEqual( + body.uuid, + uuid5( + self._UUIDv4, + f'{email_message_id} - body_multipart - 0 - ' + f'email-body - {body.value}' + ) + ) + def _check_file_fields(self, misp_object, observable_object, object_id): self.assertEqual(len(misp_object.attributes), 6) md5, sha1, sha256, filename, encoding, size = misp_object.attributes diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 3c95e5b2..9e0489fc 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -400,6 +400,86 @@ } } ] +_EMAIL_MESSAGE_OBJECTS = [ + { + "type": "observed-data", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "objects": { + "0": { + "type": "email-message", + "is_multipart": True, + "received_lines": [ + "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)" + ], + "content_type": "multipart/mixed", + "date": "2016-06-19T14:20:40.000Z", + "from_ref": "1", + "to_refs": ["2"], + "cc_refs": ["3"], + "subject": "Check out this picture of a cat!", + "additional_header_fields": { + "Content-Disposition": "inline", + "X-Mailer": "Mutt/1.5.23", + "X-Originating-IP": "198.51.100.3" + }, + "body_multipart": [ + { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!" + }, + { + "content_type": "image/png", + "content_disposition": "attachment; filename=\"tabby.png\"", + "body_raw_ref": "4" + }, + { + "content_type": "application/zip", + "content_disposition": "attachment; filename=\"tabby_pics.zip\"", + "body_raw_ref": "5" + } + ] + }, + "1": { + "type": "email-addr", + "value": "jdoe@example.com", + "display_name": "John Doe" + }, + "2": { + "type": "email-addr", + "value": "bob@example.com", + "display_name": "Bob Smith" + }, + "3": { + "type": "email-addr", + "value": "mary@example.com", + "display_name": "Mary Smith" + }, + "4": { + "type": "artifact", + "mime_type": "image/jpeg", + "payload_bin": "iVBORw0KGgoAAAANSUhEUgAAADQAAAAkCAYAAADGrhlwAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSIQQIICAl9CaISAkgJYQWQHoRbIQkQCgxBoKKHV1UcO0iAjZ0VUSxA2JH7CyKvS8WFJR1sWBX3qSArvvK9+b75s5//znznzPnztx7BwD6CZ5EkoNqApArzpfGhgQwxySnMEldgADogAyGAhseP0/Cjo6OALAMtH8v724ARN5edZRr/bP/vxYtgTCPDwASDXGaII+fC/EBAPAqvkSaDwBRzltMyZfIMaxARwoDhHihHGcocZUcpynxHoVNfCwH4hYAyOo8njQDAI3LkGcW8DOghkYvxM5igUgMAJ0JsW9u7iQBxKkQ20IbCcRyfVbaDzoZf9NMG9Tk8TIGsXIuikIOFOVJcnjT/s90/O+SmyMb8GENq3qmNDRWPmeYt1vZk8LlWB3iHnFaZBTE2hB/EAkU9hCj1ExZaILSHjXi53FgzoAexM4CXmA4xEYQB4tzIiNUfFq6KJgLMVwh6FRRPjceYn2IFwrzguJUNhulk2JVvtD6dCmHreLP8aQKv3JfD2TZCWyV/utMIVelj2kUZsYnQUyF2LJAlBgJsQbETnnZceEqm1GFmZzIARupLFYevyXEsUJxSIBSHytIlwbHquxLcvMG5ottzBRxI1V4X35mfKgyP1gLn6eIH84FuywUsxMGdIR5YyIG5iIQBgYp5451CcUJcSqdD5L8gFjlWJwqyYlW2ePmwpwQOW8OsWteQZxqLJ6YDxekUh9Pl+RHxyvjxAuzeGHRynjwZSACcEAgYAIZrGlgEsgCoraehh54p+wJBjwgBRlACBxVzMCIJEWPGF7jQCH4EyIhyBscF6DoFYICyH8dZJVXR5Cu6C1QjMgGTyHOBeEgB97LFKPEg94SwRPIiP7hnQcrH8abA6u8/9/zA+x3hg2ZCBUjG/DIpA9YEoOIgcRQYjDRDjfEfXFvPAJe/WF1wVm458A8vtsTnhLaCY8I1wkdhNsTRUXSn6IcDTqgfrAqF2k/5gK3hppueADuA9WhMq6HGwJH3BX6YeN+0LMbZDmquOVZYf6k/bcZ/PA0VHYUZwpKGULxp9j+PFLDXsNtUEWe6x/zo4w1bTDfnMGen/1zfsi+ALbhP1tiC7H92FnsJHYeO4I1ACZ2HGvEWrGjcjy4up4oVteAt1hFPNlQR/QPfwNPVp7JPOda527nL8q+fOFU+TsacCZJpklFGZn5TDb8IgiZXDHfaRjTxdnFFQD590X5+noTo/huIHqt37l5fwDgc7y/v//wdy7sOAB7PeD2P/Sds2XBT4caAOcO8WXSAiWHyy8E+Jagw51mAEyABbCF83EB7sAb+IMgEAaiQDxIBhNg9JlwnUvBFDADzAXFoBQsA6tBBdgANoPtYBfYBxrAEXASnAEXwWVwHdyFq6cTvAC94B34jCAICaEhDMQAMUWsEAfEBWEhvkgQEoHEIslIKpKBiBEZMgOZh5QiK5AKZBNSg+xFDiEnkfNIO3IbeYh0I6+RTyiGqqM6qDFqjQ5HWSgbDUfj0fFoBjoZLUTno0vQcrQa3YnWoyfRi+h1tAN9gfZhAFPD9DAzzBFjYRwsCkvB0jEpNgsrwcqwaqwOa4LP+SrWgfVgH3EizsCZuCNcwaF4As7HJ+Oz8MV4Bb4dr8db8Kv4Q7wX/0agEYwIDgQvApcwhpBBmEIoJpQRthIOEk7DvdRJeEckEvWINkQPuBeTiVnE6cTFxHXE3cQTxHbiY2IfiUQyIDmQfEhRJB4pn1RMWkvaSTpOukLqJH0gq5FNyS7kYHIKWUwuIpeRd5CPka+Qn5E/UzQpVhQvShRFQJlGWUrZQmmiXKJ0Uj5Ttag2VB9qPDWLOpdaTq2jnqbeo75RU1MzV/NUi1ETqc1RK1fbo3ZO7aHaR3VtdXt1jvo4dZn6EvVt6ifUb6u/odFo1jR/Wgotn7aEVkM7RXtA+6DB0HDS4GoINGZrVGrUa1zReEmn0K3obPoEeiG9jL6ffoneo0nRtNbkaPI0Z2lWah7SvKnZp8XQGqEVpZWrtVhrh9Z5rS5tkra1dpC2QHu+9mbtU9qPGRjDgsFh8BnzGFsYpxmdOkQdGx2uTpZOqc4unTadXl1tXVfdRN2pupW6R3U79DA9az2uXo7eUr19ejf0Pg0xHsIeIhyyaEjdkCtD3usP1ffXF+qX6O/Wv67/yYBpEGSQbbDcoMHgviFuaG8YYzjFcL3hacOeoTpDvYfyh5YM3Tf0jhFqZG8UazTdaLNRq1GfsYlxiLHEeK3xKeMeEz0Tf5Msk1Umx0y6TRmmvqYi01Wmx02fM3WZbGYOs5zZwuw1MzILNZOZbTJrM/tsbmOeYF5kvtv8vgXVgmWRbrHKotmi19LUcrTlDMtayztWFCuWVabVGquzVu+tbayTrBdYN1h32ejbcG0KbWpt7tnSbP1sJ9tW216zI9qx7LLt1tldtkft3ewz7SvtLzmgDu4OIod1Du3DCMM8h4mHVQ+76ajuyHYscKx1fOik5xThVOTU4PRyuOXwlOHLh58d/s3ZzTnHeYvz3RHaI8JGFI1oGvHaxd6F71Lpcm0kbWTwyNkjG0e+cnVwFbqud73lxnAb7bbArdntq7uHu9S9zr3bw9Ij1aPK4yZLhxXNWsw650nwDPCc7XnE86OXu1e+1z6vv7wdvbO9d3h3jbIZJRy1ZdRjH3Mfns8mnw5fpm+q70bfDj8zP55ftd8jfwt/gf9W/2dsO3YWeyf7ZYBzgDTgYMB7jhdnJudEIBYYElgS2BakHZQQVBH0INg8OCO4Nrg3xC1kesiJUEJoeOjy0JtcYy6fW8PtDfMImxnWEq4eHhdeEf4owj5CGtE0Gh0dNnrl6HuRVpHiyIYoEMWNWhl1P9omenL04RhiTHRMZczT2BGxM2LPxjHiJsbtiHsXHxC/NP5ugm2CLKE5kZ44LrEm8X1SYNKKpI4xw8fMHHMx2TBZlNyYQkpJTNma0jc2aOzqsZ3j3MYVj7sx3mb81PHnJxhOyJlwdCJ9Im/i/lRCalLqjtQvvCheNa8vjZtWldbL5/DX8F8I/AWrBN1CH+EK4bN0n/QV6V0ZPhkrM7oz/TLLMntEHFGF6FVWaNaGrPfZUdnbsvtzknJ255JzU3MPibXF2eKWSSaTpk5qlzhIiiUdk70mr57cKw2Xbs1D8sbnNebrwB/5Vpmt7BfZwwLfgsqCD1MSp+yfqjVVPLV1mv20RdOeFQYX/jYdn86f3jzDbMbcGQ9nsmdumoXMSpvVPNti9vzZnXNC5myfS52bPff3IueiFUVv5yXNa5pvPH/O/Me/hPxSW6xRLC2+ucB7wYaF+ELRwrZFIxetXfStRFByodS5tKz0y2L+4gu/jvi1/Nf+JelL2pa6L12/jLhMvOzGcr/l21dorShc8Xjl6JX1q5irSla9XT1x9fky17INa6hrZGs6yiPKG9darl229ktFZsX1yoDK3VVGVYuq3q8TrLuy3n993QbjDaUbPm0Ubby1KWRTfbV1ddlm4uaCzU+3JG45+xvrt5qthltLt37dJt7WsT12e0uNR03NDqMdS2vRWllt985xOy/vCtzVWOdYt2m33u7SPWCPbM/zval7b+wL39e8n7W/7oDVgaqDjIMl9Uj9tPrehsyGjsbkxvZDYYeam7ybDh52OrztiNmRyqO6R5ceox6bf6z/eOHxvhOSEz0nM04+bp7YfPfUmFPXWmJa2k6Hnz53JvjMqbPss8fP+Zw7ct7r/KELrAsNF90v1re6tR783e33g23ubfWXPC41Xva83NQ+qv3YFb8rJ68GXj1zjXvt4vXI6+03Em7cujnuZsctwa2u2zm3X90puPP57px7hHsl9zXvlz0welD9h90fuzvcO44+DHzY+iju0d3H/McvnuQ9+dI5/yntadkz02c1XS5dR7qDuy8/H/u884Xkxeee4j+1/qx6afvywF/+f7X2juntfCV91f968RuDN9veur5t7ovue/Au993n9yUfDD5s/8j6ePZT0qdnn6d8IX0p/2r3telb+Ld7/bn9/RKelKf4FcBgRdPTAXi9DQBaMgAMeD6jjlWe/xQFUZ5ZFQj8J6w8IyqKOwB18P89pgf+3dwEYM8WePyC+vRxAETTAIj3BOjIkYN14KymOFfKCxGeAzZGfU3LTQP/pijPnD/E/XML5Kqu4Of2X0krfGlwjnGBAAAAimVYSWZNTQAqAAAACAAEARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAh2kABAAAAAEAAABOAAAAAAAAAJAAAAABAAAAkAAAAAEAA5KGAAcAAAASAAAAeKACAAQAAAABAAAANKADAAQAAAABAAAAJAAAAABBU0NJSQAAAFNjcmVlbnNob3SHQ+rGAAAACXBIWXMAABYlAAAWJQFJUiTwAAAB1GlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj4zNjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj41MjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgr5FDsvAAAAHGlET1QAAAACAAAAAAAAABIAAAAoAAAAEgAAABIAAAHM67+vYAAAAZhJREFUWAnsVb2KwkAQHks7BRVR4jsIYpUiWKYRkTxACkuxEysVi4CFjU8RH8DWN7BOZZEiCIYQEPwDw5y73MZsyF57yZ0DYXfm2wzzzWy+5PBl8Ics9yGU8ml+JpTyAUGmJnS73cB1XWg0GuK+ElHIgu12O5QkCYvFIuq6LiwZhEjKgF6vR8kQQuSxLCuxwswQGg6HHKHT6ZRtQrZt42AwwE6ng5vNJpEMCWZKFMRK8Eb+L6Hz+Qz7/R4cxwHP86BcLkO9Xod2uw35fP7dotiOyOzxeKTRWq0GpVIpduLtkrwkP7FqtQqVSiUEL5cLHA4H6hcKBbF0Cy/jN+D7Po7HY+6DZEpDViKl8/kcr9drYqrFYhG+u1wuE8+w4Gq1Cs/OZjMWput2uw2xfr/PYVHnR5V7dQRbrVaYKEokvlcUBV+TiOam+9QQejweKMsyR4aQm06nVGUmkwk2m00O73a7GAQBRyo1hNbrNVesYRhcocwZjUbcOdM0GUTX1BCKdl/TNHw+n1yhzLnf7/TfwK6gqqoM+hVCXwAAAP///HYhaQAAAo9JREFU7VZPqHFBFJ9vY2XBVllirZTNy8IKZWVhQULZPgs7C2ShFAspewsLsiB/SmxISpSkyFKRIiR/Cum+O77u+e747vO8l2Lxpm5zzpzfnHN+Z+aeexHFMfr9PiUUCuEZDAYcqH9LtVoNsHjfbDYDo9/vB1swGIR1LiEcDgPW6/USkEKhADaDwUDY2ApiK4xcLBZhM07weDwyJs4ZE2AXoNVqAe4lCMXjcUjw7e0NkrsliMVi2IMLwoyXIBSNRiE5o9HI5HZzxsSZU0omk4B9CUKJRAKSk8vlkNxnwvl8BjwmVS6XAfodQqFQCPw89B3CCTHVxvNut4MEuYTRaETgO50OwNiEPB4PrHMJLpcL/DyU0Hg8BseYUKPR4IoPa9lslsBvNhuwxWIxsNlsNljnEnQ6HWAfSggH02g04Bxfu/V6zZUDNZ1OKalUClir1UrgqtUq2HDj2O/3hJ1R2u024HARH04ok8kQAUwmE7VarZj4lxm3azZxnEi9Xicw8/mc8ON2u6nT6URgut0upVAoCNxPCf3BntEnw2KxoHw+T1iVSiWSSCSo1+shOhHCZrfbEf1iE2tY0Wq1qNlswrpMJkMqlQrxeDw0HA5RpVIBGyO8v78jn8/HqIj+FCCz2XzR1Wo1SqfTYCMEolRXyna7pWhSROXwKXA9TqeTOhwOVx7+qovF4r8TuPaB234gEADfPz0hzj8Fdla4JadSKYquCgRjJ6PX66lcLsfewinjRuNwOIj3jfGDux++zpFIBGLg7sgepVIJbLe+jTevHHGUtLJcLhHdBBDdxhGfz0cikQgJBIJr2Jf6ZDK5+KEJIbpRXK7el5vuBHyL0J0+nwr7JfTU8t8R/PeE7ijSUyEf9xqMU4B4MecAAAAASUVORK5CYII=", + "hashes": { + "SHA-256": "effb46bba03f6c8aea5c653f9cf984f170dcdd3bbbe2ff6843c3e5da0e698766" + } + }, + "5": { + "type": "file", + "name": "tabby_pics.zip", + "magic_number_hex": "504B0304", + "hashes": { + "SHA-256": "fe90a7e910cb3a4739bed9180e807e93fa70c90f25a8915476f5e4bfbac681db" + } + } + } + } +] _FILE_OBJECTS = [ { "type": "observed-data", @@ -1520,6 +1600,11 @@ def get_bundle_with_domain_ip_objects(cls): def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + + @classmethod + def get_bundle_with_email_message_objects(cls): + return cls.__assemble_bundle(*_EMAIL_MESSAGE_OBJECTS) + @classmethod def get_bundle_with_file_objects(cls): observed_data = deepcopy(_FILE_OBJECTS) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 40299d4e..3b402953 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -274,6 +274,37 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_email_artifact_object( + self, misp_object, observed_data, artifact_id): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, artifact_id) + self._check_email_artifact_object_fields( + misp_object, observed_data.objects[artifact_id], + f'{observed_data.id} - {artifact_id}' + ) + + def _check_email_file_object(self, misp_object, observed_data, file_id): + self.assertEqual(misp_object.name, 'file') + self._check_misp_object_fields(misp_object, observed_data, file_id) + self._check_email_file_object_fields( + misp_object, observed_data.objects[file_id], + f'{observed_data.id} - {file_id}' + ) + + def _check_email_object(self, misp_object, observed_data, + email_message_id, from_id, to_id, cc_id): + self.assertEqual(misp_object.name, 'email') + self._check_misp_object_fields( + misp_object, observed_data, email_message_id + ) + object_id = f'{observed_data.id} - {email_message_id}' + self._check_email_object_fields( + misp_object, observed_data.objects[email_message_id], + observed_data.objects[from_id], observed_data.objects[to_id], + observed_data.objects[cc_id], object_id, f'{object_id} - {from_id}', + f'{object_id} - {to_id}', f'{object_id} - {cc_id}' + ) + def _check_file_and_pe_objects(self, observed_data, file_object, pe_object, *sections): self.assertEqual(file_object.name, 'file') @@ -640,6 +671,26 @@ def test_stix20_bundle_with_email_address_objects(self): ) self._check_email_address_attribute(observed_data3, ss_address) + def test_stix20_bundle_with_email_message_objects(self): + bundle = TestExternalSTIX20Bundles.get_bundle_with_email_message_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, report, observed_data = bundle.objects + misp_objects = self._check_misp_event_features(event, report) + self.assertEqual(len(misp_objects), 3) + email_object, artifact_object, file_object = misp_objects + self._check_email_object(email_object, observed_data, '0', '1', '2', '3') + email_references = email_object.references + self.assertEqual(len(email_references), 2) + artifact_reference, file_reference = email_references + self.assertEqual(artifact_reference.referenced_uuid, artifact_object.uuid) + self.assertEqual(artifact_reference.relationship_type, 'contains') + self.assertEqual(file_reference.referenced_uuid, file_object.uuid) + self.assertEqual(file_reference.relationship_type, 'contains') + self._check_email_artifact_object(artifact_object, observed_data, '4') + self._check_email_file_object(file_object, observed_data, '5') + def test_stix20_bundle_with_file_objects(self): bundle = TestExternalSTIX20Bundles.get_bundle_with_file_objects() self.parser.load_stix_bundle(bundle) diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index c93ced72..71c70637 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -505,6 +505,105 @@ "value": "donald.duck@gmail.com" } ] +_EMAIL_MESSAGE_OBJECTS = [ + { + "type": "observed-data", + "spec_version": "2.1", + "id": "observed-data--3cd23a7b-a099-49df-b397-189018311d4e", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa952", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-11-25T16:22:00.000Z", + "first_observed": "2020-10-25T16:22:00Z", + "last_observed": "2020-11-25T16:22:00Z", + "number_observed": 1, + "object_refs": [ + "email-message--cf9b4b7f-14c8-5955-8065-020e0316b559", + "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed", + "email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868", + "email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194", + "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5", + "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5", + ] + }, + { + "type": "email-message", + "spec_version": "2.1", + "id": "email-message--cf9b4b7f-14c8-5955-8065-020e0316b559", + "is_multipart": True, + "received_lines": [ + "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)" + ], + "content_type": "multipart/mixed", + "date": "2016-06-19T14:20:40.000Z", + "from_ref": "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed", + "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"], + "cc_refs": ["email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194"], + "subject": "Check out this picture of a cat!", + "additional_header_fields": { + "Content-Disposition": "inline", + "X-Mailer": "Mutt/1.5.23", + "X-Originating-IP": "198.51.100.3" + }, + "body_multipart": [ + { + "content_type": "text/plain; charset=utf-8", + "content_disposition": "inline", + "body": "Cats are funny!" + }, + { + "content_type": "image/png", + "content_disposition": "attachment; filename=\"tabby.png\"", + "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5" + }, + { + "content_type": "application/zip", + "content_disposition": "attachment; filename=\"tabby_pics.zip\"", + "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5" + } + ] + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed", + "value": "jdoe@example.com", + "display_name": "John Doe" + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868", + "value": "bob@example.com", + "display_name": "Bob Smith" + }, + { + "type": "email-addr", + "spec_version": "2.1", + "id": "email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194", + "value": "mary@example.com", + "display_name": "Mary Smith" + }, + { + "type": "artifact", + "spec_version": "2.1", + "id": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5", + "mime_type": "image/jpeg", + "payload_bin": "iVBORw0KGgoAAAANSUhEUgAAADQAAAAkCAYAAADGrhlwAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSIQQIICAl9CaISAkgJYQWQHoRbIQkQCgxBoKKHV1UcO0iAjZ0VUSxA2JH7CyKvS8WFJR1sWBX3qSArvvK9+b75s5//znznzPnztx7BwD6CZ5EkoNqApArzpfGhgQwxySnMEldgADogAyGAhseP0/Cjo6OALAMtH8v724ARN5edZRr/bP/vxYtgTCPDwASDXGaII+fC/EBAPAqvkSaDwBRzltMyZfIMaxARwoDhHihHGcocZUcpynxHoVNfCwH4hYAyOo8njQDAI3LkGcW8DOghkYvxM5igUgMAJ0JsW9u7iQBxKkQ20IbCcRyfVbaDzoZf9NMG9Tk8TIGsXIuikIOFOVJcnjT/s90/O+SmyMb8GENq3qmNDRWPmeYt1vZk8LlWB3iHnFaZBTE2hB/EAkU9hCj1ExZaILSHjXi53FgzoAexM4CXmA4xEYQB4tzIiNUfFq6KJgLMVwh6FRRPjceYn2IFwrzguJUNhulk2JVvtD6dCmHreLP8aQKv3JfD2TZCWyV/utMIVelj2kUZsYnQUyF2LJAlBgJsQbETnnZceEqm1GFmZzIARupLFYevyXEsUJxSIBSHytIlwbHquxLcvMG5ottzBRxI1V4X35mfKgyP1gLn6eIH84FuywUsxMGdIR5YyIG5iIQBgYp5451CcUJcSqdD5L8gFjlWJwqyYlW2ePmwpwQOW8OsWteQZxqLJ6YDxekUh9Pl+RHxyvjxAuzeGHRynjwZSACcEAgYAIZrGlgEsgCoraehh54p+wJBjwgBRlACBxVzMCIJEWPGF7jQCH4EyIhyBscF6DoFYICyH8dZJVXR5Cu6C1QjMgGTyHOBeEgB97LFKPEg94SwRPIiP7hnQcrH8abA6u8/9/zA+x3hg2ZCBUjG/DIpA9YEoOIgcRQYjDRDjfEfXFvPAJe/WF1wVm458A8vtsTnhLaCY8I1wkdhNsTRUXSn6IcDTqgfrAqF2k/5gK3hppueADuA9WhMq6HGwJH3BX6YeN+0LMbZDmquOVZYf6k/bcZ/PA0VHYUZwpKGULxp9j+PFLDXsNtUEWe6x/zo4w1bTDfnMGen/1zfsi+ALbhP1tiC7H92FnsJHYeO4I1ACZ2HGvEWrGjcjy4up4oVteAt1hFPNlQR/QPfwNPVp7JPOda527nL8q+fOFU+TsacCZJpklFGZn5TDb8IgiZXDHfaRjTxdnFFQD590X5+noTo/huIHqt37l5fwDgc7y/v//wdy7sOAB7PeD2P/Sds2XBT4caAOcO8WXSAiWHyy8E+Jagw51mAEyABbCF83EB7sAb+IMgEAaiQDxIBhNg9JlwnUvBFDADzAXFoBQsA6tBBdgANoPtYBfYBxrAEXASnAEXwWVwHdyFq6cTvAC94B34jCAICaEhDMQAMUWsEAfEBWEhvkgQEoHEIslIKpKBiBEZMgOZh5QiK5AKZBNSg+xFDiEnkfNIO3IbeYh0I6+RTyiGqqM6qDFqjQ5HWSgbDUfj0fFoBjoZLUTno0vQcrQa3YnWoyfRi+h1tAN9gfZhAFPD9DAzzBFjYRwsCkvB0jEpNgsrwcqwaqwOa4LP+SrWgfVgH3EizsCZuCNcwaF4As7HJ+Oz8MV4Bb4dr8db8Kv4Q7wX/0agEYwIDgQvApcwhpBBmEIoJpQRthIOEk7DvdRJeEckEvWINkQPuBeTiVnE6cTFxHXE3cQTxHbiY2IfiUQyIDmQfEhRJB4pn1RMWkvaSTpOukLqJH0gq5FNyS7kYHIKWUwuIpeRd5CPka+Qn5E/UzQpVhQvShRFQJlGWUrZQmmiXKJ0Uj5Ttag2VB9qPDWLOpdaTq2jnqbeo75RU1MzV/NUi1ETqc1RK1fbo3ZO7aHaR3VtdXt1jvo4dZn6EvVt6ifUb6u/odFo1jR/Wgotn7aEVkM7RXtA+6DB0HDS4GoINGZrVGrUa1zReEmn0K3obPoEeiG9jL6ffoneo0nRtNbkaPI0Z2lWah7SvKnZp8XQGqEVpZWrtVhrh9Z5rS5tkra1dpC2QHu+9mbtU9qPGRjDgsFh8BnzGFsYpxmdOkQdGx2uTpZOqc4unTadXl1tXVfdRN2pupW6R3U79DA9az2uXo7eUr19ejf0Pg0xHsIeIhyyaEjdkCtD3usP1ffXF+qX6O/Wv67/yYBpEGSQbbDcoMHgviFuaG8YYzjFcL3hacOeoTpDvYfyh5YM3Tf0jhFqZG8UazTdaLNRq1GfsYlxiLHEeK3xKeMeEz0Tf5Msk1Umx0y6TRmmvqYi01Wmx02fM3WZbGYOs5zZwuw1MzILNZOZbTJrM/tsbmOeYF5kvtv8vgXVgmWRbrHKotmi19LUcrTlDMtayztWFCuWVabVGquzVu+tbayTrBdYN1h32ejbcG0KbWpt7tnSbP1sJ9tW216zI9qx7LLt1tldtkft3ewz7SvtLzmgDu4OIod1Du3DCMM8h4mHVQ+76ajuyHYscKx1fOik5xThVOTU4PRyuOXwlOHLh58d/s3ZzTnHeYvz3RHaI8JGFI1oGvHaxd6F71Lpcm0kbWTwyNkjG0e+cnVwFbqud73lxnAb7bbArdntq7uHu9S9zr3bw9Ij1aPK4yZLhxXNWsw650nwDPCc7XnE86OXu1e+1z6vv7wdvbO9d3h3jbIZJRy1ZdRjH3Mfns8mnw5fpm+q70bfDj8zP55ftd8jfwt/gf9W/2dsO3YWeyf7ZYBzgDTgYMB7jhdnJudEIBYYElgS2BakHZQQVBH0INg8OCO4Nrg3xC1kesiJUEJoeOjy0JtcYy6fW8PtDfMImxnWEq4eHhdeEf4owj5CGtE0Gh0dNnrl6HuRVpHiyIYoEMWNWhl1P9omenL04RhiTHRMZczT2BGxM2LPxjHiJsbtiHsXHxC/NP5ugm2CLKE5kZ44LrEm8X1SYNKKpI4xw8fMHHMx2TBZlNyYQkpJTNma0jc2aOzqsZ3j3MYVj7sx3mb81PHnJxhOyJlwdCJ9Im/i/lRCalLqjtQvvCheNa8vjZtWldbL5/DX8F8I/AWrBN1CH+EK4bN0n/QV6V0ZPhkrM7oz/TLLMntEHFGF6FVWaNaGrPfZUdnbsvtzknJ255JzU3MPibXF2eKWSSaTpk5qlzhIiiUdk70mr57cKw2Xbs1D8sbnNebrwB/5Vpmt7BfZwwLfgsqCD1MSp+yfqjVVPLV1mv20RdOeFQYX/jYdn86f3jzDbMbcGQ9nsmdumoXMSpvVPNti9vzZnXNC5myfS52bPff3IueiFUVv5yXNa5pvPH/O/Me/hPxSW6xRLC2+ucB7wYaF+ELRwrZFIxetXfStRFByodS5tKz0y2L+4gu/jvi1/Nf+JelL2pa6L12/jLhMvOzGcr/l21dorShc8Xjl6JX1q5irSla9XT1x9fky17INa6hrZGs6yiPKG9darl229ktFZsX1yoDK3VVGVYuq3q8TrLuy3n993QbjDaUbPm0Ubby1KWRTfbV1ddlm4uaCzU+3JG45+xvrt5qthltLt37dJt7WsT12e0uNR03NDqMdS2vRWllt985xOy/vCtzVWOdYt2m33u7SPWCPbM/zval7b+wL39e8n7W/7oDVgaqDjIMl9Uj9tPrehsyGjsbkxvZDYYeam7ybDh52OrztiNmRyqO6R5ceox6bf6z/eOHxvhOSEz0nM04+bp7YfPfUmFPXWmJa2k6Hnz53JvjMqbPss8fP+Zw7ct7r/KELrAsNF90v1re6tR783e33g23ubfWXPC41Xva83NQ+qv3YFb8rJ68GXj1zjXvt4vXI6+03Em7cujnuZsctwa2u2zm3X90puPP57px7hHsl9zXvlz0welD9h90fuzvcO44+DHzY+iju0d3H/McvnuQ9+dI5/yntadkz02c1XS5dR7qDuy8/H/u884Xkxeee4j+1/qx6afvywF/+f7X2juntfCV91f968RuDN9veur5t7ovue/Au993n9yUfDD5s/8j6ePZT0qdnn6d8IX0p/2r3telb+Ld7/bn9/RKelKf4FcBgRdPTAXi9DQBaMgAMeD6jjlWe/xQFUZ5ZFQj8J6w8IyqKOwB18P89pgf+3dwEYM8WePyC+vRxAETTAIj3BOjIkYN14KymOFfKCxGeAzZGfU3LTQP/pijPnD/E/XML5Kqu4Of2X0krfGlwjnGBAAAAimVYSWZNTQAqAAAACAAEARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAh2kABAAAAAEAAABOAAAAAAAAAJAAAAABAAAAkAAAAAEAA5KGAAcAAAASAAAAeKACAAQAAAABAAAANKADAAQAAAABAAAAJAAAAABBU0NJSQAAAFNjcmVlbnNob3SHQ+rGAAAACXBIWXMAABYlAAAWJQFJUiTwAAAB1GlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj4zNjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj41MjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgr5FDsvAAAAHGlET1QAAAACAAAAAAAAABIAAAAoAAAAEgAAABIAAAHM67+vYAAAAZhJREFUWAnsVb2KwkAQHks7BRVR4jsIYpUiWKYRkTxACkuxEysVi4CFjU8RH8DWN7BOZZEiCIYQEPwDw5y73MZsyF57yZ0DYXfm2wzzzWy+5PBl8Ics9yGU8ml+JpTyAUGmJnS73cB1XWg0GuK+ElHIgu12O5QkCYvFIuq6LiwZhEjKgF6vR8kQQuSxLCuxwswQGg6HHKHT6ZRtQrZt42AwwE6ng5vNJpEMCWZKFMRK8Eb+L6Hz+Qz7/R4cxwHP86BcLkO9Xod2uw35fP7dotiOyOzxeKTRWq0GpVIpduLtkrwkP7FqtQqVSiUEL5cLHA4H6hcKBbF0Cy/jN+D7Po7HY+6DZEpDViKl8/kcr9drYqrFYhG+u1wuE8+w4Gq1Cs/OZjMWput2uw2xfr/PYVHnR5V7dQRbrVaYKEokvlcUBV+TiOam+9QQejweKMsyR4aQm06nVGUmkwk2m00O73a7GAQBRyo1hNbrNVesYRhcocwZjUbcOdM0GUTX1BCKdl/TNHw+n1yhzLnf7/TfwK6gqqoM+hVCXwAAAP///HYhaQAAAo9JREFU7VZPqHFBFJ9vY2XBVllirZTNy8IKZWVhQULZPgs7C2ShFAspewsLsiB/SmxISpSkyFKRIiR/Cum+O77u+e747vO8l2Lxpm5zzpzfnHN+Z+aeexHFMfr9PiUUCuEZDAYcqH9LtVoNsHjfbDYDo9/vB1swGIR1LiEcDgPW6/USkEKhADaDwUDY2ApiK4xcLBZhM07weDwyJs4ZE2AXoNVqAe4lCMXjcUjw7e0NkrsliMVi2IMLwoyXIBSNRiE5o9HI5HZzxsSZU0omk4B9CUKJRAKSk8vlkNxnwvl8BjwmVS6XAfodQqFQCPw89B3CCTHVxvNut4MEuYTRaETgO50OwNiEPB4PrHMJLpcL/DyU0Hg8BseYUKPR4IoPa9lslsBvNhuwxWIxsNlsNljnEnQ6HWAfSggH02g04Bxfu/V6zZUDNZ1OKalUClir1UrgqtUq2HDj2O/3hJ1R2u024HARH04ok8kQAUwmE7VarZj4lxm3azZxnEi9Xicw8/mc8ON2u6nT6URgut0upVAoCNxPCf3BntEnw2KxoHw+T1iVSiWSSCSo1+shOhHCZrfbEf1iE2tY0Wq1qNlswrpMJkMqlQrxeDw0HA5RpVIBGyO8v78jn8/HqIj+FCCz2XzR1Wo1SqfTYCMEolRXyna7pWhSROXwKXA9TqeTOhwOVx7+qovF4r8TuPaB234gEADfPz0hzj8Fdla4JadSKYquCgRjJ6PX66lcLsfewinjRuNwOIj3jfGDux++zpFIBGLg7sgepVIJbLe+jTevHHGUtLJcLhHdBBDdxhGfz0cikQgJBIJr2Jf6ZDK5+KEJIbpRXK7el5vuBHyL0J0+nwr7JfTU8t8R/PeE7ijSUyEf9xqMU4B4MecAAAAASUVORK5CYII=", + "hashes": { + "SHA-256": "effb46bba03f6c8aea5c653f9cf984f170dcdd3bbbe2ff6843c3e5da0e698766" + } + }, + { + "type": "file", + "spec_version": "2.1", + "id": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5", + "name": "tabby_pics.zip", + "magic_number_hex": "504B0304", + "hashes": { + "SHA-256": "fe90a7e910cb3a4739bed9180e807e93fa70c90f25a8915476f5e4bfbac681db" + } + } +] _FILE_OBJECTS = [ { "type": "observed-data", @@ -1800,6 +1899,10 @@ def get_bundle_with_domain_ip_objects(cls): def get_bundle_with_email_address_attributes(cls): return cls.__assemble_bundle(*_EMAIL_ADDRESS_ATTRIBUTES) + @classmethod + def get_bundle_with_email_message_objects(cls): + return cls.__assemble_bundle(*_EMAIL_MESSAGE_OBJECTS) + @classmethod def get_bundle_with_file_objects(cls): observables = deepcopy(_FILE_OBJECTS) diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 3faa619d..5f3d6fe2 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -278,6 +278,25 @@ def _check_email_address_attribute_with_display_name( self.assertEqual(display_name.type, 'email-dst-display-name') self.assertEqual(display_name.value, email_address.display_name) + def _check_email_artifact_object(self, misp_object, observed_data, artifact): + self.assertEqual(misp_object.name, 'artifact') + self._check_misp_object_fields(misp_object, observed_data, artifact.id) + self._check_email_artifact_object_fields(misp_object, artifact, artifact.id) + + def _check_email_file_object(self, misp_object, observed_data, _file): + self.assertEqual(misp_object.name, 'file') + self._check_misp_object_fields(misp_object, observed_data, _file.id) + self._check_email_file_object_fields(misp_object, _file, _file.id) + + def _check_email_object(self, misp_object, observed_data, email_message, + from_address, to_address, cc_address): + self.assertEqual(misp_object.name, 'email') + self._check_misp_object_fields(misp_object, observed_data, email_message.id) + self._check_email_object_fields( + misp_object, email_message, from_address, to_address, cc_address, + email_message.id, from_address.id, to_address.id, cc_address.id + ) + def _check_file_and_pe_objects(self, observed_data, observable_object, file_object, pe_object, *sections): self.assertEqual(file_object.name, 'file') @@ -617,6 +636,26 @@ def test_stix21_bundle_with_email_address_attributes(self): ) self._check_email_address_attribute(od3, ss_address, ea4) + def test_stix21_bundle_with_email_message_objects(self): + bundle = TestExternalSTIX21Bundles.get_bundle_with_email_message_objects() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle() + event = self.parser.misp_event + _, grouping, observed_data, message, ea1, ea2, ea3, artifact, _file = bundle.objects + misp_objects = self._check_misp_event_features_from_grouping(event, grouping) + self.assertEqual(len(misp_objects), 3) + email_object, artifact_object, file_object = misp_objects + self._check_email_object(email_object, observed_data, message, ea1, ea2, ea3) + email_references = email_object.references + self.assertEqual(len(email_references), 2) + artifact_reference, file_reference = email_references + self.assertEqual(artifact_reference.referenced_uuid, artifact_object.uuid) + self.assertEqual(artifact_reference.relationship_type, 'contains') + self.assertEqual(file_reference.referenced_uuid, file_object.uuid) + self.assertEqual(file_reference.relationship_type, 'contains') + self._check_email_artifact_object(artifact_object, observed_data, artifact) + self._check_email_file_object(file_object, observed_data, _file) + def test_stix21_bundle_with_file_objects(self): bundle = TestExternalSTIX21Bundles.get_bundle_with_file_objects() self.parser.load_stix_bundle(bundle) From 21c2b94ba0716e5959a9f71a7785355f1d8eb56f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 6 Jun 2024 15:24:13 +0200 Subject: [PATCH 135/137] fix: [stix2 import] Merged missing conflicts --- .../stix2misp/stix2_to_misp.py | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index edfa6a1b..84532eb6 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1170,10 +1170,6 @@ def _add_misp_attribute(self, attribute: dict, stix_object: _SDO_TYPING) -> MISPAttribute: misp_attribute = MISPAttribute() misp_attribute.from_dict(**attribute) -<<<<<<< HEAD - for tag in self._handle_tags_from_stix_fields(stix_object): - misp_attribute.add_tag(tag) -======= for marking in self._handle_tags_from_stix_fields(stix_object): if isinstance(marking, str): misp_attribute.add_tag(marking) @@ -1187,16 +1183,10 @@ def _add_misp_attribute(self, attribute: dict, if marking.get('tags'): for tag in marking['tags']: misp_attribute.add_tag(tag) ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f return self.misp_event.add_attribute(**misp_attribute) def _add_misp_object(self, misp_object: MISPObject, stix_object: _SDO_TYPING) -> MISPObject: -<<<<<<< HEAD - for tag in self._handle_tags_from_stix_fields(stix_object): - for attribute in misp_object.attributes: - attribute.add_tag(tag) -======= for marking in self._handle_tags_from_stix_fields(stix_object): if isinstance(marking, str): for attribute in misp_object.attributes: @@ -1213,7 +1203,6 @@ def _add_misp_object(self, misp_object: MISPObject, for tag in marking['tags']: for attribute in misp_object.attributes: attribute.add_tag(tag) ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f return self.misp_event.add_object(misp_object) def _create_generic_event(self) -> MISPEvent: @@ -1259,10 +1248,6 @@ def _handle_tags_from_stix_fields(self, stix_object: _SDO_TYPING): marking_definition = self._get_stix_object(marking_ref) except ObjectTypeLoadingError as error: if self._is_tlp_marking(marking_ref): -<<<<<<< HEAD - marking = self._get_stix_object(marking_ref) -======= ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f yield self._get_stix_object(marking_ref) else: self._object_type_loading_error(error) @@ -1290,14 +1275,11 @@ def _fetch_tags_from_labels( if label.lower() != 'threat-report'): misp_feature.add_tag(label) -<<<<<<< HEAD -======= def _handle_creator(self, reference: str) -> str: if reference in getattr(self, '_identity', {}): return self._identity[reference].name return self._mapping.identity_references(reference) or 'misp-stix' ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f def _is_tlp_marking(self, marking_ref: str) -> bool: for marking in (TLP_WHITE, TLP_GREEN, TLP_AMBER, TLP_RED): if marking_ref == marking.id: @@ -1323,18 +1305,6 @@ def _parse_confidence_level(confidence_level: int) -> str: return 'misp:confidence-level="rarely-confident"' return 'misp:confidence-level="unconfident"' -<<<<<<< HEAD - def _parse_marking_definition( - self, marking_definition: _MARKING_DEFINITION_TYPING) -> str: - if hasattr(marking_definition, 'definition_type'): - definition_type = marking_definition.definition_type - definition = marking_definition.definition[definition_type] - return f"{definition_type}:{definition}" - if hasattr(marking_definition, 'name'): - # should be TLP 2.0 definition - return marking_definition.name.lower() - raise MarkingDefinitionLoadingError(marking_definition.id) -======= def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: misp_object = { 'timestamp': self._timestamp_from_date(stix_object.modified) @@ -1348,7 +1318,6 @@ def _parse_timeline(self, stix_object: _SDO_TYPING) -> dict: if hasattr(stix_object, last) and getattr(stix_object, last): misp_object['last_seen'] = getattr(stix_object, last) return misp_object ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f @staticmethod def _populate_object_attributes( From 4476e845d280dec8c0b11e9bfe0625df1296034f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 6 Jun 2024 16:49:46 +0200 Subject: [PATCH 136/137] fix: [stix2 import] Post Observed Data Converter merge clean up and reassembling --- .../stix2misp/converters/__init__.py | 2 +- .../stix2_observed_data_converter.py | 13 - .../stix2misp/converters/stix2converter.py | 18 +- .../stix2misp/external_stix2_to_misp.py | 1660 +---------------- .../stix2misp/internal_stix2_to_misp.py | 15 - .../stix2misp/stix2_mapping.py | 39 +- .../stix2misp/stix2_to_misp.py | 54 +- 7 files changed, 54 insertions(+), 1747 deletions(-) diff --git a/misp_stix_converter/stix2misp/converters/__init__.py b/misp_stix_converter/stix2misp/converters/__init__.py index db62a993..eafa1998 100644 --- a/misp_stix_converter/stix2misp/converters/__init__.py +++ b/misp_stix_converter/stix2misp/converters/__init__.py @@ -21,7 +21,7 @@ from .stix2_note_converter import STIX2NoteConverter # noqa from .stix2_observable_objects_converter import STIX2ObservableObjectConverter # noqa from .stix2_observed_data_converter import ( # noqa - InternalSTIX2ObservedDataConverter) + ExternalSTIX2ObservedDataConverter, InternalSTIX2ObservedDataConverter) from .stix2_threat_actor_converter import( # noqa ExternalSTIX2ThreatActorConverter, InternalSTIX2ThreatActorConverter) from .stix2_tool_converter import ( # noqa diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 0a4c115c..e35c06aa 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -3529,16 +3529,3 @@ def _fetch_observables_with_id_v21( reference: self.main_parser._observable[reference] for reference in observed_data.object_refs } - - @staticmethod - def _handle_external_references(external_references: list) -> dict: - meta = defaultdict(list) - for reference in external_references: - if reference.get('url'): - meta['refs'].append(reference['url']) - feature = 'aliases' if reference.get('source_name') == 'cve' else 'external_id' - if reference.get('external_id'): - meta[feature].append(reference['external_id']) - if 'external_id' in meta and len(meta['external_id']) == 1: - meta['external_id'] = meta.pop('external_id')[0] - return meta diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index b7b3feb0..f288e0bb 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -8,9 +8,11 @@ from datetime import datetime from pymisp import AbstractMISP, MISPGalaxyCluster, MISPObject from stix2.v20.sdo import ( - AttackPattern as AttackPattern_v20, Malware as Malware_v20) + AttackPattern as AttackPattern_v20, Malware as Malware_v20, + ObservedData as ObservedData_v20) from stix2.v21.sdo import ( - AttackPattern as AttackPattern_v21, Malware as Malware_v21) + AttackPattern as AttackPattern_v21, Malware as Malware_v21, + ObservedData as ObservedData_v21) from typing import Iterator, Optional, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: @@ -28,7 +30,8 @@ ] _SDO_TYPING = Union[ AttackPattern_v20, AttackPattern_v21, - Malware_v20, Malware_v21 + Malware_v20, Malware_v21, + ObservedData_v20, ObservedData_v21 ] @@ -208,6 +211,13 @@ def parse(self, stix_object_ref: str): # GALAXIES PARSING METHODS # ############################################################################ + def _check_existing_galaxy_name(self, stix_object_name: str) -> Union[list, None]: + if stix_object_name in self.synonyms_mapping: + return self.synonyms_mapping[stix_object_name] + for name, tag_names in self.synonyms_mapping.items(): + if stix_object_name in name: + return tag_names + def _create_cluster_args( self, stix_object: _GALAXY_OBJECTS_TYPING, galaxy_type: str, description: Optional[str] = None, @@ -303,7 +313,7 @@ def _parse_galaxy_as_container(self, stix_object: _GALAXY_OBJECTS_TYPING, def _parse_galaxy_as_tag_names(self, stix_object: _GALAXY_OBJECTS_TYPING, object_type: Union[str, None]) -> dict: name = stix_object.name - tag_names = self.main_parser._check_existing_galaxy_name(name) + tag_names = self._check_existing_galaxy_name(name) if tag_names is None: tag_names = [ f'misp-galaxy:{object_type or stix_object.type}="{name}"' diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index bc50e61f..876ba519 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -1,131 +1,22 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import ipaddress -import re -from .exceptions import ( - InvalidSTIXPatternError, UnknownParsingFunctionError, - UnknownObservableMappingError, UnknownPatternMappingError, - UnknownPatternTypeError, UnknownStixObjectTypeError) +from .exceptions import UnknownParsingFunctionError, UnknownStixObjectTypeError from .external_stix2_mapping import ExternalSTIX2toMISPMapping -from .importparser import _INDICATOR_TYPING from .converters import ( ExternalSTIX2AttackPatternConverter, ExternalSTIX2CampaignConverter, ExternalSTIX2CourseOfActionConverter, ExternalSTIX2IdentityConverter, ExternalSTIX2IndicatorConverter, ExternalSTIX2IntrusionSetConverter, ExternalSTIX2LocationConverter, ExternalSTIX2MalwareAnalysisConverter, - ExternalSTIX2MalwareConverter, ExternalSTIX2ThreatActorConverter, - ExternalSTIX2ToolConverter, ExternalSTIX2VulnerabilityConverter, - STIX2ObservableObjectConverter) -from .stix2_to_misp import ( - STIX2toMISPParser, _COURSE_OF_ACTION_TYPING, _GALAXY_OBJECTS_TYPING, - _IDENTITY_TYPING, _NETWORK_TRAFFIC_TYPING, _OBSERVABLE_TYPING, - _OBSERVED_DATA_TYPING, _SDO_TYPING, _VULNERABILITY_TYPING) + ExternalSTIX2MalwareConverter, ExternalSTIX2ObservedDataConverter, + ExternalSTIX2ThreatActorConverter, ExternalSTIX2ToolConverter, + ExternalSTIX2VulnerabilityConverter, STIX2ObservableObjectConverter) +from .stix2_to_misp import STIX2toMISPParser, _OBSERVABLE_TYPING from collections import defaultdict -from pymisp import MISPGalaxy, MISPGalaxyCluster, MISPObject -from stix2.v20.observables import ( - AutonomousSystem as AutonomousSystem_v20, Directory as Directory_v20, - DomainName as DomainName_v20, EmailAddress as EmailAddress_v20, - EmailMessage as EmailMessage_v20, File as File_v20, - IPv4Address as IPv4Address_v20, IPv6Address as IPv6Address_v20, - MACAddress as MACAddress_v20, Mutex as Mutex_v20, - NetworkTraffic as NetworkTraffic_v20, Process as Process_v20, - URL as URL_v20, WindowsPEBinaryExt as WindowsPEBinaryExt_v20, - WindowsRegistryKey as WindowsRegistryKey_v20, - X509Certificate as X509Certificate_v20) -from stix2.v20.sdo import ( - CourseOfAction as CourseOfAction_v20, Vulnerability as Vulnerability_v20) -from stix2.v21.observables import ( - AutonomousSystem as AutonomousSystem_v21, Directory as Directory_v21, - DomainName as DomainName_v21, EmailAddress as EmailAddress_v21, - EmailMessage as EmailMessage_v21, File as File_v21, - IPv4Address as IPv4Address_v21, IPv6Address as IPv6Address_v21, - MACAddress as MACAddress_v21, Mutex as Mutex_v21, - NetworkTraffic as NetworkTraffic_v21, Process as Process_v21, - URL as URL_v21, WindowsPEBinaryExt as WindowsPEBinaryExt_v21, - WindowsRegistryKey as WindowsRegistryKey_v21, - X509Certificate as X509Certificate_v21) -from stix2.v21.sdo import ( - CourseOfAction as CourseOfAction_v21, Indicator as Indicator_v21, Location, - ObservedData as ObservedData_v21, Vulnerability as Vulnerability_v21) -from typing import Optional, Tuple, Union +from typing import Optional, Union MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' -# Useful lists -_observable_skip_properties = ( - 'content_ref', 'content_type', 'decryption_key', 'defanged', - 'encryption_algorithm', 'extensions', 'id', 'is_encrypted', 'is_multipart', - 'magic_number_hex', 'received_lines', 'sender_ref', 'spec_version', 'type' -) - -# Typing definitions -_AUTONOMOUS_SYSTEM_TYPING = Union[ - AutonomousSystem_v20, AutonomousSystem_v21 -] -_DIRECTORY_TYPING = Union[ - Directory_v20, Directory_v21 -] -_DOMAIN_NAME_TYPING = Union[ - DomainName_v20, DomainName_v21 -] -_EMAIL_ADDRESS_TYPING = Union[ - EmailAddress_v20, EmailAddress_v21 -] -_EMAIL_MESSAGE_TYPING = Union[ - EmailMessage_v20, EmailMessage_v21 -] -_FILE_TYPING = Union[ - File_v20, File_v21 -] -_GENERIC_SDO_TYPING = Union[ - CourseOfAction_v20, - CourseOfAction_v21, - Vulnerability_v20, - Vulnerability_v21 -] -_OBSERVABLE_OBJECTS_TYPING = Union[ - Directory_v20, Directory_v21, - DomainName_v20, DomainName_v21, - EmailAddress_v20, EmailAddress_v21, - EmailMessage_v20, EmailMessage_v21, - File_v20, File_v21, - IPv4Address_v20, IPv4Address_v21, - IPv6Address_v20, IPv6Address_v21, - MACAddress_v20, MACAddress_v21, - Mutex_v20, Mutex_v21, - NetworkTraffic_v20, NetworkTraffic_v21, - Process_v20, Process_v21, - URL_v20, URL_v21, - WindowsPEBinaryExt_v20, WindowsPEBinaryExt_v21, - WindowsRegistryKey_v20, WindowsRegistryKey_v21, - X509Certificate_v20, X509Certificate_v21 -] -_IP_ADDRESS_TYPING = Union[ - IPv4Address_v20, IPv4Address_v21, - IPv6Address_v20, IPv6Address_v21 -] -_MAC_ADDRESS_TYPING = Union[ - MACAddress_v20, MACAddress_v21 -] -_MUTEX_TYPING = Union[ - Mutex_v20, Mutex_v21 -] -_PE_EXTENSION_TYPING = Union[ - WindowsPEBinaryExt_v20, WindowsPEBinaryExt_v21 -] -_PROCESS_TYPING = Union[ - Process_v20, Process_v21 -] -_REGISTRY_KEY_TYPING = Union[ - WindowsRegistryKey_v20, WindowsRegistryKey_v21 -] -_URL_TYPING = Union[ - URL_v20, URL_v21 -] -_X509_TYPING = Union[ - X509Certificate_v20, X509Certificate_v21 -] class ExternalSTIX2toMISPParser(STIX2toMISPParser): @@ -153,6 +44,7 @@ def __init__(self, distribution: Optional[int] = 0, self._malware_analysis_parser: ExternalSTIX2MalwareAnalysisConverter self._malware_parser: ExternalSTIX2MalwareConverter self._observable_object_parser: STIX2ObservableObjectConverter + self._observed_data_parser: ExternalSTIX2ObservedDataConverter self._threat_actor_parser: ExternalSTIX2ThreatActorConverter self._tool_parser: ExternalSTIX2ToolConverter self._vulnerability_parser: ExternalSTIX2VulnerabilityConverter @@ -208,6 +100,9 @@ def _set_malware_parser(self): def _set_observable_object_parser(self): self._observable_object_parser = STIX2ObservableObjectConverter(self) + def _set_observed_data_parser(self): + self._observed_data_parser = ExternalSTIX2ObservedDataConverter(self) + def _set_threat_actor_parser(self): self._threat_actor_parser = ExternalSTIX2ThreatActorConverter(self) @@ -233,72 +128,6 @@ def _load_observable_object(self, observable: _OBSERVABLE_TYPING): # MAIN STIX OBJECTS PARSING METHODS. # ############################################################################ - @staticmethod - def _fetch_identity_object_name(identity: _IDENTITY_TYPING) -> str: - if getattr(identity, 'identity_class', None) == 'organization': - return 'organization' - return 'identity' - - def _get_attributes_from_generic_SDO( - self, stix_object: _GENERIC_SDO_TYPING, mapping: str): - for feature, attribute in getattr(self._mapping, mapping)().items(): - if hasattr(stix_object, feature): - yield {'value': getattr(stix_object, feature), **attribute} - - def _handle_import_case(self, stix_object: _SDO_TYPING, attributes: list, - name: str, *force_object: Tuple[str]): - """ - After we extracted attributes from a STIX object (Indicator pattern, - Observable object, Vulnerability fields, etc.), we want to know if it is - appropriate to import one single attribute or to create a MISP object with - an association of attributes included in a given object template. - - :param stix_object: The STIX object we convert into a MISP data structure - :param attributes: The attribute(s) extracted from the STIX object - :param name: The MISP object name or MISP attribute type to be used for the - converted MISP feature - :param force_object: List of object_relation values that force the MISP - object creation over a MISP attribute, if at least one of the attribute - has a matching object_relation field - """ - if self._handle_object_forcing(attributes, force_object): - self._handle_object_case(stix_object, attributes, name) - else: - self._add_misp_attribute( - dict( - self._create_attribute_dict(stix_object), **attributes[0] - ), - stix_object - ) - - def _handle_object_case( - self, stix_object: _SDO_TYPING, attributes: list, name: str): - """ - The attributes we generated from data converted from STIX are considered - as part of an object template. - - :param stix_object: The STIX object we convert to a MISP object - :param attributes: The attributes extracted from the STIX object - :param name: The MISP object name - """ - misp_object = self._create_misp_object(name, stix_object) - tags = tuple(self._handle_tags_from_stix_fields(stix_object)) - if tags: - for attribute in attributes: - misp_attribute = misp_object.add_attribute(**attribute) - for tag in tags: - misp_attribute.add_tag(tag) - return self.misp_event.add_object(misp_object) - for attribute in attributes: - misp_object.add_attribute(**attribute) - return self.misp_event.add_object(misp_object) - - @staticmethod - def _handle_object_forcing(attributes: list, force_object: tuple) -> bool: - if len(attributes) > 1: - return True - return attributes[0]['object_relation'] in force_object - def _handle_object_refs(self, object_refs: list): for object_ref in object_refs: object_type = object_ref.split('--')[0] @@ -317,40 +146,6 @@ def _handle_object_refs(self, object_refs: list): except UnknownParsingFunctionError as error: self._unknown_parsing_function_error(error) - def _handle_observables_mapping(self, observable_mapping: set) -> str: - """ - Simple Observable object types handling function. - We check if the Observable object types are actually mapped with a - parsing function and return it, or raise an exception. - - :param observable_mapping: The observable types in a set - :returns: The feature used to identify the appropriate parsing function - :raises: Exception when the observable types are not known - """ - to_call = '_'.join(sorted(observable_mapping)) - mapping = self._mapping.observable_mapping(to_call) - if mapping is None: - raise UnknownObservableMappingError(to_call) - return mapping - - def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: - """ - Mapping between an indicator pattern and the function used to parse it and - convert it into a MISP attribute or object. - - :param indicator: The indicator - :return: The parsing function name to convert the indicator into a MISP - attribute or object, using its pattern - """ - if isinstance(indicator, Indicator_v21) and indicator.pattern_type != 'stix': - try: - return f'_parse_{indicator.pattern_type}_pattern' - except KeyError: - raise UnknownPatternTypeError(indicator.pattern_type) - if self._is_pattern_too_complex(indicator.pattern): - return '_create_stix_pattern_object' - return '_parse_stix_pattern' - def _handle_unparsed_content(self): if not hasattr(self, '_observable'): return super()._handle_unparsed_content() @@ -383,1438 +178,3 @@ def _parse_loaded_features(self): for observable in self._observable.values(): observable['used'][self.misp_event.uuid] = False super()._parse_loaded_features() - - def _parse_observable_objects(self, observed_data: _OBSERVED_DATA_TYPING): - """ - Observed Data with embedded `objects` field parsing. - Depending on the Observable object types, we call the appropriate - parsing function. - - :param observed_data: The Observed Data object - """ - observable_types = set( - observable['type'] for observable in observed_data.objects.values() - ) - mapping = self._handle_observables_mapping(observable_types) - feature = f'_parse_{mapping}_observable_objects' - try: - parser = getattr(self, feature) - except AttributeError: - raise UnknownParsingFunctionError(feature) - parser(observed_data, 'objects') - - def _parse_observable_refs(self, observed_data: ObservedData_v21): - """ - Observed Data with `object_refs` field parsing. - The observable types are extracted first. Depending on the types, we - look for the referenced Observable objects and parse them. - - :param observed_data: The Observed Data object - """ - observable_types = set( - reference.split('--')[0] for reference in observed_data.object_refs - ) - mapping = self._handle_observables_mapping(observable_types) - feature = f'_parse_{mapping}_observable_objects' - try: - parser = getattr(self, feature) - except AttributeError: - raise UnknownParsingFunctionError(feature) - parser(observed_data, 'object_refs') - - def _parse_observed_data(self, observed_data_ref: str): - """ - Observed Data parsing function. - We want first to find out which parsing function is the best depending - whether there is an `object_refs` field or the embedded `objects`. - Then we call the appropriate parsing function - - :param observed_data: The Observed Data object - """ - observed_data = self._get_stix_object(observed_data_ref) - try: - if hasattr(observed_data, 'object_refs'): - self._parse_observable_refs(observed_data) - else: - self._parse_observable_objects(observed_data) - except UnknownObservableMappingError as observable_types: - self._observable_mapping_error(observed_data.id, observable_types) - - ################################################################################ - # OBSERVABLE OBJECTS PARSING FUNCTIONS # - ################################################################################ - - def _check_email_observable_fields( - self, email_message: _EMAIL_MESSAGE_TYPING) -> bool: - fields = tuple(self._get_populated_properties(email_message)) - if len(fields) > 1: - return True - length = 0 - for field, values in getattr( - email_message, 'additional_header_fields', {} - ).items(): - if field in self._mapping.email_additional_header_fields_mapping(): - length += len(values) if isinstance(values, list) else 1 - return length > 1 - - def _check_file_observable_fields(self, file_object: _FILE_TYPING) -> bool: - if 'windows-pebinary-ext' in getattr(file_object, 'extensions', {}): - return True - fields = [ - field for field in self._mapping.file_object_mapping() - if hasattr(file_object, field) - ] - if len(fields) > 1: - return True - return len(getattr(file_object, 'hashes', {})) > 1 - - def _check_registry_key_observable_fields( - self, registry_key: _REGISTRY_KEY_TYPING) -> bool: - if 'values' in registry_key.properties_populated(): - if len(registry_key['values']) > 1: - return True - value = registry_key['values'][0] - return 'name' in value or 'data_type' in value - return False - - def _create_attribute_from_observable_object( - self, attribute_type: str, value: str, - observed_data_id: str, object_id: str) -> dict: - return { - 'type': attribute_type, - 'value': value, - 'uuid': self._create_v5_uuid( - f'{observed_data_id} - {object_id} - {attribute_type} - {value}' - ), - 'comment': f'Original Observed Data ID: {observed_data_id}' - } - - def _create_attribute_from_observable_object_with_id( - self, attribute_type: str, value: str, - observable_object_id: str, observed_data_id: str) -> dict: - return { - 'type': attribute_type, - 'value': value, - 'uuid': self._sanitise_uuid(observable_object_id), - 'comment': f'Original Observed Data ID: {observed_data_id}' - } - - def _create_attribute_from_single_observable_object( - self, attribute_type: str, value: str, observed_data_id: str) -> dict: - return { - 'type': attribute_type, - 'value': value, - 'uuid': self._create_v5_uuid( - f'{observed_data_id} - {attribute_type} - {value}' - ), - 'comment': f'Original Observed Data ID: {observed_data_id}' - } - - def _fetch_observable_object_refs( - self, observed_data: _OBSERVED_DATA_TYPING): - for reference in observed_data.object_refs: - self._observable[reference]['used'][self.misp_event.uuid] = True - yield self._observable[reference]['observable'] - - def _fetch_observable_object_refs_with_id( - self, observed_data: _OBSERVED_DATA_TYPING): - for reference in observed_data.object_refs: - self._observable[reference]['used'][self.misp_event.uuid] = True - yield reference, self._observable[reference]['observable'] - - @staticmethod - def _fetch_observable_objects( - observed_data: _OBSERVED_DATA_TYPING): - for observable_object in observed_data.objects.values(): - yield observable_object - - @staticmethod - def _fetch_observable_objects_with_id( - observed_data: _OBSERVED_DATA_TYPING): - yield from observed_data.objects.items() - - def _fill_observable_object_attribute( - self, reference: str, observed_data_id: str) -> dict: - return { - 'uuid': self._create_v5_uuid(reference), - 'comment': f'Original Observed Data ID: {observed_data_id}' - } - - @staticmethod - def _filter_observable_objects( - observable_objects: dict, *object_types: Tuple[str]): - for object_id, observable_object in observable_objects.items(): - if observable_object.type in object_types: - yield object_id - - def _force_observable_as_object( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, - object_type: str) -> bool: - fields = getattr(self._mapping, f'{object_type}_object_fields')() - if any(hasattr(observable_object, field) for field in fields): - return True - return getattr(self, f'_check_{object_type}_observable_fields')( - observable_object - ) - - def _handle_observable_object_attribute( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, object_id: str, - observed_data_id: str, references: str) -> dict: - if hasattr(observable_object, 'id'): - return self._fill_observable_object_attribute( - f'{observable_object.id} - {references}', observed_data_id - ) - return self._fill_observable_object_attribute( - f'{observed_data_id} - {object_id} - {references}', - observed_data_id - ) - - def _handle_observable_attribute( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, feature: str, - attribute_type: str, observed_data_id: str, object_id: str) -> dict: - value = getattr(observable_object, feature) - if hasattr(observable_object, 'id'): - return self._create_attribute_from_observable_object_with_id( - attribute_type, value, observable_object.id, observed_data_id - ) - return self._create_attribute_from_observable_object( - attribute_type, value, observed_data_id, object_id - ) - - def _handle_observable_special_attribute( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, value: str, - attribute_type: str, observed_data_id: str, object_id: str) -> dict: - if hasattr(observable_object, 'id'): - return self._create_attribute_from_observable_object_with_id( - attribute_type, value, observable_object.id, observed_data_id - ) - return self._create_attribute_from_observable_object( - attribute_type, value, observed_data_id, object_id - ) - - def _handle_single_observable_attribute( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, feature: str, - attribute_type: str, observed_data_id: str) -> dict: - value = getattr(observable_object, feature) - if hasattr(observable_object, 'id'): - return self._create_attribute_from_observable_object_with_id( - attribute_type, value, observable_object.id, observed_data_id - ) - return self._create_attribute_from_single_observable_object( - attribute_type, value, observed_data_id - ) - - def _handle_single_observable_special_attribute( - self, observable_object: _OBSERVABLE_OBJECTS_TYPING, value: str, - attribute_type: str, observed_data_id: str) -> dict: - if hasattr(observable_object, 'id'): - return self._create_attribute_from_observable_object_with_id( - attribute_type, value, observable_object.id, observed_data_id - ) - return self._create_attribute_from_single_observable_object( - attribute_type, value, observed_data_id - ) - - def _has_observable_types( - self, observable_objects: dict, *object_types: Tuple[str]) -> bool: - object_ids = self._filter_observable_objects( - observable_objects, *object_types - ) - return next(object_ids, None) is not None - - def _parse_as_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_as_observables(observed_data, asset) - else: - self._parse_as_single_observable(observed_data, asset) - - def _parse_as_observables( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - autonomous_systems = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, autonomous_system in autonomous_systems.items(): - self._parse_autonomous_system_observables( - autonomous_system, observed_data, object_id - ) - - def _parse_as_single_observable( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - autonomous_system = next( - getattr(self, f'_fetch_observable_{asset}')(observed_data) - ) - self._parse_autonomous_system_single_observable( - autonomous_system, observed_data - ) - - def _parse_asn_object_attributes( - self, autonomous_system: _AUTONOMOUS_SYSTEM_TYPING, object_id: str, - observed_data_id: str, misp_object: MISPObject) -> MISPObject: - asn_attribute = ('asn', f'AS{autonomous_system.number}') - misp_object.add_attribute( - *asn_attribute, - **self._handle_observable_object_attribute( - autonomous_system, object_id, observed_data_id, - ' - '.join(asn_attribute) - ) - ) - if hasattr(autonomous_system, 'name'): - description_attribute = ('description', autonomous_system.name) - misp_object.add_attribute( - *description_attribute, - **self._handle_observable_object_attribute( - autonomous_system, object_id, observed_data_id, - ' - '.join(description_attribute) - ) - ) - - def _parse_asn_observable_object( - self, autonomous_system: _AUTONOMOUS_SYSTEM_TYPING, - object_id: str, observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object_from_observable( - 'asn', autonomous_system, object_id, observed_data - ) - self._parse_asn_object_attributes( - autonomous_system, object_id.split(' - ')[0], - observed_data.id, misp_object - ) - return misp_object - - def _parse_asn_observable_objects( - self, observed_data: ObservedData_v21, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - if self._has_observable_types( - observable_objects, 'ipv4-addr', 'ipv6-addr'): - self._parse_asn_observables(observable_objects, observed_data) - else: - self._parse_autonomous_system_observable_objects( - observable_objects, observed_data - ) - - def _parse_asn_observables( - self, observable_objects: dict, observed_data: _OBSERVED_DATA_TYPING): - as_ids = tuple( - self._filter_observable_objects( - observable_objects, 'autonomous-system' - ) - ) - feature = 'observable' if len(as_ids) > 1 else 'single_observable' - for as_id in as_ids: - autonomous_system = observable_objects[as_id] - references = tuple( - ref for ref, observable_object in observable_objects.items() - if observable_object.type in ('ipv4-addr', 'ipv6-addr') - and as_id in getattr(observable_object, 'belongs_to_refs', []) - ) - if references: - reference = f"{as_id} - {' - '.join(sorted(references))}" - misp_object = self._parse_asn_observable_object( - autonomous_system, reference, observed_data - ) - for ip_id in references: - ip_address = observable_objects[ip_id] - misp_object.add_attribute( - 'subnet-announced', ip_address.value, - **self._handle_observable_object_attribute( - ip_address, ip_id, observed_data.id, - f'subnet-announced - {ip_address.value}' - ) - ) - self._add_misp_object(misp_object, observed_data) - continue - getattr(self, f'_parse_autonomous_system_{feature}_object')( - autonomous_system, observed_data, as_id - ) - - def _parse_autonomous_system_observable_objects( - self, autonomous_systems: dict, observed_data: _OBSERVED_DATA_TYPING): - if len(autonomous_systems) > 1: - for object_id, autonomous_system in autonomous_systems.items(): - self._parse_autonomous_system_observables( - autonomous_system, observed_data, object_id - ) - else: - self._parse_autonomous_system_single_observable( - autonomous_systems.values()[0], observed_data - ) - - def _parse_autonomous_system_observables( - self, autonomous_system: _OBSERVABLE_OBJECTS_TYPING, - observed_data: _OBSERVED_DATA_TYPING, object_id: str): - if hasattr(autonomous_system, 'name'): - self._add_misp_object( - self._parse_asn_observable_object( - autonomous_system, object_id, observed_data - ), - observed_data - ) - else: - self._add_misp_attribute( - self._handle_observable_special_attribute( - autonomous_system, f'AS{autonomous_system.number}', - 'AS', observed_data.id, object_id - ), - observed_data - ) - - def _parse_autonomous_system_single_observable( - self, autonomous_system: _OBSERVABLE_OBJECTS_TYPING, - observed_data: _OBSERVED_DATA_TYPING, - object_id: Optional[str] = '0'): - if hasattr(autonomous_system, 'name'): - self._add_misp_object( - self._parse_asn_observable_object( - autonomous_system, object_id, observed_data - ), - observed_data - ) - else: - self._add_misp_attribute( - self._handle_single_observable_special_attribute( - autonomous_system, f'AS{autonomous_system.number}', - 'AS', observed_data.id - ), - observed_data - ) - - def _parse_directory_observable_object( - self, directory: _DIRECTORY_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - reference = getattr( - directory, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'directory', directory, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'directory_object_mapping', directory, misp_object, - reference, observed_data.id - ) - return self._add_misp_object(misp_object, observed_data) - - def _parse_directory_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - directories = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, directory in directories.items(): - self._parse_directory_observable_object( - directory, object_id, observed_data - ) - - def _parse_directory_observables( - self, object_id: str, observable_objects: dict, mapping: dict, - feature: str, observed_data: _OBSERVED_DATA_TYPING) -> str: - directory = observable_objects[object_id] - misp_object = self._parse_directory_observable_object( - directory, object_id, observed_data - ) - if hasattr(directory, 'contains_refs'): - for contains_ref in directory.contains_refs: - if contains_ref in mapping: - misp_object.add_reference(mapping[contains_ref], 'contains') - continue - if contains_ref in observable_objects: - object_type = observable_objects[contains_ref].type - contains_uuid = getattr( - self, f'_parse_{object_type}_observables' - )( - contains_ref, observable_objects, mapping, - feature, observed_data - ) - mapping[contains_ref] = contains_uuid - misp_object.add_reference(contains_uuid, 'contains') - return misp_object.uuid - - def _parse_domain_ip_observables( - self, object_id: str, observable_objects: dict, mapping: dict, - feature: str, observed_data: _OBSERVED_DATA_TYPING) -> str: - domain = observable_objects.pop(object_id) - if hasattr(domain, 'resolves_to_refs'): - domain_reference = getattr( - domain, 'id', f'{observed_data.id} - {object_id}' - ) - domain_ip_object = self._create_misp_object_from_observable( - 'domain-ip', domain, object_id, observed_data - ) - domain_ip_object.add_attribute( - 'domain', domain.value, - **self._fill_observable_object_attribute( - f'{domain_reference} - domain - {domain.value}', - observed_data.id - ) - ) - references = { - ref: observable_objects[ref] for ref in domain.resolve_to_refs - } - for ip_id in self._filter_observable_objects( - references, 'ipv4-addr', 'ipv6-addr'): - ip_address = references[ip_id] - ip_ref = getattr(ip_address, 'id', ip_id) - attribute = domain_ip_object.add_attribute( - 'ip', ip_address.value, - f'{domain_reference} - {ip_ref} - ip - {ip_address.value}', - observed_data.id - ) - mapping[ip_id] = attribute.uuid - misp_object = self._add_misp_object(domain_ip_object, observed_data) - if any(ref.type == 'domain-name' for ref in references.values()): - for domain_id in self._filter_observable_objects( - references, 'domain-name'): - if domain_id in mapping: - misp_object.add_reference( - mapping[domain_id], 'resolves-to' - ) - continue - domain_uuid = self._parse_domain_ip_observables( - domain_id, observable_objects, mapping, - feature, observed_data - ) - mapping[domain_id] = domain_uuid - misp_object.add_reference(domain_uuid, 'resolves-to') - return misp_object.uuid - attribute = self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - domain, 'value', 'domain', observed_data.id, object_id - ), - observed_data - ) - return attribute.uuid - - def _parse_domain_ip_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - domain_ids = tuple( - self._filter_observable_objects(observable_objects, 'domain-name') - ) - feature = 'observable' if len(observable_objects) > 1 else 'single_observable' - mapping = {} - for domain_id in domain_ids: - if domain_id not in mapping: - self._parse_domain_ip_observables( - domain_id, observable_objects, mapping, - feature, observed_data - ) - ip_ids = self._filter_observable_objects( - observable_objects, 'ipv4-addr', 'ipv6-addr' - ) - for ip_id in ip_ids: - if ip_id not in mapping: - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - observable_objects[ip_id], 'value', 'ip-dst', - observed_data.id, ip_id - ), - observed_data - ) - - def _parse_domain_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_generic_observables( - observed_data, 'value', 'domain', asset - ) - else: - self._parse_generic_single_observable( - observed_data, 'value', 'domain', asset - ) - - def _parse_email_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset): - if len(observed_data.objects) > 1: - self._parse_email_address_observables(observed_data, asset) - else: - self._parse_email_address_single_observable(observed_data, asset) - - def _parse_email_address_observables( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - email_addresses = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, email_address in email_addresses.items(): - if hasattr(email_address, 'display_name'): - self._add_misp_attribute( - self._create_attribute_from_observable_object( - 'email-dst', email_address.value, - observed_data.id, object_id - ), - observed_data - ) - self._add_misp_attribute( - self._create_attribute_from_observable_object( - 'email-dst-display-name', email_address.display_name, - observed_data.id, object_id - ), - observed_data - ) - continue - self._add_misp_attribute( - self._handle_observable_attribute( - email_address, 'value', 'email-dst', - observed_data.id, object_id - ), - observed_data - ) - - def _parse_email_address_single_observable( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - email_address = next( - getattr(self, f'_fetch_observable_{asset}')(observed_data) - ) - if hasattr(email_address, 'display_name'): - self._add_misp_attribute( - self._create_attribute_from_single_observable_object( - 'email-dst', email_address.value, observed_data.id - ), - observed_data - ) - self._add_misp_attribute( - self._create_attribute_from_single_observable_object( - 'email-dst-display-name', email_address.display_name, - observed_data.id - ), - observed_data - ) - else: - self._add_misp_attribute( - self._handle_single_observable_attribute( - email_address, 'value', 'email-dst', observed_data.id - ), - observed_data - ) - - def _parse_email_observable_object( - self, email_message: _EMAIL_MESSAGE_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING, observable_objects: dict): - reference = getattr( - email_message, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'email', email_message, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'email_object_mapping', email_message, misp_object, - reference, observed_data.id - ) - if getattr(email_message, 'from_ref', None) in observable_objects: - self._parse_email_observable_object_reference( - misp_object, observable_objects[email_message.from_ref], 'from', - email_message.from_ref, observed_data.id - ) - for feature in ('to', 'cc', 'bcc'): - if getattr(email_message, f'{feature}_refs', None) in observable_objects: - for address_ref in getattr(email_message, f'{feature}_refs'): - self._parse_email_observable_object_reference( - misp_object, observable_objects[address_ref], feature, - address_ref, observed_data.id - ) - if hasattr(email_message, 'additional_header_fields'): - for field, mapping in self._mapping.email_additional_header_fields_mapping().items(): - if field not in email_message.additional_header_fields: - continue - relation = mapping['object_relation'] - values = email_message.additional_header_fields[field] - if isinstance(values, list): - for index, value in enumerate(values): - misp_object.add_attribute( - **{ - 'value': value, **mapping, - **self._fill_observable_object_attribute( - f'{reference} - {index} - {relation}' - f' - {value}', observed_data.id - ) - } - ) - else: - misp_object.add_attribute( - **{ - 'value': values, **mapping, - **self._fill_observable_object_attribute( - f'{reference} - {relation} - {values}', - observed_data.id - ) - } - ) - self._add_misp_object(misp_object, observed_data) - - def _parse_email_observable_object_reference( - self, misp_object: MISPObject, address: _EMAIL_ADDRESS_TYPING, - relation: str, object_id: str, observed_data_id: str): - reference = getattr( - address, 'id', f'{observed_data_id} - {object_id}' - ) - misp_object.add_attribute( - relation, address.value, - **self._fill_observable_object_attribute( - f'{reference} - {relation} - {address.value}', - observed_data_id - ) - ) - if hasattr(address, 'display_name'): - misp_object.add_attribute( - f'{relation}-display-name', address.display_name, - **self._fill_observable_object_attribute( - f'{reference} - {relation}-display-name - {address.display_name}', - observed_data_id - ) - ) - - def _parse_email_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - email_ids = tuple( - self._filter_observable_objects(observable_objects, 'email-message') - ) - feature = 'observable' if len(email_ids) > 1 else 'single_observable' - for email_id in email_ids: - email_message = observable_objects[email_id] - if self._force_observable_as_object(email_message, 'email'): - self._parse_email_observable_object( - email_message, email_id, observed_data, observable_objects - ) - continue - for field in self._get_populated_properties(email_message): - attribute = self._mapping.email_message_mapping(field) - if attribute is not None: - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - email_message, field, attribute['type'], - observed_data.id - ), - observed_data - ) - break - # The only remaining field that is supported in the conversion - # mapping at this point should be `additional_header_fields` - if field == 'additional_header_fields': - header_fields = email_message.additional_header_fields - if 'Reply-To' in header_fields: - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - header_fields, 'Reply-To', 'email-reply-to', - observed_data.id - ), - observed_data - ) - break - if 'X-Mailer' in header_fields: - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - header_fields, 'X-Mailer', 'email-x-mailer', - observed_data.id - ), - observed_data - ) - break - - def _parse_file_observable_object( - self, file_object: _FILE_TYPING, object_id: str, references: dict, - reference: str, observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object_from_observable( - 'file', file_object, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'file_object_mapping', file_object, misp_object, - reference, observed_data.id - ) - if hasattr(file_object, 'hashes'): - self._populate_object_attributes_from_observable( - 'file_hashes_object_mapping', file_object.hashes, misp_object, - reference, observed_data.id - ) - if hasattr(file_object, 'parent_directory_ref'): - directory = references[file_object.parent_directory_ref] - misp_object.add_attribute( - 'path', directory.path, - **self._fill_observable_object_attribute( - f'{reference} - path - {directory.path}', - observed_data.id - ) - ) - # content_ref still to parse... - return self._add_misp_object(misp_object, observed_data) - - def _parse_file_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - file_ids = tuple( - self._filter_observable_objects(observable_objects, 'file') - ) - feature = 'observable' if len(file_ids) > 1 else 'single_observable' - mapping = {} - for object_id, observable_object in observable_objects.items(): - if object_id not in mapping: - object_type = observable_object.type - mapping[object_id] = getattr( - self, f'_parse_{object_type}_observables' - )( - object_id, observable_objects, mapping, - feature, observed_data - ) - - def _parse_file_observables( - self, object_id: str, observable_objects: dict, mapping: dict, - feature: str, observed_data: _OBSERVED_DATA_TYPING) -> str: - file_object = observable_objects[object_id] - if self._force_observable_as_object(file_object, 'file'): - reference = getattr( - file_object, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._parse_file_observable_object( - file_object, object_id, observable_objects, - reference, observed_data - ) - if hasattr(file_object, 'extensions'): - if 'windows-pebinary-ext' in file_object.extensions: - misp_object.add_reference( - self._parse_file_pe_extension( - file_object.extensions['windows-pebinary-ext'], - reference, observed_data - ), - 'includes' - ) - if hasattr(file_object, 'contains_refs'): - for contains_ref in file_object.contains_refs: - if contains_ref not in observable_objects: - continue - if contains_ref in mapping: - misp_object.add_reference( - mapping[contains_ref], 'contains' - ) - continue - contained_object_uuid = self._parse_file_observables( - contains_ref, observable_objects, mapping, - feature, observed_data - ) - mapping[contains_ref] = contained_object_uuid - misp_object.add_reference( - contained_object_uuid, 'contains' - ) - return misp_object.uuid - for field in self._get_populated_properties(file_object): - attribute = self._mapping.file_object_mapping(field) - if attribute is not None: - misp_attribute = self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - file_object, field, attribute['type'], observed_data.id - ), - observed_data - ) - return misp_attribute.uuid - attribute = self._mapping.file_hashes_object_mapping(field) - if attribute is not None: - misp_attribute = self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - file_object, field, attribute['type'], observed_data.id - ), - observed_data - ) - return misp_attribute.uuid - - def _parse_file_pe_extension( - self, extension: _PE_EXTENSION_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> str: - reference = f'{object_id} - {extension._type}' - misp_object = self._create_misp_object_from_observable_without_id( - 'pe', reference, observed_data - ) - self._populate_object_attributes_from_observable( - 'pe_object_mapping', extension, misp_object, - reference, observed_data.id - ) - if hasattr(extension, 'optional_header'): - self._populate_object_attributes_from_observable( - 'pe_optional_header_object_mapping', extension, misp_object, - reference, observed_data.id - ) - pe_object = self._add_misp_object(misp_object, observed_data) - if hasattr(extension, 'sections'): - for section_id, section in enumerate(extension.sections): - section_reference = f'{reference} - {section_id}' - section_object = self._create_misp_object_from_observable_without_id( - 'pe-section', section_reference, observed_data - ) - self._populate_object_attributes_from_observable( - 'pe_section_object_mapping', section, section_object, - section_reference, observed_data.id - ) - if hasattr(section, 'hashes'): - self._populate_object_attributes_from_observable( - 'file_hashes_object_mapping', section.hashes, section_object, - section_reference, observed_data.id - ) - self._add_misp_object(section_object, observed_data) - pe_object.add_reference(section_object.uuid, 'includes') - return pe_object.uuid - - def _parse_generic_observables( - self, observed_data: _OBSERVED_DATA_TYPING, feature: str, - attribute_type: str, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, observable_object in observable_objects.items(): - self._add_misp_attribute( - self._handle_observable_attribute( - observable_object, feature, attribute_type, - observed_data.id, object_id - ), - observed_data - ) - - def _parse_generic_single_observable( - self, observed_data: _OBSERVED_DATA_TYPING, feature: str, - attribute_type: str, asset: str): - observable_object = next( - getattr(self, f'_fetch_observable_{asset}')(observed_data) - ) - self._add_misp_attribute( - self._handle_single_observable_attribute( - observable_object, feature, attribute_type, observed_data.id - ), - observed_data - ) - - def _parse_ip_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_generic_observables( - observed_data, 'value', 'ip-dst', asset - ) - else: - self._parse_generic_single_observable( - observed_data, 'value', 'ip-dst', asset - ) - - def _parse_mac_address_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_generic_observables( - observed_data, 'value', 'mac-address', asset - ) - else: - self._parse_generic_single_observable( - observed_data, 'value', 'mac-address', asset - ) - - def _parse_mutex_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_generic_observables( - observed_data, 'name', 'mutex', asset - ) - else: - self._parse_generic_single_observable( - observed_data, 'name', 'mutex', asset - ) - - def _parse_network_connection_observable_object( - self, network_traffic: _NETWORK_TRAFFIC_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - reference = getattr( - network_traffic, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._parse_network_traffic_observable_object( - 'network-connection', network_traffic, object_id, - observed_data, reference - ) - for index, protocol in enumerate(network_traffic.protocols): - layer = self._mapping.connection_protocols(protocol) - if layer is not None: - misp_object.add_attribute( - f'layer{layer}-protocol', protocol, - **self._fill_observable_object_attribute( - f'{reference} - {index} - layer{layer}' - f'-protocol - {protocol}', - observed_data.id - ) - ) - return misp_object - - def _parse_network_socket_observable_object( - self, network_traffic: _NETWORK_TRAFFIC_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - reference = getattr( - network_traffic, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._parse_network_traffic_observable_object( - 'network-socket', network_traffic, object_id, - observed_data, reference - ) - socket_extension = network_traffic.extensions['socket-ext'] - self._populate_object_attributes_from_observable( - 'network_socket_extension_object_mapping', socket_extension, - misp_object, reference, observed_data.id - ) - for index, protocol in enumerate(network_traffic.protocols): - misp_object.add_attribute( - 'protocol', protocol, - **self._fill_observable_object_attribute( - f'{reference} - {index} - protocol - {protocol}', - observed_data.id - ) - ) - for feature in ('blocking', 'listening'): - if getattr(socket_extension, f'is_{feature}', False): - misp_object.add_attribute( - 'state', feature, - **self._fill_observable_object_attribute( - f'{reference} - state - {feature}', - observed_data.id - ) - ) - return misp_object - - @staticmethod - def _parse_network_traffic_observable_fields( - network_traffic: _NETWORK_TRAFFIC_TYPING) -> str: - if getattr(network_traffic, 'extensions', {}).get('socket-ext'): - return 'network_socket' - return 'network_connection' - - def _parse_network_traffic_observable_object( - self, name: str, network_traffic: _NETWORK_TRAFFIC_TYPING, - object_id: str, observed_data: _OBSERVED_DATA_TYPING, - reference: str) -> MISPObject: - misp_object = self._create_misp_object_from_observable( - name, network_traffic, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'network_traffic_object_mapping', network_traffic, - misp_object, reference, observed_data.id - ) - return misp_object - - def _parse_network_traffic_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - network_traffic_ids = tuple( - self._filter_observable_objects(observable_objects, 'netork-traffic') - ) - mapping = {} - for nt_id in network_traffic_ids: - if nt_id not in mapping: - mapping[nt_id] = self._parse_network_traffic_observables( - nt_id, observable_objects, mapping, observed_data - ) - - def _parse_network_traffic_observables( - self, object_id: str, observable_objects: dict, mapping: dict, - observed_data: _OBSERVED_DATA_TYPING) -> str: - network_traffic = observable_objects[object_id] - name = self._parse_network_traffic_observable_fields(network_traffic) - misp_object = getattr(self, f'_parse_{name}_observable_object')( - network_traffic, object_id, observed_data - ) - for asset in ('src', 'dst'): - if hasattr(network_traffic, f'{asset}_ref'): - referenced_id = getattr(network_traffic, f'{asset}_ref') - referenced_object = observable_objects[referenced_id] - reference = getattr( - referenced_object, 'id', - f'{observed_data.id} - {referenced_id}' - ) - relation = getattr( - self._mapping, f'{name}_object_reference_mapping' - )( - f"{referenced_object.type.split('-')[0]}-{asset}" - ) - if relation is None: - continue - misp_object.add_attribute( - relation, referenced_object.value, - **self._fill_observable_object_attribute( - f"{reference} - {relation} - {referenced_object.value}", - - ) - ) - misp_object = self._add_misp_object(misp_object, observed_data) - if hasattr(network_traffic, 'encapsulates_refs'): - for referenced_id in network_traffic.encapsulates_refs: - if referenced_id in mapping: - misp_object.add_reference( - mapping[referenced_id], 'encapsulates' - ) - continue - referenced_uuid = self._parse_network_traffic_observables( - referenced_id, observable_objects, mapping, observed_data - ) - mapping[referenced_id] = referenced_uuid - misp_object.add_reference(referenced_uuid, 'encapsulates') - if hasattr(network_traffic, 'encapsulated_by_ref'): - referenced_id = network_traffic.encapsulated_by_ref - if referenced_id in mapping: - misp_object.add_reference( - mapping[referenced_id], 'encapsulated-by' - ) - return misp_object.uuid - referenced_uuid = self._parse_network_traffic_observables( - referenced_id, observable_objects, mapping, observed_data - ) - mapping[referenced_id] = referenced_uuid - misp_object.add_reference(referenced_uuid, 'encapsulated-by') - return misp_object.uuid - - def _parse_process_observable_object( - self, process: _PROCESS_TYPING, process_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - reference = getattr(process, 'id', f'{observed_data.id} - {process_id}') - misp_object = self._create_misp_object_from_observable( - 'process', process, process_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'process_object_mapping', process, misp_object, - reference, observed_data.id - ) - if hasattr(process, 'environment_variables'): - value = ' '.join( - f'{key} {value}' for key, value in - process.environment_variables.items() - ) - misp_object.add_attribute( - 'args', value, - **self._fill_observable_object_attribute( - f'{reference} - args - {value}', observed_data.id - ) - ) - elif hasattr(process, 'arguments'): - value = ' '.join(process.arguments) - misp_object.add_attribute( - 'args', value, - **self._fill_observable_object_attribute( - f'{reference} - args - {value}', observed_data.id - ) - ) - return self._add_misp_object(misp_object, observed_data) - - def _parse_process_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - observable_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - process_ids = tuple( - self._filter_observable_objects(observable_objects, 'process') - ) - mapping = {} - for process_id in process_ids: - if process_id not in mapping: - mapping[process_id] = self._parse_process_observables( - process_id, observable_objects, mapping, observed_data - ) - - def _parse_process_observables( - self, process_id: str, observable_objects: dict, mapping: dict, - observed_data: _OBSERVED_DATA_TYPING) -> str: - process = observable_objects[process_id] - misp_object = self._parse_process_observable_object( - process, process_id, observed_data - ) - if hasattr(process, 'parent_ref'): - parent_ref = process.parent_ref - if parent_ref in observable_objects: - if parent_ref in mapping: - misp_object.add_reference(mapping[parent_ref], 'child-of') - else: - parent_uuid = self._parse_process_observables( - parent_ref, observable_objects, mapping, observed_data - ) - mapping[parent_ref] = parent_uuid - misp_object.add_reference(parent_uuid, 'child-of') - if hasattr(process, 'child_refs'): - for child_ref in process.child_refs: - if child_ref not in observable_objects: - continue - if child_ref in mapping: - misp_object.add_reference(mapping[child_ref], 'parent-of') - continue - child_uuid = self._parse_process_observables( - child_ref, observable_objects, mapping, observed_data - ) - mapping[child_ref] = child_uuid - misp_object.add_reference(child_uuid, 'parent-of') - if hasattr(process, 'image_ref'): - image_ref = process.image_ref - if image_ref in observable_objects: - if image_ref in mapping: - misp_object.add_reference(mapping[image_ref], 'executes') - else: - image_uuid = self._parse_file_observables( - image_ref, observable_objects, mapping, - 'observable', observed_data - ) - mapping[image_ref] = image_uuid - misp_object.add_reference(image_uuid, 'executes') - return misp_object.uuid - - def _parse_registry_key_observable_object( - self, registry_key: _REGISTRY_KEY_TYPING, object_id: str, - observed_data: _OBSERVED_DATA_TYPING): - reference = getattr( - registry_key, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'registry-key', registry_key, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'registry_key_object_mapping', registry_key, misp_object, - reference, observed_data.id - ) - if 'values' not in registry_key.properties_populated(): - return self._add_misp_object(misp_object, observed_data) - if len(registry_key['values']) == 1: - self._populate_object_attributes_from_observable( - 'registry_key_values_object_mapping', registry_key['values'], - misp_object, reference, observed_data.id - ) - return self._add_misp_object(misp_object, observed_data) - registry_key_object = self._add_misp_object(misp_object, observed_data) - for index, registry_value in enumerate(registry_key['values']): - value_reference = f'{reference} - values - {index}' - value_object = self._create_misp_object('registry-key-value') - value_object.from_dict( - uuid=self._create_v5_uuid(value_reference), - comment=f'Original Observed Data ID: {observed_data.id}', - **self._parse_timeline(observed_data) - ) - self._populate_object_attributes_from_observable( - 'registry_key_values_object_mapping', registry_value, - value_object, value_reference, observed_data.id - ) - self._add_misp_object(value_object, observed_data) - registry_key_object.add_reference(value_object.uuid, 'contains') - - def _parse_registry_key_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - registry_keys = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - feature = 'observable' if len(registry_keys) > 1 else 'single_observable' - for object_id, registry_key in registry_keys.items(): - if self._force_observable_as_object(registry_key, 'registry_key'): - self._parse_registry_key_observable_object( - registry_key, object_id, observed_data - ) - continue - if 'values' in registry_key.properties_populated(): - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_special_attribute')( - registry_key, - f"{registry_key.key}|{registry_key['values'][0]['data']}", - 'regkey|value', observed_data.id - ), - observed_data - ) - continue - # Potential exception here is the registry key object has no key - self._add_misp_attribute( - getattr(self, f'_handle_{feature}_attribute')( - registry_key, 'key', 'regkey', observed_data.id - ), - observed_data - ) - - def _parse_software_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - softwares = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, software in softwares.items(): - reference = getattr( - software, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'software', software, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'software_object_mapping', software, misp_object, - reference, observed_data.id - ) - if hasattr(software, 'languages'): - for index, language in enumerate(software.languages): - misp_object.add_attribute( - 'language', language, - **self._fill_observable_object_attribute( - f'{reference} - languages - {index}', - observed_data.id - ) - ) - self._add_misp_object(misp_object, observed_data) - - def _parse_url_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - if len(getattr(observed_data, asset)) > 1: - self._parse_generic_observables( - observed_data, 'value', 'url', asset - ) - else: - self._parse_generic_single_observable( - observed_data, 'value', 'url', asset - ) - - def _parse_user_account_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - user_accounts = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, user_account in user_accounts.items(): - reference = getattr( - user_account, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'user-account', user_account, object_id, observed_data - ) - self._populate_object_attributes_from_observable( - 'user_account_object_mapping', user_account, misp_object, - reference, observed_data.id - ) - if 'unix-account-ext' in getattr(user_account, 'extensions', {}): - self._populate_object_attributes_from_observable( - 'user_account_unix_extension_object_mapping', - user_account.extensions['unix-account-ext'], - misp_object, reference, observed_data.id - ) - self._add_misp_object(misp_object, observed_data) - - def _parse_x509_observable_objects( - self, observed_data: _OBSERVED_DATA_TYPING, asset: str): - x509_objects = dict( - getattr(self, f'_fetch_observable_{asset}_with_id')(observed_data) - ) - for object_id, x509_object in x509_objects.items(): - reference = getattr( - x509_object, 'id', f'{observed_data.id} - {object_id}' - ) - misp_object = self._create_misp_object_from_observable( - 'x509-certificate', x509_object, object_id, observed_data - ) - if hasattr(x509_object, 'hashes'): - self._populate_object_attributes_from_observable( - 'x509_hashes_object_mapping', x509_object.hashes, - misp_object, reference, observed_data.id - ) - self._populate_object_attributes_from_observable( - 'x509_object_mapping', x509_object, misp_object, - reference, observed_data.id - ) - self._add_misp_object(misp_object, observed_data) - - def _populate_object_attributes_from_observable( - self, feature: str, observable_object: _OBSERVABLE_OBJECTS_TYPING, - misp_object: MISPObject, reference: str, observed_data_id: str): - for field, attribute in getattr(self._mapping, feature)().items(): - if observable_object.get(field): - value = getattr( - observable_object, field, observable_object[field] - ) - misp_object.add_attribute( - **{ - 'value': value, **attribute, - **self._fill_observable_object_attribute( - f"{reference} - {attribute['object_relation']}" - f' - {value}', observed_data_id - ) - } - ) - - ################################################################################ - # MISP DATA STRUCTURES CREATION FUNCTIONS. # - ################################################################################ - - def _create_attribute_dict(self, stix_object: _SDO_TYPING) -> dict: - return super()._create_attribute_dict(stix_object) - - def _create_misp_object_from_observable( - self, name: str, observable_object: _OBSERVABLE_OBJECTS_TYPING, - object_id: str, observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - if hasattr(observable_object, 'id'): - return self._create_misp_object_from_observable_with_id( - name, observable_object.id, observed_data - ) - return self._create_misp_object_from_observable_without_id( - name, object_id, observed_data - ) - - def _create_misp_object_from_observable_with_id( - self, name: str, observable_object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object(name) - self._sanitise_object_uuid(misp_object, observable_object_id) - misp_object.from_dict(**self._parse_timeline(observed_data)) - return misp_object - - def _create_misp_object_from_observable_without_id( - self, name: str, object_id: str, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - misp_object = self._create_misp_object(name) - misp_object.from_dict( - uuid=self._create_v5_uuid(f'{observed_data.id} - {object_id}'), - comment=f'Original Observed Data ID: {observed_data.id}', - **self._parse_timeline(observed_data) - ) - return misp_object - - def _create_misp_object_from_single_observable( - self, name: str, observable_object: _OBSERVABLE_OBJECTS_TYPING, - observed_data: _OBSERVED_DATA_TYPING) -> MISPObject: - if hasattr(observable_object, 'id'): - return self._create_misp_object_from_observable_with_id( - name, observable_object.id, observed_data - ) - return self._create_misp_object(name, stix_object=observed_data) - - ################################################################################ - # UTILITY FUNCTIONS. # - ################################################################################ - - def _check_existing_galaxy_name(self, stix_object_name: str) -> Union[list, None]: - if stix_object_name in self.synonyms_mapping: - return self.synonyms_mapping[stix_object_name] - for name, tag_names in self.synonyms_mapping.items(): - if stix_object_name in name: - return tag_names - - @staticmethod - def _extract_types_from_observables(observed_data: _OBSERVED_DATA_TYPING) -> tuple: - if hasattr(observed_data, 'object_refs'): - return 'refs', [ - ref.split('--')[0] for ref in observed_data.object_refs - ] - return 'objects', [ - observable.type for observable in observed_data.objects.values() - ] - - @staticmethod - def _get_populated_properties(observable_object: _OBSERVABLE_OBJECTS_TYPING): - for field in observable_object.properties_populated(): - if field not in _observable_skip_properties: - yield field - - @staticmethod - def _handle_external_references(external_references: list) -> dict: - meta = defaultdict(list) - for reference in external_references: - if reference.get('url'): - meta['refs'].append(reference['url']) - if reference.get('external_id'): - meta['external_id'].append(reference['external_id']) - if 'external_id' in meta and len(meta['external_id']) == 1: - meta['external_id'] = meta.pop('external_id')[0] - return meta - - def _is_pattern_too_complex(self, pattern: str) -> bool: - if any(keyword in pattern for keyword in self._mapping.pattern_forbidden_relations()): - return True - if ' AND ' in pattern and ' OR ' in pattern: - return True - return False diff --git a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py index 4775b277..d746f5ed 100644 --- a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py @@ -59,15 +59,12 @@ def note_parser(self) -> STIX2NoteConverter: if not hasattr(self, '_note_parser'): self._set_note_parser() return self._note_parser -<<<<<<< HEAD -======= @property def observed_data_parser(self) -> InternalSTIX2ObservedDataConverter: return getattr( self, '_observed_data_parser', self._set_observed_data_parser() ) ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f def _set_attack_pattern_parser(self) -> InternalSTIX2AttackPatternConverter: self._attack_pattern_parser = InternalSTIX2AttackPatternConverter(self) @@ -101,13 +98,10 @@ def _set_malware_parser(self) -> InternalSTIX2MalwareConverter: def _set_note_parser(self) -> STIX2NoteConverter: self._note_parser = STIX2NoteConverter(self) -<<<<<<< HEAD -======= def _set_observed_data_parser(self) -> InternalSTIX2ObservedDataConverter: self._observed_data_parser = InternalSTIX2ObservedDataConverter(self) return self._observed_data_parser ->>>>>>> b2faea3254a8a98e89bba0a10a511bb1e426696f def _set_threat_actor_parser(self) -> InternalSTIX2ThreatActorConverter: self._threat_actor_parser = InternalSTIX2ThreatActorConverter(self) @@ -186,12 +180,3 @@ def _handle_object_refs(self, object_refs: list): self._unknown_stix_object_type_error(error) except UnknownParsingFunctionError as error: self._unknown_parsing_function_error(error) - - def _parse_custom_object(self, custom_object_ref: str): - self.custom_object_parser.parse(custom_object_ref) - - def _parse_note(self, note_ref: str): - self.note_parser.parse(note_ref) - - def _parse_observed_data(self, observed_data_ref: str): - self.observed_data_parser.parse(observed_data_ref) diff --git a/misp_stix_converter/stix2misp/stix2_mapping.py b/misp_stix_converter/stix2misp/stix2_mapping.py index 58d741f9..1ad4eeb4 100644 --- a/misp_stix_converter/stix2misp/stix2_mapping.py +++ b/misp_stix_converter/stix2misp/stix2_mapping.py @@ -82,27 +82,24 @@ class STIX2toMISPMapping(metaclass=ABCMeta): ) __stix_to_misp_mapping = Mapping( **{ - 'attack-pattern': '_parse_attack_pattern', - 'campaign': '_parse_campaign', - 'course-of-action': '_parse_course_of_action', - # 'grouping': '_parse_grouping', - 'identity': '_parse_identity', - 'indicator': '_parse_indicator', - 'intrusion-set': '_parse_intrusion_set', - 'location': '_parse_location', - 'malware': '_parse_malware', - 'malware-analysis': '_parse_malware_analysis', - # 'marking-definition': '_parse_marking_definition', - 'note': '_parse_note', - 'observed-data': '_parse_observed_data', - # 'report': '_parse_report', - 'sighting': '_parse_sighting', - 'threat-actor': '_parse_threat_actor', - 'tool': '_parse_tool', - 'vulnerability': '_parse_vulnerability', - 'x-misp-attribute': '_parse_custom_object', - 'x-misp-galaxy-cluster': '_parse_custom_object', - 'x-misp-object': '_parse_custom_object' + 'attack-pattern': 'attack_pattern_parser', + 'campaign': 'campaign_parser', + 'course-of-action': 'course_of_action_parser', + 'identity': 'identity_parser', + 'indicator': 'indicator_parser', + 'intrusion-set': 'intrusion_set_parser', + 'location': 'location_parser', + 'malware': 'malware_parser', + 'malware-analysis': 'malware_analysis_parser', + 'note': 'note_parser', + 'observed-data': 'observed_data_parser', + 'sighting': 'sighting_parser', + 'threat-actor': 'threat_actor_parser', + 'tool': 'tool_parser', + 'vulnerability': 'vulnerability_parser', + 'x-misp-attribute': 'custom_object_parser', + 'x-misp-galaxy-cluster': 'custom_object_parser', + 'x-misp-object': 'custom_object_parser' } ) __timeline_mapping = Mapping( diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 84532eb6..e654045f 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from os import walk import sys import time from .converters import ( @@ -14,6 +13,7 @@ ExternalSTIX2LocationConverter, InternalSTIX2LocationConverter, ExternalSTIX2MalwareConverter, InternalSTIX2AttackPatternConverter, InternalSTIX2MalwareAnalysisConverter, InternalSTIX2MalwareConverter, + ExternalSTIX2ObservedDataConverter, InternalSTIX2ObservedDataConverter, ExternalSTIX2ThreatActorConverter, InternalSTIX2ThreatActorConverter, ExternalSTIX2ToolConverter, InternalSTIX2ToolConverter, ExternalSTIX2VulnerabilityConverter, InternalSTIX2VulnerabilityConverter) @@ -33,12 +33,11 @@ from collections import defaultdict from datetime import datetime from pymisp import ( - AbstractMISP, MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, + MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, MISPObject, MISPSighting) from stix2 import TLP_AMBER, TLP_GREEN, TLP_RED, TLP_WHITE from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v20.common import MarkingDefinition as MarkingDefinition_v20 -from stix2.v20.observables import NetworkTraffic as NetworkTraffic_v20 from stix2.v20.sdo import ( AttackPattern as AttackPattern_v20, Campaign as Campaign_v20, CourseOfAction as CourseOfAction_v20, CustomObject as CustomObject_v20, @@ -88,7 +87,6 @@ '_tool', '_vulnerability' ) -_MISP_OBJECTS_PATH = AbstractMISP().misp_objects_path # Typing _OBSERVABLE_TYPING = Union[ @@ -162,8 +160,8 @@ _MISP_FEATURES_TYPING = Union[ MISPAttribute, MISPEvent, MISPObject ] -_NETWORK_TRAFFIC_TYPING = Union[ - NetworkTraffic_v20, NetworkTraffic_v21 +_OBSERVED_DATA_PARSER_TYPING = Union[ + ExternalSTIX2ObservedDataConverter, InternalSTIX2ObservedDataConverter ] _OBSERVED_DATA_TYPING = Union[ ObservedData_v20, ObservedData_v21 @@ -363,6 +361,12 @@ def misp_events(self) -> Union[list, MISPEvent]: self, '_STIX2toMISPParser__misp_events', self.__misp_event ) + @property + def observed_data_parser(self) -> _OBSERVED_DATA_PARSER_TYPING: + if not hasattr(self, '_observed_data_parser'): + self._set_observed_data_parser() + return self._observed_data_parser + @property def single_event(self) -> bool: return self.__single_event @@ -581,7 +585,7 @@ def _handle_object(self, object_type: str, object_ref: str): except AttributeError: raise UnknownParsingFunctionError(feature) try: - parser(object_ref) + parser.parse(object_ref) except ObjectRefLoadingError as error: self._object_ref_loading_error(error) except ObjectTypeLoadingError as error: @@ -635,9 +639,6 @@ def _misp_event_from_report(self, report: _REPORT_TYPING) -> MISPEvent: misp_event.published = False return misp_event - def _parse_attack_pattern(self, attack_pattern_ref: str): - self.attack_pattern_parser.parse(attack_pattern_ref) - def _parse_bundle_with_multiple_reports(self): if self.single_event: self.__misp_event = self._create_generic_event() @@ -683,12 +684,6 @@ def _parse_bundle_with_single_report(self): self._parse_bundle_with_no_report() self._handle_unparsed_content() - def _parse_campaign(self, campaign_ref: str): - self.campaign_parser.parse(campaign_ref) - - def _parse_course_of_action(self, course_of_action_ref: str): - self.course_of_action_parser.parse(course_of_action_ref) - def _parse_galaxies_as_container(self): clusters = defaultdict(list) for cluster in self._clusters.values(): @@ -709,15 +704,6 @@ def _parse_galaxies_as_tag_names(self): for tag in tags['tag_names']: self.misp_event.add_tag(tag) - def _parse_identity(self, identity_ref: str): - self.identity_parser.parse(identity_ref) - - def _parse_indicator(self, indicator_ref: str): - self.indicator_parser.parse(indicator_ref) - - def _parse_intrusion_set(self, intrusion_set_ref: str): - self.intrusion_set_parser.parse(intrusion_set_ref) - def _parse_loaded_features(self): for feature in _LOADED_FEATURES: if hasattr(self, feature): @@ -730,24 +716,6 @@ def _parse_loaded_features(self): except UnknownParsingFunctionError as error: self._unknown_parsing_function_error(error) - def _parse_location(self, location_ref: str): - self.location_parser.parse(location_ref) - - def _parse_malware(self, malware_ref: str): - self.malware_parser.parse(malware_ref) - - def _parse_malware_analysis(self, malware_analysis_ref: str): - self.malware_analysis_parser.parse(malware_analysis_ref) - - def _parse_threat_actor(self, threat_actor_ref: str): - self.threat_actor_parser.parse(threat_actor_ref) - - def _parse_tool(self, tool_ref: str): - self.tool_parser.parse(tool_ref) - - def _parse_vulnerability(self, vulnerability_ref: str): - self.vulnerability_parser.parse(vulnerability_ref) - ############################################################################ # MARKING DEFINITIONS PARSING METHODS. # ############################################################################ From ec608d3539c7f7937c12cb6431b8e35e72778d2d Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 6 Jun 2024 17:11:25 +0200 Subject: [PATCH 137/137] fix: [stix2 import] Making Python 3.8 & 3.9 happy with the typing --- .../stix2misp/converters/stix2_observed_data_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index e35c06aa..ad6705fe 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -2204,7 +2204,7 @@ def _create_misp_object_from_observable_object_ref( def _handle_misp_object_fields( - self, misp_object: MISPAttribute | MISPObject, + self, misp_object: Union[MISPAttribute, MISPObject], observed_data: ObservedData_v21): time_fields = self._parse_timeline(observed_data) for field in ('timestamp', 'last_seen'):