From 50b322e8015f1ace15ffb1caf46a41b17e228364 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 25 May 2023 22:59:17 +0200 Subject: [PATCH 01/36] fix: [stix2 import] Catching parsing issues that appear while the STIX file is loaded --- misp_stix_converter/misp_stix_converter.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 01b3647e..8d4413b6 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -26,7 +26,8 @@ STIXHeader, STIXPackage) from stix.core.ttps import TTPs from stix2.base import STIXJSONEncoder -from stix2.parsing import parse as stix2_parser +from stix2.exceptions import InvalidValueError +from stix2.parsing import parse as stix2_parser, ParseError from stix2.v20 import Bundle as Bundle_v20 from stix2.v21 import Bundle as Bundle_v21 from typing import List, Optional, Union @@ -490,11 +491,15 @@ def stix_1_to_misp( def stix_2_to_misp( filename: _files_type, output_filename: Optional[_files_type]=None): - with open(filename, 'rt', encoding='utf-8') as f: - bundle = stix2_parser( - f.read(), allow_custom=True, interoperability=True - ) - stix_parser = InternalSTIX2toMISPParser() if _from_misp(bundle.objects) else ExternalSTIX2toMISPParser() + try: + with open(filename, 'rt', encoding='utf-8') as f: + bundle = stix2_parser( + f.read(), allow_custom=True, interoperability=True + ) + except (ParseError, InvalidValueError) as error: + return {'errors': [error.__str__()]} + from_misp = _from_misp(bundle.objects) + stix_parser = InternalSTIX2toMISPParser() if from_misp else ExternalSTIX2toMISPParser() stix_parser.load_stix_bundle(bundle) stix_parser.parse_stix_bundle() if output_filename is None: From 40a175f5fdf0a3866427cbcbf7e06a5b8bf2a10c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 30 May 2023 11:17:21 +0200 Subject: [PATCH 02/36] fix: [stix2 import] Fixed the observable registry key values parsing in case of a single key imported as `regkey|value` attribute --- misp_stix_converter/stix2misp/external_stix2_to_misp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 4ba8f86e..d5b47696 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -1832,11 +1832,11 @@ def _parse_registry_key_observable_objects( registry_key, object_id, observed_data ) continue - if hasattr(registry_key, 'values'): + if 'values' in registry_key.properties_populated(): self._add_misp_attribute( getattr(self, f'_handle_{feature}_special_attribute')( registry_key, - f"{registry_key.key}|{registry_key.values[0]['data']}", + f"{registry_key.key}|{registry_key['values'][0]['data']}", 'regkey|value', observed_data.id ), observed_data From ffb0edf6df2d5fbdb2b7010032a092c96b0a35ef Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 30 May 2023 11:36:05 +0200 Subject: [PATCH 03/36] fix: [stix2 import] Added missing `campaign` type in the list of STIX object types to look for --- misp_stix_converter/stix2misp/stix2_to_misp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index ce478f6e..3672310c 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -49,6 +49,7 @@ _LOADED_FEATURES = ( '_attack_pattern', + '_campaign', '_course_of_action', '_custom_attribute', '_custom_object', From cf5060044f8ffa90dd4b6f7739fac447452543b2 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 31 May 2023 10:36:01 +0200 Subject: [PATCH 04/36] fix: [stix2 import] Fixed external STIX 2 `email-message` observable & pattern mapping --- misp_stix_converter/stix2misp/external_stix2_mapping.py | 4 ++-- misp_stix_converter/stix2misp/external_stix2_to_misp.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/stix2misp/external_stix2_mapping.py b/misp_stix_converter/stix2misp/external_stix2_mapping.py index 1833b6a4..70eb66bc 100644 --- a/misp_stix_converter/stix2misp/external_stix2_mapping.py +++ b/misp_stix_converter/stix2misp/external_stix2_mapping.py @@ -22,7 +22,7 @@ class ExternalSTIX2toMISPMapping(STIX2toMISPMapping): ' WITHIN ' ) - # MAIN STIX OBJECTS MAPPING + # MAIN STIX OBJECTS MAPPING __observable_mapping = Mapping( **{ 'autonomous-system': 'as', @@ -534,7 +534,7 @@ def email_address_pattern_mapping(cls, field) -> Union[dict, None]: return cls.__email_address_pattern_mapping.get(field) @classmethod - def email_message_pattern_mapping(cls, field) -> Union[dict, None]: + def email_message_mapping(cls, field) -> Union[dict, None]: return cls.__email_object_mapping.get(field) @classmethod diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index d5b47696..3a97d059 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -1307,7 +1307,7 @@ def _parse_email_observable_objects( ) continue for field in self._get_populated_properties(email_message): - attribute = self._mapping.email_object_mapping(field) + attribute = self._mapping.email_message_mapping(field) if attribute is not None: self._add_misp_attribute( getattr(self, f'_handle_{feature}_attribute')( @@ -2104,7 +2104,7 @@ def _parse_email_message_pattern( for keys, assertion, value in pattern.comparisons['email-message']: if assertion != '=': continue - attribute = self._mapping.email_message_pattern_mapping(keys[0]) + attribute = self._mapping.email_message_mapping(keys[0]) if attribute is not None: attributes.append({'value': value, **attribute}) else: From dc0d92d489de64a1803cf2ac4f829236b5520255 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 1 Jun 2023 20:55:57 +0200 Subject: [PATCH 05/36] fix: [stix2 import] Better handling of the `single_event` variable inside of the STIX 2 to MISP parser --- misp_stix_converter/stix2misp/stix2_to_misp.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 3672310c..1723238d 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -208,7 +208,7 @@ def parse_stix_bundle(self, single_event: Optional[bool] = False): ) as error: self._critical_error(error) - def parse_stix_content(self, filename: str): + def parse_stix_content(self, filename: str, single_event: Optional[bool] = False): try: with open(filename, 'rt', encoding='utf-8') as f: bundle = stix2_parser( @@ -218,7 +218,7 @@ def parse_stix_content(self, filename: str): sys.exit(exception) self.load_stix_bundle(bundle) del bundle - self.parse_stix_bundle() + self.parse_stix_bundle(single_event) ################################################################################ # PROPERTIES # @@ -501,11 +501,13 @@ def _handle_misp_event_tags( self._fetch_tags_from_labels(misp_event, stix_object.labels) def _misp_event_from_grouping(self, grouping: Grouping) -> MISPEvent: + self.__single_event = True misp_event = self._create_misp_event(grouping) misp_event.published = False return misp_event def _misp_event_from_report(self, report: _REPORT_TYPING) -> MISPEvent: + self.__single_event = True misp_event = self._create_misp_event(report) if report.published != report.modified: misp_event.published = True @@ -546,6 +548,7 @@ def _parse_bundle_with_multiple_reports(self): self.__misp_events = events def _parse_bundle_with_no_report(self): + self.__single_event = True self.__misp_event = self._create_generic_event() for feature in _LOADED_FEATURES: if hasattr(self, feature): From 640f397fc1fd3d9406c257f9973490abd3e2325f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 2 Jun 2023 16:39:05 +0200 Subject: [PATCH 06/36] chg: [stix2 import] Added more result details and arguments to the `stix_2_to_misp` helper that converts a STIX file to MISP format - We added all the arguments needed in both the declaration of the STIX 2 to MISP parser and the stix bundle parsing call - We have a more detailed return message that gives not only a success message, but also the errors and warnings --- misp_stix_converter/misp_stix_converter.py | 44 +++++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 8d4413b6..7a75ad6f 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -489,24 +489,48 @@ def stix_1_to_misp( return 1 -def stix_2_to_misp( - filename: _files_type, output_filename: Optional[_files_type]=None): +def stix_2_to_misp(filename: _files_type, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + output_dir: Optional[_files_type]=None, + output_name: Optional[_files_type]=None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False): + if isinstance(filename, str): + filename = Path(filename).resolve() try: with open(filename, 'rt', encoding='utf-8') as f: bundle = stix2_parser( f.read(), allow_custom=True, interoperability=True ) except (ParseError, InvalidValueError) as error: - return {'errors': [error.__str__()]} + return {'errors': [f'{filename} - {error.__str__()}']} from_misp = _from_misp(bundle.objects) - stix_parser = InternalSTIX2toMISPParser() if from_misp else ExternalSTIX2toMISPParser() + parser = InternalSTIX2toMISPParser if from_misp else ExternalSTIX2toMISPParser + stix_parser = parser(distribution, sharing_group_id, galaxies_as_tags) stix_parser.load_stix_bundle(bundle) - stix_parser.parse_stix_bundle() - if output_filename is None: - output_filename = f'{filename}.out' - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(stix_parser.misp_event.to_json(indent=4)) - return 1 + stix_parser.parse_stix_bundle(single_event) + traceback = {'success': 1} + for feature in ('errors', 'warnings'): + brol = getattr(stix_parser, feature) + if brol: + traceback[feature] = brol + if output_dir is None: + output_dir = filename.parent + if stix_parser.single_event: + if output_name is None: + output_name = output_dir / f'{filename.name}.out' + with open(output_name, 'wt', encoding='utf-8') as f: + f.write(stix_parser.misp_event.to_json(indent=4)) + traceback['results'] = [output_name] + return traceback + traceback['results'] = [] + for misp_event in stix_parser.misp_events: + output = output_dir / f'{filename.name}.{misp_event.uuid}.misp.out' + with open(output, 'wt', encoding='utf-8') as f: + f.write(misp_event.to_json(indent=4)) + traceback['results'].append(output) + return traceback ################################################################################ From ca0ec4ce1cb65984c12d9b8a9558f446bdb1ea78 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 2 Jun 2023 16:47:31 +0200 Subject: [PATCH 07/36] chg: [stix2 import] Added the `debug` argument to the `stix_2_to_misp` helper - We return the error and warning messages only when the `debug` flag is set --- misp_stix_converter/misp_stix_converter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 7a75ad6f..f5033cca 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -489,7 +489,7 @@ def stix_1_to_misp( return 1 -def stix_2_to_misp(filename: _files_type, +def stix_2_to_misp(filename: _files_type, debug: Optional[bool] = False, distribution: Optional[int] = 0, galaxies_as_tags: Optional[bool] = False, output_dir: Optional[_files_type]=None, @@ -511,10 +511,11 @@ def stix_2_to_misp(filename: _files_type, stix_parser.load_stix_bundle(bundle) stix_parser.parse_stix_bundle(single_event) traceback = {'success': 1} - for feature in ('errors', 'warnings'): - brol = getattr(stix_parser, feature) - if brol: - traceback[feature] = brol + if debug: + for feature in ('errors', 'warnings'): + brol = getattr(stix_parser, feature) + if brol: + traceback[feature] = brol if output_dir is None: output_dir = filename.parent if stix_parser.single_event: From 2b46537ac16ca699f68ac3152768a553d3a4df0f Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Mon, 5 Jun 2023 22:50:57 +0200 Subject: [PATCH 08/36] wip: [stix1 export] Enhanced the STIX 1 export helper features --- misp_stix_converter/misp_stix_converter.py | 491 ++++++++++++--------- 1 file changed, 284 insertions(+), 207 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index f5033cca..1e4f94ef 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -55,7 +55,6 @@ # MISP to STIX MAIN FUNCTIONS. # ################################################################################ - class AttributeCollectionHandler(): def __init__(self, return_format): self.__return_format = return_format @@ -246,10 +245,15 @@ def ttps_header(self): def misp_attribute_collection_to_stix1( - output_filename: _files_type, *input_files: List[_files_type], - return_format: str=_STIX1_default_format, - version: str=_STIX1_default_version, in_memory: bool=False, - namespace: str=_default_namespace, org: str=_default_org): + input_files: List[_files_type], debug: Optional[bool] = False, + return_format: Optional[str] = _STIX1_default_format, + namespace: Optional[str] = _default_namespace, + org: Optional[str] = _default_org, + version: Optional[str] = _STIX1_default_version, + in_memory: Optional[bool] = False, + single_output: Optional[bool] = False, + output_dir: Optional[_files_type] = None, + output_name: Optional[_files_type] = None): if return_format not in _STIX1_valid_formats: return_format = _STIX1_default_format if version not in _STIX1_valid_versions: @@ -258,113 +262,204 @@ def misp_attribute_collection_to_stix1( org = re.sub('[\W]+', '', org.replace(" ", "_")) parser = MISPtoSTIX1AttributesParser(org, version) if len(input_files) == 1: - parser.parse_json_content(input_files[0]) - return _write_raw_stix( - parser.stix_package, output_filename, namespace, org, return_format - ) - if in_memory: - package = _create_stix_package(org, version) - for filename in input_files: + try: + filename = input_files[0] + if isinstance(filename, str): + filename = Path(filename).resolve() parser.parse_json_content(filename) - current = parser.stix_package - for campaign in current.campaigns: - package.add_campaign(campaign) - for course_of_action in current.courses_of_action: - package.add_course_of_action(course_of_action) - for exploit_target in current.exploit_targets: - package.add_exploit_target(exploit_target) - for indicator in current.indicators: - package.add_indicator(indicator) - for observable in current.observables: - package.add_observable(observable) - for threat_actor in current.threat_actors: - package.add_threat_actor(threat_actor) - if current.ttps is not None: - for ttp in current.ttps: - package.add_ttp(ttp) - return _write_raw_stix( - package, output_filename, namespace, org, return_format - ) - current_path = Path(output_filename).parent.resolve() - handler = AttributeCollectionHandler(return_format) - header, _, footer = stix1_attributes_framing( - namespace, org, return_format, version - ) - for input_file in input_files: - parser.parse_json_content(input_file) - current = parser.stix_package - for feature in _STIX1_features: - values = getattr(current, feature) - if values is not None and values: - content = globals()[f'_get_{feature}'](values, return_format) - if not content: - continue - filename = getattr(handler, feature) - if filename is None: - setattr(handler, feature, uuid4()) + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + _write_raw_stix( + parser.stix_package, name, namespace, org, return_format + ) + return _generate_traceback(debug, parser, name) + except Exception as exception: + return {'fails': [f'{filename} - {exception.__str__()}']} + traceback = defaultdict(list) + if single_output: + if in_memory: + package = _create_stix_package(org, version) + name = _check_filename( + Path(__file__).resolve().parent / 'tmp', + f'{package.id_}.stix1.{return_format}', + output_dir, output_name + ) + for filename in input_files: + try: + parser.parse_json_content(filename) + current = parser.stix_package + for campaign in current.campaigns: + package.add_campaign(campaign) + for course_of_action in current.courses_of_action: + package.add_course_of_action(course_of_action) + for exploit_target in current.exploit_targets: + package.add_exploit_target(exploit_target) + for indicator in current.indicators: + package.add_indicator(indicator) + for observable in current.observables: + package.add_observable(observable) + for threat_actor in current.threat_actors: + package.add_threat_actor(threat_actor) + if current.ttps is not None: + for ttp in current.ttps: + package.add_ttp(ttp) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if any(filename not in traceback['fails'] for filename in input_files): + _write_raw_stix( + package, name, namespace, org, return_format + ) + traceback.update(_generate_traceback(debug, parser, name)) + return traceback + handler = AttributeCollectionHandler(return_format) + tmp_path = name.parent + for filename in input_files: + try: + parser.parse_json_content(filename) + package = parser.stix_package + for feature in _STIX1_features: + values = getattr(current, feature) + if values: + content = globals()[f'_get_{feature}'](values, return_format) + if not content: + continue + filename = getattr(handler, feature) + if filename is None: + setattr(handler, feature, uuid4()) + filename = getattr(handler, feature) + with open(tmp_path / filename, 'wt', encoding='utf-8') as f: + current_header = getattr(handler, f'{feature}_header') + f.write(f'{current_header}{content}') + continue + with open(tmp_path / filename, 'at', encoding='utf-8') as f: + f.write(content) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if any(filename not in traceback['fails'] for filename in input_files): + header, _, footer = stix1_attributes_framing( + namespace, org, return_format, version + ) + with open(name, 'wt', encoding='utf-8') as result: + result.write(header) + actual_features = handler.features + for feature in actual_features: filename = getattr(handler, feature) - with open(current_path / filename, 'wt', encoding='utf-8') as f: - current_header = getattr(handler, f'{feature}_header') - f.write(f'{current_header}{content}') - continue - with open(current_path / filename, 'at', encoding='utf-8') as f: - f.write(content) - with open(output_filename, 'wt', encoding='utf-8') as result: - result.write(header) - actual_features = handler.features - for feature in actual_features: - filename = getattr(handler, feature) - if filename is not None: - with open(current_path / filename, 'rt', encoding='utf-8') as current: - content = current.read() if return_format == 'xml' else current.read()[:-2] - current_footer = getattr(handler, f'{feature}_footer') - if return_format == 'json' and feature == actual_features[-1]: - current_footer = current_footer[:-2] - result.write(f'{content}{current_footer}') - os.remove(current_path / filename) - result.write(footer) - return 1 + if filename is not None: + with open(tmp_path / filename, 'rt', encoding='utf-8') as current: + content = current.read() if return_format == 'xml' else current.read()[:-2] + current_footer = getattr(handler, f'{feature}_footer') + if return_format == 'json' and feature == actual_features[-1]: + current_footer = current_footer[:-2] + result.write(f'{content}{current_footer}') + os.remove(tmp_path / filename) + result.write(footer) + traceback.update(_generate_traceback(debug, parser, name)) + return traceback + output_names = [] + for filename in input_files: + try: + if isinstance(filename, str): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_output( + filename.parent, f'{filename.name}.out', output_dir + ) + _write_raw_stix( + parser.stix_package, name, namespace, org, return_format + ) + output_names.append(name) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if output_names: + traceback.update(_generate_traceback(debug, parser, *output_names)) + return traceback def misp_event_collection_to_stix1( - output_filename: _files_type, *input_files: List[_files_type], - return_format: str=_STIX1_default_format, - version: str=_STIX1_default_version, in_memory: bool=False, - namespace: str=_default_namespace, org: str=_default_org): + input_files: List[_files_type], debug: Optional[bool] = False, + return_format: Optional[str] = _STIX1_default_format, + namespace: Optional[str] = _default_namespace, + org: Optional[str] = _default_org, + version: Optional[str] = _STIX1_default_version, + in_memory: Optional[bool] = False, + single_output: Optional[bool] = False, + output_dir: Optional[_files_type] = None, + output_name: Optional[_files_type] = None): if return_format not in _STIX1_valid_formats: return_format = _STIX1_default_format if version not in _STIX1_valid_versions: version = _STIX1_default_version if org != _default_org: org = re.sub('[\W]+', '', org.replace(" ", "_")) + _write_args = (namespace, org, return_format) + traceback = defaultdict(list) parser = MISPtoSTIX1EventsParser(org, version) - if in_memory or len(input_files) == 1: - package = _create_stix_package(org, version) - for filename in input_files: - parser.parse_json_content(filename) - if parser.stix_package.related_packages is not None: - for related_package in parser.stix_package.related_packages: - package.add_related_package(related_package) - else: - package.add_related_package(parser.stix_package) - return _write_raw_stix( - package, output_filename, namespace, org, return_format + if single_output: + name = _check_output( + Path(__file__).resolve().parent / 'tmp', + f'{package.id_}.stix1.{return_format}', output_dir ) - header, separator, footer = stix1_framing( - namespace, org, return_format, version - ) - parser.parse_json_content(input_files[0]) - content = _get_events(parser.stix_package, return_format) - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(f'{header}{content}') - for filename in input_files[1:]: - parser.parse_json_content(filename) - content = _get_events(parser.stix_package, return_format) - with open(output_filename, 'at', encoding='utf-8') as f: - f.write(f'{separator}{content}') - with open(output_filename, 'at', encoding='utf-8') as f: - f.write(footer) - return 1 + if in_memory: + package = _create_stix_package(org, version) + for filename in input_files: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + if parser.stix_package.related_packages is not None: + for related_package in parser.stix_package.related_packages: + package.add_related_package(related_package) + else: + package.add_related_package(parser.stix_package) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if any(filename not in traceback.get('fails', []) for filename in input_files): + _write_raw_stix(package, name, *_write_args) + return traceback + header, separator, footer = stix1_framing( + namespace, org, return_format, version + ) + filename = input_files[0] + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + content = _get_events(parser.stix_package, return_format) + with open(name, 'wt', encoding='utf-8') as f: + f.write(f'{header}{content}') + except Exception as exception: + traceback['fails'].append(filename) + for filename in input_files[1:]: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + content = _get_events(parser.stix_package, return_format) + with open(name, 'at', encoding='utf-8') as f: + f.write(f'{separator}{content}') + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + with open(name, 'at', encoding='utf-8') as f: + f.write(footer) + traceback.update(_generate_traceback(debug, parser, name)) + return traceback + output_names = [] + for filename in input_files: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_output( + filename.parent, f'{filename.name}.out', output_dir + ) + _write_raw_stix(parser.stix_package, name, *_write_args) + output_names.append(name) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if output_names: + traceback.update(_generate_traceback(debug, parser, *output_names)) + return traceback def misp_collection_to_stix2_0( @@ -426,24 +521,33 @@ def misp_collection_to_stix2_1( def misp_to_stix1( - filename: _files_type, return_format: str, version: str, - namespace=_default_namespace, org=_default_org, - output_filename: Optional[_files_type]=None): - if output_filename is None: - output_filename = f'{filename}.out' + filename: _files_type, debug: Optional[bool] = False, + return_format: Optional[str] = _STIX1_default_format, + namespace: Optional[str] = _default_namespace, + org: Optional[str] = _default_org, + version: Optional[str] = _STIX1_default_version, + output_dir: Optional[_files_type] = None, + output_name: Optional[_files_type] = None): + if return_format not in _STIX1_valid_formats: + return_format = _STIX1_default_format + if version not in _STIX1_valid_versions: + version = _STIX1_default_version if org != _default_org: org = re.sub('[\W]+', '', org.replace(" ", "_")) - package = _create_stix_package(org, version) parser = MISPtoSTIX1EventsParser(org, version) - parser.parse_json_content(filename) - if parser.stix_package.related_packages is not None: - for related_package in parser.stix_package.related_packages: - package.add_related_package(related_package) - else: - package.add_related_package(parser.stix_package) - return _write_raw_stix( - package, output_filename, namespace, org, return_format - ) + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + _write_raw_stix( + parser.stix_package, name, namespace, org, return_format + ) + return _generate_traceback(debug, parser, name) + except Exception as exception: + return {'fails': f'{filename} - {exception.__str__()}'} def misp_to_stix2_0( @@ -838,100 +942,26 @@ def _handle_output_filename(stix_args): def _misp_to_stix(stix_args): + collection_args = { + 'in_memory': stix_args.in_memory, + 'single_ouput': stix_args.single_output + } if stix_args.version in ('1.1.1', '1.2'): - if stix_args.feature == 'attribute': - if len(stix_args.file) == 1: - output_filename = _handle_output_filename(stix_args) - status = misp_attribute_collection_to_stix1( - output_filename, - stix_args.file[0], - return_format = stix_args.format, - version = stix_args.version, - in_memory = not stix_args.tmp_files, - namespace = stix_args.namespace, - org = stix_args.org - ) - if status != 1: - sys.exit( - f'Error while processing {stix_args.file[0]}' - f' - status code = {status}' - ) - return output_filename - if stix_args.single_output: - output = stix_args.output_dir / f'{uuid4()}.stix1.{stix_args.format}' - status = misp_attribute_collection_to_stix1( - output, - *stix_args.file, - return_format = stix_args.format, - version = stix_args.version, - in_memory = not stix_args.tmp_files, - namespace = stix_args.namespace, - org = stix_args.org - ) - if status != 1: - sys.exit(f'Error while processing your files - status code = {status}') - return output - results = [] - for filename in stix_args.file: - output = _handle_output_dir(stix_args, filename) - status = misp_attribute_collection_to_stix1( - output, - filename, - return_format = stix_args.format, - version = stix_args.version, - in_memory = not stix_args.tmp_files, - namespace = stix_args.namespace, - org = stix_args.org - ) - if status == 1: - results.append(output) - else: - print(f'Error while processing {filename} - status code = {status}', file=sys.stderr) - return results - if len(stix_args.file) == 1: - output_filename = _handle_output_filename(stix_args) - filename = stix_args.file[0] - status = misp_to_stix1( - filename, - stix_args.format, - stix_args.version, - namespace=stix_args.namespace, - org=stix_args.org, - output_filename=output_filename - ) - if status != 1: - sys.exit(f'Error while processing {filename} - status code = {status}') - return output_filename - if stix_args.single_output: - output = stix_args.output_dir / f'{uuid4()}.stix1.{stix_args.format}' - status = misp_event_collection_to_stix1( - output, - *stix_args.file, - return_format=stix_args.format, - version=stix_args.version, - in_memory=not stix_args.tmp_files, - namespace=stix_args.namespace, - org=stix_args.org - ) - if status != 1: - sys.exit(f'Error while processing your files - status code = {status}') - return output - results = [] - for filename in stix_args.file: - output = _handle_output_dir(stix_args, filename) - status = misp_to_stix1( - filename, - stix_args.format, - stix_args.version, - namespace=stix_args.namespace, - version=stix_args.version, - output_filename=output + stix1_args = { + 'return_format': stix_args.format, 'namespace': stix_args.namespace, + 'in_memory': stix_args.in_memory, 'version': stix_args.version, + 'output_dir': stix_args.output_dir, 'org': stix_args.org, + 'output_name': stix_args.output_name, 'debug': stix_args.debug + } + if stix_args.level == 'attribute': + return misp_attribute_collection_to_stix1( + *stix_args.file, **collection_args, **stix1_args ) - if status == 1: - results.append(output) - else: - print(f'Error while processing {filename} - status code = {status}', file=sys.stderr) - return results, 1 + if len(stix_args.file) == 1: + return misp_to_stix1(stix_args.file[0], **stix1_args) + return misp_event_collection_to_stix1( + *stix_args.file, **collection_args, **stix1_args + ) if len(stix_args.file) == 1: filename = stix_args.file[0] output_filename = _handle_output_filename(stix_args) @@ -976,4 +1006,51 @@ def _stix_to_misp(stix_args): output_filename = _handle_output_filename(stix_args) method(stix_args.file[0], output_filename=output_filename) return output_filename - return _process_files(stix_args, method) \ No newline at end of file + return _process_files(stix_args, method) + + +################################################################################ +# UTILITY FUNCTIONS. # +################################################################################ + +def _check_filename(default_dir: Path, default_name: str, + output_dir: _files_type, output_name: _files_type) -> Path: + if output_name is None: + return _check_output(default_dir, default_name, output_dir) + if not isinstance(output_name, Path): + output_name = Path(output_name).resolve() + if output_name.is_dir(): + return output_name / default_name + return output_name + + +def _check_output( + default_dir: Path, default_name: str, output_dir: _files_type) -> Path: + if output_dir is None: + return default_dir / default_name + if not isinstance(output_dir, Path): + output_dir = Path(output_dir).resolve() + if output_dir.is_file(): + return output_dir + return output_dir / default_name + + +def _check_output_dir(default_dir: Path, output_dir: _files_type) -> Path: + if output_dir is None: + return default_dir + if not isinstance(output_dir, Path): + output_dir = Path(output_dir).resolve() + if output_dir.is_file(): + return output_dir.parent + return output_dir + + +def _generate_traceback(debug: bool, parser, *output_names: List[Path]) -> dict: + traceback = {'success': 1} + if debug: + for feature in ('errors', 'warnings'): + brol = getattr(parser, feature) + if brol: + traceback[feature] = brol + traceback['results'] = output_names + return traceback \ No newline at end of file From 610d5ad2d6ee8b5d5d27890b53b4c1f19556f5c8 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 6 Jun 2023 22:24:37 +0200 Subject: [PATCH 09/36] fix: [stix1 export] Making sure we avoid exceptions with the fails catching on traceback messages --- misp_stix_converter/misp_stix_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 1e4f94ef..eede20a2 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -306,7 +306,7 @@ def misp_attribute_collection_to_stix1( package.add_ttp(ttp) except Exception as exception: traceback['fails'].append(f'{filename} - {exception.__str__()}') - if any(filename not in traceback['fails'] for filename in input_files): + if any(filename not in traceback.get('fails', []) for filename in input_files): _write_raw_stix( package, name, namespace, org, return_format ) @@ -336,7 +336,7 @@ def misp_attribute_collection_to_stix1( f.write(content) except Exception as exception: traceback['fails'].append(f'{filename} - {exception.__str__()}') - if any(filename not in traceback['fails'] for filename in input_files): + if any(filename not in traceback.get('fails', []) for filename in input_files): header, _, footer = stix1_attributes_framing( namespace, org, return_format, version ) From 2c8feb977b462dc7b6f2368619d76408633d59f1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 6 Jun 2023 22:32:03 +0200 Subject: [PATCH 10/36] fix: [stix1 export] Fixed name for the result STIX 1 event collections export & added a missing traceback --- misp_stix_converter/misp_stix_converter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index eede20a2..5cdcbece 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -396,9 +396,10 @@ def misp_event_collection_to_stix1( traceback = defaultdict(list) parser = MISPtoSTIX1EventsParser(org, version) if single_output: - name = _check_output( + name = _check_filename( Path(__file__).resolve().parent / 'tmp', - f'{package.id_}.stix1.{return_format}', output_dir + f'{package.id_}.stix1.{return_format}', + output_dir, output_name ) if in_memory: package = _create_stix_package(org, version) @@ -416,6 +417,7 @@ def misp_event_collection_to_stix1( traceback['fails'].append(f'{filename} - {exception.__str__()}') if any(filename not in traceback.get('fails', []) for filename in input_files): _write_raw_stix(package, name, *_write_args) + traceback.update(_generate_traceback(debug, parser, name)) return traceback header, separator, footer = stix1_framing( namespace, org, return_format, version From 89ee9d78dd1d67e909ce7fd6c901dc247ebdd01c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 6 Jun 2023 22:34:15 +0200 Subject: [PATCH 11/36] fix: [stix1 export] Added a use case to support the use of the events collection export even with a single file --- misp_stix_converter/misp_stix_converter.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 5cdcbece..ba5ecf6d 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -393,8 +393,21 @@ def misp_event_collection_to_stix1( if org != _default_org: org = re.sub('[\W]+', '', org.replace(" ", "_")) _write_args = (namespace, org, return_format) - traceback = defaultdict(list) parser = MISPtoSTIX1EventsParser(org, version) + if len(input_files) == 1: + filename = input_files[0] + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + _write_raw_stix(parser.stix_package, name, *_write_args) + return _generate_traceback(debug, parser, name) + except Exception as exception: + return {'fails': f'{filename} - {exception.__str__()}'} + traceback = defaultdict(list) if single_output: name = _check_filename( Path(__file__).resolve().parent / 'tmp', From 27c9bd3762636348e7ae46a4ecdbfe68097d4cd1 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Tue, 6 Jun 2023 22:42:37 +0200 Subject: [PATCH 12/36] wip: [stix2 export] Enhanced STIX 2 export helpers --- misp_stix_converter/misp_stix_converter.py | 220 ++++++++++++--------- 1 file changed, 129 insertions(+), 91 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index ba5ecf6d..4c20b801 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -48,7 +48,9 @@ ) _STIX1_valid_formats = ('json', 'xml') _STIX1_valid_versions = ('1.1.1', '1.2') +_STIX2_default_version = '2.1' _STIX2_event_types = ('grouping', 'report') +_STIX2_valid_versions = ('2.0', '2.1') ################################################################################ @@ -477,62 +479,111 @@ def misp_event_collection_to_stix1( return traceback -def misp_collection_to_stix2_0( - output_filename: _files_type, *input_files: List[_files_type], - in_memory: bool=False): - parser = MISPtoSTIX20Parser() - if in_memory or len(input_files) == 1: - for filename in input_files: +def misp_collection_to_stix2( + input_files: List[_files_type], debug: Optional[bool] = False, + version: Optional[str] = _STIX2_default_version, + in_memory: Optional[bool] = False, + single_output: Optional[bool] = False, + output_dir: Optional[_files_type] = None, + output_name: Optional[_files_type] = None): + if version not in _STIX2_valid_versions: + version = _STIX2_default_version + parser = MISPtoSTIX21Parser() if version == '2.1' else MISPtoSTIX20Parser() + if len(input_files) == 1: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() parser.parse_json_content(filename) - objects = parser.stix_objects - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write( - json.dumps(Bundle_v20(objects), cls=STIXJSONEncoder, indent=4) + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name ) - return 1 - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write( - f'{json.dumps(Bundle_v20(), cls=STIXJSONEncoder, indent=4)[:-2]},' - '\n "objects": [\n' + with open(name, 'wt', encoding='utf-8') as f: + f.write(parser.bundle.serialize(indent=4)) + return _generate_traceback(debug, parser, name) + except Exception as exception: + return {'fails': f'{filename} - {exception.__str__()}'} + traceback = defaultdict(list) + if single_output: + if in_memory: + for filename in input_files: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if any(filename not in traceback.get('fails', []) for filename in input_files): + bundle = parser.bundle + name = _check_filename( + Path(__file__).resolve().parent / 'tmp', + f"{bundle.id.split('--'[1])}.stix" + f"{version.replace('.', '')}.json", + output_dir, output_name + ) + with open(name, 'wt', encoding='utf-8') as f: + f.write(bundle.serialize(indent=4)) + traceback.update(_generate_traceback(debug, parser, name)) + return traceback + bundle = Bundle_v21() if version == '2.1' else Bundle_v20() + name = _check_filename( + Path(__file__).resolve().parent / 'tmp', + f"{bundle.id.split('--')[1]}.stix{version.replace('.', '')}.json", + output_dir, output_name ) - for filename in input_files[:-1]: - parser.parse_json_content(filename) - with open(output_filename, 'at', encoding='utf-8') as f: - f.write(f'{json.dumps([parser.fetch_stix_objects], cls=STIXJSONEncoder, indent=4)[8:-8]},\n') - parser.parse_json_content(input_files[-1]) - with open(output_filename, 'at', encoding='utf-8') as f: - footer = ' ]\n}' - f.write(f'{json.dumps([parser.stix_objects], cls=STIXJSONEncoder, indent=4)[8:-8]}\n{footer}') - return 1 - - -def misp_collection_to_stix2_1( - output_filename: _files_type, *input_files: List[_files_type], - in_memory: bool=False): - parser = MISPtoSTIX21Parser() - if in_memory or len(input_files) == 1: - for filename in input_files: + with open(name, 'wt', encoding='utf-8') as f: + f.write(f'{bundle.serialize(indent=4)[:-2]},\n "objects": [\n') + written = False + try: + filename = input_files[0] + if not isinstance(filename, Path): + filename = Path(filename).resolve() parser.parse_json_content(filename) - objects = parser.stix_objects - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write( - json.dumps(Bundle_v21(objects), cls=STIXJSONEncoder, indent=4) + stix_objects = json.dumps( + [parser.fetch_stix_objects], cls=STIXJSONEncoder, indent=4 ) - return 1 - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write( - f'{json.dumps(Bundle_v21(), cls=STIXJSONEncoder, indent=4)[:-2]},' - '\n "objects": [\n' - ) - for filename in input_files[:-1]: - parser.parse_json_content(filename) - with open(output_filename, 'at', encoding='utf-8') as f: - f.write(f'{json.dumps([parser.fetch_stix_objects], cls=STIXJSONEncoder, indent=4)[8:-8]},\n') - parser.parse_json_content(input_files[-1]) - with open(output_filename, 'at', encoding='utf-8') as f: - footer = ' ]\n}' - f.write(f'{json.dumps([parser.stix_objects], cls=STIXJSONEncoder, indent=4)[8:-8]}\n{footer}') - return 1 + with open(name, 'at', encoding='utf-8') as f: + f.write(stix_objects[8:-8]) + written = True + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + for filename in input_files[1:]: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + stix_objects = json.dumps( + [parser.fetch_stix_objects], cls=STIXJSONEncoder, indent=4 + ) + separator = ',\n' if written else '' + with open(name, 'at', encoding='utf-8') as f: + f.write(f"{separator}{stix_objects[8:-8]}") + written = True + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if written: + with open(name, 'at', encoding='utf-8') as f: + f.write(' ]\n}') + traceback.update(_generate_traceback(debug, parser, name)) + else: + name.remove() + return traceback + output_names = [] + for filename in input_files: + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_output( + filename.parent, f'{filename.name}.out', output_dir + ) + with open(name, 'wt', encoding='utf-8') as f: + f.write(parser.bundle.serialize(indent=4)) + output_names.append(name) + except Exception as exception: + traceback['fails'].append(f'{filename} - {exception.__str__()}') + if output_names: + traceback.update(_generate_traceback(debug, parser, *output_names)) + return traceback def misp_to_stix1( @@ -565,26 +616,25 @@ def misp_to_stix1( return {'fails': f'{filename} - {exception.__str__()}'} -def misp_to_stix2_0( - filename: _files_type, output_filename: Optional[_files_type]=None): - if output_filename is None: - output_filename = f'{filename}.out' - parser = MISPtoSTIX20Parser() - parser.parse_json_content(filename) - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(json.dumps(parser.bundle, cls=STIXJSONEncoder, indent=4)) - return 1 - - -def misp_to_stix2_1( - filename: _files_type, output_filename: Optional[_files_type]=None): - if output_filename is None: - output_filename = f'{filename}.out' - parser = MISPtoSTIX21Parser() - parser.parse_json_content(filename) - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(json.dumps(parser.bundle, cls=STIXJSONEncoder, indent=4)) - return 1 +def misp_to_stix2(filename: _files_type, debug: Optional[bool] = False, + version: Optional[str] = _STIX2_default_version, + output_dir: Optional[_files_type] = None, + output_name: Optional[_files_type] = None): + if version not in _STIX2_valid_versions: + version = _STIX2_default_version + parser = MISPtoSTIX21Parser() if version == '2.1' else MISPtoSTIX20Parser() + try: + if not isinstance(filename, Path): + filename = Path(filename).resolve() + parser.parse_json_content(filename) + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + with open(name, 'wt', encoding='utf-8') as f: + f.write(json.dumps(parser.bundle, cls=STIXJSONEncoder, indent=4)) + return _generate_traceback(debug, parser, name) + except Exception as exception: + return {'fails': f'{filename} - {exception.__str__()}'} ################################################################################ @@ -977,27 +1027,15 @@ def _misp_to_stix(stix_args): return misp_event_collection_to_stix1( *stix_args.file, **collection_args, **stix1_args ) + stix2_args = { + 'debug': stix_args.debug, 'output_dir': stix_args.output_dir, + 'output_name': stix_args.output_name, 'version': stix_args.version + } if len(stix_args.file) == 1: - filename = stix_args.file[0] - output_filename = _handle_output_filename(stix_args) - args = (filename, output_filename) - status = misp_to_stix2_0(*args) if stix_args.version == '2.0' else misp_to_stix2_1(*args) - if status != 1: - sys.exit(f'Error while processing {filename} - status code = {status}') - return output_filename - if stix_args.single_output: - output = stix_args.output_dir / f"{uuid4()}.stix{stix_args.version.replace('.', '')}.json" - method = misp_collection_to_stix2_0 if stix_args.version == '2.0' else misp_collection_to_stix2_1 - status = method( - output, - *stix_args.file, - in_memory = not stix_args.tmp_files - ) - if status != 1: - sys.exit(f'Error while processing your files - status code = {status}') - return output - method = misp_to_stix2_0 if stix_args.version == '2.0' else misp_to_stix2_1 - return _process_files(stix_args, method) + return misp_to_stix2(stix_args.file[0]) + return misp_collection_to_stix2( + stix_args.file, **collection_args, **stix2_args + ) def _process_files(stix_args, method): From 7dce3f56327816dca1a766ce812e0569a6a64a9b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 7 Jun 2023 10:37:46 +0200 Subject: [PATCH 13/36] wip: [stix2 import] Enhanced STIX 2 import helper --- misp_stix_converter/misp_stix_converter.py | 81 +++++++--------------- 1 file changed, 24 insertions(+), 57 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 4c20b801..eefd10ba 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -679,28 +679,22 @@ def stix_2_to_misp(filename: _files_type, debug: Optional[bool] = False, stix_parser = parser(distribution, sharing_group_id, galaxies_as_tags) stix_parser.load_stix_bundle(bundle) stix_parser.parse_stix_bundle(single_event) - traceback = {'success': 1} - if debug: - for feature in ('errors', 'warnings'): - brol = getattr(stix_parser, feature) - if brol: - traceback[feature] = brol if output_dir is None: output_dir = filename.parent if stix_parser.single_event: - if output_name is None: - output_name = output_dir / f'{filename.name}.out' - with open(output_name, 'wt', encoding='utf-8') as f: + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + with open(name, 'wt', encoding='utf-8') as f: f.write(stix_parser.misp_event.to_json(indent=4)) - traceback['results'] = [output_name] - return traceback - traceback['results'] = [] + return _generate_traceback(debug, parser, name) + output_names = [] for misp_event in stix_parser.misp_events: output = output_dir / f'{filename.name}.{misp_event.uuid}.misp.out' with open(output, 'wt', encoding='utf-8') as f: f.write(misp_event.to_json(indent=4)) - traceback['results'].append(output) - return traceback + output_names.append(output) + return _generate_traceback(debug, parser, *output_names) ################################################################################ @@ -987,25 +981,12 @@ def _write_raw_stix( else: with open(filename, 'wt', encoding='utf-8') as f: f.write(json.dumps(package.to_dict(), indent=4)) - return 1 ################################################################################ # COMMAND LINE FUNCTIONS # ################################################################################ -def _handle_output_dir(stix_args, filename): - if stix_args.output_dir is None: - return f'{filename}.out' - return stix_args.output_dir / f'{filename.name}.out' - - -def _handle_output_filename(stix_args): - if stix_args.output_name is None: - return f'{stix_args.file[0]}.out' - return stix_args.output_name - - def _misp_to_stix(stix_args): collection_args = { 'in_memory': stix_args.in_memory, @@ -1038,30 +1019,26 @@ def _misp_to_stix(stix_args): ) -def _process_files(stix_args, method): - results = [] +def _stix_to_misp(stix_args): + method = stix_2_to_misp if stix_args.version == '2' else stix_1_to_misp + results = defaultdict(list) for filename in stix_args.file: - output_filename = _handle_output_dir(stix_args, filename) - status = method(filename, output_filename=output_filename) - if status == 1: - results.append(output_filename) - else: - print( - f'Error while processing {filename} - status code = {status}', - file=sys.stderr - ) + traceback = method( + filename, debug=stix_args.debug, + distribution=stix_args.distribution, + galaxies_as_tags=stix_args.galaxies_as_tags, + output_dir=stix_args.output_dir, + output_name=stix_args.output_name, + sharing_group_id=stix_args.sharing_group, + single_event=stix_args.single_output + ) + if traceback.pop('success', 0) == 1: + results.update(traceback) + continue + results['fails'].extend(traceback['errors']) return results -def _stix_to_misp(stix_args): - method = stix_2_to_misp if stix_args.version in ('2.0', '2.1') else stix_1_to_misp - if len(stix_args.file) == 1: - output_filename = _handle_output_filename(stix_args) - method(stix_args.file[0], output_filename=output_filename) - return output_filename - return _process_files(stix_args, method) - - ################################################################################ # UTILITY FUNCTIONS. # ################################################################################ @@ -1088,16 +1065,6 @@ def _check_output( return output_dir / default_name -def _check_output_dir(default_dir: Path, output_dir: _files_type) -> Path: - if output_dir is None: - return default_dir - if not isinstance(output_dir, Path): - output_dir = Path(output_dir).resolve() - if output_dir.is_file(): - return output_dir.parent - return output_dir - - def _generate_traceback(debug: bool, parser, *output_names: List[Path]) -> dict: traceback = {'success': 1} if debug: From 9ffeae9b4aa940008297cfac642e8f7dac19453c Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 7 Jun 2023 10:46:10 +0200 Subject: [PATCH 14/36] fix: [stix1 export] Fixed arguments passed to the MISP collections export to STIX 1 --- misp_stix_converter/misp_stix_converter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index eefd10ba..032f0da5 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -994,19 +994,19 @@ def _misp_to_stix(stix_args): } if stix_args.version in ('1.1.1', '1.2'): stix1_args = { - 'return_format': stix_args.format, 'namespace': stix_args.namespace, - 'in_memory': stix_args.in_memory, 'version': stix_args.version, - 'output_dir': stix_args.output_dir, 'org': stix_args.org, - 'output_name': stix_args.output_name, 'debug': stix_args.debug + 'debug': stix_args.debug, 'return_format': stix_args.format, + 'version': stix_args.version, 'namespace': stix_args.namespace, + 'org': stix_args.org, 'output_dir': stix_args.output_dir, + 'output_name': stix_args.output_name } if stix_args.level == 'attribute': return misp_attribute_collection_to_stix1( - *stix_args.file, **collection_args, **stix1_args + stix_args.file, **collection_args, **stix1_args ) if len(stix_args.file) == 1: return misp_to_stix1(stix_args.file[0], **stix1_args) return misp_event_collection_to_stix1( - *stix_args.file, **collection_args, **stix1_args + stix_args.file, **collection_args, **stix1_args ) stix2_args = { 'debug': stix_args.debug, 'output_dir': stix_args.output_dir, From 949fe4c29279c78229d0b522b2054bf08f25fb60 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 7 Jun 2023 15:52:52 +0200 Subject: [PATCH 15/36] fix: [stix2 export] Fixed some statements in the MISP collections export to STIX 2 helper - Including fixes on: - the single file handling (regarding the single file name) - the default directory for collections export results - the input files argument of the function --- misp_stix_converter/misp_stix_converter.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 032f0da5..c8808d32 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -480,7 +480,7 @@ def misp_event_collection_to_stix1( def misp_collection_to_stix2( - input_files: List[_files_type], debug: Optional[bool] = False, + *input_files: List[_files_type], debug: Optional[bool] = False, version: Optional[str] = _STIX2_default_version, in_memory: Optional[bool] = False, single_output: Optional[bool] = False, @@ -490,6 +490,7 @@ def misp_collection_to_stix2( version = _STIX2_default_version parser = MISPtoSTIX21Parser() if version == '2.1' else MISPtoSTIX20Parser() if len(input_files) == 1: + filename = input_files[0] try: if not isinstance(filename, Path): filename = Path(filename).resolve() @@ -515,8 +516,8 @@ def misp_collection_to_stix2( if any(filename not in traceback.get('fails', []) for filename in input_files): bundle = parser.bundle name = _check_filename( - Path(__file__).resolve().parent / 'tmp', - f"{bundle.id.split('--'[1])}.stix" + Path(__file__).resolve().parents[1] / 'tmp', + f"{bundle.id.split('--')[1]}.stix" f"{version.replace('.', '')}.json", output_dir, output_name ) @@ -526,7 +527,7 @@ def misp_collection_to_stix2( return traceback bundle = Bundle_v21() if version == '2.1' else Bundle_v20() name = _check_filename( - Path(__file__).resolve().parent / 'tmp', + Path(__file__).resolve().parents[1] / 'tmp', f"{bundle.id.split('--')[1]}.stix{version.replace('.', '')}.json", output_dir, output_name ) From d4872cd4a88ea3043a342fc46956ceb3d405c983 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 7 Jun 2023 19:57:46 +0200 Subject: [PATCH 16/36] fix: [stix2 export] Returning the result files in a traceback message as list --- misp_stix_converter/misp_stix_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index c8808d32..602133e9 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -1073,5 +1073,5 @@ def _generate_traceback(debug: bool, parser, *output_names: List[Path]) -> dict: brol = getattr(parser, feature) if brol: traceback[feature] = brol - traceback['results'] = output_names + traceback['results'] = list(output_names) return traceback \ No newline at end of file From ff6f86856d4d182dca0a9df4d7ad971fde35eac2 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Wed, 7 Jun 2023 20:04:08 +0200 Subject: [PATCH 17/36] chg: [tests] Added new tests and changes on the collections export as STIX 2 --- ...nt_stix20.json => test_event1_stix20.json} | 0 ...nt_stix21.json => test_event1_stix21.json} | 0 tests/test_event2_stix20.json | 89 +++++++++++++++++ tests/test_event2_stix21.json | 97 +++++++++++++++++++ tests/test_stix20_export.py | 86 ++++++++++++---- tests/test_stix21_export.py | 95 ++++++++++++++---- 6 files changed, 331 insertions(+), 36 deletions(-) rename tests/{test_event_stix20.json => test_event1_stix20.json} (100%) rename tests/{test_event_stix21.json => test_event1_stix21.json} (100%) create mode 100644 tests/test_event2_stix20.json create mode 100644 tests/test_event2_stix21.json diff --git a/tests/test_event_stix20.json b/tests/test_event1_stix20.json similarity index 100% rename from tests/test_event_stix20.json rename to tests/test_event1_stix20.json diff --git a/tests/test_event_stix21.json b/tests/test_event1_stix21.json similarity index 100% rename from tests/test_event_stix21.json rename to tests/test_event1_stix21.json diff --git a/tests/test_event2_stix20.json b/tests/test_event2_stix20.json new file mode 100644 index 00000000..37f3ac08 --- /dev/null +++ b/tests/test_event2_stix20.json @@ -0,0 +1,89 @@ +{ + "type": "bundle", + "id": "bundle--71356f2b-eefe-48db-a93a-4aa2233e3a59", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-Project", + "identity_class": "organization" + }, + { + "type": "report", + "id": "report--50bf87fc-d134-499f-9e26-806cbe89ed37", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-STIX-Converter test event with domain|ip attribute", + "published": "2020-10-25T16:22:00Z", + "object_refs": [ + "indicator--726f90e2-2ad6-44a2-b345-c05e55e850e5" + ], + "labels": [ + "Threat-Report", + "misp:tool=\"MISP-STIX-Converter\"" + ] + }, + { + "type": "indicator", + "id": "indicator--726f90e2-2ad6-44a2-b345-c05e55e850e5", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "description": "Domain|ip test attribute", + "pattern": "[domain-name:value = 'circl.lu' AND domain-name:resolves_to_refs[*].value = '149.13.33.14']", + "valid_from": "2020-10-25T16:22:00Z", + "kill_chain_phases": [ + { + "kill_chain_name": "misp-category", + "phase_name": "Network activity" + } + ], + "labels": [ + "misp:type=\"domain|ip\"", + "misp:category=\"Network activity\"", + "misp:to_ids=\"True\"" + ] + }, + { + "type": "report", + "id": "report--71356f2b-eefe-48db-a93a-4aa2233e3a59", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-STIX-Converter test event with filename attribute", + "published": "2020-10-25T16:22:00Z", + "object_refs": [ + "indicator--02de0847-dccf-481a-a96a-6a7654328658" + ], + "labels": [ + "Threat-Report", + "misp:tool=\"MISP-STIX-Converter\"" + ] + }, + { + "type": "indicator", + "id": "indicator--02de0847-dccf-481a-a96a-6a7654328658", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "description": "Filename test attribute", + "pattern": "[file:name = 'test_file_name']", + "valid_from": "2020-10-25T16:22:00Z", + "kill_chain_phases": [ + { + "kill_chain_name": "misp-category", + "phase_name": "Payload delivery" + } + ], + "labels": [ + "misp:type=\"filename\"", + "misp:category=\"Payload delivery\"", + "misp:to_ids=\"True\"" + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_event2_stix21.json b/tests/test_event2_stix21.json new file mode 100644 index 00000000..acffafa5 --- /dev/null +++ b/tests/test_event2_stix21.json @@ -0,0 +1,97 @@ +{ + "type": "bundle", + "id": "bundle--71356f2b-eefe-48db-a93a-4aa2233e3a59", + "objects": [ + { + "type": "identity", + "spec_version": "2.1", + "id": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-Project", + "identity_class": "organization" + }, + { + "type": "grouping", + "spec_version": "2.1", + "id": "grouping--50bf87fc-d134-499f-9e26-806cbe89ed37", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-STIX-Converter test event with domain|ip attribute", + "context": "suspicious-activity", + "object_refs": [ + "indicator--726f90e2-2ad6-44a2-b345-c05e55e850e5" + ], + "labels": [ + "Threat-Report", + "misp:tool=\"MISP-STIX-Converter\"" + ] + }, + { + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--726f90e2-2ad6-44a2-b345-c05e55e850e5", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "description": "Domain|ip test attribute", + "pattern": "[domain-name:value = 'circl.lu' AND domain-name:resolves_to_refs[*].value = '149.13.33.14']", + "pattern_type": "stix", + "pattern_version": "2.1", + "valid_from": "2020-10-25T16:22:00Z", + "kill_chain_phases": [ + { + "kill_chain_name": "misp-category", + "phase_name": "Network activity" + } + ], + "labels": [ + "misp:type=\"domain|ip\"", + "misp:category=\"Network activity\"", + "misp:to_ids=\"True\"" + ] + }, + { + "type": "grouping", + "spec_version": "2.1", + "id": "grouping--71356f2b-eefe-48db-a93a-4aa2233e3a59", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "name": "MISP-STIX-Converter test event with filename attribute", + "context": "suspicious-activity", + "object_refs": [ + "indicator--02de0847-dccf-481a-a96a-6a7654328658" + ], + "labels": [ + "Threat-Report", + "misp:tool=\"MISP-STIX-Converter\"" + ] + }, + { + "type": "indicator", + "spec_version": "2.1", + "id": "indicator--02de0847-dccf-481a-a96a-6a7654328658", + "created_by_ref": "identity--a0c22599-9e58-4da4-96ac-7051603fa951", + "created": "2020-10-25T16:22:00.000Z", + "modified": "2020-10-25T16:22:00.000Z", + "description": "Filename test attribute", + "pattern": "[file:name = 'test_file_name']", + "pattern_type": "stix", + "pattern_version": "2.1", + "valid_from": "2020-10-25T16:22:00Z", + "kill_chain_phases": [ + { + "kill_chain_name": "misp-category", + "phase_name": "Payload delivery" + } + ], + "labels": [ + "misp:type=\"filename\"", + "misp:category=\"Payload delivery\"", + "misp:to_ids=\"True\"" + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index e69ce2d6..29dff104 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -3,8 +3,8 @@ from datetime import datetime from misp_stix_converter import ( - MISPtoSTIX20Mapping, MISPtoSTIX20Parser, misp_collection_to_stix2_0, - misp_to_stix2_0) + MISPtoSTIX20Mapping, MISPtoSTIX20Parser, misp_collection_to_stix2, + misp_to_stix2) from pymisp import MISPAttribute, MISPEvent from .test_events import * from .update_documentation import ( @@ -4776,30 +4776,80 @@ def test_tool(self): class TestCollectionSTIX20Export(TestCollectionSTIX2Export): def test_attributes_collection(self): name = 'test_attributes_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix20.json' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix20.json' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] - self.assertEqual(misp_collection_to_stix2_0(output_file, *input_files), 1) - self._check_stix2_results_export(to_test_name, reference_name) - self.assertEqual(misp_collection_to_stix2_0(output_file, *input_files, in_memory=True), 1) - self._check_stix2_results_export(to_test_name, reference_name) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.0', single_output=True, + output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.0', in_memory=True, + single_output=True, output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) def test_events_collection(self): name = 'test_events_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix20.json' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix20.json' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] - self.assertEqual(misp_collection_to_stix2_0(output_file, *input_files), 1) - self._check_stix2_results_export(to_test_name, reference_name) - self.assertEqual(misp_collection_to_stix2_0(output_file, *input_files, in_memory=True), 1) - self._check_stix2_results_export(to_test_name, reference_name) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.0', single_output=True, + output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.0', in_memory=True, + single_output=True, output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2(*input_files, version='2.0'), + { + 'success': 1, + 'results': [ + self._current_path / f'{name}_{n}.json.out' for n in (1, 2) + ] + } + ) + for n in (1, 2): + self._check_stix2_results_export( + self._current_path / f'{name}_{n}.json.out', + self._current_path / f'test_event{n}_stix20.json' + ) + def test_event_export(self): name = 'test_events_collection_1.json' - self.assertEqual(misp_to_stix2_0(self._current_path / name), 1) - self._check_stix2_results_export(f'{name}.out', 'test_event_stix20.json') + filename = self._current_path / name + output_file = self._current_path / f'{name}.out' + reference_file = self._current_path / 'test_event1_stix20.json' + self.assertEqual( + misp_to_stix2(filename, version='2.0'), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + filename, version='2.0' + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) class TestFeedSTIX20Export(TestSTIX2Export): diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index f83e635c..c89afdde 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -3,8 +3,8 @@ from datetime import datetime from misp_stix_converter import ( - MISPtoSTIX21Mapping, MISPtoSTIX21Parser, misp_collection_to_stix2_1, - misp_to_stix2_1) + MISPtoSTIX21Mapping, MISPtoSTIX21Parser, misp_collection_to_stix2, + misp_to_stix2) from pymisp import MISPAttribute, MISPEvent from .test_events import * from .update_documentation import ( @@ -6108,30 +6108,89 @@ def test_tool(self): class TestCollectionSTIX21Export(TestCollectionSTIX2Export): def test_attributes_collection(self): name = 'test_attributes_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix21.json' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix21.json' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] - self.assertEqual(misp_collection_to_stix2_1(output_file, *input_files), 1) - self._check_stix2_results_export(to_test_name, reference_name) - self.assertEqual(misp_collection_to_stix2_1(output_file, *input_files, in_memory=True), 1) - self._check_stix2_results_export(to_test_name, reference_name) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.1', single_output=True, + output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.1', in_memory=True, + single_output=True, output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2(*input_files, version='2.1'), + { + 'success': 1, + 'results': [ + self._current_path / f'{name}_{n}.json.out' for n in (1, 2) + ] + } + ) def test_events_collection(self): name = 'test_events_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix21.json' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix21.json' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] - self.assertEqual(misp_collection_to_stix2_1(output_file, *input_files), 1) - self._check_stix2_results_export(to_test_name, reference_name) - self.assertEqual(misp_collection_to_stix2_1(output_file, *input_files, in_memory=True), 1) - self._check_stix2_results_export(to_test_name, reference_name) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.1', single_output=True, + output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + *input_files, version='2.1', in_memory=True, + single_output=True, output_name=output_file + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2(*input_files, version='2.1'), + { + 'success': 1, + 'results': [ + self._current_path / f'{name}_{n}.json.out' for n in (1, 2) + ] + } + ) + for n in (1, 2): + self._check_stix2_results_export( + self._current_path / f'{name}_{n}.json.out', + self._current_path / f'test_event{n}_stix21.json' + ) + def test_event_export(self): name = 'test_events_collection_1.json' - self.assertEqual(misp_to_stix2_1(self._current_path / name), 1) - self._check_stix2_results_export(f'{name}.out', 'test_event_stix21.json') + filename = self._current_path / name + output_file = self._current_path / f'{name}.out' + reference_file = self._current_path / 'test_event1_stix21.json' + self.assertEqual( + misp_to_stix2(filename, version='2.1'), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) + self.assertEqual( + misp_collection_to_stix2( + filename, version='2.1' + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix2_results_export(output_file, reference_file) class TestFeedSTIX21Export(TestSTIX2Export): From 4b1ca509bb85f95f37df6f582dd8cb260131f2ca Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Thu, 8 Jun 2023 19:55:41 +0200 Subject: [PATCH 18/36] fix: [stix export] Made STIX framing methods more modular --- misp_stix_converter/misp2stix/framing.py | 34 ++++++++++++++----- ...vent_stix11.xml => test_event1_stix11.xml} | 0 ...vent_stix12.xml => test_event1_stix12.xml} | 0 3 files changed, 25 insertions(+), 9 deletions(-) rename tests/{test_event_stix11.xml => test_event1_stix11.xml} (100%) rename tests/{test_event_stix12.xml => test_event1_stix12.xml} (100%) diff --git a/misp_stix_converter/misp2stix/framing.py b/misp_stix_converter/misp2stix/framing.py index 8f191867..b87d32bf 100644 --- a/misp_stix_converter/misp2stix/framing.py +++ b/misp_stix_converter/misp2stix/framing.py @@ -7,23 +7,38 @@ from mixbox import idgen from mixbox.namespaces import Namespace from stix.core import STIXHeader, STIXPackage -from typing import Optional -from uuid import uuid4 +from typing import Optional, Union +from uuid import uuid4, UUID from .stix1_mapping import NS_DICT, SCHEMALOC_DICT json_footer = ']}\n' +_UUID_typing = Union[UUID, str] -def stix1_attributes_framing(namespace: str, orgname: str, return_format: str, version: str) -> tuple: - stix_package = _stix_package(orgname, version) +def stix1_attributes_framing(namespace: str, orgname: str, return_format: str, + version: str) -> tuple: + stix_package = _create_stix_package(orgname, version) + return _stix1_attributes_framing( + namespace, orgname, return_format, stix_package + ) + + +def _stix1_attributes_framing(namespace: str, orgname: str, return_format: str, + stix_package: STIXPackage) -> tuple: if return_format == 'xml': namespaces = _handle_namespaces(namespace, orgname) return _stix_xml_attributes_framing(stix_package, namespaces) return _stix_json_attributes_framing(stix_package) -def stix1_framing(namespace: str, orgname: str, return_format: str, version: str) -> tuple: - stix_package = _stix_package(orgname, version) +def stix1_framing(namespace: str, orgname: str, return_format: str, + version: str) -> tuple: + stix_package = _create_stix_package(orgname, version) + return _stix1_framing(namespace, orgname, return_format, stix_package) + + +def _stix1_framing(namespace: str, orgname: str, return_format: str, + stix_package: STIXPackage) -> tuple: if return_format == 'xml': namespaces = _handle_namespaces(namespace, orgname) return _stix_xml_framing(stix_package, namespaces) @@ -35,14 +50,14 @@ def stix_xml_separator(): return f" \n <{header}>\n" -def stix20_framing(uuid: Optional[str] = None) -> tuple: +def stix20_framing(uuid: Optional[_UUID_typing] = None) -> tuple: header = '{"type": "bundle", "spec_version": "2.0", "id":' if uuid is None: uuid = uuid4() return f'{header} "bundle--{uuid}", "objects": [', ', ', json_footer -def stix21_framing(uuid: Optional[str] = None) -> tuple: +def stix21_framing(uuid: Optional[_UUID_typing] = None) -> tuple: header = '{"type": "bundle", "id":' if uuid is None: uuid = uuid4() @@ -72,7 +87,8 @@ def _stix_json_framing(stix_package: STIXPackage) -> tuple: return header, ', ', ']}}' -def _stix_package(orgname: str, version: str, uuid: Optional[str] = None) -> STIXPackage: +def _create_stix_package(orgname: str, version: str, + uuid: Optional[_UUID_typing] = None) -> STIXPackage: parsed_orgname = re.sub('[\W]+', '', orgname.replace(' ', '_')) if uuid is None: uuid = uuid4() diff --git a/tests/test_event_stix11.xml b/tests/test_event1_stix11.xml similarity index 100% rename from tests/test_event_stix11.xml rename to tests/test_event1_stix11.xml diff --git a/tests/test_event_stix12.xml b/tests/test_event1_stix12.xml similarity index 100% rename from tests/test_event_stix12.xml rename to tests/test_event1_stix12.xml From b10c9d76e82a0dcaa02128238db51552c75a81c4 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 00:39:19 +0200 Subject: [PATCH 19/36] fix: [stix1 export] Fixed the creation process of the STIX package used to serve as container for related packages --- misp_stix_converter/misp2stix/misp_to_stix1.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/misp2stix/misp_to_stix1.py b/misp_stix_converter/misp2stix/misp_to_stix1.py index e2fa4e3b..49daf4bb 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix1.py +++ b/misp_stix_converter/misp2stix/misp_to_stix1.py @@ -4,8 +4,9 @@ import json import re import socket -from .stix1_mapping import MISPtoSTIX1Mapping from .exportparser import MISPtoSTIXParser +from .framing import _create_stix_package as _stix_package +from .stix1_mapping import MISPtoSTIX1Mapping from abc import ABCMeta from base64 import b64encode from collections import defaultdict @@ -306,9 +307,9 @@ def _parse_file_attribute(self, attribute: dict): self._handle_attribute(attribute, observable) def _parse_hash_attribute(self, attribute: dict): - hash = self._parse_hash_value(attribute['type'], attribute['value']) + _hash = self._parse_hash_value(attribute['type'], attribute['value']) file_object = File() - file_object.add_hash(hash) + file_object.add_hash(_hash) observable = self._create_observable(file_object, attribute['uuid'], 'File') self._handle_attribute(attribute, observable) @@ -1155,7 +1156,7 @@ def parse_json_content(self, filename): with open(filename, 'rt', encoding='utf-8') as f: json_content = json.loads(f.read()) if json_content.get('response'): - package = STIXPackage() + package = _stix_package(self._orgname, self._version, header=False) for event in json_content['response']: self.parse_misp_event(event) package.add_related_package(self._stix_package) From af92ed584014c80fadc9c16f1f9170d0d318d853 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 00:41:13 +0200 Subject: [PATCH 20/36] fix: [stix1 export] Added option to generate a Package with no header --- misp_stix_converter/misp2stix/framing.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/misp_stix_converter/misp2stix/framing.py b/misp_stix_converter/misp2stix/framing.py index b87d32bf..2d668895 100644 --- a/misp_stix_converter/misp2stix/framing.py +++ b/misp_stix_converter/misp2stix/framing.py @@ -87,8 +87,9 @@ def _stix_json_framing(stix_package: STIXPackage) -> tuple: return header, ', ', ']}}' -def _create_stix_package(orgname: str, version: str, - uuid: Optional[_UUID_typing] = None) -> STIXPackage: +def _create_stix_package( + orgname: str, version: str, header: Optional[bool] = True, + uuid: Optional[_UUID_typing] = None) -> STIXPackage: parsed_orgname = re.sub('[\W]+', '', orgname.replace(' ', '_')) if uuid is None: uuid = uuid4() @@ -97,10 +98,11 @@ def _create_stix_package(orgname: str, version: str, timestamp=datetime.datetime.now() ) stix_package.version = version - stix_header = STIXHeader() - stix_header.title = f"Export from {orgname}'s MISP" - stix_header.package_intents = 'Threat Report' - stix_package.stix_header = stix_header + if header: + stix_header = STIXHeader() + stix_header.title = f"Export from {orgname}'s MISP" + stix_header.package_intents = 'Threat Report' + stix_package.stix_header = stix_header return stix_package From 95354ba28e554806e96936fb0dbf0f0dc7775986 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 01:14:46 +0200 Subject: [PATCH 21/36] fix: [stix1 export] Handling cases when there is no STIX header - In this specific case, the STIX package in XML format is a single xml tag with the included `/` closing character... so we remove it - ( JSON >>>>> XML definitely :) ) --- misp_stix_converter/misp2stix/framing.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misp_stix_converter/misp2stix/framing.py b/misp_stix_converter/misp2stix/framing.py index 2d668895..79a8f616 100644 --- a/misp_stix_converter/misp2stix/framing.py +++ b/misp_stix_converter/misp2stix/framing.py @@ -117,6 +117,8 @@ def _stix_xml_framing(stix_package: STIXPackage, namespaces: dict) -> tuple: s_related = "stix:Related_Package" header = stix_package.to_xml(auto_namespace=False, ns_dict=namespaces, schemaloc_dict=SCHEMALOC_DICT) header = header.decode() + if header.endswith('/>\n'): + header = f'{header[:-3]}>\n' header = f"{header.replace(s_stix, '')} <{s_related}s>\n <{s_related}>\n" footer = f" \n \n{s_stix}" separator = f" \n <{s_related}>\n" From 080da624c7d66811afcb4822a56a0c7568d7ba84 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 01:24:53 +0200 Subject: [PATCH 22/36] fix: [stix1 export] Reusing methods from the framing to generate packages (& handling namespaces) --- misp_stix_converter/misp_stix_converter.py | 83 ++++++++-------------- 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 602133e9..c1ff090c 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -5,7 +5,9 @@ import os import re import sys -from .misp2stix.framing import stix1_attributes_framing, stix1_framing +from .misp2stix.framing import ( + _stix1_attributes_framing, _stix1_framing, _handle_namespaces, + _create_stix_package) from .misp2stix.misp_to_stix1 import ( MISPtoSTIX1AttributesParser, MISPtoSTIX1EventsParser) from .misp2stix.misp_to_stix20 import MISPtoSTIX20Parser @@ -22,8 +24,7 @@ Namespace, NamespaceNotFoundError, register_namespace) from pathlib import Path from stix.core import ( - Campaigns, CoursesOfAction, Indicators, ThreatActors, - STIXHeader, STIXPackage) + Campaigns, CoursesOfAction, Indicators, ThreatActors, STIXPackage) from stix.core.ttps import TTPs from stix2.base import STIXJSONEncoder from stix2.exceptions import InvalidValueError @@ -260,8 +261,6 @@ def misp_attribute_collection_to_stix1( return_format = _STIX1_default_format if version not in _STIX1_valid_versions: version = _STIX1_default_version - if org != _default_org: - org = re.sub('[\W]+', '', org.replace(" ", "_")) parser = MISPtoSTIX1AttributesParser(org, version) if len(input_files) == 1: try: @@ -280,37 +279,37 @@ def misp_attribute_collection_to_stix1( return {'fails': [f'{filename} - {exception.__str__()}']} traceback = defaultdict(list) if single_output: + stix_package = _create_stix_package(org, version) + name = _check_filename( + Path(__file__).resolve().parent / 'tmp', + f'{stix_package.id_}.stix1.{return_format}', + output_dir, output_name + ) if in_memory: - package = _create_stix_package(org, version) - name = _check_filename( - Path(__file__).resolve().parent / 'tmp', - f'{package.id_}.stix1.{return_format}', - output_dir, output_name - ) for filename in input_files: try: parser.parse_json_content(filename) current = parser.stix_package for campaign in current.campaigns: - package.add_campaign(campaign) + stix_package.add_campaign(campaign) for course_of_action in current.courses_of_action: - package.add_course_of_action(course_of_action) + stix_package.add_course_of_action(course_of_action) for exploit_target in current.exploit_targets: - package.add_exploit_target(exploit_target) + stix_package.add_exploit_target(exploit_target) for indicator in current.indicators: - package.add_indicator(indicator) + stix_package.add_indicator(indicator) for observable in current.observables: - package.add_observable(observable) + stix_package.add_observable(observable) for threat_actor in current.threat_actors: - package.add_threat_actor(threat_actor) + stix_package.add_threat_actor(threat_actor) if current.ttps is not None: for ttp in current.ttps: - package.add_ttp(ttp) + stix_package.add_ttp(ttp) except Exception as exception: traceback['fails'].append(f'{filename} - {exception.__str__()}') if any(filename not in traceback.get('fails', []) for filename in input_files): _write_raw_stix( - package, name, namespace, org, return_format + stix_package, name, namespace, org, return_format ) traceback.update(_generate_traceback(debug, parser, name)) return traceback @@ -321,7 +320,7 @@ def misp_attribute_collection_to_stix1( parser.parse_json_content(filename) package = parser.stix_package for feature in _STIX1_features: - values = getattr(current, feature) + values = getattr(package, feature) if values: content = globals()[f'_get_{feature}'](values, return_format) if not content: @@ -339,8 +338,8 @@ def misp_attribute_collection_to_stix1( except Exception as exception: traceback['fails'].append(f'{filename} - {exception.__str__()}') if any(filename not in traceback.get('fails', []) for filename in input_files): - header, _, footer = stix1_attributes_framing( - namespace, org, return_format, version + header, _, footer = _stix1_attributes_framing( + namespace, org, return_format, stix_package ) with open(name, 'wt', encoding='utf-8') as result: result.write(header) @@ -392,8 +391,6 @@ def misp_event_collection_to_stix1( return_format = _STIX1_default_format if version not in _STIX1_valid_versions: version = _STIX1_default_version - if org != _default_org: - org = re.sub('[\W]+', '', org.replace(" ", "_")) _write_args = (namespace, org, return_format) parser = MISPtoSTIX1EventsParser(org, version) if len(input_files) == 1: @@ -411,13 +408,13 @@ def misp_event_collection_to_stix1( return {'fails': f'{filename} - {exception.__str__()}'} traceback = defaultdict(list) if single_output: + stix_package = _create_stix_package(org, version, header=False) name = _check_filename( Path(__file__).resolve().parent / 'tmp', - f'{package.id_}.stix1.{return_format}', + f'{stix_package.id_}.stix1.{return_format}', output_dir, output_name ) if in_memory: - package = _create_stix_package(org, version) for filename in input_files: try: if not isinstance(filename, Path): @@ -425,17 +422,17 @@ def misp_event_collection_to_stix1( parser.parse_json_content(filename) if parser.stix_package.related_packages is not None: for related_package in parser.stix_package.related_packages: - package.add_related_package(related_package) + stix_package.add_related_package(related_package) else: - package.add_related_package(parser.stix_package) + stix_package.add_related_package(parser.stix_package) except Exception as exception: traceback['fails'].append(f'{filename} - {exception.__str__()}') if any(filename not in traceback.get('fails', []) for filename in input_files): - _write_raw_stix(package, name, *_write_args) + _write_raw_stix(stix_package, name, *_write_args) traceback.update(_generate_traceback(debug, parser, name)) return traceback - header, separator, footer = stix1_framing( - namespace, org, return_format, version + header, separator, footer = _stix1_framing( + namespace, org, return_format, stix_package ) filename = input_files[0] try: @@ -599,8 +596,6 @@ def misp_to_stix1( return_format = _STIX1_default_format if version not in _STIX1_valid_versions: version = _STIX1_default_version - if org != _default_org: - org = re.sub('[\W]+', '', org.replace(" ", "_")) parser = MISPtoSTIX1EventsParser(org, version) try: if not isinstance(filename, Path): @@ -698,21 +693,6 @@ def stix_2_to_misp(filename: _files_type, debug: Optional[bool] = False, return _generate_traceback(debug, parser, *output_names) -################################################################################ -# STIX PACKAGE CREATION HELPERS. # -################################################################################ - -def _create_stix_package(orgname: str, version: str) -> STIXPackage: - package = STIXPackage() - package.version = version - header = STIXHeader() - header.title = f"Export from {orgname}'s MISP" - header.package_intents = "Threat Report" - package.stix_header = header - package.id_ = f"{orgname}:Package-{uuid4()}" - return package - - ################################################################################ # STIX CONTENT LOADING FUNCTIONS # ################################################################################ @@ -965,12 +945,7 @@ def _write_raw_stix( package: STIXPackage, filename: _files_type, namespace: str, org: str, return_format: str) -> bool: if return_format == 'xml': - namespaces = namespaces = {namespace: org} - namespaces.update(NS_DICT) - try: - idgen.set_id_namespace(Namespace(namespace, org)) - except TypeError: - idgen.set_id_namespace(Namespace(namespace, org, "MISP")) + namespaces = _handle_namespaces(namespace, org) with open(filename, 'wb') as f: f.write( package.to_xml( From 3429d14def6d75c3e841b172027c48fe727624df Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 01:25:46 +0200 Subject: [PATCH 23/36] fix; [stix1 export] Fixed the input files argument for the collections export as STIX 1 helpers --- misp_stix_converter/misp_stix_converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index c1ff090c..3b32dbcb 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -248,7 +248,7 @@ def ttps_header(self): def misp_attribute_collection_to_stix1( - input_files: List[_files_type], debug: Optional[bool] = False, + *input_files: List[_files_type], debug: Optional[bool] = False, return_format: Optional[str] = _STIX1_default_format, namespace: Optional[str] = _default_namespace, org: Optional[str] = _default_org, @@ -378,7 +378,7 @@ def misp_attribute_collection_to_stix1( def misp_event_collection_to_stix1( - input_files: List[_files_type], debug: Optional[bool] = False, + *input_files: List[_files_type], debug: Optional[bool] = False, return_format: Optional[str] = _STIX1_default_format, namespace: Optional[str] = _default_namespace, org: Optional[str] = _default_org, From d147f5a5746676cedb33163e445ed1bea2390f03 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 01:26:44 +0200 Subject: [PATCH 24/36] fix: [stix1 export] Fixed Package header writting for methods used to replicate the MISP pagination - used with collections export helpers --- misp_stix_converter/misp_stix_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 3b32dbcb..4bf4005f 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -806,7 +806,7 @@ def _get_courses_of_action_header(return_format: str = 'xml') -> str: def _get_events(package: STIXPackage, return_format: str = 'xml') -> str: if return_format == 'xml': if package.related_packages is not None: - length = 96 + len(package.id_) + len(package.version) + length = 135 + len(package.id_) + len(package.version) return package.to_xml(include_namespaces=False).decode()[length:-82] content = '\n '.join( package.to_xml(include_namespaces=False).decode().split('\n') From 489cc1d59d594000e9d0076c25e5bdfd372adc2e Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 9 Jun 2023 09:22:42 +0200 Subject: [PATCH 25/36] fix: [import] added missing import --- misp_stix_converter/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 3045ffa2..bfdb87d9 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -11,7 +11,7 @@ # STIX 1 special halpers from .misp_stix_converter import ( _get_campaigns, _get_courses_of_action, _get_events, _get_indicators, - _get_observables, _get_threat_actors, _get_ttps) + _get_observables, _get_threat_actors, _get_ttps, _from_misp) # STIX 1 footers from .misp_stix_converter import ( _get_campaigns_footer, _get_courses_of_action_footer, _get_indicators_footer, From 144498834e506ac6cff68baee4433de8613de050 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:07:22 +0200 Subject: [PATCH 26/36] chg: [tests] Updated STIX 1 export sample result files --- tests/test_event1_stix11.xml | 6 +- tests/test_event2_stix11.xml | 192 +++++++++++++++++++++++++++++++++++ tests/test_event2_stix12.xml | 192 +++++++++++++++++++++++++++++++++++ 3 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 tests/test_event2_stix11.xml create mode 100644 tests/test_event2_stix12.xml diff --git a/tests/test_event1_stix11.xml b/tests/test_event1_stix11.xml index 08d75551..c8ad37cd 100644 --- a/tests/test_event1_stix11.xml +++ b/tests/test_event1_stix11.xml @@ -52,11 +52,7 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:xs="http://www.w3.org/2001/XMLSchema" - id="example:Package-63cdbac3-e5a1-4879-a9c0-24a219b3bffc" version="1.1.1"> - - Export from MISP's MISP - Threat Report - + id="MISP:STIXPackage-63cdbac3-e5a1-4879-a9c0-24a219b3bffc" version="1.1.1"> diff --git a/tests/test_event2_stix11.xml b/tests/test_event2_stix11.xml new file mode 100644 index 00000000..341ef6f0 --- /dev/null +++ b/tests/test_event2_stix11.xml @@ -0,0 +1,192 @@ + + + + + + Export from MISP-Project's MISP + Threat Report + + + + MISP-STIX-Converter test event with domain|ip attribute + + 2020-10-25T00:00:00+00:00 + + + + MISP-Project + + + + + Network activity + + Network activity: circl.lu|149.13.33.14 (MISP Attribute) + Domain Watchlist + Domain|ip test attribute + + + + + + + circl.lu + + + + + + + 149.13.33.14 + + + + + + + High + Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none + + + + MISP-Project + + + + + + + + Event Threat Level: Medium + + + MISP Tag: misp:tool="MISP-STIX-Converter" + + + + + MISP-Project + + + + + + + + + + Export from MISP-Project's MISP + Threat Report + + + + MISP-STIX-Converter test event with filename attribute + + 2020-10-25T00:00:00+00:00 + + + + MISP-Project + + + + + Payload delivery + + Payload delivery: test_file_name (MISP Attribute) + File Hash Watchlist + Filename test attribute + + + + + test_file_name + + + + + High + Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none + + + + MISP-Project + + + + + + + + Event Threat Level: Medium + + + MISP Tag: misp:tool="MISP-STIX-Converter" + + + + + MISP-Project + + + + + + + + diff --git a/tests/test_event2_stix12.xml b/tests/test_event2_stix12.xml new file mode 100644 index 00000000..864ecb61 --- /dev/null +++ b/tests/test_event2_stix12.xml @@ -0,0 +1,192 @@ + + + + + + Export from MISP-Project's MISP + Threat Report + + + + MISP-STIX-Converter test event with domain|ip attribute + + 2020-10-25T00:00:00+00:00 + + + + MISP-Project + + + + + Network activity + + Network activity: circl.lu|149.13.33.14 (MISP Attribute) + Domain Watchlist + Domain|ip test attribute + + + + + + + circl.lu + + + + + + + 149.13.33.14 + + + + + + + High + Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none + + + + MISP-Project + + + + + + + + Event Threat Level: Medium + + + MISP Tag: misp:tool="MISP-STIX-Converter" + + + + + MISP-Project + + + + + + + + + + Export from MISP-Project's MISP + Threat Report + + + + MISP-STIX-Converter test event with filename attribute + + 2020-10-25T00:00:00+00:00 + + + + MISP-Project + + + + + Payload delivery + + Payload delivery: test_file_name (MISP Attribute) + File Hash Watchlist + Filename test attribute + + + + + test_file_name + + + + + High + Derived from MISP's IDS flag. If an attribute is marked for IDS exports, the confidence will be high, otherwise none + + + + MISP-Project + + + + + + + + Event Threat Level: Medium + + + MISP Tag: misp:tool="MISP-STIX-Converter" + + + + + MISP-Project + + + + + + + + From 0e7a20c49af085d5fe9c6d2a659c8c8553627b16 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:09:30 +0200 Subject: [PATCH 27/36] fix: [tests] Updated tests for STIX 1 export helpers --- tests/_test_stix_export.py | 20 ++--- tests/test_stix1_export.py | 176 +++++++++++++++++++++++-------------- 2 files changed, 118 insertions(+), 78 deletions(-) diff --git a/tests/_test_stix_export.py b/tests/_test_stix_export.py index 3de17d30..6ae53e98 100644 --- a/tests/_test_stix_export.py +++ b/tests/_test_stix_export.py @@ -25,15 +25,15 @@ def tearDown(self): class TestCollectionSTIX1Export(TestCollectionSTIXExport): - def _check_stix1_collection_export_results(self, to_test_name, reference_name): - to_test = STIXPackage.from_xml(str(self._current_path / to_test_name)).to_dict() - reference = STIXPackage.from_xml(str(self._current_path / reference_name)).to_dict() + def _check_stix1_collection_export_results(self, to_test_file, reference_file): + to_test = STIXPackage.from_xml(to_test_file).to_dict() + reference = STIXPackage.from_xml(reference_file).to_dict() self.__recursive_feature_tests(reference, to_test, exclude=('id', 'timestamp')) - def _check_stix1_export_results(self, to_test_name, reference_name): - to_test = STIXPackage.from_xml(str(self._current_path / to_test_name)).to_dict() - reference = STIXPackage.from_xml(str(self._current_path / reference_name)).to_dict() - self.__recursive_feature_tests(reference, to_test, exclude=('id',)) + def _check_stix1_export_results(self, to_test_file, reference_file): + to_test = STIXPackage.from_xml(to_test_file).to_dict() + reference = STIXPackage.from_xml(reference_file).to_dict() + self.__recursive_feature_tests(reference, to_test, exclude=('id', 'timestamp')) def __check_observables(self, reference_observables, observables_to_test): for reference_observable, observable_to_test in zip(reference_observables, observables_to_test): @@ -68,10 +68,10 @@ def __recursive_feature_tests(self, reference, to_test, exclude=tuple()): class TestCollectionSTIX2Export(TestCollectionSTIXExport): - def _check_stix2_results_export(self, to_test_name, reference_name): - with open(self._current_path / to_test_name, 'rt', encoding='utf-8') as f: + def _check_stix2_results_export(self, to_test_file, reference_file): + with open(to_test_file, 'rt', encoding='utf-8') as f: to_test = json.loads(f.read()) - with open(self._current_path / reference_name, 'rt', encoding='utf-8') as f: + with open(reference_file, 'rt', encoding='utf-8') as f: reference = json.loads(f.read()) self.assertEqual(reference['objects'], to_test['objects']) diff --git a/tests/test_stix1_export.py b/tests/test_stix1_export.py index 88dc7477..6a68440f 100644 --- a/tests/test_stix1_export.py +++ b/tests/test_stix1_export.py @@ -4127,122 +4127,162 @@ def test_event_with_vulnerability_galaxy(self): class TestCollectionStix1Export(TestCollectionSTIX1Export): def test_attribute_collection_export_11(self): name = 'test_attributes_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix11.xml' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix11.xml' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] self.assertEqual( misp_attribute_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.1.1', - in_memory=True + *input_files, return_format='xml', version='1.1.1', + in_memory=True, single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_export_results(to_test_name, reference_name) + self._check_stix1_export_results(output_file, reference_file) self.assertEqual( misp_attribute_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.1.1' + *input_files, return_format='xml', version='1.1.1', + single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_export_results(to_test_name, reference_name) + self._check_stix1_export_results(output_file, reference_file) def test_attribute_collection_export_12(self): name = 'test_attributes_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix12.xml' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix12.xml' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] self.assertEqual( misp_attribute_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.2', - in_memory=True + *input_files, return_format='xml', version='1.2', + in_memory=True, single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_export_results(to_test_name, reference_name) + self._check_stix1_export_results(output_file, reference_file) self.assertEqual( misp_attribute_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.2' + *input_files, return_format='xml', version='1.2', + single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_export_results(to_test_name, reference_name) + self._check_stix1_export_results(output_file, reference_file) def test_event_collection_export_11(self): name = 'test_events_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix11.xml' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix11.xml' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] self.assertEqual( misp_event_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.1.1' + *input_files, return_format='xml', version='1.1.1', + single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_collection_export_results(to_test_name, reference_name) + self._check_stix1_collection_export_results(output_file, reference_file) self.assertEqual( misp_event_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.1.1', - in_memory=True + *input_files, return_format='xml', version='1.1.1', + in_memory=True, single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_collection_export_results(to_test_name, reference_name) + self._check_stix1_collection_export_results(output_file, reference_file) + self.assertEqual( + misp_event_collection_to_stix1( + *input_files, return_format='xml', version='1.1.1' + ), + { + 'success': 1, + 'results': [ + self._current_path / f'{name}_{n}.json.out' for n in (1, 2) + ] + } + ) + for n in (1, 2): + with open(self._current_path / f'{name}_{n}.json.out', 'r') as i: + with open(f'ntm_collection{n}_11.xml', 'w') as o: + o.write(i.read()) + self._check_stix1_export_results( + self._current_path / f'{name}_{n}.json.out', + self._current_path / f'test_event{n}_stix11.xml' + ) def test_event_collection_export_12(self): name = 'test_events_collection' - to_test_name = f'{name}.json.out' - reference_name = f'{name}_stix12.xml' - output_file = self._current_path / to_test_name + output_file = self._current_path / f'{name}.json.out' + reference_file = self._current_path / f'{name}_stix12.xml' input_files = [self._current_path / f'{name}_{n}.json' for n in (1, 2)] self.assertEqual( misp_event_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.2' + *input_files, return_format='xml', version='1.2', + single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_collection_export_results(to_test_name, reference_name) + self._check_stix1_collection_export_results(output_file, reference_file) self.assertEqual( misp_event_collection_to_stix1( - output_file, - *input_files, - return_format='xml', - version='1.2', - in_memory=True + *input_files, return_format='xml', version='1.2', + in_memory=True, single_output=True, output_name=output_file ), - 1 + {'success': 1, 'results': [output_file]} ) - self._check_stix1_collection_export_results(to_test_name, reference_name) + self._check_stix1_collection_export_results(output_file, reference_file) + self.assertEqual( + misp_event_collection_to_stix1( + *input_files, return_format='xml', version='1.2' + ), + { + 'success': 1, + 'results': [ + self._current_path / f'{name}_{n}.json.out' for n in (1, 2) + ] + } + ) + for n in (1, 2): + with open(self._current_path / f'{name}_{n}.json.out', 'r') as i: + with open(f'ntm_collection{n}_12.xml', 'w') as o: + o.write(i.read()) + self._check_stix1_export_results( + self._current_path / f'{name}_{n}.json.out', + self._current_path / f'test_event{n}_stix12.xml' + ) def test_event_export_11(self): name = 'test_events_collection_1.json' - self.assertEqual(misp_to_stix1(self._current_path / name, 'xml', '1.1.1'), 1) - self._check_stix1_export_results(f'{name}.out', 'test_event_stix11.xml') + filename = self._current_path / name + output_file = self._current_path / f'{name}.out' + reference_file = self._current_path / 'test_event1_stix11.xml' + self.assertEqual( + misp_to_stix1(filename, return_format='xml', version='1.1.1'), + {'success': 1, 'results': [output_file]} + ) + self._check_stix1_export_results(output_file, reference_file) + self.assertEqual( + misp_event_collection_to_stix1( + filename, return_format='xml', version='1.1.1' + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix1_export_results(output_file, reference_file) def test_event_export_12(self): name = 'test_events_collection_1.json' - self.assertEqual(misp_to_stix1(self._current_path / name, 'xml', '1.2'), 1) - self._check_stix1_export_results(f'{name}.out', 'test_event_stix12.xml') + filename = self._current_path / name + output_file = self._current_path / f'{name}.out' + reference_file = self._current_path / 'test_event1_stix12.xml' + self.assertEqual( + misp_to_stix1(filename, return_format='xml', version='1.2'), + {'success': 1, 'results': [output_file]} + ) + self._check_stix1_export_results(output_file, reference_file) + self.assertEqual( + misp_event_collection_to_stix1( + filename, return_format='xml', version='1.2' + ), + {'success': 1, 'results': [output_file]} + ) + self._check_stix1_export_results(output_file, reference_file) \ No newline at end of file From 00972511b6b6d77ac47949f75ed8dc333b728c89 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:38:33 +0200 Subject: [PATCH 28/36] fix; [stix export] Fixed fail messages as the command line feature wants lists --- misp_stix_converter/misp_stix_converter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 4bf4005f..6f639112 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -405,7 +405,7 @@ def misp_event_collection_to_stix1( _write_raw_stix(parser.stix_package, name, *_write_args) return _generate_traceback(debug, parser, name) except Exception as exception: - return {'fails': f'{filename} - {exception.__str__()}'} + return {'fails': [f'{filename} - {exception.__str__()}']} traceback = defaultdict(list) if single_output: stix_package = _create_stix_package(org, version, header=False) @@ -499,7 +499,7 @@ def misp_collection_to_stix2( f.write(parser.bundle.serialize(indent=4)) return _generate_traceback(debug, parser, name) except Exception as exception: - return {'fails': f'{filename} - {exception.__str__()}'} + return {'fails': [f'{filename} - {exception.__str__()}']} traceback = defaultdict(list) if single_output: if in_memory: @@ -609,7 +609,7 @@ def misp_to_stix1( ) return _generate_traceback(debug, parser, name) except Exception as exception: - return {'fails': f'{filename} - {exception.__str__()}'} + return {'fails': [f'{filename} - {exception.__str__()}']} def misp_to_stix2(filename: _files_type, debug: Optional[bool] = False, @@ -630,7 +630,7 @@ def misp_to_stix2(filename: _files_type, debug: Optional[bool] = False, f.write(json.dumps(parser.bundle, cls=STIXJSONEncoder, indent=4)) return _generate_traceback(debug, parser, name) except Exception as exception: - return {'fails': f'{filename} - {exception.__str__()}'} + return {'fails': [f'{filename} - {exception.__str__()}']} ################################################################################ From 464014338616f393dca2c2dbef02b2fdab4b38ca Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:39:41 +0200 Subject: [PATCH 29/36] fix: [stix2 export] Fixed footer for collections export as STIX 2 --- misp_stix_converter/misp_stix_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 6f639112..740e5b8d 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -560,7 +560,7 @@ def misp_collection_to_stix2( traceback['fails'].append(f'{filename} - {exception.__str__()}') if written: with open(name, 'at', encoding='utf-8') as f: - f.write(' ]\n}') + f.write('\n ]\n}') traceback.update(_generate_traceback(debug, parser, name)) else: name.remove() From ae986b9fb8ca9411486b485fe6d70988ca84c577 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:40:46 +0200 Subject: [PATCH 30/36] fix: [stix export] Fixed arguments to give from the command line feature to the STIX export helpers --- misp_stix_converter/misp_stix_converter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 740e5b8d..ab4c7cb8 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -966,7 +966,7 @@ def _write_raw_stix( def _misp_to_stix(stix_args): collection_args = { 'in_memory': stix_args.in_memory, - 'single_ouput': stix_args.single_output + 'single_output': stix_args.single_output } if stix_args.version in ('1.1.1', '1.2'): stix1_args = { @@ -977,12 +977,12 @@ def _misp_to_stix(stix_args): } if stix_args.level == 'attribute': return misp_attribute_collection_to_stix1( - stix_args.file, **collection_args, **stix1_args + *stix_args.file, **collection_args, **stix1_args ) if len(stix_args.file) == 1: return misp_to_stix1(stix_args.file[0], **stix1_args) return misp_event_collection_to_stix1( - stix_args.file, **collection_args, **stix1_args + *stix_args.file, **collection_args, **stix1_args ) stix2_args = { 'debug': stix_args.debug, 'output_dir': stix_args.output_dir, @@ -991,7 +991,7 @@ def _misp_to_stix(stix_args): if len(stix_args.file) == 1: return misp_to_stix2(stix_args.file[0]) return misp_collection_to_stix2( - stix_args.file, **collection_args, **stix2_args + *stix_args.file, **collection_args, **stix2_args ) From dcb48ec95fad4748e442602567ee06a07cd13c9b Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:42:17 +0200 Subject: [PATCH 31/36] fix: [misp_stix_converter] Fixed helpers import - using the method names recently changed --- misp_stix_converter/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 3045ffa2..a736dfe8 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -5,9 +5,9 @@ from .misp2stix import * # Helpers from .misp_stix_converter import ( - _from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2_0, - misp_collection_to_stix2_1, misp_event_collection_to_stix1, misp_to_stix1, - misp_to_stix2_0, misp_to_stix2_1, stix_1_to_misp, stix_2_to_misp) + _from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2, + misp_event_collection_to_stix1, misp_to_stix1, misp_to_stix2, + stix_1_to_misp, stix_2_to_misp) # STIX 1 special halpers from .misp_stix_converter import ( _get_campaigns, _get_courses_of_action, _get_events, _get_indicators, From 4a7ea084543f20f7c87ab48d0a549d43e100706e Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:43:23 +0200 Subject: [PATCH 32/36] chg: [misp_stix_converter] Changes on the command line feature - Cleaner separation between the 2 main features, export & import, as well as cleaner arguments in general - Better handling of the messages returned by the helper methods that are call by the command line feature --- misp_stix_converter/__init__.py | 144 +++++++++++++++++++++++--------- 1 file changed, 106 insertions(+), 38 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index a736dfe8..5c146bdc 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -28,43 +28,51 @@ def main(): parser = argparse.ArgumentParser(description='Convert MISP <-> STIX') - - feature_parser = parser.add_mutually_exclusive_group(required=True) - feature_parser.add_argument( - '-e', '--export', action='store_true', help='Export MISP to STIX.' + parser.add_argument( + '--debug', action='store_true', help='Show errors and warnings' ) - feature_parser.add_argument( - '-i', '--import', action='store_true', help='Import STIX to MISP.' + + # SUBPARSERS TO SEPARATE THE 2 MAIN FEATURES + subparsers = parser.add_subparsers( + title='Main feature', dest='feature', required=True ) - parser.add_argument( - '-v', '--version', choices=['1.1.1', '1.2', '2.0', '2.1'], - required=True, help='STIX version.' + # EXPORT SUBPARSER + export_parser = subparsers.add_parser( + 'export', help='Export MISP to STIX - try ' + '`misp_stix_converter export -h` for more help.' ) - parser.add_argument( + export_parser.add_argument( '-f', '--file', nargs='+', type=Path, required=True, help='Path to the file(s) to convert.' ) - parser.add_argument( + export_parser.add_argument( + '-v', '--version', choices=['1.1.1', '1.2', '2.0', '2.1'], + required=True, help='STIX specific version.' + ) + export_parser.add_argument( '-s', '--single_output', action='store_true', help='Produce only one result file (in case of multiple input file).' ) - parser.add_argument( - '-t', '--tmp_files', action='store_true', - help='Store result in file (in case of multiple result files) ' - 'instead of keeping it in memory only.' - ) - parser.add_argument( - '-o', '--output_name', type=Path, help='Output file name' + export_parser.add_argument( + '-m', '--in_memory', action='store_true', + help='Store result in memory (in case of multiple result files) ' + 'instead of storing it in tmp files.' ) - parser.add_argument( + export_parser.add_argument( '--output_dir', type=Path, - help='Output path for the conversion results.' + help='Output path - used in the case of multiple input files when the ' + '`single_output` argument is not used.' ) - - stix1_parser = parser.add_argument_group('STIX 1 specific parameters') + export_parser.add_argument( + '-o', '--output_name', type=Path, + help='Output file name - used in the case of a single input file or ' + 'when the `single_output` argument is used.' + ) + # STIX 1 EXPORT SPECIFIC ARGUMENTS + stix1_parser = export_parser.add_argument_group('STIX 1 specific arguments') stix1_parser.add_argument( - '--feature', default='event', choices=['attribute', 'event'], + '--level', default='event', choices=['attribute', 'event'], help='MISP data structure level.' ) stix1_parser.add_argument( @@ -79,22 +87,82 @@ def main(): '-org', default='MISP', help='Organisation name to be used in the STIX 1 header.' ) + export_parser.set_defaults(func=_misp_to_stix) + + # IMPORT SUBPARSER + import_parser = subparsers.add_parser( + 'import', help='Import STIX to MISP - try ' + '`misp_stix_converter import -h` for more help.' + ) + import_parser.add_argument( + '-f', '--file', nargs='+', type=Path, required=True, + help='Path to the file(s) to convert.' + ) + import_parser.add_argument( + '-v', '--version', choices=['1', '2'], + required=True, help='STIX major version.' + ) + import_parser.add_argument( + '-s', '--single_output', action='store_true', + help='Produce only one MISP event per STIX file' + '(in case of multiple Report, Grouping or Incident objects).' + ) + import_parser.add_argument( + '-o', '--output_name', type=Path, + help='Output file name - used in the case of a single input file or ' + 'when the `single_output` argument is used.' + ) + import_parser.add_argument( + '--output_dir', type=Path, + help='Output path - used in the case of multiple input files when the ' + '`single_output` argument is not used.' + ) + import_parser.add_argument( + '-d', '--distribution', type=int, default=0, + help='Distribution level for the imported MIPS content.' + ) + import_parser.add_argument( + '-sg', '--sharing_group', type=int, default=None, + help='Sharing group ID when distribution is 4.' + ) + import_parser.add_argument( + '--galaxies_as_tags', action='store_true', + help='Import MISP Galaxies as tag names instead of the standard Galaxy format.' + ) + import_parser.set_defaults(func=_stix_to_misp) stix_args = parser.parse_args() if len(stix_args.file) > 1 and stix_args.single_output and stix_args.output_dir is None: stix_args.output_dir = Path(__file__).parents[1] / 'tmp' - - results = _misp_to_stix(stix_args) if stix_args.export else _stix_to_misp(stix_args) - if isinstance(results, list): - files = '\n - '.join(str(result) for result in results) - print( - 'Successfully processed your ' - f"{'files' if len(results) > 1 else 'file'}. Results available in:" - f"\n - {files}" - ) - else: - print( - 'Successfully processed your ' - f"{'files' if len(stix_args.file) > 1 else 'file'}. " - f"Results available in {results}" - ) + feature = 'MISP to STIX' if stix_args.feature == 'export' else 'STIX to MISP' + try: + traceback = stix_args.func(stix_args) + for field in ('errors', 'warnings'): + if field in traceback: + messages = '\n - '.join(traceback[field]) + print(f'{field.capitalize()} encountered during the ' + f'{feature} conversion process:\n - {messages}') + if 'fails' in traceback: + fails = '\n - '.join(traceback['fails']) + print('Failed parsing the following - and the related error ' + f'message:\n - {fails}') + if 'results' in traceback: + results = traceback['results'] + if isinstance(results, list): + files = '\n - '.join(str(result) for result in results) + print( + 'Successfully processed your ' + f"{'files' if len(results) > 1 else 'file'}. Results " + f"available in:\n - {files}" + ) + else: + print( + 'Successfully processed your ' + f"{'files' if len(stix_args.file) > 1 else 'file'}. " + f"Results available in {results}" + ) + else: + print(f'No result from the {feature} conversion.') + except Exception as exception: + print(f'Breaking exception encountered during the {feature} conversion ' + f'process: {exception.__str__()}') \ No newline at end of file From b917550f141ddcad88acefdccc18c6f6d7342f26 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:46:45 +0200 Subject: [PATCH 33/36] fix: [tests] Fixed STIX 1 export result samples --- tests/test_event1_stix12.xml | 6 +----- tests/test_events_collection_stix11.xml | 6 +----- tests/test_events_collection_stix12.xml | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/tests/test_event1_stix12.xml b/tests/test_event1_stix12.xml index 5fcea96d..df0a84fe 100644 --- a/tests/test_event1_stix12.xml +++ b/tests/test_event1_stix12.xml @@ -52,11 +52,7 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:xs="http://www.w3.org/2001/XMLSchema" - id="MISP:Package-fc2914a3-adc9-4a9b-9a06-e543f2cc14bf" version="1.2"> - - Export from MISP's MISP - Threat Report - + id="MISP:STIXPackage-fc2914a3-adc9-4a9b-9a06-e543f2cc14bf" version="1.2"> diff --git a/tests/test_events_collection_stix11.xml b/tests/test_events_collection_stix11.xml index 17aa6775..8856a21c 100644 --- a/tests/test_events_collection_stix11.xml +++ b/tests/test_events_collection_stix11.xml @@ -52,11 +52,7 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:xs="http://www.w3.org/2001/XMLSchema" - id="MISP:Package-0c467501-2514-462f-90d6-3ea04bb0e721" version="1.1.1" timestamp="2020-10-25T16:22:00Z"> - - Export from MISP's MISP - Threat Report - + id="MISP:STIXPackage-0c467501-2514-462f-90d6-3ea04bb0e721" version="1.1.1" timestamp="2020-10-25T16:22:00Z"> diff --git a/tests/test_events_collection_stix12.xml b/tests/test_events_collection_stix12.xml index 161db7bd..4ba5220e 100644 --- a/tests/test_events_collection_stix12.xml +++ b/tests/test_events_collection_stix12.xml @@ -52,11 +52,7 @@ xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:xs="http://www.w3.org/2001/XMLSchema" - id="MISP:Package-0c467501-2514-462f-90d6-3ea04bb0e721" version="1.2" timestamp="2020-10-25T16:22:00Z"> - - Export from MISP's MISP - Threat Report - + id="MISP:STIXPackage-0c467501-2514-462f-90d6-3ea04bb0e721" version="1.2" timestamp="2020-10-25T16:22:00Z"> From 487e5cc488e4dce47f4eb56e984e87bbaf8a29d9 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 10:48:54 +0200 Subject: [PATCH 34/36] fix: [tests] Removed unused imports --- tests/test_external_stix20_import.py | 2 -- tests/test_external_stix21_import.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 8787abeb..edea8ddc 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json -from uuid import uuid5 from .test_external_stix20_bundles import TestExternalSTIX20Bundles from ._test_stix import TestSTIX21 from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 8789590d..06b93f85 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -1,8 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import json -from uuid import uuid5 from .test_external_stix21_bundles import TestExternalSTIX21Bundles from ._test_stix import TestSTIX21 from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import From 26404dcc63fe262e080488b6e65c0055d6e1f9a3 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 11:00:05 +0200 Subject: [PATCH 35/36] chg: [package] Bumped version --- misp_stix_converter/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 34b45423..1bd2d490 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.4.170' +__version__ = '2.4.172' import argparse from .misp_stix_mapping import Mapping diff --git a/pyproject.toml b/pyproject.toml index 969fb741..6a6a9cf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "misp-stix" -version = "2.4.170" +version = "2.4.172" description = "Python scripts used by MISP to export MISP format into STIX and to import STIX into MISP format." authors = ["Christian Studer "] maintainers = ["Christian Studer "] From 167183de889fc8e888450200650c9491480e4b29 Mon Sep 17 00:00:00 2001 From: Christian Studer Date: Fri, 9 Jun 2023 11:00:58 +0200 Subject: [PATCH 36/36] chg: [poetry] Updated dependencies --- poetry.lock | 63 +++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0be72b11..5f05a80d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,14 +32,14 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "certifi" -version = "2022.12.7" +version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, ] [[package]] @@ -158,21 +158,21 @@ python-dateutil = "*" [[package]] name = "deprecated" -version = "1.2.13" +version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, ] [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "exceptiongroup" @@ -494,14 +494,14 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "publicsuffixlist" -version = "0.9.4" +version = "0.10.0.20230608" description = "publicsuffixlist implement" category = "main" optional = false python-versions = ">=2.6" files = [ - {file = "publicsuffixlist-0.9.4-py2.py3-none-any.whl", hash = "sha256:107fc6f729b042ef31dceb1573ef16547407c9034667724589aa9b20f9f23920"}, - {file = "publicsuffixlist-0.9.4.tar.gz", hash = "sha256:0340e98f2ac656913c9c2b0a7ab109844f854be40e1977a22bff0ab0e72bdbfc"}, + {file = "publicsuffixlist-0.10.0.20230608-py2.py3-none-any.whl", hash = "sha256:4b641e8367a45443165cf56fdab405faa67ca7a21bbe3e0449e1f2ed4f744836"}, + {file = "publicsuffixlist-0.10.0.20230608.tar.gz", hash = "sha256:84cb4e189317d151aab86c50517fca48800b4f46ce459681132a194832cb0657"}, ] [package.extras] @@ -534,30 +534,30 @@ files = [ [[package]] name = "pymisp" -version = "2.4.170.1" +version = "2.4.171" description = "Python API for MISP." category = "main" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "pymisp-2.4.170.1-py3-none-any.whl", hash = "sha256:f324b30940854b5aeac5d62028da444cd0ac23df808c54e448d7c97a30fbb207"}, - {file = "pymisp-2.4.170.1.tar.gz", hash = "sha256:45c060a250a9cb275797b7c7dcc1ba7c6880ec1075fbf98e4ef273959ddf1a94"}, + {file = "pymisp-2.4.171-py3-none-any.whl", hash = "sha256:9410f44cd811f91d523593b5353e0fd37057ba1b1cf856a4050454cbb8a5b788"}, + {file = "pymisp-2.4.171.tar.gz", hash = "sha256:6dca1dd7be6e63c423eb48e84e587cb776db545de3385dc4875ecc55192a7863"}, ] [package.dependencies] deprecated = ">=1.2.13,<2.0.0" jsonschema = ">=4.17.3,<5.0.0" -publicsuffixlist = ">=0.9.4,<0.10.0" +publicsuffixlist = ">=0.10.0.20230506,<0.11.0.0" python-dateutil = ">=2.8.2,<3.0.0" -requests = ">=2.28.2,<3.0.0" +requests = ">=2.30.0,<3.0.0" [package.extras] -brotli = ["urllib3[brotli] (>=1.26.15,<2.0.0)"] +brotli = ["urllib3[brotli]"] docs = ["recommonmark (>=0.7.1,<0.8.0)", "sphinx-autodoc-typehints (>=1.23.0,<2.0.0)"] -email = ["RTFDE (>=0.0.2,<0.0.3)", "extract_msg (==0.40)", "oletools (>=0.60.1,<0.61.0)"] +email = ["RTFDE (>=0.0.2,<0.0.3)", "extract_msg (>=0.41.1,<0.42.0)", "oletools (>=0.60.1,<0.61.0)"] fileobjects = ["lief (>=0.13.0,<0.14.0)", "pydeep2 (>=0.5.1,<0.6.0)", "python-magic (>=0.4.27,<0.5.0)"] -openioc = ["beautifulsoup4 (==4.11.1)"] -pdfexport = ["reportlab (>=3.6.12,<4.0.0)"] +openioc = ["beautifulsoup4 (>=4.12.2,<5.0.0)"] +pdfexport = ["reportlab (>=4.0.0,<5.0.0)"] url = ["pyfaup (>=1.2,<2.0)"] virustotal = ["validators (>=0.20.0,<0.21.0)"] @@ -650,21 +650,21 @@ files = [ [[package]] name = "requests" -version = "2.28.2" +version = "2.31.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -830,20 +830,21 @@ files = [ [[package]] name = "urllib3" -version = "1.26.15" +version = "2.0.3" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" files = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, + {file = "urllib3-2.0.3-py3-none-any.whl", hash = "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1"}, + {file = "urllib3-2.0.3.tar.gz", hash = "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "weakrefmethod"