From 7a9519880265c3180f8c608bcd8d5b85df9c980d Mon Sep 17 00:00:00 2001 From: robinwenzel <53773625+robinwenzel@users.noreply.github.com> Date: Mon, 7 Oct 2019 11:49:05 +0200 Subject: [PATCH] Tiffy 1.0.2 (#4) * 1.0.2 Changes added threshold setting for IDS flag in config tagging for attributes derived from c2-server and malware observations added removed unused config values base_severity and base_confidence Bugfixes Fixed Bug with logging when no log_path was provided --- CHANGELOG.md | 9 +++ README.md | 11 ++++ TIELoader.py | 4 +- helpers/MISPHelper.py | 21 ++++++- model/__init__.py | 1 + model/config.py | 27 ++++----- model/tags.py | 119 +++++++++++++++++++++++++++++++++++++ settings/config.sample.yml | 3 +- settings/tags.sample.yml | 18 ++++++ tiffy.py | 7 ++- 10 files changed, 195 insertions(+), 25 deletions(-) create mode 100755 model/tags.py create mode 100755 settings/tags.sample.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ab1e4..f830df3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # tiffy Changelog +### 1.0.2 + #### Changes + - added threshold setting for IDS flag in config + - tagging for attributes derived from TIE observations added + - removed unused config values base_severity and base_confidence + + #### Bugfixes + - Fixed Bug with logging when no log_path was provided + ### 1.0.1 #### Changes - simplified Config File format, added possibility to use environment variables for configuration (see readme) diff --git a/README.md b/README.md index c66d5d3..dcc2223 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,16 @@ to the base event. Double quotes need to be escaped. If no tags are passed `TLP: $ ./tiffy.py --event-tags {\"name\":\"tlp:amber\"} ``` +## Setting Tags for Attributes + +You can assign Tags to attributes using the `settings/tags.yml` file. Currently only attributes derived from TIE +`c2-server` and `malware` observations can be tagged. Add your desired Tag to the list of tags as seen in the sample +file. The format for a new entry is: + +``` +- name: tag:name:here +``` + ## Setting the Output Format You can choose the output format of the feed. Currently only MISP-JSON is supported but more formats will follow. @@ -205,6 +215,7 @@ at least the required variables. | TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY | 2 | | IoC will get this severity if it is added | | TIFFY_CONF_MISP_EVENTS_PUBLISHED | false | | IoC will get published in MISP | | TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS | false | | Set IDS flag for this IoC | +| TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD| 90 | 90 | Confidence Threshold at or above which attributes should get IDS flag | | TIFFY_PARAM_TIE_SEEN_FIRST | | YYYY-MM-DD | Download only IoC which are first seen at ... and newer | | TIFFY_PARAM_TIE_SEEN_LAST | | YYYY-MM-DD | Download only IoC which are last seen at ... and older | | TIFFY_PARAM_TIE_ACTOR | | example1,example2 | Download only IoC with this actor | diff --git a/TIELoader.py b/TIELoader.py index a605473..98b6200 100644 --- a/TIELoader.py +++ b/TIELoader.py @@ -25,7 +25,7 @@ def get_csv_value(field, src): class TIELoader: @staticmethod - def start(out_format, conf, tags, category, actor, family, source, first_seen, last_seen, min_confidence, + def start(out_format, conf, tags, attr_tags, category, actor, family, source, first_seen, last_seen, min_confidence, min_severity, max_confindence, max_severity, proxy_tie_addr, no_filter=False, disable_cert_verify=False): # Building Auth Header @@ -154,7 +154,7 @@ def start(out_format, conf, tags, category, actor, family, source, first_seen, l if out_format == 'MISP': # Serialize event as MISP Event - event, attr_hashes = MISPHelper.generate_MISP_Event(deduplicated_observations, conf, tags) + event, attr_hashes = MISPHelper.generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags) event_json = event.to_json() event_from_json = json.loads(event_json) event_from_json['publish_timestamp'] = str(event_from_json['publish_timestamp']) diff --git a/helpers/MISPHelper.py b/helpers/MISPHelper.py index 074053a..b5efb01 100644 --- a/helpers/MISPHelper.py +++ b/helpers/MISPHelper.py @@ -7,7 +7,7 @@ from pymisp import MISPEvent, MISPOrganisation, MISPAttribute -def generate_MISP_Event(deduplicated_observations, conf, tags): +def generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags): dt = datetime.now() event = MISPEvent() @@ -34,8 +34,25 @@ def generate_MISP_Event(deduplicated_observations, conf, tags): misp_attr['timestamp'] = dt.strftime("%s") misp_attr.type = get_Attribute_Type(attr) misp_attr.value = get_MISP_Fitted_Value(attr["value"], misp_attr.type) + if 'c2-server' in attr['categories'] and attr_tags.c2tags: + misp_attr['Tag'] = attr_tags.c2tags + if 'malware' in attr['categories'] and attr_tags.malwaretags: + misp_attr['Tag'] = attr_tags.malwaretags + if 'espionage' in attr['categories'] and attr_tags.espionagetags: + misp_attr['Tag'] = attr_tags.espionagetags + if 'bot' in attr['categories'] and attr_tags.bottags: + misp_attr['Tag'] = attr_tags.bottags + if 'whitelist' in attr['categories'] and attr_tags.whitelisttags: + misp_attr['Tag'] = attr_tags.whitelisttags + if 'cybercrime' in attr['categories'] and attr_tags.cybercrimetags: + misp_attr['Tag'] = attr_tags.cybercrimetags + if 'phishing' in attr['categories'] and attr_tags.phishingtags: + misp_attr['Tag'] = attr_tags.phishingtags misp_attr.category = get_Attribute_Category(attr) - misp_attr.to_ids = conf.attr_to_ids + if conf.attr_to_ids and attr['min_confidence'] >= conf.attr_to_ids_threshold: + misp_attr.to_ids = True + else: + misp_attr.to_ids = False misp_attr['comment'] = 'categories: ' + str(attr['categories']) + ' actors: ' + str(attr['actors']) + \ ' families: ' + str(attr['families']) + ' sources: ' + str(attr['sources']) + \ ' severity: ' + str(attr['max_severity']) + \ diff --git a/model/__init__.py b/model/__init__.py index cca5d9b..3b8b2f5 100755 --- a/model/__init__.py +++ b/model/__init__.py @@ -1 +1,2 @@ from .config import Config +from .tags import Tags diff --git a/model/config.py b/model/config.py index 307f135..1e05952 100644 --- a/model/config.py +++ b/model/config.py @@ -17,8 +17,7 @@ def __init__(self): self.__Event_Base_Threat_Level = 3 self.__Event_Published = False self.__Attr_ToIDS = True - self.__Base_Confidence = 60 - self.__Base_Severity = 1 + self.__Attr_ToIDS_threshold = 90 # --- Getter @property @@ -50,12 +49,8 @@ def attr_to_ids(self): return self.__Attr_ToIDS @property - def base_confidence(self): - return self.__Base_Confidence - - @property - def base_severity(self): - return self.__Base_Severity + def attr_to_ids_threshold(self): + return self.__Attr_ToIDS_threshold # --- Setter @@ -87,13 +82,9 @@ def event_published(self, value): def attr_to_ids(self, value): self.__Attr_ToIDS = value - @base_confidence.setter - def base_confidence(self, value): - self.__Base_Confidence = value - - @base_severity.setter - def base_severity(self, value): - self.__Base_Severity = value + @attr_to_ids_threshold.setter + def attr_to_ids_threshold(self, value): + self.__Attr_ToIDS_threshold = value @staticmethod def parse(configfile): @@ -174,6 +165,10 @@ def parse(configfile): conf.attr_to_ids = bool(os.environ.get('TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS')) else: conf.attr_to_ids = Config.get_config_value_optional(attr_vals, "to_ids", "True") + if 'TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD' in os.environ: + conf.attr_to_ids_threshold = int(os.environ.get('TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD')) + else: + conf.attr_to_ids_threshold = Config.get_config_value_optional(attr_vals, "to_ids_threshold", 90) else: Config.raise_error_critical("Could not find attributes values ") else: @@ -194,7 +189,7 @@ def parseFromEnv(): config.base_severity = os.environ['TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY'] config.event_published = os.environ['TIFFY_CONF_MISP_EVENTS_PUBLISHED'] config.attr_to_ids = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS'] - config.attr_tagging = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TAGGING'] + config.attr_to_ids_threshold = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD'] return config except KeyError: diff --git a/model/tags.py b/model/tags.py new file mode 100755 index 0000000..0d46890 --- /dev/null +++ b/model/tags.py @@ -0,0 +1,119 @@ +""" +DCSO TIE2MISP Parser +Copyright (c) 2017, DCSO GmbH +""" +import yaml +import warnings + + +class Tags: + def __init__(self): + self.__c2tags = list() + self.__malwaretags = list() + self.__espionagetags = list() + self.__bottags = list() + self.__whitelisttags = list() + self.__cybercrimetags = list() + self.__phishingtags = list() + + @property + def c2tags(self): + return self.__c2tags + + @property + def malwaretags(self): + return self.__malwaretags + + @property + def espionagetags(self): + return self.__espionagetags + + @property + def bottags(self): + return self.__bottags + + @property + def whitelisttags(self): + return self.__whitelisttags + + @property + def cybercrimetags(self): + return self.__cybercrimetags + + @property + def phishingtags(self): + return self.__phishingtags + + @c2tags.setter + def c2tags(self, value): + self.__c2tags = value + + @malwaretags.setter + def malwaretags(self, value): + self.__malwaretags = value + + @espionagetags.setter + def espionagetags(self, value): + self.__espionagetags = value + + @bottags.setter + def bottags(self, value): + self.__bottags = value + + @whitelisttags.setter + def whitelisttags(self, value): + self.__whitelisttags = value + + @cybercrimetags.setter + def cybercrimetags(self, value): + self.__cybercrimetags = value + + @phishingtags.setter + def phishingtags(self, value): + self.__phishingtags = value + + @staticmethod + def parse(tagfile): + + tags = Tags() + + # Load Config + tag_file = open(tagfile, "r", encoding="utf-8") + raw_tags = yaml.load(tag_file, Loader=yaml.FullLoader) + + if "c2_tags" in raw_tags: + c2tags = raw_tags["c2_tags"] + for tag in c2tags: + tags.c2tags.append(tag) + + if "malware_tags" in raw_tags: + malwaretags = raw_tags["malware_tags"] + for tag in malwaretags: + tags.malwaretags.append(tag) + + if "espionage_tags" in raw_tags: + espionagetags = raw_tags["espionage_tags"] + for tag in espionagetags: + tags.espionagetags.append(tag) + + if "bot_tags" in raw_tags: + bottags = raw_tags["bot_tags"] + for tag in bottags: + tags.bottags.append(tag) + + if "whitelist_tags" in raw_tags: + whitelisttags = raw_tags["whitelist_tags"] + for tag in whitelisttags: + tags.whitelisttags.append(tag) + + if "cybercrime_tags" in raw_tags: + cybercrimetags = raw_tags["cybercrime_tags"] + for tag in cybercrimetags: + tags.cybercrimetags.append(tag) + + if "phishing_tags" in raw_tags: + phishingtags = raw_tags["phishing_tags"] + for tag in phishingtags: + tags.phishingtags.append(tag) + + return tags diff --git a/settings/config.sample.yml b/settings/config.sample.yml index 8a47ba6..2d564ad 100755 --- a/settings/config.sample.yml +++ b/settings/config.sample.yml @@ -6,12 +6,11 @@ MISP: events: base_threat_level: 3 published: False - base_confidence: 60 - base_severity: 1 org: name: "ACME" uuid: "5804adw2-12fe-1234-34av-07lk82aw012a" attributes: to_ids: False + to_ids_threshold: 90 \ No newline at end of file diff --git a/settings/tags.sample.yml b/settings/tags.sample.yml new file mode 100755 index 0000000..1bfc752 --- /dev/null +++ b/settings/tags.sample.yml @@ -0,0 +1,18 @@ +# If you don't need some tags just remove them or comment them out +c2_tags: + - name: veris:action:malware:variety="C2" + - name: veris:confidence="High" + - name: tlp:amber +malware_tags: + - name: veris:action:malware:variety="C2" + - name: veris:confidence="High" +espionage_tags: + - name: espionage:tag +bot_tags: + - name: bot:tag +whitelist_tags: + - name: whitelist:tag +cybercrime_tags: + - name: cybercrime:tag +phishing_tags: + - name: phishing:tag \ No newline at end of file diff --git a/tiffy.py b/tiffy.py index 9b73bfd..5a3110a 100755 --- a/tiffy.py +++ b/tiffy.py @@ -36,7 +36,7 @@ from click.testing import CliRunner from TIELoader import TIELoader -from model import Config +from model import Config, Tags # CTRL+C Handler -------------------------------------------------------------- @@ -127,7 +127,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out loglvl = 'INFO' if log_file_path is None: log_file_path = sys.path[0] - TIELoader.init_logger(log_file_path, "tiffy.py", loglvl, disable_console_log, disable_file_log) + TIELoader.init_logger(log_file_path, "tiffy.py.log", loglvl, disable_console_log, disable_file_log) try: # Check date arguments @@ -217,6 +217,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out try: # Load config and tags conf = Config.parse("settings/config.yml") + attr_tags = Tags.parse("settings/tags.yml") logging.info("Powering up flux capacitor. Starting up tiffy.") logging.info("#### Start new TIE-Query ####") @@ -244,7 +245,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out logging.debug("conf.org_name: " + str(conf.org_name)) logging.debug("log_file_path: " + str(log_file_path)) - TIELoader.start(output_format, conf, event_tags, category, actor, family, source, given_first_seen_date, + TIELoader.start(output_format, conf, event_tags, attr_tags, category, actor, family, source, given_first_seen_date, given_last_seen_date, min_confidence, min_severity, max_confidence, max_severity, proxy_tie_addr, no_filter, disable_cert_verify)