Tiffy 1.0.2 (#4)

* 1.0.2

Changes

added threshold setting for IDS flag in config
tagging for attributes derived from c2-server and malware observations added
removed unused config values base_severity and base_confidence

Bugfixes

Fixed Bug with logging when no log_path was provided

CHANGELOG.md

@@ -1,5 +1,14 @@
 # tiffy Changelog
 
+### 1.0.2
+   #### Changes
+   - added threshold setting for IDS flag in config
+   - tagging for attributes derived from TIE observations added
+   - removed unused config values base_severity and base_confidence
+
+   #### Bugfixes
+   - Fixed Bug with logging when no log_path was provided
+
 ### 1.0.1
    #### Changes
    - simplified Config File format, added possibility to use environment variables for configuration (see readme)

README.md

@@ -121,6 +121,16 @@ to the base event. Double quotes need to be escaped. If no tags are passed `TLP:
 $ ./tiffy.py --event-tags {\"name\":\"tlp:amber\"}
 ```
 
+## Setting Tags for Attributes
+
+You can assign Tags to attributes using the `settings/tags.yml` file. Currently only attributes derived from TIE
+`c2-server` and `malware` observations can be tagged. Add your desired Tag to the list of tags as seen in the sample
+file. The format for a new entry is:
+
+```
+- name: tag:name:here
+```
+
 ## Setting the Output Format
 
 You can choose the output format of the feed. Currently only MISP-JSON is supported but more formats will follow.
@@ -205,6 +215,7 @@ at least the required variables.
 | TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY     | 2                    |                          | IoC will get this severity if it is added                         |
 | TIFFY_CONF_MISP_EVENTS_PUBLISHED         | false                |                          | IoC will get published in MISP                                    |
 | TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS        | false                |                          | Set IDS flag for this IoC                                         |
+| TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD| 90                 |  90                      | Confidence Threshold at or above which attributes should get IDS flag                                        |
 | TIFFY_PARAM_TIE_SEEN_FIRST               |                      | YYYY-MM-DD               | Download only IoC which are first seen at ... and newer           |
 | TIFFY_PARAM_TIE_SEEN_LAST                |                      | YYYY-MM-DD               | Download only IoC which are last seen at ... and older            |
 | TIFFY_PARAM_TIE_ACTOR                    |                      | example1,example2        | Download only IoC with this actor                                 |

TIELoader.py

@@ -25,7 +25,7 @@ def get_csv_value(field, src):
 class TIELoader:
 
     @staticmethod
-    def start(out_format, conf, tags, category, actor, family, source, first_seen, last_seen, min_confidence,
+    def start(out_format, conf, tags, attr_tags, category, actor, family, source, first_seen, last_seen, min_confidence,
               min_severity, max_confindence, max_severity, proxy_tie_addr, no_filter=False, disable_cert_verify=False):
 
         # Building Auth Header
@@ -154,7 +154,7 @@ def start(out_format, conf, tags, category, actor, family, source, first_seen, l
         if out_format == 'MISP':
             # Serialize event as MISP Event
 
-            event, attr_hashes = MISPHelper.generate_MISP_Event(deduplicated_observations, conf, tags)
+            event, attr_hashes = MISPHelper.generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags)
             event_json = event.to_json()
             event_from_json = json.loads(event_json)
             event_from_json['publish_timestamp'] = str(event_from_json['publish_timestamp'])

helpers/MISPHelper.py

@@ -7,7 +7,7 @@
 from pymisp import MISPEvent, MISPOrganisation, MISPAttribute
 
 
-def generate_MISP_Event(deduplicated_observations, conf, tags):
+def generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags):
     dt = datetime.now()
 
     event = MISPEvent()
@@ -34,8 +34,25 @@ def generate_MISP_Event(deduplicated_observations, conf, tags):
         misp_attr['timestamp'] = dt.strftime("%s")
         misp_attr.type = get_Attribute_Type(attr)
         misp_attr.value = get_MISP_Fitted_Value(attr["value"], misp_attr.type)
+        if 'c2-server' in attr['categories'] and attr_tags.c2tags:
+            misp_attr['Tag'] = attr_tags.c2tags
+        if 'malware' in attr['categories'] and attr_tags.malwaretags:
+            misp_attr['Tag'] = attr_tags.malwaretags
+        if 'espionage' in attr['categories'] and attr_tags.espionagetags:
+            misp_attr['Tag'] = attr_tags.espionagetags
+        if 'bot' in attr['categories'] and attr_tags.bottags:
+            misp_attr['Tag'] = attr_tags.bottags
+        if 'whitelist' in attr['categories'] and attr_tags.whitelisttags:
+            misp_attr['Tag'] = attr_tags.whitelisttags
+        if 'cybercrime' in attr['categories'] and attr_tags.cybercrimetags:
+            misp_attr['Tag'] = attr_tags.cybercrimetags
+        if 'phishing' in attr['categories'] and attr_tags.phishingtags:
+            misp_attr['Tag'] = attr_tags.phishingtags
         misp_attr.category = get_Attribute_Category(attr)
-        misp_attr.to_ids = conf.attr_to_ids
+        if conf.attr_to_ids and attr['min_confidence'] >= conf.attr_to_ids_threshold:
+            misp_attr.to_ids = True
+        else:
+            misp_attr.to_ids = False
         misp_attr['comment'] = 'categories: ' + str(attr['categories']) + ' actors: ' + str(attr['actors']) + \
                                ' families: ' + str(attr['families']) + ' sources: ' + str(attr['sources']) + \
                                ' severity: ' + str(attr['max_severity']) + \

model/__init__.py

@@ -1 +1,2 @@
 from .config import Config
+from .tags import Tags

model/config.py

@@ -17,8 +17,7 @@ def __init__(self):
         self.__Event_Base_Threat_Level = 3
         self.__Event_Published = False
         self.__Attr_ToIDS = True
-        self.__Base_Confidence = 60
-        self.__Base_Severity = 1
+        self.__Attr_ToIDS_threshold = 90
 
     # --- Getter
     @property
@@ -50,12 +49,8 @@ def attr_to_ids(self):
         return self.__Attr_ToIDS
 
     @property
-    def base_confidence(self):
-        return self.__Base_Confidence
-
-    @property
-    def base_severity(self):
-        return self.__Base_Severity
+    def attr_to_ids_threshold(self):
+        return self.__Attr_ToIDS_threshold
 
     # --- Setter
 
@@ -87,13 +82,9 @@ def event_published(self, value):
     def attr_to_ids(self, value):
         self.__Attr_ToIDS = value
 
-    @base_confidence.setter
-    def base_confidence(self, value):
-        self.__Base_Confidence = value
-
-    @base_severity.setter
-    def base_severity(self, value):
-        self.__Base_Severity = value
+    @attr_to_ids_threshold.setter
+    def attr_to_ids_threshold(self, value):
+        self.__Attr_ToIDS_threshold = value
 
     @staticmethod
     def parse(configfile):
@@ -174,6 +165,10 @@ def parse(configfile):
                     conf.attr_to_ids = bool(os.environ.get('TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS'))
                 else:
                     conf.attr_to_ids = Config.get_config_value_optional(attr_vals, "to_ids", "True")
+                if 'TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD' in os.environ:
+                    conf.attr_to_ids_threshold = int(os.environ.get('TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD'))
+                else:
+                    conf.attr_to_ids_threshold = Config.get_config_value_optional(attr_vals, "to_ids_threshold", 90)
             else:
                 Config.raise_error_critical("Could not find attributes values ")
         else:
@@ -194,7 +189,7 @@ def parseFromEnv():
             config.base_severity = os.environ['TIFFY_CONF_MISP_EVENTS_BASE_SEVERITY']
             config.event_published = os.environ['TIFFY_CONF_MISP_EVENTS_PUBLISHED']
             config.attr_to_ids = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS']
-            config.attr_tagging = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TAGGING']
+            config.attr_to_ids_threshold = os.environ['TIFFY_CONF_MISP_ATTRIBUTES_TO_IDS_THRESHOLD']
 
             return config
         except KeyError:

model/tags.py

@@ -0,0 +1,119 @@
+"""
+DCSO TIE2MISP Parser
+Copyright (c) 2017, DCSO GmbH
+"""
+import yaml
+import warnings
+
+
+class Tags:
+    def __init__(self):
+        self.__c2tags = list()
+        self.__malwaretags = list()
+        self.__espionagetags = list()
+        self.__bottags = list()
+        self.__whitelisttags = list()
+        self.__cybercrimetags = list()
+        self.__phishingtags = list()
+
+    @property
+    def c2tags(self):
+        return self.__c2tags
+
+    @property
+    def malwaretags(self):
+        return self.__malwaretags
+
+    @property
+    def espionagetags(self):
+        return self.__espionagetags
+
+    @property
+    def bottags(self):
+        return self.__bottags
+
+    @property
+    def whitelisttags(self):
+        return self.__whitelisttags
+
+    @property
+    def cybercrimetags(self):
+        return self.__cybercrimetags
+
+    @property
+    def phishingtags(self):
+        return self.__phishingtags
+
+    @c2tags.setter
+    def c2tags(self, value):
+        self.__c2tags = value
+
+    @malwaretags.setter
+    def malwaretags(self, value):
+        self.__malwaretags = value
+
+    @espionagetags.setter
+    def espionagetags(self, value):
+        self.__espionagetags = value
+
+    @bottags.setter
+    def bottags(self, value):
+        self.__bottags = value
+
+    @whitelisttags.setter
+    def whitelisttags(self, value):
+        self.__whitelisttags = value
+
+    @cybercrimetags.setter
+    def cybercrimetags(self, value):
+        self.__cybercrimetags = value
+
+    @phishingtags.setter
+    def phishingtags(self, value):
+        self.__phishingtags = value
+
+    @staticmethod
+    def parse(tagfile):
+
+        tags = Tags()
+
+        # Load Config
+        tag_file = open(tagfile, "r", encoding="utf-8")
+        raw_tags = yaml.load(tag_file, Loader=yaml.FullLoader)
+
+        if "c2_tags" in raw_tags:
+            c2tags = raw_tags["c2_tags"]
+            for tag in c2tags:
+                tags.c2tags.append(tag)
+
+        if "malware_tags" in raw_tags:
+            malwaretags = raw_tags["malware_tags"]
+            for tag in malwaretags:
+                tags.malwaretags.append(tag)
+
+        if "espionage_tags" in raw_tags:
+            espionagetags = raw_tags["espionage_tags"]
+            for tag in espionagetags:
+                tags.espionagetags.append(tag)
+
+        if "bot_tags" in raw_tags:
+            bottags = raw_tags["bot_tags"]
+            for tag in bottags:
+                tags.bottags.append(tag)
+
+        if "whitelist_tags" in raw_tags:
+            whitelisttags = raw_tags["whitelist_tags"]
+            for tag in whitelisttags:
+                tags.whitelisttags.append(tag)
+
+        if "cybercrime_tags" in raw_tags:
+            cybercrimetags = raw_tags["cybercrime_tags"]
+            for tag in cybercrimetags:
+                tags.cybercrimetags.append(tag)
+
+        if "phishing_tags" in raw_tags:
+            phishingtags = raw_tags["phishing_tags"]
+            for tag in phishingtags:
+                tags.phishingtags.append(tag)
+
+        return tags

settings/config.sample.yml

@@ -6,12 +6,11 @@ MISP:
   events:
     base_threat_level: 3
     published: False
-    base_confidence: 60
-    base_severity: 1
   org:
     name: "ACME"
     uuid: "5804adw2-12fe-1234-34av-07lk82aw012a"
   attributes:
     to_ids: False
+    to_ids_threshold: 90
 
 

settings/tags.sample.yml

@@ -0,0 +1,18 @@
+# If you don't need some tags just remove them or comment them out
+c2_tags:
+  - name: veris:action:malware:variety="C2"
+  - name: veris:confidence="High"
+  - name: tlp:amber
+malware_tags:
+  - name: veris:action:malware:variety="C2"
+  - name: veris:confidence="High"
+espionage_tags:
+  - name: espionage:tag
+bot_tags:
+  - name: bot:tag
+whitelist_tags:
+  - name: whitelist:tag
+cybercrime_tags:
+  - name: cybercrime:tag
+phishing_tags:
+  - name: phishing:tag

tiffy.py

@@ -36,7 +36,7 @@
 from click.testing import CliRunner
 
 from TIELoader import TIELoader
-from model import Config
+from model import Config, Tags
 
 
 # CTRL+C Handler --------------------------------------------------------------
@@ -127,7 +127,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out
         loglvl = 'INFO'
     if log_file_path is None:
         log_file_path = sys.path[0]
-    TIELoader.init_logger(log_file_path, "tiffy.py", loglvl, disable_console_log, disable_file_log)
+    TIELoader.init_logger(log_file_path, "tiffy.py.log", loglvl, disable_console_log, disable_file_log)
     try:
 
         # Check date arguments
@@ -217,6 +217,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out
         try:
             # Load config and tags
             conf = Config.parse("settings/config.yml")
+            attr_tags = Tags.parse("settings/tags.yml")
 
             logging.info("Powering up flux capacitor. Starting up tiffy.")
             logging.info("#### Start new TIE-Query ####")
@@ -244,7 +245,7 @@ def init(category, actor, family, source, first_seen, last_seen, event_tags, out
             logging.debug("conf.org_name: " + str(conf.org_name))
             logging.debug("log_file_path: " + str(log_file_path))
 
-            TIELoader.start(output_format, conf, event_tags, category, actor, family, source, given_first_seen_date,
+            TIELoader.start(output_format, conf, event_tags, attr_tags, category, actor, family, source, given_first_seen_date,
                             given_last_seen_date, min_confidence, min_severity, max_confidence, max_severity,
                             proxy_tie_addr, no_filter, disable_cert_verify)