set observed-data created and modified times to be the current time w…

…hen translating data source results to STIX (#169)
opencybersecurityalliance · Sep 13, 2019 · 3e4242f · 3e4242f
1 parent caa3457
commit 3e4242f
Show file tree

Hide file tree

Showing 9 changed files with 153 additions and 234 deletions.
diff --git a/stix_shifter/stix_translation/src/json_to_stix/json_to_stix_translator.py b/stix_shifter/stix_translation/src/json_to_stix/json_to_stix_translator.py
@@ -3,6 +3,7 @@
 
 from . import observable
 from stix2validator import validate_instance, print_results
+from datetime import datetime
 
 
 # convert JSON data to STIX object using map_data and transformers
@@ -78,10 +79,9 @@ def _add_property(obj, key, stix_value, group=False):
                 child_obj[prop] = {}
             child_obj = child_obj[prop]
 
-
         if split_key[-1] not in child_obj.keys():
             child_obj[split_key[-1]] = stix_value
-        elif group is True: # Mapping of multiple data fields to single STIX object field. Ex: Network Protocols
+        elif group is True:  # Mapping of multiple data fields to single STIX object field. Ex: Network Protocols
             if (isinstance(child_obj[split_key[-1]], list)):
                 child_obj[split_key[-1]].extend(stix_value)                      # append to existing list
 
@@ -98,9 +98,9 @@ def _handle_cybox_key_def(key_to_add, observation, stix_value, obj_name_map, obj
         """
         obj_type, obj_prop = key_to_add.split('.', 1)
         objs_dir = observation['objects']
-        
+
         if obj_name in obj_name_map:
-                obj = objs_dir[obj_name_map[obj_name]]
+            obj = objs_dir[obj_name_map[obj_name]]
         else:
             obj = {'type': obj_type}
             obj_dir_key = str(len(objs_dir))
@@ -128,19 +128,19 @@ def _valid_stix_value(props_map, key, stix_value):
                 return False
         return True
 
-    def _transform(self,object_map,observation,ds_map,ds_key,obj):
+    def _transform(self, object_map, observation, ds_map, ds_key, obj):
 
-        to_map = obj[ ds_key ]
+        to_map = obj[ds_key]
 
         if ds_key not in ds_map:
             print('{} is not found in map, skipping'.format(ds_key))
             return
 
-        if isinstance( to_map, dict ):
+        if isinstance(to_map, dict):
             print('{} is complex; descending'.format(to_map))
             # If the object is complex we must descend into the map on both sides
             for key in to_map.keys():
-                self._transform(object_map,observation,ds_map[ds_key],key,to_map)
+                self._transform(object_map, observation, ds_map[ds_key], key, to_map)
             return
 
         generic_hash_key = ''
@@ -215,13 +215,15 @@ def transform(self, obj):
             'id': stix_type + '--' + str(uuid.uuid4()),
             'type': stix_type,
             'created_by_ref': self.identity_id,
+            'created': "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
+            'modified': "{}Z".format(datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3]),
             'objects': {}
         }
 
         # create normal type objects
-        if isinstance(obj,dict):
+        if isinstance(obj, dict):
             for ds_key in obj.keys():
-                self._transform(object_map,observation,ds_map,ds_key,obj)
+                self._transform(object_map, observation, ds_map, ds_key, obj)
         else:
             print("Not a dict: {}".format(obj))
 

diff --git a/stix_shifter/stix_translation/src/modules/car/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/car/json/to_stix_map.json
@@ -17,18 +17,6 @@
       "cybox": false
     }
   ],
-  "created": [
-    {
-      "key": "created",
-      "cybox": false
-    }
-  ],
-  "modified": [
-    {
-      "key": "modified",
-      "cybox": false
-    }
-  ],
   "process.pid": [
     {
       "key": "process.pid",
@@ -268,4 +256,4 @@
       "references": "payload"
     }
   ]
-}
+}
diff --git a/stix_shifter/stix_translation/src/modules/carbonblack/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/carbonblack/json/to_stix_map.json
@@ -15,14 +15,6 @@
       "key": "process.created",
       "object": "process"
     },
-    {
-      "key": "created",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "cybox": false
@@ -66,7 +58,7 @@
     "key": "process.pid",
     "object": "process"
   },
-  "parent_pid" : [
+  "parent_pid": [
     {
       "key": "process.pid",
       "object": "parent_process"
@@ -85,9 +77,9 @@
     "key": "file.hashes.MD5",
     "object": "parent_file"
   },
-  "cmdline" : {
-      "key": "process.command_line",
-      "object": "process"
+  "cmdline": {
+    "key": "process.command_line",
+    "object": "process"
   },
   "comms_ip": [
     {
@@ -103,7 +95,9 @@
     {
       "key": "process.opened_connection_refs",
       "object": "process",
-      "references": [ "traffic" ]
+      "references": [
+        "traffic"
+      ]
     }
   ],
   "interface_ip": [
@@ -126,14 +120,6 @@
       "key": "file.created",
       "object": "binary_file"
     },
-    {
-      "key": "created",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "cybox": false
@@ -155,4 +141,4 @@
     "key": "file.hashes.MD5",
     "object": "binary_file"
   }
-}
+}
diff --git a/stix_shifter/stix_translation/src/modules/csa/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/csa/json/to_stix_map.json
@@ -43,16 +43,6 @@
     "transformer": "ToInteger"
   },
   "starttime": [
-    {
-      "key": "created",
-      "transformer": "EpochSecondsToTimestamp",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "transformer": "EpochSecondsToTimestamp",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "transformer": "EpochSecondsToTimestamp",
@@ -115,4 +105,4 @@
     "object": "nt",
     "transformer": "ToLowercaseArray"
   }
-}
+}
diff --git a/stix_shifter/stix_translation/src/modules/dummy/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/dummy/json/to_stix_map.json
@@ -60,16 +60,6 @@
     "transformer": "ToInteger"
   },
   "StartTime": [
-    {
-      "key": "created",
-      "transformer": "EpochToTimestamp",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "transformer": "EpochToTimestamp",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "transformer": "EpochToTimestamp",
@@ -122,4 +112,4 @@
       }
     ]
   }
-}
+}
diff --git a/stix_shifter/stix_translation/src/modules/elastic_ecs/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/elastic_ecs/json/to_stix_map.json
@@ -1,13 +1,5 @@
 {
   "@timestamp": [
-    {
-      "key": "created",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "cybox": false
@@ -20,8 +12,8 @@
   "source": {
     "ip": [
       {
-      "key": "ipv4-addr.value",
-      "object": "src_ip"
+        "key": "ipv4-addr.value",
+        "object": "src_ip"
       },
       {
         "key": "ipv6-addr.value",
@@ -58,8 +50,8 @@
   "client": {
     "ip": [
       {
-      "key": "ipv4-addr.value",
-      "object": "src_ip"
+        "key": "ipv4-addr.value",
+        "object": "src_ip"
       },
       {
         "key": "ipv6-addr.value",
@@ -96,8 +88,8 @@
   "destination": {
     "ip": [
       {
-      "key": "ipv4-addr.value",
-      "object": "dst_ip"
+        "key": "ipv4-addr.value",
+        "object": "dst_ip"
       },
       {
         "key": "ipv6-addr.value",
@@ -134,8 +126,8 @@
   "server": {
     "ip": [
       {
-      "key": "ipv4-addr.value",
-      "object": "dst_ip"
+        "key": "ipv4-addr.value",
+        "object": "dst_ip"
       },
       {
         "key": "ipv6-addr.value",
@@ -173,19 +165,19 @@
     "transport": {
       "key": "network-traffic.protocols",
       "object": "nt",
-      "group" : "True",
+      "group": "True",
       "transformer": "ToLowercaseArray"
     },
     "type": {
       "key": "network-traffic.protocols",
       "object": "nt",
-      "group" : "True",
+      "group": "True",
       "transformer": "ToLowercaseArray"
     },
     "protocol": {
       "key": "network-traffic.protocols",
       "object": "nt",
-      "group" : "True",
+      "group": "True",
       "transformer": "ToLowercaseArray"
     }
   },
@@ -204,12 +196,10 @@
     }
   },
   "process": {
-    "executable":
-    {
+    "executable": {
       "key": "process.command_line",
       "object": "proc"
-    }
-    ,
+    },
     "start": {
       "key": "process.created",
       "object": "proc"
@@ -221,8 +211,8 @@
     },
     "name": [
       {
-      "key": "file.name",
-      "object": "file"
+        "key": "file.name",
+        "object": "file"
       },
       {
         "key": "process.image_ref",
@@ -234,7 +224,7 @@
   "user": {
     "name": [
       {
-        "key":"user-account.user_id",
+        "key": "user-account.user_id",
         "object": "user"
       },
       {
@@ -244,6 +234,4 @@
       }
     ]
   }
-}
-
-
+}
diff --git a/stix_shifter/stix_translation/src/modules/qradar/json/to_stix_map.json b/stix_shifter/stix_translation/src/modules/qradar/json/to_stix_map.json
@@ -43,16 +43,6 @@
     "transformer": "ToInteger"
   },
   "starttime": [
-    {
-      "key": "created",
-      "transformer": "EpochToTimestamp",
-      "cybox": false
-    },
-    {
-      "key": "modified",
-      "transformer": "EpochToTimestamp",
-      "cybox": false
-    },
     {
       "key": "first_observed",
       "transformer": "EpochToTimestamp",
@@ -172,4 +162,4 @@
     "key": "domain-name.value",
     "transformer": "ToDomainName"
   }
-}
+}