work in progress for idaholab#329, improve extracted_files download page

mmguero-dev · Feb 1, 2024 · 5ee1bc7 · 5ee1bc7
1 parent 2b9abe0
commit 5ee1bc7
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 9 deletions.
diff --git a/file-monitor/supervisord.conf b/file-monitor/supervisord.conf
@@ -155,6 +155,7 @@ command=/usr/local/bin/extracted_files_http_server.py
   --zip %(ENV_EXTRACTED_FILE_HTTP_SERVER_ZIP)s
   --recursive %(ENV_EXTRACTED_FILE_HTTP_SERVER_RECURSIVE)s
   --directory /zeek/extract_files
+  --malcolm
 autostart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s
 autorestart=%(ENV_EXTRACTED_FILE_HTTP_SERVER_ENABLE)s
 startsecs=0

diff --git a/shared/bin/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py
@@ -79,6 +79,11 @@ def do_GET(self):
         fullpath = self.translate_path(self.path)
         fileBaseName = os.path.basename(fullpath)
 
+        carvedFileRegex = re.compile(
+            r'^(?P<source>[^-]+)-(?P<fuid>F[a-zA-Z0-9]+|unknown)-(?P<uid>C[a-zA-Z0-9]+|unknown)-(?P<timestamp>\d+)(?P<ext>\..+)?$'
+        )
+        carvedFileRegexAlt = re.compile(r'^(?P<uid>C[a-zA-Z0-9]+)_(?P<fuid>F[a-zA-Z0-9]+)')
+
         if os.path.isdir(fullpath):
             # directory listing
             # SimpleHTTPRequestHandler.do_GET(self)
@@ -97,39 +102,69 @@ def do_GET(self):
                 hr()
                 with div(id='listing'):
                     with table().add(tbody()):
-                        tr().add(th("Name"), th("Type" if args.magic else "Extension"), th("Size"))
+                        t = tr()
+                        t.add(th("Name"), th("Type" if args.magic else "Extension"), th("Size"))
+                        if args.malcolm:
+                            t.add(
+                                th("Source", style="text-align: center;"),
+                                th("UID", style="text-align: center;"),
+                                th("FUID", style="text-align: center;"),
+                                th("Timestamp", style="text-align: center;"),
+                            )
                         if fileBaseName != '.':
                             tr().add(
                                 td(a('..', href=f'..')),
-                                td("Directory"),
+                                td("Directory", style="text-align: center;"),
                                 td(''),
                             )
                         for dirpath, dirnames, filenames in os.walk(fullpath):
                             for dirname in sorted(dirnames, key=natural_sort_key):
                                 try:
+                                    t = tr()
                                     child = os.path.join(dirpath, dirname)
-                                    tr().add(
+                                    t.add(
                                         td(a(dirname, href=f'{dirname}/')),
                                         td("Directory"),
                                         td(''),
                                     )
-                                except:
-                                    pass
+                                except Exception as e:
+                                    eprint(f'Error with directory "{dirname}"": {e}')
                             for filename in sorted(filenames, key=natural_sort_key):
                                 try:
+                                    t = tr()
                                     child = os.path.join(dirpath, filename)
                                     fileinfo = (
                                         magic.from_file(child, mime=True)
                                         if args.magic
                                         else os.path.splitext(filename)[1]
                                     )
-                                    tr().add(
+                                    t.add(
                                         td(a(filename, href=f'{filename}')),
                                         td(fileinfo),
-                                        td(sizeof_fmt(os.path.getsize(child))),
+                                        td(sizeof_fmt(os.path.getsize(child)), style="text-align: right;"),
                                     )
-                                except:
-                                    pass
+                                    if args.malcolm:
+                                        fmatch = carvedFileRegex.search(filename)
+                                        if fmatch is None:
+                                            fmatch = carvedFileRegexAlt.search(filename)
+                                        if fmatch is not None:
+                                            timestampStr = fmatch.groupdict().get('timestamp', '')
+                                            try:
+                                                timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S').isoformat()
+                                            except Exception as te:
+                                                if timestampStr:
+                                                    eprint(f'Error with time "{timestampStr}": {te}')
+                                                timestamp = timestampStr
+                                            t.add(
+                                                td(fmatch.groupdict().get('source', ''), style="text-align: center;"),
+                                                td(fmatch.groupdict().get('uid', '')),
+                                                td(fmatch.groupdict().get('fuid', '')),
+                                                td(timestamp, style="text-align: center;"),
+                                            )
+                                        else:
+                                            eprint(f'nope on {filename}')
+                                except Exception as e:
+                                    eprint(f'Error with file "{filename}": {e}')
                             break
                 hr()
 
@@ -214,6 +249,7 @@ def main():
     defaultZip = os.getenv('EXTRACTED_FILE_HTTP_SERVER_ZIP', 'false')
     defaultRecursive = os.getenv('EXTRACTED_FILE_HTTP_SERVER_RECURSIVE', 'false')
     defaultMagic = os.getenv('EXTRACTED_FILE_HTTP_SERVER_MAGIC', 'false')
+    defaultMalcolm = os.getenv('EXTRACTED_FILE_HTTP_SERVER_MALCOLM', 'false')
     defaultPort = int(os.getenv('EXTRACTED_FILE_HTTP_SERVER_PORT', 8440))
     defaultKey = os.getenv('EXTRACTED_FILE_HTTP_SERVER_KEY', 'infected')
     defaultDir = os.getenv('EXTRACTED_FILE_HTTP_SERVER_PATH', orig_path)
@@ -292,6 +328,16 @@ def main():
         metavar='true|false',
         help=f"Recursively look for requested file if not found",
     )
+    parser.add_argument(
+        '--malcolm',
+        dest='malcolm',
+        type=str2bool,
+        nargs='?',
+        const=True,
+        default=defaultMalcolm,
+        metavar='true|false',
+        help=f"Include columns for Zeek-extracted files in Malcolm",
+    )
     try:
         parser.error = parser.exit
         args = parser.parse_args()