Merge branch 'v1.6.3'

Remove Cymon.io dasboard
Temporarily pull raw data from Pastebin as alternative to psbdmp archive API endpoint
Return "no data" for failed scans in urlscan.io
Fix URLhaus hash search functionality
Remove phishing_catcher_external.yaml
Added time constraint to Phishing Kit Tracker
Added phishingkittracker git to pull down all the lookup tables

README.md

@@ -44,9 +44,6 @@ Click **[HERE](https://github.com/ecstatic-nobel/OSweep/wiki/Setup)** to get sta
 **CyberCrime Tracker - Dashboard**
 ![CyberCrime Tracker - Dashboard](https://raw.githubusercontent.com/ecstatic-nobel/OSweep/master/static/assets/cybercrimeTracker_dashboard.png)  
 
-**Cymon - Dashboard**
-![Cymon - Dashboard](https://raw.githubusercontent.com/ecstatic-nobel/OSweep/master/static/assets/cymon_dashboard.png)  
-
 **GreyNoise - Dashboard**  
 ![GreyNoise - Dashboard](https://raw.githubusercontent.com/ecstatic-nobel/OSweep/master/static/assets/greynoise_dashboard.png)  
 

bin/commons.py

@@ -30,6 +30,11 @@ def create_session():
     })
     return session
 
+def get_apidomain(api):
+    """Return the API domain."""
+    if api == "hybrid-analysis":
+        return config.hybrid_analysis_api_domain
+
 def get_apikey(api):
     """Return the API key."""
     if api == "greynoise":

bin/hybrid_analysis.py

@@ -25,7 +25,7 @@
 import commons
 
 
-api = "https://www.hybrid-analysis.com/api/v2/search/{}".lower()
+api = "https://{}/api/v2/search/{}".lower()
 
 def process_iocs(results):
     """Return data formatted for Splunk from Hybrid-Analysis."""
@@ -64,23 +64,24 @@ def process_iocs(results):
             param = sys.argv[2]
             provided_iocs = sys.argv[3:]
 
-    session = commons.create_session()
-    api_key = commons.get_apikey("hybrid-analysis")
+    session    = commons.create_session()
+    api_domain = commons.get_apidomain("hybrid-analysis")
+    api_key    = commons.get_apikey("hybrid-analysis")
     splunk_table = []
 
     for provided_ioc in set(provided_iocs):
         provided_ioc = commons.deobfuscate_string(provided_ioc)
         provided_ioc = provided_ioc.lower()
 
-        ioc_dicts = query_hybridanalysis(endpoint, param, provided_ioc, api_key, session)
+        ioc_dicts = query_hybridanalysis(endpoint, param, provided_ioc, api_domain, api_key, session)
 
         for ioc_dict in ioc_dicts:
             splunk_table.append(ioc_dict)
 
     session.close()
     return splunk_table
 
-def query_hybridanalysis(endpoint, param, provided_ioc, api_key, session):
+def query_hybridanalysis(endpoint, param, provided_ioc, api_domain, api_key, session):
     """ """
     ioc_dicts = []
 
@@ -89,7 +90,7 @@ def query_hybridanalysis(endpoint, param, provided_ioc, api_key, session):
         "Accept":"application/json", 
         "User-Agent":"Falcon Sandbox"
     })
-    resp = session.post(api.format(endpoint), data={param:provided_ioc}, timeout=180)
+    resp = session.post(api.format(api_domain, endpoint), data={param:provided_ioc}, timeout=180)
 
     if resp.status_code == 200 and resp.content != '':
         results = resp.json()

bin/phishing_catcher.py

@@ -78,19 +78,9 @@ def write_file(file_contents, file_path):
 
 def process_iocs(results):
     """Return data formatted for Splunk."""
-    with open("phishing_catcher_suspicious.yaml", "r") as s, open("phishing_catcher_external.yaml", "r") as e:
+    with open("phishing_catcher_suspicious.yaml", "r") as s:
         global suspicious
         suspicious = yaml.safe_load(s)
-        external   = yaml.safe_load(e)
-
-    if external["override_suspicious"] is True:
-        suspicious = external
-    else:
-        if external["keywords"] is not None:
-            suspicious["keywords"].update(external["keywords"])
-
-        if external["tlds"] is not None:
-            suspicious["tlds"].update(external["tlds"])
 
     if results != None:
         provided_iocs = [y for x in results for y in x.values()]

bin/phishing_kit_tracker.py

@@ -15,34 +15,67 @@
 Debugger: open("/tmp/splunk_script.txt", "a").write("{}: <MSG>\n".format(<VAR>))
 """
 
+import time
 from collections import OrderedDict
+import glob
 import os
+import shutil
 import sys
+import zipfile
 
-app_home   = "{}/etc/apps/OSweep".format(os.environ['SPLUNK_HOME'])
+app_home   = "{}/etc/apps/OSweep".format(os.environ["SPLUNK_HOME"])
 tp_modules = "{}/bin/_tp_modules".format(app_home)
 sys.path.insert(0, tp_modules)
 import validators
 
 import commons
 
 
-csv = "https://raw.githubusercontent.com/neonprimetime/PhishingKitTracker/master/PhishingKitTracker.csv"
+date = time.strftime("%Y-%m")
+csv  = "https://raw.githubusercontent.com/neonprimetime/PhishingKitTracker/master/{}_PhishingKitTracker.csv"
+
+def get_project():
+    """Download the project to /tmp"""
+    session = commons.create_session()
+    project = "https://github.com/neonprimetime/PhishingKitTracker/archive/master.zip"
+    resp    = session.get(project, timeout=180)
+
+    if not (resp.status_code == 200 and resp.content != ""):
+        return
+
+    with open("/tmp/master.zip", "wb") as repo:
+        repo.write(resp.content)
+
+    repo_zip = zipfile.ZipFile("/tmp/master.zip", "r")
+    repo_zip.extractall("/tmp/")
+    repo_zip.close()
+
+    # Remove current files
+    for csv in glob.glob("/{}/etc/apps/OSweep/lookups/2*_PhishingKitTracker.csv".format(os.environ["SPLUNK_HOME"])):
+        os.remove(csv)
+
+    # Add new files
+    for csv in glob.glob("/tmp/PhishingKitTracker-master/2*_PhishingKitTracker.csv"):
+        shutil.move(csv, "/{}/etc/apps/OSweep/lookups".format(os.environ["SPLUNK_HOME"]))
+
+    os.remove("/tmp/master.zip")
+    shutil.rmtree("/tmp/PhishingKitTracker-master")
+    return
 
 def get_feed():
     """Return the latest report summaries from the feed."""
     session   = commons.create_session()
-    data_feed = check_project(session)
+    data_feed = get_file(session)
 
     if data_feed == None:
         return
     return data_feed
 
-def check_project(session):
+def get_file(session):
     """Return a list of tags."""
-    resp = session.get(csv, timeout=180)
+    resp = session.get(csv.format(date), timeout=180)
 
-    if resp.status_code == 200 and resp.content != '':
+    if resp.status_code == 200 and resp.content != "":
         return resp.content.splitlines()
     return
 
@@ -64,6 +97,8 @@ def write_file(data_feed, file_path):
     if sys.argv[1].lower() == "feed":
         data_feed    = get_feed()
         lookup_path  = "{}/lookups".format(app_home)
-        file_path    = "{}/phishing_kit_tracker.csv".format(lookup_path)
+        file_path    = "{}/{}_PhishingKitTracker.csv".format(lookup_path, date)
 
         write_file(data_feed, file_path)
+    elif sys.argv[1].lower() == "git":
+        get_project()

bin/psbdmp.py

@@ -87,13 +87,19 @@ def psbdmp_search(provided_ioc, session):
 
 def psbdmp_dump(provided_ioc, session):
     """ """
-    base_url  = "https://psbdmp.ws/api/dump/get/{}"
+    # psbdmp.ws does not have an endpoint to the archive
+    # base_url  = "https://psbdmp.ws/api/dump/get/{}"
+    base_url  = "https://pastebin.com/raw/{}"
     url       = base_url.format(provided_ioc)
     resp      = session.get(url, timeout=180)
     psd_dicts = []
 
-    if resp.status_code == 200 and resp.json()["error"] != 1:
-        dump = resp.json()
+    # psbdmp.ws does not have an endpoint to the archive
+    # if resp.status_code == 200 and resp.json()["error"] != 1:
+    #     dump = resp.json()
+    #     psd_dicts.append(dump)
+    if resp.status_code == 200 and resp.content != "":
+        dump = {"id":provided_ioc, "data":resp.content}
         psd_dicts.append(dump)
     else:
         psd_dicts.append({"no data": provided_ioc})

bin/urlhaus.py

@@ -115,8 +115,10 @@ def process_iocs(results):
             ioc_type = "host"
         elif validators.url(provided_ioc):
             ioc_type = "url"
-        elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc):
-            ioc_type = "payload"
+        elif re.match("^[a-f\d]{32}$", provided_ioc):
+            ioc_type = "md5_hash"
+        elif re.match("^[a-f\d]{64}$", provided_ioc):
+            ioc_type = "sha256_hash"
         else:
             splunk_table.append({"invalid": provided_ioc})
             continue
@@ -132,15 +134,12 @@ def process_iocs(results):
 
 def query_urlhaus(session, provided_ioc, ioc_type):
     """ """
-    if re.match("^[a-f\d]{32}$", provided_ioc) and ioc_type == "payload":
-        data_field = "md5_hash"
-    elif re.match("^[a-f\d]{64}$", provided_ioc) and ioc_type == "payload":
-        data_field = "sha256_hash"
-    else:
-        data_field = ioc_type
+    uri_dir = ioc_type
+    if ioc_type in ["md5_hash", "sha256_hash"]:
+        uri_dir = "payload"    
 
-    api       = "https://urlhaus-api.abuse.ch/v1/{}/"
-    resp      = session.post(api.format(ioc_type), timeout=180, data={data_field: provided_ioc})
+    api  = "https://urlhaus-api.abuse.ch/v1/{}/"
+    resp = session.post(api.format(uri_dir), timeout=180, data={ioc_type: provided_ioc})
     ioc_dicts = []
 
     if resp.status_code == 200 and resp.text != "":
@@ -203,7 +202,7 @@ def query_urlhaus(session, provided_ioc, ioc_type):
                     })
 
                 ioc_dicts.append(ioc_dict)
-        elif ioc_type == "payload":
+        elif ioc_type in ["md5_hash", "sha256_hash"]:
             if len(resp_content["urls"]) == 0:
                 ioc_dicts.append({"invalid": provided_ioc})
                 return ioc_dicts

bin/urlscan.py

@@ -73,6 +73,10 @@ def process_iocs(results):
             continue
 
         for ioc_dict in ioc_dicts:
+            if "ip" not in ioc_dict:
+                splunk_table.append({"no data": provided_ioc})
+                continue
+
             splunk_table.append(ioc_dict)
 
     session.close()
@@ -135,8 +139,8 @@ def query_urlscan_file(session, provided_ioc):
 def query_urlscan(session, provided_ioc):
     """Return data from urlscan about the provided IOC."""
     query_type = sys.argv[1]
-    api    = "https://urlscan.io/api/v1/search/?size=10000&q="
-    resp   = session.get("{}{}".format(api, provided_ioc), timeout=180)
+    api  = "https://urlscan.io/api/v1/search/?size=10000&q={}"
+    resp = session.get(api.format(provided_ioc), timeout=180)
 
     if resp.status_code == 200 and "results" in resp.json().keys() and \
        len(resp.json()["results"]) > 0:

default/app.conf

@@ -11,4 +11,4 @@ label = OSweep™
 [launcher]
 author = ecstatic-nobel
 description = "Don't Just Search OSINT. Sweep It."
-version = 1.6.0
+version = 1.6.3