Improve autocalibration

db/400_blacklist.txt

@@ -1,8 +1,9 @@
 %2e%2e//google.com
 %ff
+%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
 %2e%2e;/test
 %3f/
 %C0%AE%C0%AE%C0%AF
-.%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
+../../../../../../etc/passwd
 ..;/
 cgi-bin/.%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd

db/403_blacklist.txt

@@ -1,21 +1,9 @@
-.hta
-.htaccess
-.htaccess-dev
-.htaccess-local
-.htaccess-marco
-.htaccess.BAK
-.htaccess.bak
-.htaccess.old
-.htaccess.inc
-.htaccess.txt
-.htaccess~
-.htaccess/
-.htpasswd
-.htpasswd-old
-.htpasswd.bak
-.htpasswd.inc
-.htpa55wd
-.htpasswd/
-.htpasswrd
-.htgroup
-.htusers
+%2e%2e//google.com
+%ff
+%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
+%2e%2e;/test
+%3f/
+%C0%AE%C0%AE%C0%AF
+../../../../../../etc/passwd
+..;/
+cgi-bin/.%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd

db/500_blacklist.txt

@@ -1,6 +1,8 @@
 %ff
+%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
 %3f/
 %C0%AE%C0%AE%C0%AF
 %2e%2e;/test
+../../../../../../etc/passwd
 ..;/
 

db/dicc.txt

@@ -2,6 +2,7 @@
 !.htaccess
 !.htpasswd
 %2e%2e//google.com
+%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
 %2e%2e;/test
 %3f/
 %C0%AE%C0%AE%C0%AF
@@ -38,7 +39,7 @@
 +CSCOT+/oem-customization?app=AnyConnect&type=oem&platform=..&resource-type=..&name=%2bCSCOE%2b/portal_inc.lua
 +CSCOT+/translation
 +CSCOT+/translation-table?type=mst&textdomain=/%2bCSCOE%2b/portal_inc.lua&default-language&lang=../
-.%2e/%2e%2e/%2e%2e/%2e%2e/etc/passwd
+../../../../../../etc/passwd
 ..;/
 .0
 .7z

lib/core/fuzzer.py

@@ -155,29 +155,24 @@ def __init__(
 
     def setup_scanners(self) -> None:
         # Default scanners (wildcard testers)
-        self.scanners["default"].update(
-            {
-                "index": Scanner(self._requester, path=self._base_path),
-                "random": Scanner(
-                    self._requester, path=self._base_path + WILDCARD_TEST_POINT_MARKER
-                ),
-            }
+        self.scanners["default"]["random"] = Scanner(
+            self._requester, path=self._base_path + WILDCARD_TEST_POINT_MARKER
         )
 
         if options["exclude_response"]:
             self.scanners["default"]["custom"] = Scanner(
                 self._requester, tested=self.scanners, path=options["exclude_response"]
             )
 
-        for prefix in options["prefixes"] + DEFAULT_TEST_PREFIXES:
+        for prefix in set(options["prefixes"] + DEFAULT_TEST_PREFIXES):
             self.scanners["prefixes"][prefix] = Scanner(
                 self._requester,
                 tested=self.scanners,
                 path=f"{self._base_path}{prefix}{WILDCARD_TEST_POINT_MARKER}",
                 context=f"/{self._base_path}{prefix}***",
             )
 
-        for suffix in options["suffixes"] + DEFAULT_TEST_SUFFIXES:
+        for suffix in set(options["suffixes"] + DEFAULT_TEST_SUFFIXES):
             self.scanners["suffixes"][suffix] = Scanner(
                 self._requester,
                 tested=self.scanners,

lib/core/settings.py

@@ -69,9 +69,9 @@
 
 STANDARD_PORTS = {"http": 80, "https": 443}
 
-DEFAULT_TEST_PREFIXES = (".",)
+DEFAULT_TEST_PREFIXES = (".", ".ht")
 
-DEFAULT_TEST_SUFFIXES = ("/",)
+DEFAULT_TEST_SUFFIXES = ("/", "~")
 
 DEFAULT_TOR_PROXIES = ("socks5://127.0.0.1:9050", "socks5://127.0.0.1:9150")
 

lib/utils/diff.py

@@ -50,14 +50,19 @@ def compare_to(self, content):
 
         i = -1
         splitted_content = content.split()
+        # Allow one miss, see https://github.com/maurosoria/dirsearch/issues/1279
+        misses = 0
         for pattern in self._static_patterns:
             try:
                 i = splitted_content.index(pattern, i + 1)
             except ValueError:
-                return False
+                if misses or len(self._static_patterns) < 20:
+                    return False
 
-        # The number of static patterns is not big enough to say it's a reliable method
-        if len(self._static_patterns) < 20 and len(content.split()) > len(self._base_content.split()):
+                misses += 1
+
+        # Static patterns doesn't seem to be a reliable enough method
+        if len(content.split()) > len(self._base_content.split()) and len(self._static_patterns) < 20:
             return difflib.SequenceMatcher(None, self._base_content, content).ratio() > 0.75
 
         return True