Refactored the wildcard redirect regex generator

lib/core/scanner.py

@@ -17,8 +17,8 @@
 #  Author: Mauro Soria
 
 import re
-from difflib import SequenceMatcher
 
+from urllib.parse import unquote
 from lib.utils import RandomUtils
 from thirdparty.sqlmap import DynamicContentParser
 
@@ -37,6 +37,7 @@ def __init__(self, requester, calibration=None, suffix=None, prefix=None):
         self.redirectRegExp = None
         self.invalidStatus = None
         self.dynamicParser = None
+        self.sign = None
         self.ratio = 0.98
         self.setup()
 
@@ -81,23 +82,29 @@ def setup(self):
         if baseRatio < self.ratio:
             self.ratio = baseRatio
 
-    def generateRedirectRegExp(self, firstLocation, secondLocation):
-        sm = SequenceMatcher(None, firstLocation, secondLocation)
-        marks = []
-
-        for blocks in sm.get_matching_blocks():
-            i = blocks[0]
-            n = blocks[2]
-            # Empty block
-
-            if n == 0:
-                continue
-
-            mark = firstLocation[i:i + n]
-            marks.append(mark)
-
-        regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks)))
-        return regexp
+    def generateRedirectRegExp(self, firstLoc, firstPath, secondLoc, secondPath):
+        # Use a unique sign to locate where the path gets reflected in the redirect
+        self.sign = RandomUtils.randString(n=20)
+        firstLoc = firstLoc.replace(firstPath, self.sign)
+        secondLoc = secondLoc.replace(secondPath, self.sign)
+        regExpStart = "^"
+        regExpEnd = "$"
+
+        for f, s in zip(firstLoc, secondLoc):
+            if f == s:
+                regExpStart += re.escape(f)
+            else:
+                regExpStart += ".*"
+                break
+
+        if regExpStart.endswith(".*"):
+            for f, s in zip(firstLoc[::-1], secondLoc[::-1]):
+                if f == s:
+                    regExpEnd = re.escape(f) + regExpEnd
+                else:
+                    break
+
+        return unquote(regExpStart + regExpEnd)
 
     def scan(self, path, response):
         if self.invalidStatus == response.status == 404:
@@ -107,7 +114,17 @@ def scan(self, path, response):
             return True
 
         if self.redirectRegExp and response.redirect:
-            redirectToInvalid = re.match(self.redirectRegExp, response.redirect)
+            path = re.escape(unquote(path))
+            # A lot of times, '#' or '?' will be removed in the redirect, cause false positives
+            for char in ["\\#", "\\?"]:
+                if char in path:
+                    path = path.replace(char, "(|" + char) + ")"
+
+            redirectRegExp = self.redirectRegExp.replace(self.sign, path)
+
+            # Redirect sometimes encodes/decodes characters in URL, which may confuse the
+            # rule check and make noise in the output, so we need to unquote() everything
+            redirectToInvalid = re.match(redirectRegExp, unquote(response.redirect))
 
             # If redirection doesn't match the rule, mark as found
             if redirectToInvalid is None: