Merge pull request #32 from codingo/timk-fuzzy-logic

Timk fuzzy logic implementation

Author: timkent

README.md

@@ -5,7 +5,7 @@ A virtual host scanner that can be used with pivot tools, detect catch-all scena
 
 ## Key Benefits
 
-* Quickly highlight unqiue content in catch-all scenarios
+* Quickly highlight unique content in catch-all scenarios
 * Locate the outliers in catch-all scenarios where results have dynamic content on the page (such as the time)
 * Identify aliases by tweaking the unique depth of matches
 * Wordlist supports standard words and a variable to input a base hostname (for e.g. dev.%s from the wordlist would be run as dev.BASE_HOST)
@@ -66,5 +66,5 @@ $ cat bank.htb | VHostScan.py -t 10.10.10.29 -
 ### STDIN and WordList
 You can still specify a wordlist to use along with stdin. In these cases wordlist information will be appended to stdin. For example:
 ```bash
-$ cat vhostname | VhostScan.py -t localhost -w ./wordlists/wordlist.txt -
+$ echo -e 'a.example.com\b.example.com' | VhostScan.py -t localhost -w ./wordlists/wordlist.txt -
 ```

VHostScan.py

@@ -27,6 +27,7 @@ def main():
     parser.add_argument('--ignore-content-length', dest='ignore_content_length', type=int, help='Ignore content lengths of specificed amount (default 0).', default=0)
     parser.add_argument('--unique-depth', dest='unique_depth', type=int, help='Show likely matches of page content that is found x times (default 1).', default=1)
     parser.add_argument("--ssl", dest="ssl",   action="store_true", help="If set then connections will be made over HTTPS instead of HTTP (default http).", default=False)
+    parser.add_argument("--fuzzy-logic", dest="fuzzy_logic", action="store_true", help="If set then fuzzy match will be performed against unique hosts (default off).", default=False)
     parser.add_argument("--waf", dest="add_waf_bypass_headers",   action="store_true", help="If set then simple WAF bypass headers will be sent.", default=False)
     parser.add_argument("-oN",   dest="output_normal", help="Normal output printed to a file when the -oN option is specified with a filename argument." )
     parser.add_argument("-", dest="stdin", action="store_true", help="By passing a blank '-' you tell VHostScan to expect input from stdin (pipe).", default=False)
@@ -77,13 +78,16 @@ def main():
         print("[>] Ignoring Content length: %s" % (arguments.ignore_content_length))
 
     scanner = virtual_host_scanner( arguments.target_hosts, arguments.base_host, wordlist, arguments.port, arguments.real_port, arguments.ssl, 
-                                    arguments.unique_depth, arguments.ignore_http_codes, arguments.ignore_content_length, arguments.add_waf_bypass_headers)
+                                    arguments.unique_depth, arguments.ignore_http_codes, arguments.ignore_content_length, arguments.fuzzy_logic, arguments.add_waf_bypass_headers)
 
     scanner.scan()
-    output = output_helper(scanner)
+    output = output_helper(scanner, arguments)
 
     print(output.output_normal_likely())
 
+    if(arguments.fuzzy_logic):
+        print(output.output_fuzzy())
+
     if(arguments.output_normal):
         output.write_normal(arguments.output_normal)
         print("\n[+] Writing normal ouptut to %s" % arguments.output_normal)

lib/core/__version__.py

@@ -2,5 +2,5 @@
 # |V|H|o|s|t|S|c|a|n|  Developed by @codingo_ & @__timk
 # +-+-+-+-+-+-+-+-+-+  https://github.com/codingo/VHostScan
 
-__version__ = '0.7'
+__version__ = '1.0'
 

lib/core/discovered_host.py

@@ -7,4 +7,5 @@ def __init__(self):
         self.hostname = ''
         self.response_code = 0
         self.hash = ''
-        self.keys = []
+        self.keys = []
+        self.content = b''

lib/core/virtual_host_scanner.py

@@ -5,6 +5,7 @@
 from lib.core.discovered_host import *
 
 
+
 class virtual_host_scanner(object):
     """Virtual host scanning class
     
@@ -19,7 +20,7 @@ class virtual_host_scanner(object):
         output: folder to write output file to
     """
 
-    def __init__(self, target, base_host, wordlist, port=80, real_port=80, ssl=False, unique_depth=1, ignore_http_codes='404', ignore_content_length=0, add_waf_bypass_headers=False):
+    def __init__(self, target, base_host, wordlist, port=80, real_port=80, ssl=False, unique_depth=1, ignore_http_codes='404', ignore_content_length=0, fuzzy_logic=False, add_waf_bypass_headers=False):
         self.target = target
         self.base_host = base_host
         self.port = int(port)
@@ -29,6 +30,7 @@ def __init__(self, target, base_host, wordlist, port=80, real_port=80, ssl=False
         self.wordlist = wordlist
         self.unique_depth = unique_depth
         self.ssl = ssl
+        self.fuzzy_logic = fuzzy_logic
         self.add_waf_bypass_headers = add_waf_bypass_headers
 
         # this can be made redundant in future with better exceptions
@@ -87,6 +89,7 @@ def scan(self):
             host.hostname = hostname
             host.response_code = res.status_code
             host.hash = page_hash
+            host.content = res.content
 
             for key, val in res.headers.items():
                 output += '  {}: {}\n'.format(key, val)
@@ -118,4 +121,4 @@ def likely_matches(self):
         segmented_data = dataframe.groupby("val_col").filter(lambda x: len(x) <= self.unique_depth)
         matches = ((segmented_data["key_col"].values).tolist())
 
-        return matches
+        return matches

lib/helpers/output_helper.py

@@ -1,18 +1,32 @@
 from lib.core.discovered_host import *
 from lib.helpers.file_helper import *
 import time
+from fuzzywuzzy import fuzz
+import itertools
+import numpy as np
+
 
 class output_helper(object):
-    def __init__(self, scanner):
+    def __init__(self, scanner, arguments):
         self.scanner = scanner
+        self.arguments = arguments
 
     def write_normal(self, filename):
 
         file = file_helper(filename)
 
         # todo: finish check_directory (needs regex to split out filename)
         # file.check_directory(filename)
-        file.write_file(self.generate_header() + self.output_normal_likely() + self.output_normal_detail())
+
+        output = self.generate_header()
+        output += self.output_normal_likely()
+        
+        if(self.arguments.fuzzy_logic):
+            output += self.output_fuzzy()
+        
+        output += self.output_normal_detail()
+        
+        file.write_file(output)
 
     def output_normal_likely(self):
         uniques = False
@@ -28,15 +42,30 @@ def output_normal_likely(self):
         else:
             return "\n[!] No matches with a unique count of {} or less.".format(depth)
 
+
+    def output_fuzzy(self):
+        output = "\n\n[+] Match similarity using fuzzy logic:"
+        request_hashes = {}
+        
+        for host in self.scanner.hosts:
+            request_hashes[host.hash] = host.content
+        
+        for a, b in itertools.combinations(request_hashes.keys(), 2):
+            output += "\n\t[>] {} is {}% similar to {}".format(a, fuzz.ratio(request_hashes[a], request_hashes[b]), b)
+        
+        return output
+
+
     def output_normal_detail(self):
         output = "\n\n[+] Full scan results"
 
-        for p in self.scanner.hosts: 
-            output += "\n\n{} (Code: {}) hash: {}".format(str(p.hostname), str(p.response_code), str(p.hash))
-            for key in p.keys: output += "\n\t{}".format(key)
+        for host in self.scanner.hosts: 
+            output += "\n\n{} (Code: {}) hash: {}".format(str(host.hostname), str(host.response_code), str(host.hash))
+            for key in host.keys: output += "\n\t{}".format(key)
 
         return output
 
+
     def generate_header(self):
         output = "VHostScanner Log: {} {}\n".format(time.strftime("%d/%m/%Y"), time.strftime("%H:%M:%S"))
         output += "\tTarget: {}\n\tBase Host: {}\n\tPort: {}".format(self.scanner.target, self.scanner.base_host, self.scanner.port)

wordlists/testing.txt

@@ -0,0 +1,3 @@
+lol
+intranet.example.com
+dev.example.com