Land rapid7#1949, make dirbusting optional for crawler

Author: Tod Beardsley

lib/anemone/core.rb

@@ -53,7 +53,8 @@ class Core
       # accept cookies from the server and send them back?
       :accept_cookies => false,
       # skip any link with a query string? e.g. http://foo.com/?u=user
-      :skip_query_strings => false
+      :skip_query_strings => false,
+      :dirbust  => true
     }
 
     # Create setter methods for all options to be called from the crawl block

lib/anemone/page.rb

@@ -55,6 +55,7 @@ def initialize(url, params = {})
       @url = url
       @data = OpenStruct.new
 
+      @dirbust = params[:dirbust]
       @code = params[:code]
       @headers = params[:headers] || {}
       @headers['content-type'] ||= ['']
@@ -83,7 +84,10 @@ def self.extractors
 
     def run_extractors
       return [] if !doc
-      self.class.extractors.map { |e| e.new( self ).run rescue next }.flatten.
+      self.class.extractors.map do |e|
+	      next if e == Extractors::Dirbuster && !dirbust?
+	      e.new( self ).run rescue next
+      end.flatten.
           compact.map do |p|
               abs = to_absolute( URI( p ) ) rescue next
               !in_domain?( abs ) ? nil : abs
@@ -181,6 +185,10 @@ def to_absolute(link)
       return absolute
     end
 
+    def dirbust?
+	    @dirbust
+    end
+
     #
     # Returns +true+ if *uri* is in the same domain as the page, returns
     # +false+ otherwise

lib/anemone/rex_http.rb

@@ -46,7 +46,9 @@ def fetch_pages(url, referer = nil, depth = nil)
                                       :referer => referer,
                                       :depth => depth,
                                       :redirect_to => redirect_to,
-                                      :response_time => response_time)
+                                      :response_time => response_time,
+                                      :dirbust => @opts[:dirbust]
+          )
           # Store the associated raw HTTP request
           page.request = response.request
 		  pages << page

lib/msf/core/auxiliary/crawler.rb

@@ -32,6 +32,7 @@ def initialize(info = {})
 
 		register_advanced_options(
 			[
+				OptBool.new('RUN_DIRBUSTER', [ false, 'The maximum number of pages to crawl per URL', true]),
 				OptInt.new('RequestTimeout', [false, 'The maximum number of seconds to wait for a reply', 15]),
 				OptInt.new('RedirectLimit', [false, 'The maximum number of redirects for a single request', 5]),
 				OptInt.new('RetryLimit', [false, 'The maximum number of attempts for a single request', 5]),
@@ -173,6 +174,10 @@ def max_crawl_threads
 		datastore['MAX_THREADS']
 	end
 
+	def dirbust?
+		datastore['RUN_DIRBUSTER']
+	end
+
 	# Scrub links that end in these extensions. If more or less is
 	# desired by a particular module, this should get redefined.
 	def get_link_filter
@@ -275,6 +280,7 @@ def crawler_options(t)
 		opts[:framework]           = framework
 		opts[:module]              = self
 		opts[:timeout]             = get_connection_timeout
+		opts[:dirbust]             = dirbust?
 
 		if (t[:headers] and t[:headers].length > 0)
 			opts[:inject_headers] = t[:headers]