Add invisible auth to web crawler

the anemone web crawler now properly supports our invisible auth scheme
for rex http.

Author: David Maloney

lib/anemone/rex_http.rb

@@ -188,7 +188,9 @@ def connection(url)
 			context,
 			url.scheme == "https",
 			'SSLv23',
-			@opts[:proxies]
+			@opts[:proxies],
+                    @opts[:username],
+                    @opts[:password]
 		)
 
 		conn.set_config(

lib/msf/core/auxiliary/crawler.rb

@@ -22,7 +22,9 @@ def initialize(info = {})
 				Opt::Proxies,
 				OptInt.new('MAX_PAGES', [ true, 'The maximum number of pages to crawl per URL', 500]),
 				OptInt.new('MAX_MINUTES', [ true, 'The maximum number of minutes to spend on each URL', 5]),
-				OptInt.new('MAX_THREADS', [ true, 'The maximum number of concurrent requests', 4])
+				OptInt.new('MAX_THREADS', [ true, 'The maximum number of concurrent requests', 4]),
+				OptString.new('USERNAME', [false, 'The HTTP username to specify for authentication']),
+				OptString.new('PASSWORD', [false, 'The HTTP password to specify for authentication'])
 			], self.class
 		)
 
@@ -34,8 +36,6 @@ def initialize(info = {})
 				OptString.new('UserAgent', [true, 'The User-Agent header to use for all requests',
 					"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
 				]),
-				OptString.new('BasicAuthUser', [false, 'The HTTP username to specify for basic authentication']),
-				OptString.new('BasicAuthPass', [false, 'The HTTP password to specify for basic authentication']),
 				OptString.new('HTTPAdditionalHeaders', [false, "A list of additional headers to send (separated by \\x01)"]),
 				OptString.new('HTTPCookie', [false, "A HTTP cookie header to send with each request"]),
 				OptBool.new('SSL', [ false, 'Negotiate SSL for outgoing connections', false]),
@@ -118,8 +118,9 @@ def run
 			:info     => ""
 		})
 
-		if datastore['BasicAuthUser']
-			t[:http_basic_auth] = [ "#{datastore['BasicAuthUser']}:#{datastore['BasicAuthPass']}" ].pack("m*").gsub(/\s+/, '')
+		if datastore['USERNAME'] and datastore['USERNAME'] != ''
+			t[:username] = datastore['USERNAME'].to_s
+			t[:password] = datastore['PASSWORD'].to_s
 		end
 
 		if datastore['HTTPCookie']
@@ -278,9 +279,8 @@ def crawler_options(t)
 			opts[:cookies] = t[:cookies]
 		end
 
-		if t[:http_basic_auth]
-			opts[:http_basic_auth] = t[:http_basic_auth]
-		end
+		opts[:username] = t[:username] || ''
+		opts[:password] =t[:password] || ''
 
 		opts
 	end