Merge pull request rapid7#20173 from adfoster-r7/update-web-crawlers-to-support-http-logging

Update web crawlers to support HTTP logging

Author: adfoster-r7

lib/anemone/rex_http.rb

@@ -191,8 +191,9 @@ def connection(url)
       url.scheme == "https",
       @opts[:ssl_version],
       @opts[:proxies],
-                    @opts[:username],
-                    @opts[:password]
+      @opts[:username],
+      @opts[:password],
+      subscriber: @opts[:http_subscriber]
     )
 
     conn.set_config(

lib/msf/core/auxiliary/http_crawler.rb

@@ -283,6 +283,7 @@ def crawler_options(t)
     opts[:module]              = self
     opts[:timeout]             = get_connection_timeout
     opts[:dirbust]             = dirbust?
+    opts[:http_subscriber] = Rex::Proto::Http::HttpLoggerSubscriber.new(logger: self)
 
     if (t[:headers] and t[:headers].length > 0)
       opts[:inject_headers] = t[:headers]

lib/msf/core/auxiliary/web/http.rb

@@ -113,7 +113,8 @@ def connect
       'Auto',
       nil,
       username,
-      password
+      password,
+      subscriber: opts[:http_subscriber]
     )
 
     c.set_config({

lib/rex/proto/http/client.rb

@@ -19,9 +19,8 @@ class Client
 
         #
         # Creates a new client instance
-        # @param http_trace_proc_request [Proc] A proc object passed to log HTTP requests if HTTP-Trace is set
-        # @param http_trace_proc_response [Proc] A proc object passed to log HTTP responses if HTTP-Trace is set
         #
+        # @param [Rex::Proto::Http::HttpSubscriber] subscriber A subscriber to Http requests/responses
         def initialize(host, port = 80, context = {}, ssl = nil, ssl_version = nil, proxies = nil, username = '', password = '', kerberos_authenticator: nil, comm: nil, subscriber: nil, sslkeylogfile: nil)
           self.hostname = host
           self.port = port.to_i

modules/auxiliary/scanner/http/crawler.rb

@@ -18,6 +18,9 @@ def initialize
 
     register_advanced_options([
       OptString.new('ExcludePathPatterns', [false, 'Newline-separated list of path patterns to ignore (\'*\' is a wildcard)']),
+      OptBool.new('HttpTrace', [false, 'Show the raw HTTP requests and responses', false]),
+      OptBool.new('HttpTraceHeadersOnly', [false, 'Show HTTP headers only in HttpTrace', false]),
+      OptString.new('HttpTraceColors', [false, 'HTTP request and response colors for HttpTrace (unset to disable)', 'red/blu']),
     ])
     @for_each_page_blocks = []
   end