Land rapid7#8735, robots.txt for HttpServer

Author: wvu

lib/msf/core/exploit/http/server.rb

@@ -43,7 +43,8 @@ def initialize(info = {})
 
     register_advanced_options([
       OptAddress.new('URIHOST', [false, 'Host to use in URI (useful for tunnels)']),
-      OptPort.new('URIPORT', [false, 'Port to use in URI (useful for tunnels)'])
+      OptPort.new('URIPORT', [false, 'Port to use in URI (useful for tunnels)']),
+      OptBool.new('SendRobots', [false, 'Return a robots.txt file if asked for one', false])
     ])
 
     # Used to keep track of resources added to the service manager by
@@ -180,7 +181,26 @@ def start_service(opts = {})
       print_status("Local IP: #{proto}://#{Rex::Socket.source_address('1.2.3.4')}:#{opts['ServerPort']}#{uopts['Path']}")
     end
 
+    if datastore['SendRobots']
+      add_robots_resource
+    end
+
     add_resource(uopts)
+
+  end
+
+  def add_robots_resource
+    proc = Proc.new do |cli, req|
+      self.cli = cli
+      send_robots(cli, req)
+    end
+
+    vprint_status('Adding hardcoded URI /robots.txt')
+    begin
+      add_resource('Path' => '/robots.txt', 'Proc' => proc)
+    rescue RuntimeError => e
+      print_warning(e.message)
+    end
   end
 
   # Set {#on_request_uri} to handle the given +uri+ in addition to the one
@@ -610,6 +630,22 @@ def send_not_found(cli)
     cli.send_response(resp_404)
   end
 
+  #
+  # Sends a canned robots.txt file
+  #
+  def send_robots(cli, request)
+    print_status('Sending robots.txt')
+    robots = create_response(200, 'Success')
+    robots['Content-Type'] = 'text/plain'
+
+    robots.body = %Q{\
+User-agent: *
+Disallow: /
+}
+
+    cli.send_response(robots)
+  end
+
 
   #
   # Returns the configured (or random, if not configured) URI path