added robots.txt support for http exploit server

lib/msf/core/exploit/http/server.rb

@@ -42,7 +42,8 @@ module Exploit::Remote::HttpServer
 
     register_advanced_options([
       OptAddress.new('URIHOST', [false, 'Host to use in URI (useful for tunnels)']),
-      OptPort.new('URIPORT', [false, 'Port to use in URI (useful for tunnels)'])
+      OptPort.new('URIPORT', [false, 'Port to use in URI (useful for tunnels)']),
+      OptBool.new('SendRobots', [ false, "Return a robots.txt file if asked for one", false]),
     ])
 
     # Used to keep track of resources added to the service manager by
@@ -179,7 +180,26 @@ module Exploit::Remote::HttpServer
       print_status("Local IP: #{proto}://#{Rex::Socket.source_address('1.2.3.4')}:#{opts['ServerPort']}#{uopts['Path']}")
     end
 
+    if datastore['SendRobots']
+      add_robots_resource()
+    end
+
     add_resource(uopts)
+
+  end
+
+  def add_robots_resource()
+    proc = Proc.new do |cli, req|
+      self.cli = cli
+      send_robots(cli, req)
+    end
+
+    vprint_status("Adding hardcoded uri /robots.txt")
+    begin
+      add_resource({'Path' => "/robots.txt", 'Proc' => proc})
+    rescue RuntimeError => e
+      print_warning(e.message)
+    end
   end
 
   # Set {#on_request_uri} to handle the given +uri+ in addition to the one
@@ -605,6 +625,22 @@ module Exploit::Remote::HttpServer
     cli.send_response(resp_404)
   end
 
+  #
+  # Sends a canned robots.txt file
+  #
+  def send_robots(cli, request)
+	print_status("sending robots.txt")
+    robots = create_response(200, 'Success')
+	robots['Content-Type'] = "text/plain"
+
+    robots.body = %Q{\
+User-agent: *
+Disallow: /
+}
+
+    cli.send_response(robots)
+  end
+
 
   #
   # Returns the configured (or random, if not configured) URI path