Updates to HTTP:

* chunked transfer support
	* generic Request factory Client.request
	* runtime configuration via Client.config

Still busted:
	* Enforcement of max_data length
	* Error handling on incomplete responses


git-svn-id: file:///home/svn/incoming/trunk@2849 4d416f70-5f16-0410-b530-b9f4589650da

lib/rex/proto/http/client.rb

@@ -30,7 +30,7 @@ class Client
 			c.stop
 		end
 	end
-
+	
 	#
 	# Initializes a GET request and returns it to the caller.
 	#
@@ -41,6 +41,8 @@ class Client
 	def initialize(host, port = 80)
 		self.hostname = host
 		self.port     = port.to_i
+		self.request_config = {}
+		self.client_config  = {}
 	end
 
 	#
@@ -50,6 +52,72 @@ class Client
 		"HTTP Client"
 	end
 
+	#
+	# Configures the Client object and the Request factory
+	#
+	def config (chash)
+		req_opts = %w{ user-agent vhost cookie proto }
+		cli_opts = %w{ max-data }	
+		chash.each_pair { |k,v| 
+			req_opts.include?(k) ? 
+				self.request_config[k] = v : self.client_config[k] = v 
+		}
+	end
+
+	#
+	# Set parameters for the Request factory
+	#
+	def request_option(k, v)
+		(v != nil) ? self.request_config[k] = v : self.request_config[k]
+	end
+	
+	#
+	# Set parameters for the actual Client
+	#
+	def client_option(k, v)
+		(v != nil) ? self.client_config[k] = v : self.client_config[k]
+	end
+	
+	#
+	# The Request factory
+	#
+	def request (chash) 
+		method = chash['method']
+		proto  = chash['proto']  || self.request_config['proto']
+		uri    = chash['uri'] 
+		
+		req    = Rex::Proto::Http::Request.new(method, uri, proto)
+		
+		#
+		# Configure the request headers using the Client configuration
+		#
+
+		if self.request_config['cookie']
+			req['Cookie'] = self.request_config['cookie']
+		end
+		
+		if self.request_config['user-agent']
+			req['User-Agent'] = self.request_config['user-agent']
+		end
+		
+		#
+		# Configure the rest of the request based on config hash
+		#		
+		req['Host'] = (self.request_config['vhost'] || self.hostname) + ':' + self.port.to_s
+		
+		# Set the request body if a data chunk has been specified
+		if chash['data']
+			req.body = chash['data']
+		end
+		
+		# Set the content-type
+		if chash['content-type']
+			req['Content-Type'] = chash['content-type']
+		end
+
+		req
+	end
+	
 	#
 	# Connects to the remote server if possible.
 	#
@@ -78,7 +146,7 @@ class Client
 	# Initializes a request by setting the host header and other cool things.
 	#
 	def init_request(req)
-		req['Host'] = "#{hostname}:#{port}"
+		req['Host'] = "#{request_config.has_key?('vhost') ? request_config['vhost'] : hostname}:#{port}"
 
 		return req
 	end
@@ -88,7 +156,8 @@ class Client
 	#
 	def send_request(req, t = -1)
 		resp = Response.new
-
+		resp.max_data = self.client_config['max-data']
+		
 		# Connect to the server
 		connect
 
@@ -119,8 +188,13 @@ class Client
 		# Close our side if we aren't pipelining
 		close if (!pipelining?)
 
-		# Returns the response to the caller
-		return (resp.completed?) ? resp : nil
+		# XXX - How should we handle this?
+		if (not resp.completed?)
+			# raise RuntimeError, resp.error, caller
+		end
+
+		# Always return the Response object back to the client
+		return resp
 	end
 
 	#
@@ -147,12 +221,14 @@ class Client
 	attr_accessor :pipeline
 	attr_accessor :local_host
 	attr_accessor :local_port
-
+	attr_accessor :client_config
+	
 protected
 
 	attr_accessor :hostname, :port
 	attr_accessor :conn
-
+	attr_accessor :request_config, :client_config
+	
 end
 
 end

lib/rex/proto/http/client.rb.ut.rb

@@ -11,13 +11,67 @@ class Rex::Proto::Http::Client::UnitTest < Test::Unit::TestCase
 
 	def test_parse
 		c = Klass.new('www.google.com')
-		r = Rex::Proto::Http::Request::Get.new('/')
+
+		# Set request factory parameters
+		c.config(
+			'vhost'      => 'www.google.com',
+			'user-agent' => 'Metasploit Framework/3.0',
+			'proto'      => '1.1',
+			'cookie'     => 'NoCookie=NotACookie'
+		)
+
+		# Set client parameters
+		c.config(
+			'max-data' => 1024 * 1024
+		)
+
+		#
+		# Request the main web pagfe
+		#
+		r = c.request(
+			'method' => 'GET',
+			'uri'    => '/'
+		)
 
 		resp = c.send_request(r)
 
 		assert_equal(200, resp.code)
 		assert_equal('OK', resp.message)
-		assert_equal('1.0', resp.proto)
+		assert_equal('1.1', resp.proto)
+
+		#
+		# Request a file that does not exist
+		#
+		r = c.request(
+			'method' => 'GET',
+			'uri'    => '/NoFileHere.404'
+		)
+
+		resp = c.send_request(r)
+		
+		assert_equal(404, resp.code)
+		assert_equal('Not Found', resp.message)
+		assert_equal('1.1', resp.proto)
+		
+		
+		#
+		# Send a POST request that results in a 302
+		#
+		c = Klass.new('beta.microsoft.com')
+		c.request_option('vhost', 'beta.microsoft.com')
+
+		r = c.request(
+			'method' => 'POST',
+			'uri'    => '/',
+			'data'   => 'var=val',
+			'content-type' => 'application/x-www-form-urlencoded'
+		)
+
+		resp = c.send_request(r)
+
+		assert_equal(302, resp.code)
+		assert_equal('Object moved', resp.message)
+		assert_equal('1.1', resp.proto)
 	end
 
 end

lib/rex/proto/http/packet.rb

@@ -4,7 +4,7 @@ module Rex
 module Proto
 module Http
 
-DefaultProtocol = '1.0'
+DefaultProtocol = '1.1'
 
 ###
 #
@@ -38,7 +38,7 @@ class Packet
 
 	def initialize()
 		self.headers = Header.new
-		self.auto_cl = false
+		self.auto_cl = true
 
 		reset
 	end
@@ -62,6 +62,7 @@ class Packet
 	# codes (Completed, Partial, or Error)
 	#
 	def parse(buf)
+
 		# Append the incoming buffer to the buffer queue.
 		self.bufq += buf
 
@@ -96,6 +97,8 @@ class Packet
 		self.state = ParseState::ProcessingHeader
 		self.headers.reset
 		self.body  = ''
+		self.transfer_chunked = nil
+		self.inside_chunk = nil
 	end
 
 	#
@@ -109,7 +112,7 @@ class Packet
 		# things are completed as it's hard to tell whether or not they really
 		# are.
 		if ((self.state == ParseState::ProcessingBody) and
-		    (self.body_bytes_left < 0))
+		    (self.body_bytes_left < 0) )
 			comp = true
 		# Or, if the parser state actually is completed, then we're good.
 		elsif (self.state == ParseState::Completed)
@@ -160,12 +163,17 @@ class Packet
 	attr_accessor :bufq
 	attr_accessor :body
 	attr_accessor :auto_cl
-
+	attr_accessor :max_data
+	attr_reader   :incomplete
+	
 protected
 
 	attr_writer   :headers
 	attr_writer   :error
+	attr_writer   :incomplete
 	attr_accessor :body_bytes_left
+	attr_accessor :transfer_chunked
+	attr_accessor :inside_chunk
 
 	##
 	#
@@ -189,9 +197,19 @@ protected
 	# Parses the header portion of the request.
 	#
 	def parse_header
+		
 		# Does the buffer queue contain the entire header?  If so, parse it and
 		# transition to the body parsing phase.
-		if ((head = self.bufq.slice!(/(.+\r\n\r\n)/m)))
+		idx = self.bufq.index(/\r*\n\r*\n/)
+		
+		if (idx == -1)
+			self.headers.from_s(self.bufq)	
+		end
+		
+		if (idx >= 0)
+			# Extract the header block
+			head = self.bufq.slice!(0, idx)
+
 			# Serialize the headers
 			self.headers.from_s(head)
 
@@ -202,30 +220,34 @@ protected
 				self.body_bytes_left = -1
 			end
 
+			if (self.headers['Transfer-Encoding'])
+				self.transfer_chunked = 1 if self.headers['Transfer-Encoding'] =~ /chunked/i
+			end
+
 			connection    = self.headers['Connection']
 			comp_on_close = false
 
 			if (connection and connection == 'close')
 				comp_on_close = true
 			end
-
+			
 			# Change states to processing the body if we have a content length of
 			# the connection type is close.
-			if ((self.body_bytes_left > 0) or
-			    (comp_on_close))
+			if ((self.body_bytes_left > 0) or (comp_on_close) or self.transfer_chunked)
 				self.state = ParseState::ProcessingBody
 			else
 				self.state = ParseState::Completed
 			end
-
-			# No command string?  Wack.
-			if (self.headers.cmd_string == nil)
-				raise RuntimeError, "Invalid command string", caller
-			end
-
-			# Allow derived classes to update the parts of the command string
-			self.update_cmd_parts(self.headers.cmd_string)
 		end
+		
+		# No command string?  Wack.
+		if (self.headers.cmd_string == nil)
+			raise RuntimeError, "Invalid command string", caller
+		end
+
+		# Allow derived classes to update the parts of the command string
+		self.update_cmd_parts(self.headers.cmd_string)
+
 	end
 
 	#
@@ -233,6 +255,30 @@ protected
 	#
 	def parse_body
 
+		# Just return if the buffer is empty
+		if (self.bufq.length == 0)
+			return
+		end
+
+		# Handle chunked transfer-encoding responses
+		if (self.transfer_chunked and self.inside_chunk != 1 and self.bufq.length) 
+			
+			# Remove any leading newlines or spaces
+			self.bufq.lstrip!
+			
+			# Extract the actual hexadecimal length value
+			clen = self.bufq.slice!(/.*\r*\n/)
+			clen.rstrip!
+			
+			self.body_bytes_left = clen.hex
+			if (self.body_bytes_left == 0)
+				self.bufq.rstrip!
+				return self.state = ParseState::Completed
+			end
+			
+			self.inside_chunk = 1
+		end
+
 		# If there are bytes remaining, slice as many as we can and append them
 		# to our body state.
 		if (self.body_bytes_left > 0)
@@ -246,10 +292,16 @@ protected
 			self.bufq  = ''
 		end
 
+		# Finish this chunk and move on to the next one
+		if (self.transfer_chunked and self.body_bytes_left == 0)
+			self.inside_chunk = 0
+			return self.parse_body
+		end
+
 		# If there are no more bytes left, then parsing has completed and we're
 		# ready to go.
-		if (self.body_bytes_left == 0)
-			self.state = ParseState::Completed
+		if (self.transfer_chunked != 1 and self.body_bytes_left <= 0)
+			return self.state = ParseState::Completed
 		end
 	end
 

lib/rex/proto/http/request.rb

@@ -44,7 +44,8 @@ class Request < Packet
 		self.method    = method
 		self.uri       = uri
 		self.uri_parts = {}
-		self.proto     = proto
+		self.proto     = proto || DefaultProtocol
+
 	end
 
 	#