added some http protocol stuff, ruby's builtin stuff sucks

git-svn-id: file:///home/svn/incoming/trunk@2823 4d416f70-5f16-0410-b530-b9f4589650da

documentation/TODO

@@ -50,3 +50,4 @@ X  - findsock handler
   - more ui wrapping
   - fix route addition/removal in stdapi server dll (mib structure issue)
   - fix interactive stream pool channels
+- dupe input instance when passing to sessions

lib/rex.rb

@@ -31,10 +31,6 @@ require 'rex/io/stream_server'
 
 # Sockets
 require 'rex/socket'
-require 'rex/socket/parameters'
-require 'rex/socket/tcp'
-require 'rex/socket/tcp_server'
-require 'rex/socket/comm/local'
 
 # Parsers
 require 'rex/parser/arguments'

lib/rex/io/stream.rb

@@ -1,3 +1,5 @@
+require 'rex/thread_safe'
+
 module Rex
 module IO
 
@@ -165,14 +167,32 @@ module Stream
 
 		buf = ""
 		lps = 0
+		eof = false
 
 		# Keep looping until there is no more data to be gotten..
 		while (has_read_data?(ltimeout) == true)
-			temp = read(def_block_size)
-			
+			# Catch EOF errors so that we can handle them properly.
+			begin
+				temp = read(def_block_size)
+			rescue EOFError
+				eof = true
+			end
+
 			# If we read zero bytes and we had data, then we've hit EOF
 			if (temp and temp.length == 0)
-				raise EOFError
+				eof = true
+			end
+
+			# If we reached EOF and there are no bytes in the buffer we've been
+			# reading into, then throw an EOF error.
+			if (eof)
+				# If we've already read at least some data, then it's time to
+				# break out and let it be processed before throwing an EOFError.
+				if (buf.length > 0)
+					break
+				else
+					raise EOFError
+				end
 			end
 
 			break if (temp == nil or temp.empty? == true)

lib/rex/post/meterpreter/ui/console.rb

@@ -80,8 +80,6 @@ class Console
 			log_error(info.to_s)
 		rescue
 			log_error("Error running command #{method}: #{$!}")
-
-			print_line("yo:#{$@.join("\n")}")
 		end
 	end
 
@@ -90,6 +88,8 @@ class Console
 
 		elog(msg, 'meterpreter')
 
+		print_error("XXX: logging stack")
+
 		dlog("Call stack:\n#{$@.join("\n")}", 'meterpreter')
 	end
 

lib/rex/proto/http.rb

@@ -0,0 +1,5 @@
+require 'rex/proto/http/packet'
+require 'rex/proto/http/request'
+require 'rex/proto/http/response'
+require 'rex/proto/http/client'
+require 'rex/proto/http/server'

lib/rex/proto/http/client.rb

@@ -0,0 +1,148 @@
+require 'rex/socket'
+require 'rex/proto/http'
+
+module Rex
+module Proto
+module Http
+
+###
+#
+# Client
+# ------
+#
+# Acts as a client to an HTTP server, sending requests and receiving
+# responses.  This is modeled somewhat after Net::HTTP.
+#
+###
+class Client
+
+	#
+	# Performs a block-based HTTP operation
+	#
+	def self.start(host, &block)
+		c = Client.new(host)
+
+		begin
+			block.call(c)
+		ensure
+			c.stop
+		end
+	end
+
+	def self.get(uri = '/', proto = DefaultProtocol)
+		return init_request(Request::Get.new(uri, proto))
+	end
+
+	def initialize(host, port = 80)
+		self.hostname = host
+		self.port     = port.to_i
+	end
+
+	#
+	# Connects to the remote server if possible.
+	#
+	def connect
+		# If we already have a connection and we aren't pipelining, close it.
+		if (self.conn and !pipelining?)
+			close
+		end
+
+		self.conn = Rex::Socket::Tcp.create(
+			'PeerHost'  => self.hostname,
+			'PeerPort'  => self.port.to_i,
+			'LocalHost' => self.local_host,
+			'LocalPort' => self.local_port)
+	end
+
+	#
+	# Closes the connection to the remote server.
+	#
+	def close
+		self.conn.close if (self.conn)
+		self.conn = nil
+	end
+
+	#
+	# Initializes a request by setting the host header and other cool things.
+	#
+	def init_request(req)
+		req['Host'] = "#{hostname}:#{port}"
+
+		return req
+	end
+
+	#
+	# Transmits a request and reads in a response
+	#
+	def send_request(req, t = -1)
+		resp = Response.new
+
+		# Connect to the server
+		connect
+
+		# Send it on over
+		conn.put(req.to_s)
+
+		# Tell the remote side if we aren't pipelining
+		conn.shutdown(::Socket::SHUT_WR) if (!pipelining?)
+
+		# Wait at most t seconds for the full response to be read in.  We only
+		# do this if t was specified as a negative value indicating an infinite
+		# wait cycle.  If t were specified as nil it would indicate that no
+		# response parsing is required.
+		timeout((t < 0) ? nil : t) {
+			# Now, read in the response until we're good to go.
+			begin
+				# Keep running until we finish parsing or EOF is reached
+				while ((rv = resp.parse(conn.get)) != Packet::ParseCode::Completed)
+					# Parsing error?  Raise an exception, our job is done.
+					if (rv == Packet::ParseCode::Error)
+						raise RuntimeError, resp.error, caller
+					end
+				end
+			rescue EOFError
+			end
+		} if (t)
+
+		# Close our side if we aren't pipelining
+		close if (!pipelining?)
+
+		# Returns the response to the caller
+		return (resp.completed?) ? resp : nil
+	end
+
+	#
+	# Cleans up any outstanding connections and other resources
+	#
+	def stop
+		close
+	end
+
+	#
+	# Returns whether or not the conn is valid.
+	#
+	def conn?
+		conn != nil
+	end
+
+	#
+	# Whether or not connections should be pipelined
+	#
+	def pipelining?
+		pipeline
+	end
+
+	attr_accessor :pipeline
+	attr_accessor :local_host
+	attr_accessor :local_port
+
+protected
+
+	attr_accessor :hostname, :port
+	attr_accessor :conn
+
+end
+
+end
+end
+end

lib/rex/proto/http/client.rb.ut.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), '..', '..', '..'))
+
+require 'test/unit'
+require 'rex/proto/http'
+
+class Rex::Proto::Http::Client::UnitTest < Test::Unit::TestCase
+
+	Klass = Rex::Proto::Http::Client
+
+	def test_parse
+		c = Klass.new('www.google.com')
+		r = Rex::Proto::Http::Request::Get.new('/')
+
+		resp = c.send_request(r)
+
+		assert_equal(200, resp.code)
+		assert_equal('OK', resp.message)
+		assert_equal('1.0', resp.proto)
+	end
+
+end

lib/rex/proto/http/header.rb

@@ -0,0 +1,116 @@
+require 'rex/proto/http'
+
+module Rex
+module Proto
+module Http
+
+###
+#
+# Header
+# ------
+#
+# Represents the logical HTTP header portion of an HTTP packet (request or
+# response).
+#
+###
+class Packet::Header < Hash
+
+	def initialize
+		self.dcase_hash = {}
+
+		reset
+	end
+
+	#
+	# Parses a header from a string.
+	#
+	def from_s(header)
+		reset
+
+		# Extract the command string
+		self.cmd_string = header.slice!(/(.+\r\n)/)
+
+		# Extract each header value pair
+		header.split(/\r\n/m).each { |str|
+			if (md = str.match(/^(.+?): (.+?)$/))
+				self[md[1]] = md[2]
+			end
+		}
+	end
+
+	#
+	# More advanced [] that does downcase comparison.
+	#
+	def [](key)
+		begin
+			if ((rv = self.fetch(key)) == nil)
+				rv = self.dcase_hash[key.downcase]	
+			end
+		rescue IndexError
+			rv = nil
+		end
+
+		return rv
+	end
+
+	#
+	# More advanced []= that does downcase storage.
+	#
+	def []=(key, value)
+		stored = false
+
+		self.each_key { |k|
+			if (k.downcase == key.downcase)
+				self.store(k, value)
+				stored = true
+			end
+		}
+
+		self.store(key, value) if (stored == false)
+		self.dcase_hash[key.downcase] = value
+	end
+
+	#
+	# Converts the header to a string.
+	#
+	def to_s(prefix = '')
+		str = prefix
+
+		each_pair { |var, val|
+			str += "#{var.to_s}: #{val.to_s}\r\n"
+		}
+
+		str += "\r\n"
+
+		return str
+	end
+
+	#
+	# Brings in from an array like yo.
+	#
+	def from_a(ary)
+		ary.each { |e|
+			self[e[0]] = e[1]
+		}
+	end
+
+	#
+	# Flushes all header pairs.
+	#
+	def reset
+		self.cmd_string = ''
+		self.clear
+		self.dcase_hash.clear
+	end
+
+	attr_accessor :cmd_string
+
+protected
+
+	attr_accessor :dcase_hash
+
+end
+
+end
+end
+end

lib/rex/proto/http/packet.rb

@@ -0,0 +1,242 @@
+require 'rex/proto/http'
+
+module Rex
+module Proto
+module Http
+
+DefaultProtocol = '1.0'
+
+###
+#
+# Packet
+# ------
+#
+# This class represents an HTTP packet.
+#
+###
+class Packet
+
+	#
+	# Parser processing codes
+	#
+	module ParseCode
+		Completed = 1
+		Partial   = 2
+		Error     = 3
+	end
+
+	#
+	# Parser states
+	#
+	module ParseState
+		ProcessingHeader = 1
+		ProcessingBody   = 2
+		Completed        = 3
+	end
+
+	require 'rex/proto/http/header'
+
+	def initialize()
+		self.headers = Header.new
+		self.auto_cl = false
+
+		reset
+	end
+
+	#
+	# Return the associated header value, if any.
+	#
+	def [](key)
+		self.headers[key]
+	end
+
+	#
+	# Set the associated header value.
+	#
+	def []=(key, value)
+		self.headers[key] = value
+	end
+
+	#
+	# Parses the supplied buffer.  Returns one of the two parser processing
+	# codes (Completed, Partial, or Error)
+	#
+	def parse(buf)
+		# Append the incoming buffer to the buffer queue.
+		self.bufq += buf
+
+		begin
+			# If we're processing headers, do that now.
+			if (self.state == ParseState::ProcessingHeader)
+				parse_header
+			end
+	
+			# If we're processing the body (possibly after having finished
+			# processing headers), do that now.
+			if (self.state == ParseState::ProcessingBody)
+				parse_body
+			end
+		rescue
+			self.error = $!
+
+			puts "#{self.error}: \n#{$@.join("\n")}"
+
+			return ParseCode::Error
+		end
+
+		# Return completed or partial to the parsing status to the caller
+		(self.state == ParseState::Completed) ? ParseCode::Completed : ParseCode::Partial
+	end
+
+	#
+	# Reset the parsing state and buffers.
+	#
+	def reset
+		self.bufq  = ''
+		self.state = ParseState::ProcessingHeader
+		self.headers.reset
+		self.body  = ''
+	end
+
+	#
+	# Returns whether or not parsing has completed.
+	#
+	def completed?
+		comp = false
+
+		# If the parser state is processing the body and there are an
+		# undetermined number of bytes left to read, we just need to say that
+		# things are completed as it's hard to tell whether or not they really
+		# are.
+		if ((self.state == ParseState::ProcessingBody) and
+		    (self.body_bytes_left < 0))
+			comp = true
+		# Or, if the parser state actually is completed, then we're good.
+		elsif (self.state == ParseState::Completed)
+			comp = true
+		end
+
+		return comp
+	end
+
+	#
+	# Converts the packet to a string
+	#
+	def to_s
+		# Update the content length field in the header with the body length.
+		if (self.body and self.auto_cl)
+			self.headers['Content-Length'] = self.body.length
+		end
+
+		str  = self.headers.to_s(cmd_string)
+		str += self.body || ''
+	end
+
+	#
+	# Converts the packet from a string
+	#
+	def from_s(str)
+		reset
+
+		parse(str)
+	end
+
+	#
+	# Returns the command string, such as:
+	#
+	# HTTP/1.0 200 OK for a response
+	#
+	# or
+	# 
+	# GET /foo HTTP/1.0 for a request
+	#
+	def cmd_string
+		self.headers.cmd_string
+	end
+
+	attr_reader   :headers
+	attr_reader   :error
+	attr_accessor :state
+	attr_accessor :bufq
+	attr_accessor :body
+	attr_accessor :auto_cl
+
+protected
+
+	attr_writer   :headers
+	attr_writer   :error
+	attr_accessor :body_bytes_left
+
+	##
+	#
+	# Overridable methods
+	#
+	##
+	
+	#
+	# Allows derived classes to split apart the command string.
+	#
+	def update_cmd_parts(str)
+	end
+
+	##
+	#
+	# Parsing
+	#
+	##
+
+	#
+	# Parses the header portion of the request.
+	#
+	def parse_header
+		# Does the buffer queue contain the entire header?  If so, parse it and
+		# transition to the body parsing phase.
+		if ((head = self.bufq.slice!(/(.+\r\n\r\n)/m)))
+			# Serialize the headers
+			self.headers.from_s(head)
+
+			# Change states to processing the body
+			self.state = ParseState::ProcessingBody
+
+			# Extract the content length, if any.
+			if (self.headers['content-length'])
+				self.body_bytes_left = self.headers['content-length'].to_i
+			else
+				self.body_bytes_left = -1
+			end
+	
+			# Allow derived classes to update the parts of the command string
+			self.update_cmd_parts(self.headers.cmd_string)
+		end
+	end
+
+	#
+	# Parses the body portion of the request.
+	#
+	def parse_body
+
+		# If there are bytes remaining, slice as many as we can and append them
+		# to our body state.
+		if (self.body_bytes_left > 0)
+			part = self.bufq.slice!(0, self.body_bytes_left)
+
+			self.body += part
+			self.body_bytes_left -= part.length
+		# Otherwise, just read it all.
+		else
+			self.body += self.bufq
+			self.bufq  = ''
+		end
+
+		# If there are no more bytes left, then parsing has completed and we're
+		# ready to go.
+		if (self.body_bytes_left == 0)
+			self.state = ParseState::Completed
+		end
+	end
+
+end
+
+end
+end
+end

lib/rex/proto/http/packet.rb.ut.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), '..', '..', '..'))
+
+require 'test/unit'
+require 'rex/proto/http'
+
+class Rex::Proto::Http::Packet::UnitTest < Test::Unit::TestCase
+
+	Klass = Rex::Proto::Http::Packet
+
+	def test_parse
+		h = Klass.new
+
+		req1 = 
+			"GET / HTTP/1.0\r\n" +
+			"Foo: Bird\r\n" +
+			"Accept: text/html\r\n" +
+			"\r\n" + 
+			"Super body"
+
+		assert_equal(Klass::ParseCode::Partial, h.parse(req1))
+		assert_equal(true, h.completed?)
+		assert_equal("Bird", h.headers['Foo'])
+		assert_equal("text/html", h.headers['Accept'])
+		assert_equal("Super body", h.body);
+		assert_equal(req1, h.to_s)
+	end
+
+	def test_to_s
+		h = Klass.new
+
+		h.headers['Foo']     = 'Fishing'
+		h.headers['Chicken'] = 47
+		h.auto_cl = true
+
+		assert_equal(
+			"Foo: Fishing\r\n" +
+			"Content-Length: 0\r\n" +
+			"Chicken: 47\r\n\r\n", h.to_s)
+	end
+
+	def test_from_s
+		h = Klass.new
+
+		h.from_s(
+			"HTTP/1.0 200 OK\r\n" +
+			"Lucifer: Beast\r\n" +
+			"HoHo: Satan\r\n" +
+			"Eat: Babies\r\n" +
+			"\r\n")
+
+		assert_equal('Babies', h['Eat'])
+		h['Eat'] = "fish"
+		assert_equal('fish', h['Eat'])
+	end
+
+end

lib/rex/proto/http/request.rb

@@ -0,0 +1,74 @@
+require 'rex/proto/http'
+
+module Rex
+module Proto
+module Http
+
+###
+#
+# Request
+# -------
+#
+# HTTP request class.
+#
+###
+class Request < Packet
+
+	##
+	#
+	# Some individual request types.
+	#
+	##
+	class Get < Request
+		def initialize(uri = '/', proto = DefaultProtocol)
+			super('GET', uri, proto)
+		end
+	end
+
+	class Post < Request
+		def initialize(uri = '/', proto = DefaultProtocol)
+			super('POST', uri, proto)
+		end
+	end
+
+	class Put < Request
+		def initialize(uri = '/', proto = DefaultProtocol)
+			super('PUT', uri, proto)
+		end
+	end
+
+	def initialize(method = 'GET', uri = '/', proto = DefaultProtocol)
+		super()
+
+		self.method = method
+		self.uri    = uri
+		self.proto  = proto
+	end
+
+	#
+	# Updates the command parts for this specific packet type.
+	#
+	def update_cmd_parts(str)
+		if (md = str.match(/^(.+?)\s+(.+?)\s+HTTP\/(.+?)\r?\n?$/))
+			self.method = md[1]
+			self.uri    = md[2]
+			self.proto  = md[3]
+		end
+	end
+
+	#
+	# Returns the command string derived from the three values
+	#
+	def cmd_string
+		"#{self.method} #{self.uri} HTTP/#{self.proto}\r\n"
+	end
+
+	attr_accessor :method
+	attr_accessor :uri
+	attr_accessor :proto
+
+end
+
+end
+end
+end

lib/rex/proto/http/request.rb.ut.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), '..', '..', '..'))
+
+require 'test/unit'
+require 'rex/proto/http'
+
+class Rex::Proto::Http::Packet::UnitTest < Test::Unit::TestCase
+
+	Klass = Rex::Proto::Http::Request
+
+	def test_to_s
+		h = Klass.new
+
+		h.headers['Foo']     = 'Fishing'
+		h.headers['Chicken'] = 47
+		h.auto_cl = true
+
+		assert_equal(
+			"GET / HTTP/1.0\r\n" +
+			"Foo: Fishing\r\n" +
+			"Content-Length: 0\r\n" +
+			"Chicken: 47\r\n\r\n", h.to_s)
+	end
+
+	def test_from_s
+		h = Klass.new
+
+		h.from_s(
+			"POST /foo HTTP/1.0\r\n" +
+			"Lucifer: Beast\r\n" +
+			"HoHo: Satan\r\n" +
+			"Eat: Babies\r\n" +
+			"\r\n")
+
+		assert_equal('POST', h.method)
+		assert_equal('/foo', h.uri)
+		assert_equal('1.0', h.proto)
+		assert_equal("POST /foo HTTP/1.0\r\n", h.cmd_string)
+		h.method = 'GET'
+		assert_equal("GET /foo HTTP/1.0\r\n", h.cmd_string)
+		assert_equal('Babies', h['Eat'])
+	end
+
+end

lib/rex/proto/http/response.rb

@@ -0,0 +1,75 @@
+require 'rex/proto/http'
+
+module Rex
+module Proto
+module Http
+
+###
+#
+# Response
+# --------
+#
+# HTTP response class.
+#
+###
+class Response < Packet
+
+	##
+	#
+	# Builtin response class wrappers.
+	#
+	## 
+
+	class OK < Response
+		def initialize(message = 'OK', proto = DefaultProtocol)
+			super(200, message, proto)
+		end
+	end
+
+	class E404 < Response
+		def initialize(message = 'File not found', proto = DefaultProtocol)
+			super(404, message, proto)
+		end
+	end
+
+	#
+	# Constructage
+	#
+	def initialize(code = 200, message = 'OK', proto = DefaultProtocol)
+		super()
+
+		self.code    = code.to_i
+		self.message = message
+		self.proto   = proto
+
+		# Default responses to auto content length on
+		self.auto_cl = true
+	end
+
+	#
+	# Updates the various parts of the HTTP response command string.
+	#
+	def update_cmd_parts(str)
+		if (md = str.match(/HTTP\/(.+?)\s+(\d+)\s?(.+?)\r?\n?$/))
+			self.message = md[3].gsub(/\r/, '')
+			self.code    = md[2].to_i
+			self.proto   = md[1]
+		end
+	end
+
+	#
+	# Returns the response based command string.
+	#
+	def cmd_string
+		"HTTP\/#{proto} #{code}#{(message and message.length > 0) ? ' ' + message : ''}\r\n"
+	end
+
+	attr_accessor :code
+	attr_accessor :message
+	attr_accessor :proto
+
+end
+
+end
+end
+end

lib/rex/proto/http/response.rb.ut.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/ruby
+
+$:.unshift(File.join(File.dirname(__FILE__), '..', '..', '..'))
+
+require 'test/unit'
+require 'rex/proto/http'
+
+class Rex::Proto::Http::Packet::UnitTest < Test::Unit::TestCase
+
+	Klass = Rex::Proto::Http::Response
+
+	def test_to_s
+		h = Klass.new
+
+		h.headers['Foo']     = 'Fishing'
+		h.headers['Chicken'] = 47
+		h.auto_cl = true
+
+		assert_equal(
+			"HTTP/1.0 200 OK\r\n" +
+			"Foo: Fishing\r\n" +
+			"Content-Length: 0\r\n" +
+			"Chicken: 47\r\n\r\n", h.to_s)
+	end
+
+	def test_from_s
+		h = Klass.new
+
+		h.from_s(
+			"HTTP/1.0 404 File not found\r\n" +
+			"Lucifer: Beast\r\n" +
+			"HoHo: Satan\r\n" +
+			"Eat: Babies\r\n" +
+			"\r\n")
+
+		assert_equal('404', h.code)
+		assert_equal('File not found', h.message)
+		assert_equal('1.0', h.proto)
+		assert_equal("HTTP/1.0 404 File not found\r\n", h.cmd_string)
+		h.code = 470
+		assert_equal("HTTP/1.0 470 File not found\r\n", h.cmd_string)
+		assert_equal('Babies', h['Eat'])
+	end
+
+end

lib/rex/socket.rb

@@ -17,6 +17,9 @@ class Socket
 	end
 
 	require 'rex/socket/parameters'
+	require 'rex/socket/tcp'
+	require 'rex/socket/tcp_server'
+	require 'rex/socket/comm/local'
 
 	##
 	#