Initial overhaul of the HTTP client api. This removes nearly all of the client evasion methods, but leaves the code in a great state to reimplement them with less issues.

git-svn-id: file:///home/svn/framework3/trunk@4222 4d416f70-5f16-0410-b530-b9f4589650da
unstable
HD Moore 2006-12-19 07:11:55 +00:00
parent 239fe1e8ae
commit 804df25240
6 changed files with 419 additions and 232 deletions

View File

@ -23,23 +23,34 @@ module Exploit::Remote::HttpClient
Opt::RPORT(80),
OptString.new('VHOST', [ false, "HTTP server virtual host" ]),
Opt::SSL,
], Exploit::Remote::HttpClient
], self.class
)
register_advanced_options(
[
OptString.new('UserAgent', [false, 'The User-Agent header to use for all requests'])
], self.class
)
register_evasion_options(
[
OptEnum.new('HTTP::uri_encode', [false, 'Enable URI encoding', 'none', ['none','hex-normal', 'hex-all', 'u-normal', 'u-all'], 'none']),
OptBool.new('HTTP::chunked', [false, 'Enable chunking of HTTP request via "Transfer-Encoding: chunked"', 'false']),
OptBool.new('HTTP::header_folding', [false, 'Enable folding of HTTP headers', 'false']),
OptBool.new('HTTP::junk_headers', [false, 'Enable insertion of random junk HTTP headers', 'false']),
OptBool.new('HTTP::junk_slashes', [false, 'Enable insertion of random junk HTTP headers', 'false']),
OptBool.new('HTTP::junk_directories', [false, 'Enable insertion of random junk directories in the URI', 'false']),
OptBool.new('HTTP::junk_params', [false, 'Enable insertion of random junk parameters', 'false']),
OptBool.new('HTTP::junk_self_referring_directories', [false, 'Enable insertion of random self referring directories (eg: /./././)', 'false']),
OptInt.new('HTTP::junk_pipeline', [true, 'Insert the specified number of junk pipeline requests', 0]),
OptBool.new('HTTP::fake_uri_end', [false, 'Add a fake end of URI (eg: /%20HTTP/1.0/../../)', 'false']),
OptBool.new('HTTP::fake_param_start', [false, 'Add a fake start of params to the URI (eg: /%3fa=b/../)', 'false']),
], Exploit::Remote::HttpClient
OptEnum.new('HTTP::uri_encode', [false, 'Enable URI encoding', 'none', ['none','hex-normal', 'hex-all', 'u-normal', 'u-all'], 'hex-normal'])
#
# Still re-implementing the following options
#
# OptBool.new('HTTP::chunked', [false, 'Enable chunking of HTTP request via "Transfer-Encoding: chunked"', 'false']),
# OptBool.new('HTTP::header_folding', [false, 'Enable folding of HTTP headers', 'false']),
# OptBool.new('HTTP::junk_headers', [false, 'Enable insertion of random junk HTTP headers', 'false']),
# OptBool.new('HTTP::junk_slashes', [false, 'Enable insertion of random junk HTTP headers', 'false']),
# OptBool.new('HTTP::junk_directories', [false, 'Enable insertion of random junk directories in the URI', 'false']),
# OptBool.new('HTTP::junk_params', [false, 'Enable insertion of random junk parameters', 'false']),
# OptBool.new('HTTP::junk_self_referring_directories', [false, 'Enable insertion of random self referring directories (eg: /./././)', 'false']),
# OptInt.new('HTTP::junk_pipeline', [true, 'Insert the specified number of junk pipeline requests', 0]),
# OptBool.new('HTTP::fake_uri_end', [false, 'Add a fake end of URI (eg: /%20HTTP/1.0/../../)', 'false']),
# OptBool.new('HTTP::fake_param_start', [false, 'Add a fake start of params to the URI (eg: /%3fa=b/../)', 'false']),
], self.class
)
end
@ -58,7 +69,11 @@ module Exploit::Remote::HttpClient
)
# Configure the HTTP client with the supplied parameter
nclient.config( { 'vhost' => datastore['VHOST'], }.update(opts) )
nclient.set_config(
'vhost' => datastore['VHOST'],
'uri_encode_mode' => datastore['HTTP::uri_encode'],
'agent' => datastore['UserAgent']
)
# If this connection is global, persist it
if (opts['global'])
@ -120,59 +135,21 @@ module Exploit::Remote::HttpClient
#
# Connects to the server, creates a request, sends the request, reads the response
#
def request(opts={}, timeout = -1)
def send_request_raw(opts={}, timeout = -1)
c = connect(opts)
if (datastore['HTTP::junk_pipeline'].to_i > 0)
c.junk_pipeline = datastore['HTTP::junk_pipeline'].to_i
end
request = c.request(opts)
if (datastore['HTTP::uri_encode'])
request.uri_encode_mode = datastore['HTTP::uri_encode']
end
if (datastore['HTTP::chunked'] == true)
request.auto_cl = false
request.transfer_chunked = true
end
if (datastore['HTTP::header_folding'] == true)
request.headers.fold = 1
end
if (datastore['HTTP::junk_headers'] == true)
request.headers.junk_headers = 1
end
if (datastore['HTTP::junk_directories'] == true)
request.junk_directories = 1
end
if (datastore['HTTP::junk_slashes'] == true)
request.junk_slashes = 1
end
if (datastore['HTTP::junk_params'] == true)
request.junk_params = 1
end
if (datastore['HTTP::fake_uri_end'] == true)
request.junk_end_of_uri = 1
end
if (datastore['HTTP::fake_param_start'] == true)
request.junk_param_start = 1
end
if (datastore['HTTP::junk_self_referring_directories'] == true)
request.junk_self_referring_directories = 1
end
c.send_request(request, opts[:timeout] ? opts[:timeout] : timeout)
r = c.request_raw(opts)
c.send_recv(r, opts[:timeout] ? opts[:timeout] : timeout)
end
#
# Connects to the server, creates a request, sends the request, reads the response
#
def send_request_cgi(opts={}, timeout = -1)
c = connect(opts)
r = c.request_cgi(opts)
c.send_recv(r, opts[:timeout] ? opts[:timeout] : timeout)
end
##
#
# Wrappers for getters

View File

@ -718,19 +718,23 @@ protected
client = Rex::Proto::Http::Client.new(server_host, server_port)
begin
# Create the CGI parameter list
vars = { 'method' => method }
opts.each_pair do |k, v|
vars[k] = xlate_param(v)
end
# Initialize the request with the POST body.
request = client.gen_post(server_uri)
request.body += "method=#{method}"
# Enumerate each option filter specified and convert it into a CGI
# parameter.
opts.each_pair { |k, v|
request.body += "&#{k}=#{xlate_param(v)}"
}
request = client.request_cgi(
'method' => 'POST',
'cgi' => server_uri,
'vars_post' => vars
)
# Send the request and grab the response.
response = client.send_request(request, 300)
response = client.send_recv(request, 300)
# Non-200 return code?
if (response.code != 200)

View File

@ -1,5 +1,6 @@
require 'rex/socket'
require 'rex/proto/http'
require 'rex/text'
module Rex
module Proto
@ -8,132 +9,148 @@ module Http
###
#
# Acts as a client to an HTTP server, sending requests and receiving
# responses. This is modeled somewhat after Net::HTTP.
# responses.
#
###
class Client
include Proto
#
# Performs a block-based HTTP operation.
# Creates a new client instance
#
def self.start(host, &block)
c = Client.new(host)
begin
block.call(c)
ensure
c.stop
end
end
#
# Initializes a GET request and returns it to the caller.
#
def gen_get(uri = '/', proto = DefaultProtocol)
return init_request(Request::Get.new(uri, proto))
end
#
# Initializes a POST request and returns it to the caller.
#
def gen_post(uri = '/', proto = DefaultProtocol)
return init_request(Request::Post.new(uri, proto))
end
def initialize(host, port = 80, context = {}, ssl = nil)
self.hostname = host
self.port = port.to_i
self.context = context
self.ssl = ssl
self.request_config = {}
self.client_config = {}
end
#
# HTTP client.
#
def alias
"HTTP Client"
end
#
# Configures the Client object and the Request factory.
#
def config(chash)
req_opts = %w{ user-agent vhost cookie proto }
cli_opts = %w{ max-data }
chash.each_pair { |k,v|
req_opts.include?(k) ?
self.request_config[k] = v : self.client_config[k] = v
self.config = {
'read_max_data' => (1024*1024*1),
'vhost' => self.hostname,
'version' => '1.1',
'agent' => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
'uri_encode_mode' => 'hex-normal',
'uri_full_url' => false
}
end
#
# Set parameters for the Request factory.
# Set configuration options
#
def request_option(k, v)
(v != nil) ? self.request_config[k] = v : self.request_config[k]
def set_config(opts = {})
opts.each_pair do |var,val|
config[var]=val
end
end
#
# Set parameters for the actual Client.
# Create an arbitrary HTTP request
#
def client_option(k, v)
(v != nil) ? self.client_config[k] = v : self.client_config[k]
end
#
# The Request factory.
#
def request(chash)
method = chash['method'] || 'GET'
proto = chash['proto'] || self.request_config['proto'] || DefaultProtocol
uri = chash['uri'] || '/'
def request_raw(opts={})
c_enc = opts['encode'] || false
c_uri = opts['uri'] || '/'
c_body = opts['body'] || ''
c_meth = opts['method'] || 'GET'
c_vers = opts['version'] || config['version'] || '1.1'
c_qs = opts['query']
c_ag = opts['agent'] || config['agent']
c_cook = opts['cookie'] || config['cookie']
c_host = opts['vhost'] || config['vhost']
c_head = opts['headers'] || config['headers'] || {}
c_conn = opts['connection']
uri = set_uri(c_uri)
req = Rex::Proto::Http::Request.new(method, uri, proto)
# pass on the junk_pipeline config
if self.junk_pipeline
req.junk_pipeline = self.junk_pipeline
end
req = ''
req += set_method(c_meth)
req += set_method_uri_spacer()
req += set_uri_prepend()
req += (c_enc ? set_encode_uri(uri) : uri)
#
# Configure the request headers using the Client configuration
#
if self.request_config['cookie']
req['Cookie'] = self.request_config['cookie']
if (c_qs)
req += '?'
req += (c_enc ? set_encode_qs(c_qs) : c_qs)
end
req += set_uri_append()
req += set_uri_version_spacer()
req += set_version(c_vers)
req += set_host_header(c_host)
req += set_agent_header(c_ag)
req += set_cookie_header(c_cook)
req += set_connection_header(c_conn)
req += set_extra_headers(c_head)
req += set_body(c_body)
if self.request_config['user-agent']
req['User-Agent'] = self.request_config['user-agent']
end
#
# Configure the rest of the request based on config hash
#
req['Host'] = (self.request_config['vhost'] || self.hostname) + ':' + self.port.to_s
# Set the request body if a data chunk has been specified
if chash['data']
req.body = chash['data']
end
# Merge headers supplied by the caller.
if chash['headers']
req.headers.merge!(chash['headers'])
end
# Set the content-type
if chash['content-type']
req['Content-Type'] = chash['content-type']
end
req
end
#
# Create a CGI compatible request
#
def request_cgi(opts={})
c_enc = opts['encode'] || false
c_cgi = opts['cgi'] || '/'
c_body = opts['body'] || ''
c_meth = opts['method'] || 'GET'
c_vers = opts['version'] || config['version'] || '1.1'
c_qs = opts['query'] || ''
c_varg = opts['vars_get'] || {}
c_varp = opts['vars_post'] || {}
c_head = opts['headers'] || config['headers'] || {}
c_type = opts['ctype'] || 'application/x-www-form-urlencoded'
c_ag = opts['agent'] || config['agent']
c_cook = opts['cookie'] || config['cookie']
c_host = opts['vhost'] || config['vhost']
c_conn = opts['connection']
c_path = opts['path_info']
uri = set_cgi(c_cgi)
qstr = c_qs
pstr = c_body
c_varg.each_pair do |var,val|
qstr << '&' if qstr.length > 0
qstr << set_encode_uri(var)
qstr << '='
qstr << set_encode_uri(val)
end
c_varp.each_pair do |var,val|
pstr << '&' if pstr.length > 0
pstr << set_encode_uri(var)
pstr << '='
pstr << set_encode_uri(val)
end
req = ''
req += set_method(c_meth)
req += set_method_uri_spacer()
req += set_uri_prepend()
req += set_encode_uri(uri)
if (qstr.length > 0)
req += '?'
req += qstr
end
req += set_path_info(c_path)
req += set_uri_append()
req += set_uri_version_spacer()
req += set_version(c_vers)
req += set_host_header(c_host)
req += set_agent_header(c_ag)
req += set_cookie_header(c_cook)
req += set_connection_header(c_conn)
req += set_extra_headers(c_head)
# TODO:
# * Implement chunked transfer
req += set_content_type_header(c_type)
req += set_content_len_header(pstr.length)
req += set_body(pstr)
req
end
#
# Connects to the remote server if possible.
#
@ -170,21 +187,18 @@ class Client
end
#
# Initializes a request by setting the host header and other cool things.
# Transmit a HTTP request and receive the response
#
def init_request(req)
req['Host'] = "#{request_config.has_key?('vhost') ? request_config['vhost'] : hostname}:#{port}"
return req
def send_recv(req, t = -1)
send_request(req)
read_response(t)
end
#
# Transmits a request and reads in a response.
#
def send_request(req, t = -1)
resp = Response.new
resp.max_data = self.client_config['max-data']
# Send a HTTP request to the server
# TODO:
# * Handle junk pipeline requests
def send_request(req)
# Connect to the server
connect
@ -192,7 +206,20 @@ class Client
req_string = req.to_s
# Send it on over
conn.put(req_string)
ret = conn.put(req)
# Tell the remote side if we aren't pipelining
conn.shutdown(::Socket::SHUT_WR) if (!pipelining?)
ret
end
#
# Read a response from the server
#
def read_response(t = -1)
resp = Response.new
resp.max_data = config['read_max_data']
# Tell the remote side if we aren't pipelining
conn.shutdown(::Socket::SHUT_WR) if (!pipelining?)
@ -288,7 +315,201 @@ class Client
def pipelining?
pipeline
end
#
# Return the encoded URI
# ['none','hex-normal', 'hex-all', 'u-normal', 'u-all']
def set_encode_uri(uri)
Rex::Text.uri_encode(uri, self.config['uri_encode_mode'])
end
#
# Return the encoded query string
#
def set_encode_qs(qs)
Rex::Text.uri_encode(uri, self.config['uri_encode_mode'])
end
#
# Return the uri
#
def set_uri(uri)
if (self.config['uri_full_url'])
url = self.ssl ? "https" : "http"
url += self.config['vhost']
url += (self.port == 80) ? "" : ":#{self.port}"
url += uri
url
else
uri
end
end
#
# Return the cgi
# TODO:
# * Implement self-referential directories
# * Implement bogus relative directories
def set_cgi(uri)
url = uri
if (self.config['uri_full_url'])
url = self.ssl ? "https" : "http"
url += self.config['vhost']
url += (self.port == 80) ? "" : ":#{self.port}"
url += uri
end
url
end
#
# Return the HTTP method string
#
def set_method(method)
# TODO:
# * Randomize case
# * Replace with random valid method
# * Replace with random invalid method
method
end
#
# Return the HTTP version string
#
def set_version(version)
# TODO:
# * Randomize case
# * Replace with random valid versions
# * Replace with random invalid versions
"HTTP/" + version + "\r\n"
end
#
# Return the HTTP seperator and body string
#
def set_body(data)
"\r\n" + data
end
#
# Return the HTTP path info
# TODO:
# * Encode path information
def set_path_info(path)
path ? path : ''
end
#
# Return the spacing between the method and uri
#
def set_method_uri_spacer
# TODO:
# * Support different space types
" "
end
#
# Return the spacing between the uri and the version
#
def set_uri_version_spacer
# TODO:
# * Support different space types
" "
end
#
# Return the padding to place before the uri
#
def set_uri_prepend
# TODO:
# * Support different padding types
""
end
#
# Return the padding to place before the uri
#
def set_uri_append
# TODO:
# * Support different padding types
""
end
#
# Return the HTTP Host header
#
def set_host_header(host)
return "" if self.config['uri_full_url']
host ||= self.config['vhost']
set_formatted_header("Host", host)
end
#
# Return the HTTP agent header
#
def set_agent_header(agent)
agent ? set_formatted_header("User-Agent", agent) : ""
end
#
# Return the HTTP cookie header
#
def set_cookie_header(cookie)
cookie ? set_formatted_header("Cookie", cookie) : ""
end
#
# Return the HTTP connection header
#
def set_connection_header(conn)
conn ? set_formatted_header("Connection", conn) : ""
end
#
# Return the content type header
#
def set_content_type_header(ctype)
set_formatted_header("Content-Type", ctype)
end
#
# Return the content length header
# TODO:
# * Ignore this if chunked encoding is set
def set_content_len_header(clen)
set_formatted_header("Content-Length", clen)
end
#
# Return a string of formatted extra headers
# TODO:
# * Implement junk header stuffing
def set_extra_headers(headers)
buf = ''
headers.each_pair do |var,val|
buf += set_formatted_header(var, val)
end
buf
end
#
# Return a formatted header string
# TODO:
# * Implement header folder
def set_formatted_header(var, val)
"#{var}: #{val}\r\n"
end
#
# The client request configuration
#
attr_accessor :config
#
# Whether or not pipelining is in use.
#
@ -302,10 +523,6 @@ class Client
#
attr_accessor :local_port
#
# Client configuration attributes.
#
attr_accessor :client_config
#
# The underlying connection.
#
attr_accessor :conn
@ -323,7 +540,7 @@ protected
attr_accessor :ssl
attr_accessor :hostname, :port # :nodoc:
attr_accessor :request_config, :client_config # :nodoc:
end

View File

@ -13,27 +13,27 @@ class Rex::Proto::Http::Client::UnitTest < Test::Unit::TestCase
c = Klass.new('www.google.com')
# Set request factory parameters
c.config(
c.set_config(
'vhost' => 'www.google.com',
'user-agent' => 'Metasploit Framework/3.0',
'proto' => '1.1',
'agent' => 'Metasploit Framework/3.0',
'version' => '1.1',
'cookie' => 'NoCookie=NotACookie'
)
# Set client parameters
c.config(
'max-data' => 1024 * 1024
c.set_config(
'read_max_data' => 1024 * 1024
)
#
# Request the main web pagfe
#
r = c.request(
r = c.request_raw(
'method' => 'GET',
'uri' => '/'
)
resp = c.send_request(r)
resp = c.send_recv(r)
assert_equal(200, resp.code)
assert_equal('OK', resp.message)
@ -42,12 +42,12 @@ class Rex::Proto::Http::Client::UnitTest < Test::Unit::TestCase
#
# Request a file that does not exist
#
r = c.request(
r = c.request_raw(
'method' => 'GET',
'uri' => '/NoFileHere.404'
)
resp = c.send_request(r)
resp = c.send_recv(r)
assert_equal(404, resp.code)
assert_equal('Not Found', resp.message)
@ -58,45 +58,34 @@ class Rex::Proto::Http::Client::UnitTest < Test::Unit::TestCase
# Send a POST request that results in a 302
#
c = Klass.new('beta.microsoft.com')
c.request_option('vhost', 'beta.microsoft.com')
c.set_config('vhost' => 'beta.microsoft.com')
r = c.request(
r = c.request_cgi(
'method' => 'POST',
'uri' => '/',
'data' => 'var=val',
'content-type' => 'application/x-www-form-urlencoded'
'cgi' => '/',
'vars_post' => { 'var' => 'val' },
'ctype' => 'application/x-www-form-urlencoded'
)
resp = c.send_request(r)
resp = c.send_recv(r)
assert_equal(302, resp.code)
assert_equal('Object moved', resp.message)
assert_equal(200, resp.code)
assert_equal('OK', resp.message)
assert_equal('1.1', resp.proto)
end
def test_ssl
c = Klass.new('www.geotrust.com', '443', {}, 'true')
c.request_option('vhost', 'www.geotrust.com')
r = c.request(
c = Klass.new('www.geotrust.com', 443, {}, true)
c.set_config('vhost' => 'www.geotrust.com')
r = c.request_raw(
'method' => 'GET',
'uri' => '/'
)
resp = c.send_request(r)
resp = c.send_recv(r)
assert_equal(200, resp.code)
assert_equal('OK', resp.message)
assert_equal('1.1', resp.proto)
c.close
end
def test_junk_pipeline
host = 'www.apache.org'
client = Klass.new(host)
client.junk_pipeline = 5
client.request_option('vhost', host)
request = client.request('method' => 'GET', 'uri' => '/no-such-uri.html')
response = client.send_request(request)
assert_equal(404, response.code, 'pipeline response')
client.close
end
end

View File

@ -19,8 +19,8 @@ class Rex::Proto::Http::Server::UnitTest < Test::Unit::TestCase
c = CliKlass.new(ListenHost, ListenPort)
1.upto(10) {
req = Rex::Proto::Http::Request::Get.new('/')
res = c.send_request(req)
req = c.request_raw('uri' => '/')
res = c.send_recv(req)
assert_not_nil(res)
assert_equal(404, res.code)
}
@ -44,8 +44,8 @@ class Rex::Proto::Http::Server::UnitTest < Test::Unit::TestCase
})
1.upto(10) {
req = Rex::Proto::Http::Request::Get.new('/foo')
res = c.send_request(req)
req = c.request_raw('uri' => '/foo')
res = c.send_recv(req)
assert_not_nil(res)
assert_equal(200, res.code)
assert_equal("Chickens everywhere", res.body)
@ -53,8 +53,8 @@ class Rex::Proto::Http::Server::UnitTest < Test::Unit::TestCase
s.remove_resource('/foo')
req = Rex::Proto::Http::Request::Get.new('/foo')
res = c.send_request(req)
req = c.request_raw('uri' => '/foo')
res = c.send_recv(req)
assert_not_nil(res)
assert_equal(404, res.code)
ensure

View File

@ -306,7 +306,7 @@ module Text
return str if mode == 'none' # fast track no encoding
all = /[^\/\\]+/
normal = /[^a-zA-Z1-9]+/
normal = /[^a-zA-Z1-9\/\/\\]+/
case mode
when 'hex-normal'