From a0b76b1979f6c8055cd5be84d54f4d9a9d24ce0b Mon Sep 17 00:00:00 2001 From: HD Moore Date: Sat, 13 Nov 2010 03:33:36 +0000 Subject: [PATCH] This merges in support for Rex sockets to anemone git-svn-id: file:///home/svn/framework3/trunk@11016 4d416f70-5f16-0410-b530-b9f4589650da --- lib/anemone.rb | 4 +- lib/anemone/core.rb | 2 +- lib/anemone/docs/MODIFIED.txt | 3 + lib/anemone/page.rb | 4 +- lib/anemone/rex_http.rb | 192 ++++++++++++++++++++++++++++++++++ 5 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 lib/anemone/rex_http.rb diff --git a/lib/anemone.rb b/lib/anemone.rb index 245e1a6ae9..34a12e4c33 100644 --- a/lib/anemone.rb +++ b/lib/anemone.rb @@ -1,6 +1,6 @@ require 'rubygems' require 'anemone/core' -# Overload the HTTP class (soon) -# require 'anemone/rex_http' +# Overload the HTTP class +require 'anemone/rex_http' diff --git a/lib/anemone/core.rb b/lib/anemone/core.rb index ac8546c1d9..a22440e8b4 100644 --- a/lib/anemone/core.rb +++ b/lib/anemone/core.rb @@ -195,7 +195,7 @@ module Anemone @pages = PageStore.new(storage) @robots = Robots.new(@opts[:user_agent]) if @opts[:obey_robots_txt] - freeze_options + # freeze_options end # diff --git a/lib/anemone/docs/MODIFIED.txt b/lib/anemone/docs/MODIFIED.txt index fc0c32151f..5547e5ae0c 100644 --- a/lib/anemone/docs/MODIFIED.txt +++ b/lib/anemone/docs/MODIFIED.txt @@ -4,3 +4,6 @@ This snapshot of Anemone has been modified for use within the Metasploit Framewo * Insert a callback to use a custom Net::HTTP factory * Insert a callback to tweak HTTP requests before they are sent * Addition of core.shutdown() to stop all background threads + * Addition of a Rex::Proto::HTTP version of anemone/http + * Handle Rex::Proto::HTTP headers (not arrays) + * Disable freeze_options() since it can recurse into the framework/module options diff --git a/lib/anemone/page.rb b/lib/anemone/page.rb index af437cb201..c9c1e0908b 100644 --- a/lib/anemone/page.rb +++ b/lib/anemone/page.rb @@ -136,7 +136,9 @@ module Anemone # The content-type returned by the HTTP request for this page # def content_type - headers['content-type'].first + res = headers['content-type'] + res = res.first if res.respond_to?(:first) + res end # diff --git a/lib/anemone/rex_http.rb b/lib/anemone/rex_http.rb new file mode 100644 index 0000000000..599fb25d3a --- /dev/null +++ b/lib/anemone/rex_http.rb @@ -0,0 +1,192 @@ +require 'rex' +require 'anemone/page' +require 'anemone/cookie_store' + + +# +# This is an alternate Anemone::HTTP implementation that uses the Metasploit Rex +# library and the Rex::Proto::Http protocol stack. +# + +module Anemone + class HTTP + # Maximum number of redirects to follow on each get_response + # REDIRECT_LIMIT = 5 + + # CookieStore for this HTTP client + attr_reader :cookie_store + + def initialize(opts = {}) + @connections = {} + @opts = opts + @cookie_store = CookieStore.new(@opts[:cookies]) + end + + # + # Fetch a single Page from the response of an HTTP request to *url*. + # Just gets the final destination page. + # + def fetch_page(url, referer = nil, depth = nil) + fetch_pages(url, referer, depth).last + end + + # + # Create new Pages from the response of an HTTP request to *url*, + # including redirects + # + def fetch_pages(url, referer = nil, depth = nil) + begin + url = URI(url) unless url.is_a?(URI) + pages = [] + get(url, referer) do |response, code, location, redirect_to, response_time| + + pages << Page.new(location, :body => response.body.dup, + :code => code, + :headers => response.headers, + :referer => referer, + :depth => depth, + :redirect_to => redirect_to, + :response_time => response_time) + end + + return pages + rescue => e + if verbose? + puts e.inspect + puts e.backtrace + end + return [Page.new(url, :error => e)] + end + end + + # + # The maximum number of redirects to follow + # + def redirect_limit + @opts[:redirect_limit] || REDIRECT_LIMIT + end + + # + # The user-agent string which will be sent with each request, + # or nil if no such option is set + # + def user_agent + @opts[:user_agent] + end + + # + # The virtual host to override the host header with, per url + # TODO: implement + # + def virtual_host(url) + url.host + end + + # + # Does this HTTP client accept cookies from the server? + # + def accept_cookies? + @opts[:accept_cookies] + end + + private + + # + # Retrieve HTTP responses for *url*, including redirects. + # Yields the response object, response code, and URI location + # for each response. + # + def get(url, referer = nil) + limit = redirect_limit + loc = url + begin + # if redirected to a relative url, merge it with the host of the original + # request url + loc = url.merge(loc) if loc.relative? + + response, response_time = get_response(loc, referer) + code = Integer(response.code) + + redirect_to = nil + if code >= 300 and code <= 310 + redirect_to = URI(response['location']).normalize + end + + yield response, code, loc, redirect_to, response_time + limit -= 1 + end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0 + end + + # + # Get an HTTPResponse for *url*, sending the appropriate User-Agent string + # + # MODIFIED: Change get_response to allow fine tuning of the HTTP request before + # it is sent to the remote system. + # + def get_response(url, referer = nil) + full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}" + + opts = {} + opts['User-Agent'] = user_agent if user_agent + opts['Referer'] = referer.to_s if referer + opts['Cookie'] = @cookie_store.to_s unless @cookie_store.empty? || (!accept_cookies? && @opts[:cookies].nil?) + + retries = 0 + begin + start = Time.now() + + response = nil + request = nil + begin + conn = connection(url) + request = conn.request_raw(opts) + response = conn.send_recv(request, @opts[:timeout] || 10 ) + rescue ::Errno::EPIPE, ::Timeout::Error + end + + finish = Time.now() + + response_time = ((finish - start) * 1000).round + @cookie_store.merge!(response['Set-Cookie']) if accept_cookies? + return response, response_time + rescue EOFError + retries += 1 + retry unless retries > (@opts[:retry_limit] || 3) + end + end + + def connection(url) + context = { } + context['Msf'] = @opts[:framework] if @opts[:framework] + context['MsfExploit'] = @opts[:module] if @opts[:module] + + conn = Rex::Proto::Http::Client.new( + url.host, + url.port.to_i, + context, + url.scheme == "https", + 'SSLv23', + @opts[:proxies] + ) + + conn.set_config( + 'vhost' => virtual_host(url), + 'agent' => user_agent + ) + + conn + end + + def verbose? + @opts[:verbose] + end + + # + # Allowed to connect to the requested url? + # + def allowed?(to_url, from_url) + to_url.host.nil? || (to_url.host == from_url.host) + end + + end +end