updated auxiliary/scanner/http/crawler to accept a callback for each page

modules/auxiliary/scanner/http/crawler.rb

@@ -22,10 +22,11 @@ class Metasploit3 < Msf::Auxiliary
 			'Name'        => 'Web Site Crawler',
 			'Version'     => '$Revision$',
 			'Description' => 'Crawl a web site and store information about what was found',
-			'Author'      => 'hdm',
+			'Author'      => %w(hdm tasos),
 			'License'     => MSF_LICENSE
 		)
 
+		@for_each_page_blocks = []
 	end
 
 =begin
@@ -35,6 +36,20 @@ class Metasploit3 < Msf::Auxiliary
 	end
 =end
 
+	def run
+		super
+
+		if form = form_from_url( @current_site, datastore['URI'] )
+			print_status((" " * 24) + "FORM: #{form[:method]} #{form[:path]}")
+			report_web_form( form )
+			self.form_count += 1
+		end
+	end
+
+	def for_each_page( &block )
+		@for_each_page_blocks << block if block_given?
+	end
+
 	#
 	# The main callback from the crawler, redefines crawler_process_page() as
 	# defined by Msf::Auxiliary::HttpCrawler
@@ -111,49 +126,12 @@ class Metasploit3 < Msf::Auxiliary
 		# Apache multiview directories
 		return if page.url.query =~ /^C=[A-Z];O=/ # Apache
 
-		# Scrub out the jsessionid appends
-		page.url.path = page.url.path.sub(/;jsessionid=[a-zA-Z0-9]+/, '')
-
-		#
-		# Continue processing forms
-		#
 		forms = []
 		form_template = { :web_site => t[:site] }
-		form  = {}.merge(form_template)
 
-		# This page has a query parameter we can test with GET parameters
-		# ex: /test.php?a=b&c=d
-		if page.url.query and not page.url.query.empty?
-			form[:method] = 'GET'
-			form[:path]   = page.url.path
-			vars = page.url.query.split('&').map{|x| x.split("=", 2) }
-			form[:params] = vars
-		end
-
-		# This is a REST-ish application with numeric parameters
-		# ex: /customers/343
-		if not form[:path] and page.url.path.to_s =~ /(.*)\/(\d+)$/
-			path_base = $1
-			path_info = $2
-			form[:method] = 'PATH'
-			form[:path]   = path_base
-			form[:params] = [['PATH', path_info]]
-			form[:query]  = page.url.query.to_s
-		end
-
-		# This is an application that uses PATH_INFO for parameters:
-		# ex:  /index.php/Main_Page/Article01
-		if not form[:path] and page.url.path.to_s =~ /(.*\/[a-z0-9A-Z]{3,256}\.[a-z0-9A-Z]{2,8})(\/.*)/
-			path_base = $1
-			path_info = $2
-			form[:method] = 'PATH'
-			form[:path]   = path_base
-			form[:params] = [['PATH', path_info]]
-			form[:query]  = page.url.query.to_s
-		end
-
-		# Done processing URI-based forms
+		if form = form_from_url( t[:site], page.url )
 			forms << form
+		end
 
 		if page.doc
 			page.doc.css("form").each do |f|
@@ -199,7 +177,20 @@ class Metasploit3 < Msf::Auxiliary
 					form[:params] << [inp['name'].to_s, inp['value'] || inp.content || '', { :type => inp['type'].to_s }]
 				end
 
-				# XXX: handle SELECT elements
+				f.css( 'select' ).each do |s|
+					value = nil
+
+					# iterate over each option to find the default value (if there is a selected one)
+					s.children.each do |opt|
+						ov = opt['value'] || opt.content
+						value = ov if opt['selected']
+					end
+
+					# set the first one as the default value if we don't already have one
+					value ||= s.children.first['value'] || s.children.first.content rescue ''
+
+					form[:params] << [ s['name'].to_s, value.to_s, [ :type => 'select'] ]
+				end
 
 				forms << form
 			end
@@ -212,5 +203,55 @@ class Metasploit3 < Msf::Auxiliary
 			report_web_form(form)
 			self.form_count += 1
 		end
+
+		@for_each_page_blocks.each { |p| p.call( page ) }
 	end
+
+	def form_from_url( website, url )
+		url = URI( url.to_s ) if !url.is_a?( URI )
+
+		# Scrub out the jsessionid appends
+		url.path = url.path.sub(/;jsessionid=[a-zA-Z0-9]+/, '')
+
+		#
+		# Continue processing forms
+		#
+		forms = []
+		form_template = { :web_site => website }
+		form  = {}.merge(form_template)
+
+		# This page has a query parameter we can test with GET parameters
+		# ex: /test.php?a=b&c=d
+		if url.query and not url.query.empty?
+			form[:method] = 'GET'
+			form[:path]   = url.path
+			vars = url.query.split('&').map{|x| x.split("=", 2) }
+			form[:params] = vars
+		end
+
+		# This is a REST-ish application with numeric parameters
+		# ex: /customers/343
+		if not form[:path] and url.path.to_s =~ /(.*)\/(\d+)$/
+			path_base = $1
+			path_info = $2
+			form[:method] = 'PATH'
+			form[:path]   = path_base
+			form[:params] = [['PATH', path_info]]
+			form[:query]  = url.query.to_s
+		end
+
+		# This is an application that uses PATH_INFO for parameters:
+		# ex:  /index.php/Main_Page/Article01
+		if not form[:path] and url.path.to_s =~ /(.*\/[a-z0-9A-Z]{3,256}\.[a-z0-9A-Z]{2,8})(\/.*)/
+			path_base = $1
+			path_info = $2
+			form[:method] = 'PATH'
+			form[:path]   = path_base
+			form[:params] = [['PATH', path_info]]
+			form[:query]  = url.query.to_s
+		end
+
+		form[:method] ? form : nil
+	end
+
 end