Using hpricot

git-svn-id: file:///home/svn/framework3/trunk@8862 4d416f70-5f16-0410-b530-b9f4589650da

data/msfcrawler/basic.rb

@@ -1,18 +1,26 @@
 require 'rubygems'
 require 'pathname'
+require 'hpricot'
 require 'uri'
 
-class CrawlerBasic < BaseParser
+class CrawlerSimple < BaseParser
 
 	def parse(request,result)
 		
-		#puts "R: #{result.body}"
+		if !result['Content-Type'].include? "text/html"
+			return
+		end
 		
-		links = result.body.to_s.scan(/href\s*=\s*[\"\'](.+?)[\"\']/)  
+		doc = Hpricot(result.body.to_s)
+		doc.search('a').each do |link|
 		
-		links.each do |link| 
+		hr = link.attributes['href']
+		
+		if hr
+		#links = result.body.to_s.scan(/href\s*=\s*[\"\'](.+?)[\"\']/) 
+		#links.each do |link| 
 			begin
-				uri = URI.parse(link[0])
+				uri = URI.parse(hr)
 			
 				tssl = false
 				if uri.scheme == "https"
@@ -23,6 +31,7 @@ class CrawlerBasic < BaseParser
 
 				if !uri.host or uri.host == nil
 					thost = request['rhost']
+					tssl = self.targetssl	
 				else
 					thost = uri.host	
 				end
@@ -40,12 +49,14 @@ class CrawlerBasic < BaseParser
 				end
 				
 				
+				
 				newp = Pathname.new(tpath)
-				if !newp.absolute?
 				oldp = Pathname.new(request['uri'])
+				if !oldp.absolute?
+					if !newp.absolute?
 						newp = oldp + newp.cleanpath
 					end
-				
+				end
 
 				hreq = {
 					'rhost'		=> thost,
@@ -57,7 +68,7 @@ class CrawlerBasic < BaseParser
 					'query'		=> uri.query
 					
 				}
-				#puts "R: #{hreq['uri']}"
+				
 				insertnewpath(hreq)
 					
 			rescue URI::InvalidURIError
@@ -67,4 +78,5 @@ class CrawlerBasic < BaseParser
 		end
 		end
 	end 
+end