From 0e78719eafa34309179cc2e8287f9605f769636a Mon Sep 17 00:00:00 2001
From: et <>
Date: Thu, 8 Apr 2010 03:40:08 +0000
Subject: [PATCH] Simple parsing modules

git-svn-id: file:///home/svn/framework3/trunk@9042 4d416f70-5f16-0410-b530-b9f4589650da
---
 data/msfcrawler/basic.rb   | 54 +-------------------------------------
 data/msfcrawler/flash.rb   | 52 +-----------------------------------
 data/msfcrawler/forms.rb   | 52 ++----------------------------------
 data/msfcrawler/objects.rb | 52 +-----------------------------------
 data/msfcrawler/scripts.rb | 52 +-----------------------------------
 5 files changed, 6 insertions(+), 256 deletions(-)

diff --git a/data/msfcrawler/basic.rb b/data/msfcrawler/basic.rb
index ed73027c78..9bc3ee0f26 100644
--- a/data/msfcrawler/basic.rb
+++ b/data/msfcrawler/basic.rb
@@ -18,60 +18,8 @@ class CrawlerSimple < BaseParser
 		
 		if hr and !hr.match(/^(\#|javascript\:)/) 
 			begin
-				uri = URI.parse(hr)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
+				hreq = urltohash('GET',hr,request['uri'],nil)				
 				
-				
-				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/plain',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-
 				insertnewpath(hreq)
 					
 			rescue URI::InvalidURIError
diff --git a/data/msfcrawler/flash.rb b/data/msfcrawler/flash.rb
index 2ae549d90f..52d3a83b7d 100644
--- a/data/msfcrawler/flash.rb
+++ b/data/msfcrawler/flash.rb
@@ -47,58 +47,8 @@ class CrawlerFlash < BaseParser
 				links.each do |link| 
 				
 					begin
-						uri = URI.parse(link[0])
-			
-						tssl = false
-						if uri.scheme == "https"
-							tssl = true
-						else
-							tssl = false
-						end
+						hreq = urltohash('GET',link[0],request['uri'],nil)
 
-						if !uri.host or uri.host == nil
-							thost = request['rhost']
-							tssl = self.targetssl	
-						else
-							thost = uri.host	
-						end
-
-						if !uri.port or uri.port == nil
-							tport = request['rport']
-						else
-							tport = uri.port
-						end
-
-						if !uri.path or uri.path == nil
-							tpath = "/"
-						else
-							tpath = uri.path
-						end
-				
-						newp = Pathname.new(tpath)
-						oldp = Pathname.new(request['uri'])
-						if !newp.absolute?
-							if oldp.to_s[-1,1] == '/'
-								newp = oldp+newp
-							else
-								if !newp.to_s.empty?
-									newp = File.join(oldp.dirname,newp)
-								end
-							end		
-						end
-
-						hreq = {
-							'rhost'		=> thost,
-							'rport'		=> tport,
-							'uri'  		=> newp.to_s,
-							'method'   	=> 'GET',
-							'ctype'		=> 'text/plain',
-							'ssl'		=> tssl,
-							'query'		=> uri.query,
-							'data'		=> nil
-						}
-				
-					
 						insertnewpath(hreq)
 					
 					rescue URI::InvalidURIError
diff --git a/data/msfcrawler/forms.rb b/data/msfcrawler/forms.rb
index f3a8e4ff7a..7590b38f77 100644
--- a/data/msfcrawler/forms.rb
+++ b/data/msfcrawler/forms.rb
@@ -47,58 +47,10 @@ class CrawlerForms < BaseParser
 	
 		
 			begin
-				uri = URI.parse(hr)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
+				hreq = urltohash(m,hr,request['uri'],data)
 				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> m,
-					'ctype'		=> 'application/x-www-form-urlencoded',
-					'ssl'		=> tssl,
-					'query'		=> m == 'GET'? data : uri.query,
-					'data'		=> m == 'GET'? nil : data
+				hreq['ctype'] = 'application/x-www-form-urlencoded'
 					
-				}
-				#puts hreq
 				insertnewpath(hreq)
 			
 					
diff --git a/data/msfcrawler/objects.rb b/data/msfcrawler/objects.rb
index ca9b706b9f..4a9e0011e6 100644
--- a/data/msfcrawler/objects.rb
+++ b/data/msfcrawler/objects.rb
@@ -20,58 +20,8 @@ class CrawlerObjects < BaseParser
 			s = obj['src']
 
 			begin
-				uri = URI.parse(s)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
+				hreq = urltohash('GET',s,request['uri'],nil)
 				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/html',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-				#puts hreq
 				insertnewpath(hreq)
 			
 					
diff --git a/data/msfcrawler/scripts.rb b/data/msfcrawler/scripts.rb
index bfcbb5fe0b..78c81fb1b2 100644
--- a/data/msfcrawler/scripts.rb
+++ b/data/msfcrawler/scripts.rb
@@ -20,58 +20,8 @@ class CrawlerScripts < BaseParser
 			s = obj['src']
 
 			begin
-				uri = URI.parse(s)
-			
-				tssl = false
-				if uri.scheme == "https"
-					tssl = true
-				else
-					tssl = false
-				end
-
-				if !uri.host or uri.host == nil
-					thost = request['rhost']
-					tssl = self.targetssl	
-				else
-					thost = uri.host	
-				end
-
-				if !uri.port or uri.port == nil
-					tport = request['rport']
-				else
-					tport = uri.port
-				end
-
-				if !uri.path or uri.path == nil
-					tpath = "/"
-				else
-					tpath = uri.path
-				end
+				hreq = urltohash('GET',s,request['uri'],nil)				
 				
-				newp = Pathname.new(tpath)
-				oldp = Pathname.new(request['uri'])
-				if !newp.absolute?
-					if oldp.to_s[-1,1] == '/'
-						newp = oldp+newp
-					else
-						if !newp.to_s.empty?
-							newp = File.join(oldp.dirname,newp)
-						end
-					end		
-				end
-				
-				hreq = {
-					'rhost'		=> thost,
-					'rport'		=> tport,
-					'uri'  		=> newp.to_s,
-					'method'   	=> 'GET',
-					'ctype'		=> 'text/html',
-					'ssl'		=> tssl,
-					'query'		=> uri.query,
-					'data'		=> nil
-					
-				}
-				#puts hreq
 				insertnewpath(hreq)