Simple parsing modules

git-svn-id: file:///home/svn/framework3/trunk@9042 4d416f70-5f16-0410-b530-b9f4589650da
unstable
et 2010-04-08 03:40:08 +00:00
parent 4853f0266c
commit 0e78719eaf
5 changed files with 6 additions and 256 deletions

View File

@ -18,59 +18,7 @@ class CrawlerSimple < BaseParser
if hr and !hr.match(/^(\#|javascript\:)/)
begin
uri = URI.parse(hr)
tssl = false
if uri.scheme == "https"
tssl = true
else
tssl = false
end
if !uri.host or uri.host == nil
thost = request['rhost']
tssl = self.targetssl
else
thost = uri.host
end
if !uri.port or uri.port == nil
tport = request['rport']
else
tport = uri.port
end
if !uri.path or uri.path == nil
tpath = "/"
else
tpath = uri.path
end
newp = Pathname.new(tpath)
oldp = Pathname.new(request['uri'])
if !newp.absolute?
if oldp.to_s[-1,1] == '/'
newp = oldp+newp
else
if !newp.to_s.empty?
newp = File.join(oldp.dirname,newp)
end
end
end
hreq = {
'rhost' => thost,
'rport' => tport,
'uri' => newp.to_s,
'method' => 'GET',
'ctype' => 'text/plain',
'ssl' => tssl,
'query' => uri.query,
'data' => nil
}
hreq = urltohash('GET',hr,request['uri'],nil)
insertnewpath(hreq)

View File

@ -47,57 +47,7 @@ class CrawlerFlash < BaseParser
links.each do |link|
begin
uri = URI.parse(link[0])
tssl = false
if uri.scheme == "https"
tssl = true
else
tssl = false
end
if !uri.host or uri.host == nil
thost = request['rhost']
tssl = self.targetssl
else
thost = uri.host
end
if !uri.port or uri.port == nil
tport = request['rport']
else
tport = uri.port
end
if !uri.path or uri.path == nil
tpath = "/"
else
tpath = uri.path
end
newp = Pathname.new(tpath)
oldp = Pathname.new(request['uri'])
if !newp.absolute?
if oldp.to_s[-1,1] == '/'
newp = oldp+newp
else
if !newp.to_s.empty?
newp = File.join(oldp.dirname,newp)
end
end
end
hreq = {
'rhost' => thost,
'rport' => tport,
'uri' => newp.to_s,
'method' => 'GET',
'ctype' => 'text/plain',
'ssl' => tssl,
'query' => uri.query,
'data' => nil
}
hreq = urltohash('GET',link[0],request['uri'],nil)
insertnewpath(hreq)

View File

@ -47,58 +47,10 @@ class CrawlerForms < BaseParser
begin
uri = URI.parse(hr)
hreq = urltohash(m,hr,request['uri'],data)
tssl = false
if uri.scheme == "https"
tssl = true
else
tssl = false
end
hreq['ctype'] = 'application/x-www-form-urlencoded'
if !uri.host or uri.host == nil
thost = request['rhost']
tssl = self.targetssl
else
thost = uri.host
end
if !uri.port or uri.port == nil
tport = request['rport']
else
tport = uri.port
end
if !uri.path or uri.path == nil
tpath = "/"
else
tpath = uri.path
end
newp = Pathname.new(tpath)
oldp = Pathname.new(request['uri'])
if !newp.absolute?
if oldp.to_s[-1,1] == '/'
newp = oldp+newp
else
if !newp.to_s.empty?
newp = File.join(oldp.dirname,newp)
end
end
end
hreq = {
'rhost' => thost,
'rport' => tport,
'uri' => newp.to_s,
'method' => m,
'ctype' => 'application/x-www-form-urlencoded',
'ssl' => tssl,
'query' => m == 'GET'? data : uri.query,
'data' => m == 'GET'? nil : data
}
#puts hreq
insertnewpath(hreq)

View File

@ -20,58 +20,8 @@ class CrawlerObjects < BaseParser
s = obj['src']
begin
uri = URI.parse(s)
hreq = urltohash('GET',s,request['uri'],nil)
tssl = false
if uri.scheme == "https"
tssl = true
else
tssl = false
end
if !uri.host or uri.host == nil
thost = request['rhost']
tssl = self.targetssl
else
thost = uri.host
end
if !uri.port or uri.port == nil
tport = request['rport']
else
tport = uri.port
end
if !uri.path or uri.path == nil
tpath = "/"
else
tpath = uri.path
end
newp = Pathname.new(tpath)
oldp = Pathname.new(request['uri'])
if !newp.absolute?
if oldp.to_s[-1,1] == '/'
newp = oldp+newp
else
if !newp.to_s.empty?
newp = File.join(oldp.dirname,newp)
end
end
end
hreq = {
'rhost' => thost,
'rport' => tport,
'uri' => newp.to_s,
'method' => 'GET',
'ctype' => 'text/html',
'ssl' => tssl,
'query' => uri.query,
'data' => nil
}
#puts hreq
insertnewpath(hreq)

View File

@ -20,58 +20,8 @@ class CrawlerScripts < BaseParser
s = obj['src']
begin
uri = URI.parse(s)
hreq = urltohash('GET',s,request['uri'],nil)
tssl = false
if uri.scheme == "https"
tssl = true
else
tssl = false
end
if !uri.host or uri.host == nil
thost = request['rhost']
tssl = self.targetssl
else
thost = uri.host
end
if !uri.port or uri.port == nil
tport = request['rport']
else
tport = uri.port
end
if !uri.path or uri.path == nil
tpath = "/"
else
tpath = uri.path
end
newp = Pathname.new(tpath)
oldp = Pathname.new(request['uri'])
if !newp.absolute?
if oldp.to_s[-1,1] == '/'
newp = oldp+newp
else
if !newp.to_s.empty?
newp = File.join(oldp.dirname,newp)
end
end
end
hreq = {
'rhost' => thost,
'rport' => tport,
'uri' => newp.to_s,
'method' => 'GET',
'ctype' => 'text/html',
'ssl' => tssl,
'query' => uri.query,
'data' => nil
}
#puts hreq
insertnewpath(hreq)