Fix condition
git-svn-id: file:///home/svn/framework3/trunk@8887 4d416f70-5f16-0410-b530-b9f4589650daunstable
parent
1265cccde7
commit
4a326135fc
|
@ -151,6 +151,7 @@ class HttpCrawler
|
|||
|
||||
hashreq = @NotViewedQueue.take(reqfilter, $taketimeout)
|
||||
|
||||
|
||||
if !@ViewedQueue.include?(hashsig(hashreq))
|
||||
@ViewedQueue[hashsig(hashreq)] = Time.now
|
||||
|
||||
|
@ -159,7 +160,7 @@ class HttpCrawler
|
|||
else
|
||||
|
||||
####
|
||||
if i < $threadnum
|
||||
if i <= $threadnum
|
||||
a.push(Thread.new {
|
||||
####
|
||||
prx = nil
|
||||
|
@ -177,8 +178,10 @@ class HttpCrawler
|
|||
)
|
||||
|
||||
|
||||
|
||||
sendreq(c,hashreq)
|
||||
|
||||
|
||||
####
|
||||
})
|
||||
|
||||
|
@ -270,8 +273,9 @@ class HttpCrawler
|
|||
@crawlermodules.each_key do |k|
|
||||
@crawlermodules[k].parse(reqopts,resp)
|
||||
end
|
||||
when 302
|
||||
when 301..302
|
||||
puts "(#{resp.code}) Redirection to: #{resp['Location']}"
|
||||
puts urltohash(resp['Location'])
|
||||
insertnewpath(urltohash(resp['Location']))
|
||||
when 404
|
||||
puts "Invalid link (404) #{reqopts['uri']}"
|
||||
|
|
Loading…
Reference in New Issue