et
0e78719eaf
git-svn-id: file:///home/svn/framework3/trunk@9042 4d416f70-5f16-0410-b530-b9f4589650da
34 lines
566 B
Ruby
34 lines
566 B
Ruby
require 'rubygems'
|
|
require 'pathname'
|
|
require 'hpricot'
|
|
require 'uri'
|
|
|
|
class CrawlerSimple < BaseParser
|
|
|
|
def parse(request,result)
|
|
|
|
if !result['Content-Type'].include? "text/html"
|
|
return
|
|
end
|
|
|
|
doc = Hpricot(result.body.to_s)
|
|
doc.search('a').each do |link|
|
|
|
|
hr = link.attributes['href']
|
|
|
|
if hr and !hr.match(/^(\#|javascript\:)/)
|
|
begin
|
|
hreq = urltohash('GET',hr,request['uri'],nil)
|
|
|
|
insertnewpath(hreq)
|
|
|
|
rescue URI::InvalidURIError
|
|
#puts "Parse error"
|
|
#puts "Error: #{link[0]}"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|