Retab all the things (except external/)

This commit is contained in:
Tab Assassin
2013-09-30 13:47:53 -05:00
parent 0ecba377f5
commit 2e8d19edcf
293 changed files with 32962 additions and 32962 deletions
+18 -18
View File
@@ -18,29 +18,29 @@ require 'uri'
class CrawlerSimple < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
doc = Hpricot(result.body.to_s)
doc.search('a').each do |link|
doc = Hpricot(result.body.to_s)
doc.search('a').each do |link|
hr = link.attributes['href']
hr = link.attributes['href']
if hr and !hr.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',hr,request['uri'],nil)
if hr and !hr.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',hr,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
end
+38 -38
View File
@@ -18,60 +18,60 @@ require 'uri'
class CrawlerForms < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
hr = ''
m = ''
hr = ''
m = ''
doc = Hpricot(result.body.to_s)
doc.search('form').each do |f|
hr = f.attributes['action']
doc = Hpricot(result.body.to_s)
doc.search('form').each do |f|
hr = f.attributes['action']
fname = f.attributes['name']
if fname.empty?
fname = "NONE"
end
fname = f.attributes['name']
if fname.empty?
fname = "NONE"
end
m = "GET"
if !f.attributes['method'].empty?
m = f.attributes['method'].upcase
end
m = "GET"
if !f.attributes['method'].empty?
m = f.attributes['method'].upcase
end
#puts "Parsing form name: #{fname} (#{m})"
#puts "Parsing form name: #{fname} (#{m})"
htmlform = Hpricot(f.inner_html)
htmlform = Hpricot(f.inner_html)
arrdata = []
arrdata = []
htmlform.search('input').each do |p|
#puts p.attributes['name']
#puts p.attributes['type']
#puts p.attributes['value']
htmlform.search('input').each do |p|
#puts p.attributes['name']
#puts p.attributes['type']
#puts p.attributes['value']
#raw_request has uri_encoding disabled as it encodes '='.
arrdata << (p.attributes['name'] + "=" + Rex::Text.uri_encode(p.attributes['value']))
end
#raw_request has uri_encoding disabled as it encodes '='.
arrdata << (p.attributes['name'] + "=" + Rex::Text.uri_encode(p.attributes['value']))
end
data = arrdata.join("&").to_s
data = arrdata.join("&").to_s
begin
hreq = urltohash(m,hr,request['uri'],data)
begin
hreq = urltohash(m,hr,request['uri'],data)
hreq['ctype'] = 'application/x-www-form-urlencoded'
hreq['ctype'] = 'application/x-www-form-urlencoded'
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
+17 -17
View File
@@ -14,28 +14,28 @@ require 'uri'
class CrawlerFrames < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
doc = Hpricot(result.body.to_s)
doc.search('iframe').each do |ifra|
doc = Hpricot(result.body.to_s)
doc.search('iframe').each do |ifra|
ir = ifra.attributes['src']
ir = ifra.attributes['src']
if ir and !ir.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',ir,request['uri'],nil)
if ir and !ir.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',ir,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Error"
end
end
end
end
rescue URI::InvalidURIError
#puts "Error"
end
end
end
end
end
+18 -18
View File
@@ -15,29 +15,29 @@ require 'uri'
class CrawlerImage < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
doc = Hpricot(result.body.to_s)
doc.search('img').each do |i|
doc = Hpricot(result.body.to_s)
doc.search('img').each do |i|
im = i.attributes['src']
im = i.attributes['src']
if im and !im.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',im,request['uri'],nil)
if im and !im.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',im,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{i[0]}"
end
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{i[0]}"
end
end
end
end
end
+18 -18
View File
@@ -15,29 +15,29 @@ require 'uri'
class CrawlerLink < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
doc = Hpricot(result.body.to_s)
doc.search('link').each do |link|
doc = Hpricot(result.body.to_s)
doc.search('link').each do |link|
hr = link.attributes['href']
hr = link.attributes['href']
if hr and !hr.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',hr,request['uri'],nil)
if hr and !hr.match(/^(\#|javascript\:)/)
begin
hreq = urltohash('GET',hr,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
end
+18 -18
View File
@@ -18,31 +18,31 @@ require 'uri'
class CrawlerObjects < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
hr = ''
m = ''
hr = ''
m = ''
doc = Hpricot(result.body.to_s)
doc.search("//object/embed").each do |obj|
doc = Hpricot(result.body.to_s)
doc.search("//object/embed").each do |obj|
s = obj['src']
s = obj['src']
begin
hreq = urltohash('GET',s,request['uri'],nil)
begin
hreq = urltohash('GET',s,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end
+18 -18
View File
@@ -18,31 +18,31 @@ require 'uri'
class CrawlerScripts < BaseParser
def parse(request,result)
def parse(request,result)
if !result['Content-Type'].include? "text/html"
return
end
if !result['Content-Type'].include? "text/html"
return
end
hr = ''
m = ''
hr = ''
m = ''
doc = Hpricot(result.body.to_s)
doc.search("//script").each do |obj|
doc = Hpricot(result.body.to_s)
doc.search("//script").each do |obj|
s = obj['src']
s = obj['src']
begin
hreq = urltohash('GET',s,request['uri'],nil)
begin
hreq = urltohash('GET',s,request['uri'],nil)
insertnewpath(hreq)
insertnewpath(hreq)
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
rescue URI::InvalidURIError
#puts "Parse error"
#puts "Error: #{link[0]}"
end
end
end
end