HD Moore
b3cc6e19b6
git-svn-id: file:///home/svn/framework3/trunk@10924 4d416f70-5f16-0410-b530-b9f4589650da
42 lines
813 B
Ruby
42 lines
813 B
Ruby
require 'anemone'
|
|
require 'optparse'
|
|
require 'ostruct'
|
|
|
|
options = OpenStruct.new
|
|
options.relative = false
|
|
|
|
begin
|
|
# make sure that the last option is a URL we can crawl
|
|
root = URI(ARGV.last)
|
|
rescue
|
|
puts <<-INFO
|
|
Usage:
|
|
anemone url-list [options] <url>
|
|
|
|
Synopsis:
|
|
Crawls a site starting at the given URL, and outputs the URL of each page
|
|
in the domain as they are encountered.
|
|
|
|
Options:
|
|
-r, --relative Output relative URLs (rather than absolute)
|
|
INFO
|
|
exit(0)
|
|
end
|
|
|
|
# parse command-line options
|
|
opts = OptionParser.new
|
|
opts.on('-r', '--relative') { options.relative = true }
|
|
opts.parse!(ARGV)
|
|
|
|
Anemone.crawl(root, :discard_page_bodies => true) do |anemone|
|
|
|
|
anemone.on_every_page do |page|
|
|
if options.relative
|
|
puts page.url.path
|
|
else
|
|
puts page.url
|
|
end
|
|
end
|
|
|
|
end
|