0.0.2

youngbrioche · Jan 10, 2011 · 911e235 · 911e235
1 parent 5f285f0
commit 911e235
Show file tree

Hide file tree

Showing 10 changed files with 88 additions and 10 deletions.
diff --git a/History.txt b/History.txt
@@ -1,4 +1,10 @@
-=== 1.0.0 / 2011-01-10
+=== 0.0.2 / 2011-01-10
+
+* 1 major enhancement
+
+  * Handle relative urls
+
+=== 0.0.1 / 2011-01-10
 
 * 1 major enhancement
 

diff --git a/README.txt b/README.txt
@@ -16,7 +16,6 @@ gem install rawler
 
 == TODO
 
-* Handle relative urls!
 * Handle https (now returns 400). See http://stackoverflow.com/questions/1719809/ruby-on-rails-https-post-bad-request
 * Export to html
 * Handle multiple urls at once

diff --git a/lib/rawler.rb b/lib/rawler.rb
@@ -6,7 +6,7 @@
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Rawler
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 
   autoload :Base, "rawler/base"
   autoload :Crawler, "rawler/crawler"

diff --git a/lib/rawler/crawler.rb b/lib/rawler/crawler.rb
@@ -12,11 +12,17 @@ def links
       content = Net::HTTP.get(URI.parse(url))
 
       doc = Nokogiri::HTML(content)
-      doc.css('a').map { |a| a['href'] }
+      doc.css('a').map { |a| absolute_url(a['href']) }
     rescue Errno::ECONNREFUSED
       $output.puts "Couldn't connect to #{url}"
       []
     end
+
+    private
+
+    def absolute_url(path)
+      URI.parse(url).merge(path.to_s).to_s
+    end
 
   end
 

diff --git a/pkg/rawler-0.0.1/History.txt b/pkg/rawler-0.0.1/History.txt
@@ -1,4 +1,10 @@
-=== 1.0.0 / 2011-01-10
+=== 0.0.2 / 2011-01-10
+
+* 1 major enhancement
+
+  * Handle relative urls
+
+=== 0.0.1 / 2011-01-10
 
 * 1 major enhancement
 

diff --git a/pkg/rawler-0.0.1/README.txt b/pkg/rawler-0.0.1/README.txt
@@ -16,7 +16,6 @@ gem install rawler
 
 == TODO
 
-* Handle relative urls!
 * Handle https (now returns 400). See http://stackoverflow.com/questions/1719809/ruby-on-rails-https-post-bad-request
 * Export to html
 * Handle multiple urls at once

diff --git a/pkg/rawler-0.0.1/lib/rawler.rb b/pkg/rawler-0.0.1/lib/rawler.rb
@@ -6,7 +6,7 @@
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module Rawler
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 
   autoload :Base, "rawler/base"
   autoload :Crawler, "rawler/crawler"

diff --git a/pkg/rawler-0.0.1/lib/rawler/crawler.rb b/pkg/rawler-0.0.1/lib/rawler/crawler.rb
@@ -12,11 +12,17 @@ def links
       content = Net::HTTP.get(URI.parse(url))
 
       doc = Nokogiri::HTML(content)
-      doc.css('a').map { |a| a['href'] }
+      doc.css('a').map { |a| absolute_url(a['href']) }
     rescue Errno::ECONNREFUSED
       $output.puts "Couldn't connect to #{url}"
       []
     end
+
+    private
+
+    def absolute_url(path)
+      URI.parse(url).merge(path.to_s).to_s
+    end
 
   end
 

diff --git a/pkg/rawler-0.0.1/spec/unit/crawler_spec.rb b/pkg/rawler-0.0.1/spec/unit/crawler_spec.rb
@@ -3,12 +3,40 @@
 describe Rawler::Crawler do
 
   it "should parse all links" do
-    url = 'http://example.com'
+    url = 'http://example.com/'
     register(url, site)
 
     Rawler::Crawler.new(url).links.should == ['http://example.com/foo', 'http://external.com/bar']
   end
 
+  it "should return an empty array when raising Errno::ECONNREFUSED" do
+    url = 'http://example.com'
+    register(url, site)
+
+    Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+
+    crawler = Rawler::Crawler.new(url).links.should == []
+  end
+
+  it "should parse relative links" do
+    url = 'http://example.com/path'
+    register(url, '<a href="/foo">foo</a>')
+
+    Rawler::Crawler.new(url).links.should == ['http://example.com/foo']
+  end
+
+  # it "should print a message when raising Errno::ECONNREFUSED" do
+  #   pending "refactor output. Don't use a global variable"
+  #   url = 'http://example.com'
+  #   register(url, site)
+  #   
+  #   Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+  #   
+  #   $stdout.should_receive(:puts).with("Couldn't connect to #{url}")
+  #   
+  #   Rawler::Crawler.new(url).links
+  # end
+
   private
 
   def site

diff --git a/spec/unit/crawler_spec.rb b/spec/unit/crawler_spec.rb
@@ -3,12 +3,40 @@
 describe Rawler::Crawler do
 
   it "should parse all links" do
-    url = 'http://example.com'
+    url = 'http://example.com/'
     register(url, site)
 
     Rawler::Crawler.new(url).links.should == ['http://example.com/foo', 'http://external.com/bar']
   end
 
+  it "should return an empty array when raising Errno::ECONNREFUSED" do
+    url = 'http://example.com'
+    register(url, site)
+
+    Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+
+    crawler = Rawler::Crawler.new(url).links.should == []
+  end
+
+  it "should parse relative links" do
+    url = 'http://example.com/path'
+    register(url, '<a href="/foo">foo</a>')
+
+    Rawler::Crawler.new(url).links.should == ['http://example.com/foo']
+  end
+
+  # it "should print a message when raising Errno::ECONNREFUSED" do
+  #   pending "refactor output. Don't use a global variable"
+  #   url = 'http://example.com'
+  #   register(url, site)
+  #   
+  #   Net::HTTP.should_receive(:get).and_raise Errno::ECONNREFUSED
+  #   
+  #   $stdout.should_receive(:puts).with("Couldn't connect to #{url}")
+  #   
+  #   Rawler::Crawler.new(url).links
+  # end
+
   private
 
   def site