Parser should ignore anchor tags without an href attribute.

youngbrioche · Feb 9, 2011 · a0a6de1 · a0a6de1
1 parent 30b2c95
commit a0a6de1
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/lib/rawler/crawler.rb b/lib/rawler/crawler.rb
@@ -16,7 +16,7 @@ def links
       response = Rawler::Request.get(url)
 
       doc = Nokogiri::HTML(response.body)
-      doc.css('a').map { |a| a['href'] }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
+      doc.css('a').map { |a| a['href'] }.select { |url| !url.nil? }.map { |url| absolute_url(url) }.select { |url| valid_url?(url) }
     rescue Errno::ECONNREFUSED
       write("Couldn't connect to #{url}")
       []

diff --git a/spec/lib/rawler/crawler_spec.rb b/spec/lib/rawler/crawler_spec.rb
@@ -85,7 +85,7 @@
     let(:url)     { 'http://example.com/path' }
     let(:crawler) { Rawler::Crawler.new(url) }
     let(:js_url)  { "javascript:fn('nbjmup;jhfs.esf{fio/dpn');" }
-    let(:content) { "<a href=\"#{js_url}\">foo</a>" }
+    let(:content) { "<a href=\"#{js_url}\">foo</a><a name=\"foo\">" }
 
     before(:each) do
       register(url, content)