Land rapid7#3537 from @pagedegeek, msfcrawler fix

Author: Tod Beardsley

Gemfile

@@ -21,8 +21,6 @@ gem 'robots'
 # Needed for some post modules
 gem 'sqlite3'
 
-gem 'hpricot', :require => 'hpricot'
-
 group :db do
   # Needed for Msf::DbManager
   gem 'activerecord', '>= 3.0.0', '< 4.0.0'

Gemfile.lock

@@ -21,7 +21,6 @@ GEM
     factory_girl (4.2.0)
       activesupport (>= 3.0.0)
     fivemat (1.2.1)
-    hpricot (0.8.6)
     i18n (0.6.5)
     json (1.8.0)
     metasploit_data_models (0.17.0)
@@ -84,7 +83,6 @@ DEPENDENCIES
   database_cleaner
   factory_girl (>= 4.1.0)
   fivemat (= 1.2.1)
-  hpricot
   json
   metasploit_data_models (= 0.17.0)
   meterpreter_bins (= 0.0.6)

data/msfcrawler/basic.rb

@@ -13,7 +13,7 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerSimple < BaseParser
@@ -24,23 +24,20 @@ def parse(request,result)
       return
     end
 
-    doc = Hpricot(result.body.to_s)
-    doc.search('a').each do |link|
-
-    hr = link.attributes['href']
-
-    if hr and !hr.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',hr,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
+    # doc = Hpricot(result.body.to_s)
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('a').each do |anchor_tag|
+      hr = anchor_tag['href']
+      if hr && !hr.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', hr, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+          #puts "Parse error"
+          #puts "Error: #{link[0]}"
+        end
       end
     end
-    end
   end
 end
 

data/msfcrawler/forms.rb

@@ -13,7 +13,7 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerForms < BaseParser
@@ -27,49 +27,30 @@ def parse(request,result)
     hr = ''
     m = ''
 
-    doc = Hpricot(result.body.to_s)
-    doc.search('form').each do |f|
-      hr = f.attributes['action']
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('form').each do |f|
+      hr = f['action']
 
-      fname = f.attributes['name']
-      if fname.empty?
-        fname = "NONE"
-      end
-
-      m = "GET"
-      if !f.attributes['method'].empty?
-        m = f.attributes['method'].upcase
-      end
+      fname = f['name']
+      fname = "NONE" if fname.empty?
 
-      #puts "Parsing form name: #{fname} (#{m})"
+      m = f['method'].empty? ? 'GET' : f['method'].upcase
 
-      htmlform = Hpricot(f.inner_html)
+      htmlform = Nokogiri::HTML(f.inner_html)
 
       arrdata = []
 
-      htmlform.search('input').each do |p|
-        #puts p.attributes['name']
-        #puts p.attributes['type']
-        #puts p.attributes['value']
-
-        #raw_request has uri_encoding disabled as it encodes '='.
-        arrdata << (p.attributes['name'] + "=" + Rex::Text.uri_encode(p.attributes['value']))
+      htmlform.css('input').each do |p|
+        arrdata << "#{p['name']}=#{Rex::Text.uri_encode(p['value'])}"
       end
 
       data = arrdata.join("&").to_s
 
-
       begin
-        hreq = urltohash(m,hr,request['uri'],data)
-
+        hreq = urltohash(m, hr, request['uri'], data)
         hreq['ctype'] = 'application/x-www-form-urlencoded'
-
         insertnewpath(hreq)
-
-
       rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
       end
     end
   end

data/msfcrawler/frames.rb

@@ -9,33 +9,29 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerFrames < BaseParser
 
   def parse(request,result)
 
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('iframe').each do |ifra|
-
-    ir = ifra.attributes['src']
-
-    if ir and !ir.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',ir,request['uri'],nil)
+    return unless result['Content-Type'].include?('text/html')
 
-        insertnewpath(hreq)
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('iframe').each do |ifra|
+      ir = ifra['src']
 
-      rescue URI::InvalidURIError
-        #puts "Error"
+      if ir && !ir.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', ir, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
+
 end
 

data/msfcrawler/image.rb

@@ -10,33 +10,26 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerImage < BaseParser
 
   def parse(request,result)
 
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('img').each do |i|
-
-    im = i.attributes['src']
-
-    if im and !im.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',im,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{i[0]}"
+    return unless result['Content-Type'].include?('text/html')
+
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('img').each do |i|
+      im = i['src']
+      if im && !im.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', im, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
 end

data/msfcrawler/link.rb

@@ -10,33 +10,25 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerLink < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('link').each do |link|
-
-    hr = link.attributes['href']
-
-    if hr and !hr.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',hr,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
+    return unless result['Content-Type'].include?('text/html')
+
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('link').each do |link|
+      hr = link['href']
+      if hr && !hr.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', hr, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
 end

data/msfcrawler/objects.rb

@@ -13,36 +13,25 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerObjects < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
+    return unless result['Content-Type'].include?('text/html') # TOOD: use MIXIN
     hr = ''
     m = ''
-
-    doc = Hpricot(result.body.to_s)
-    doc.search("//object/embed").each do |obj|
-
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.xpath("//object/embed").each do |obj|
       s = obj['src']
-
       begin
-        hreq = urltohash('GET',s,request['uri'],nil)
-
+        hreq = urltohash('GET', s, request['uri'], nil)
         insertnewpath(hreq)
-
-
       rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
       end
     end
   end
+
 end
 

data/msfcrawler/scripts.rb

@@ -13,36 +13,27 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerScripts < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
+    return unless result['Content-Type'].include? "text/html"
 
     hr = ''
     m = ''
-
-    doc = Hpricot(result.body.to_s)
-    doc.search("//script").each do |obj|
-
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.xpath("//script").each do |obj|
       s = obj['src']
-
       begin
-        hreq = urltohash('GET',s,request['uri'],nil)
-
+        hreq = urltohash('GET', s, request['uri'], nil)
         insertnewpath(hreq)
-
-
       rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
       end
     end
+
   end
+
 end