Merge remote-tracking branch 'upstream/master' into land-8056-outlook

Author: Brent Cook

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    metasploit-framework (4.14.2)
+    metasploit-framework (4.14.3)
       actionpack (~> 4.2.6)
       activerecord (~> 4.2.6)
       activesupport (~> 4.2.6)
@@ -14,7 +14,7 @@ PATH
       metasploit-concern
       metasploit-credential
       metasploit-model
-      metasploit-payloads (= 1.2.17)
+      metasploit-payloads (= 1.2.18)
       metasploit_data_models
       metasploit_payloads-mettle (= 0.1.7)
       msgpack
@@ -104,7 +104,7 @@ GEM
     bcrypt (3.1.11)
     bit-struct (0.15.0)
     builder (3.2.3)
-    capybara (2.12.1)
+    capybara (2.13.0)
       addressable
       mime-types (>= 1.16)
       nokogiri (>= 1.3.3)
@@ -145,8 +145,8 @@ GEM
     ffi (1.9.18)
     filesize (0.1.1)
     fivemat (1.3.2)
-    gherkin (4.0.0)
-    google-protobuf (3.2.0)
+    gherkin (4.1.1)
+    google-protobuf (3.2.0.2)
     googleauth (0.5.1)
       faraday (~> 0.9)
       jwt (~> 1.4)
@@ -164,7 +164,7 @@ GEM
     json (2.0.3)
     jwt (1.5.6)
     little-plugger (1.1.4)
-    logging (2.1.0)
+    logging (2.2.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.10)
     loofah (2.0.3)
@@ -190,7 +190,7 @@ GEM
       activemodel (~> 4.2.6)
       activesupport (~> 4.2.6)
       railties (~> 4.2.6)
-    metasploit-payloads (1.2.17)
+    metasploit-payloads (1.2.18)
     metasploit_data_models (2.0.14)
       activerecord (~> 4.2.6)
       activesupport (~> 4.2.6)
@@ -227,7 +227,7 @@ GEM
       pcaprub
     patch_finder (1.0.2)
     pcaprub (0.12.4)
-    pg (0.19.0)
+    pg (0.20.0)
     pg_array_parser (0.0.9)
     postgres_ext (3.0.0)
       activerecord (>= 4.0.0)
@@ -256,7 +256,7 @@ GEM
       thor (>= 0.18.1, < 2.0)
     rake (12.0.0)
     rb-readline (0.5.4)
-    recog (2.1.4)
+    recog (2.1.5)
       nokogiri
     redcarpet (3.4.0)
     rex-arch (0.1.4)
@@ -335,7 +335,7 @@ GEM
       faraday (~> 0.9)
       jwt (~> 1.5)
       multi_json (~> 1.10)
-    simplecov (0.13.0)
+    simplecov (0.14.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
@@ -350,7 +350,7 @@ GEM
       thread_safe (~> 0.1)
     tzinfo-data (1.2017.1)
       tzinfo (>= 1.0.0)
-    windows_error (0.1.0)
+    windows_error (0.1.1)
     xpath (2.0.0)
       nokogiri (~> 1.3)
     yard (0.9.8)

data/msfcrawler/basic.rb

@@ -1,28 +1,16 @@
 ##
-# $Id$
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
-##
-
-# $Revision$
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
 
 class CrawlerSimple < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
+    return unless result['Content-Type'].include?('text/html')
 
     # doc = Hpricot(result.body.to_s)
     doc = Nokogiri::HTML(result.body.to_s)

data/msfcrawler/comments.rb

@@ -0,0 +1,31 @@
+##
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
+##
+
+require 'pathname'
+require 'nokogiri'
+require 'uri'
+
+class CrawlerComments < BaseParser
+
+  def parse(request,result)
+    return unless result['Content-Type'].include?('text/html')
+
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.xpath('//comment()').each do |comment|
+      # searching for href
+      hr = /href\s*=\s*"([^"]*)"/.match(comment)
+      if hr
+        begin
+          hreq = urltohash('GET', hr[1], request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+          # ignored
+        end
+      end
+
+    end
+
+  end
+end

data/msfcrawler/forms.rb

@@ -1,46 +1,30 @@
 ##
-# $Id$
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
-##
-
-# $Revision$
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
 
 class CrawlerForms < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    hr = ''
-    m = ''
+    return unless result['Content-Type'].include?('text/html')
 
     doc = Nokogiri::HTML(result.body.to_s)
     doc.css('form').each do |f|
       hr = f['action']
 
-      fname = f['name']
-      fname = "NONE" if fname.empty?
+      # Removed because unused
+      #fname = f['name']
+      #fname = 'NONE' if fname.empty?
 
-      m = f['method'].empty? ? 'GET' : f['method'].upcase
-
-      htmlform = Nokogiri::HTML(f.inner_html)
+      m = (f['method'].empty? ? 'GET' : f['method'].upcase)
 
       arrdata = []
 
-      htmlform.css('input').each do |p|
+      f.css('input').each do |p|
         arrdata << "#{p['name']}=#{Rex::Text.uri_encode(p['value'])}"
       end
 
@@ -51,7 +35,10 @@ def parse(request,result)
         hreq['ctype'] = 'application/x-www-form-urlencoded'
         insertnewpath(hreq)
       rescue URI::InvalidURIError
+        #puts "Parse error"
+        #puts "Error: #{link[0]}"
       end
+
     end
   end
 end

data/msfcrawler/frames.rb

@@ -1,13 +1,8 @@
-
 ##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
@@ -27,6 +22,7 @@ def parse(request,result)
           hreq = urltohash('GET', ir, request['uri'], nil)
           insertnewpath(hreq)
         rescue URI::InvalidURIError
+          # ignored
         end
       end
 

data/msfcrawler/image.rb

@@ -1,14 +1,8 @@
-
 ##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-# $Revision: 9212 $
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
@@ -27,6 +21,7 @@ def parse(request,result)
           hreq = urltohash('GET', im, request['uri'], nil)
           insertnewpath(hreq)
         rescue URI::InvalidURIError
+          # ignored
         end
       end
 

data/msfcrawler/link.rb

@@ -1,14 +1,8 @@
-
 ##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-# $Revision: 9212 $
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
@@ -26,6 +20,7 @@ def parse(request,result)
           hreq = urltohash('GET', hr, request['uri'], nil)
           insertnewpath(hreq)
         rescue URI::InvalidURIError
+          # ignored
         end
       end
 

data/msfcrawler/objects.rb

@@ -1,17 +1,8 @@
 ##
-# $Id$
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
-##
-
-# $Revision$
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
@@ -29,6 +20,7 @@ def parse(request,result)
         hreq = urltohash('GET', s, request['uri'], nil)
         insertnewpath(hreq)
       rescue URI::InvalidURIError
+        # ignored
       end
     end
   end

data/msfcrawler/scripts.rb

@@ -1,17 +1,8 @@
 ##
-# $Id$
+# This module requires Metasploit: http://metasploit.com/download
+# Current source: https://github.com/rapid7/metasploit-framework
 ##
 
-##
-# This file is part of the Metasploit Framework and may be subject to
-# redistribution and commercial restrictions. Please see the Metasploit
-# Framework web site for more information on licensing and terms of use.
-# http://metasploit.com/framework/
-##
-
-# $Revision$
-
-require 'rubygems'
 require 'pathname'
 require 'nokogiri'
 require 'uri'
@@ -21,15 +12,14 @@ class CrawlerScripts < BaseParser
   def parse(request,result)
     return unless result['Content-Type'].include? "text/html"
 
-    hr = ''
-    m = ''
     doc = Nokogiri::HTML(result.body.to_s)
     doc.xpath("//script").each do |obj|
       s = obj['src']
       begin
         hreq = urltohash('GET', s, request['uri'], nil)
         insertnewpath(hreq)
       rescue URI::InvalidURIError
+        # ignored
       end
     end