Merge tag '2014072301' into staging/electro-release

Conflicts:
	Gemfile.lock
	modules/post/windows/gather/credentials/gpp.rb

This removes the active flag in the gpp.rb module.  According to Lance,
the active flag is no longer used.

Author: bturner-r7

.rubocop.yml

@@ -0,0 +1,19 @@
+LineLength:
+  Enabled: true
+  Max: 180
+
+MethodLength:
+  Enabled: true
+  Max: 100
+
+Style/ClassLength:
+  Exclude:
+    # Most modules are quite large and all contained in one class.  This is OK.
+    - 'modules/**/*'
+
+Style/NumericLiterals:
+  Enabled: false
+
+Documentation:
+  Exclude:
+    - 'modules/**/*'

CONTRIBUTING.md

@@ -33,6 +33,7 @@ and Metasploit's [Common Coding Mistakes](https://github.com/rapid7/metasploit-f
 ## Code Contributions
 
 * **Do** stick to the [Ruby style guide](https://github.com/bbatsov/ruby-style-guide).
+* Similarly, **try** to get Rubocop passing or at least relatively quiet against the files added/modified as part of your contribution
 * **Do** follow the [50/72 rule](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) for Git commit messages.
 * **Do** create a [topic branch](http://git-scm.com/book/en/Git-Branching-Branching-Workflows#Topic-Branches) to work on instead of working directly on `master`.
 

Gemfile

@@ -15,6 +15,8 @@ group :db do
 end
 
 group :development do
+  # Style/sanity checking Ruby code
+  gem 'rubocop'
   # Markdown formatting for yard
   gem 'redcarpet'
   # generating documentation

Gemfile.lock

@@ -45,6 +45,7 @@ GEM
     arel (3.0.3)
     arel-helpers (2.0.1)
       activerecord (>= 3.1.0, < 5)
+    ast (2.0.0)
     bcrypt (3.1.7)
     builder (3.0.4)
     coderay (1.1.0)
@@ -87,8 +88,12 @@ GEM
     nokogiri (1.6.3.1)
       mini_portile (= 0.6.0)
     packetfu (1.1.9)
+    parser (2.1.9)
+      ast (>= 1.1, < 3.0)
+      slop (~> 3.4, >= 3.4.5)
     pcaprub (0.11.3)
     pg (0.17.1)
+    powerpack (0.0.9)
     pry (0.10.0)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -107,6 +112,7 @@ GEM
       rake (>= 0.8.7)
       rdoc (~> 3.4)
       thor (>= 0.14.6, < 2.0)
+    rainbow (2.0.0)
     rake (10.3.2)
     rdoc (3.12.2)
       json (~> 1.4)
@@ -132,10 +138,17 @@ GEM
       rspec-core (~> 2.99.0)
       rspec-expectations (~> 2.99.0)
       rspec-mocks (~> 2.99.0)
+    rubocop (0.23.0)
+      json (>= 1.7.7, < 2)
+      parser (~> 2.1.9)
+      powerpack (~> 0.0.6)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.4)
+    ruby-progressbar (1.5.1)
+      activesupport (>= 3.0.0)
     rubyntlm (0.4.0)
     rubyzip (1.1.6)
     shoulda-matchers (2.6.2)
-      activesupport (>= 3.0.0)
     simplecov (0.5.4)
       multi_json (~> 1.0.3)
       simplecov-html (~> 0.5.3)
@@ -172,6 +185,7 @@ DEPENDENCIES
   redcarpet
   rspec (>= 2.12, < 3.0.0)
   rspec-rails (>= 2.12, < 3.0.0)
+  rubocop
   shoulda-matchers
   simplecov (= 0.5.4)
   timecop

data/msfcrawler/basic.rb

@@ -13,7 +13,7 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerSimple < BaseParser
@@ -24,23 +24,20 @@ def parse(request,result)
       return
     end
 
-    doc = Hpricot(result.body.to_s)
-    doc.search('a').each do |link|
-
-    hr = link.attributes['href']
-
-    if hr and !hr.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',hr,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
+    # doc = Hpricot(result.body.to_s)
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('a').each do |anchor_tag|
+      hr = anchor_tag['href']
+      if hr && !hr.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', hr, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+          #puts "Parse error"
+          #puts "Error: #{link[0]}"
+        end
       end
     end
-    end
   end
 end
 

data/msfcrawler/forms.rb

@@ -13,7 +13,7 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerForms < BaseParser
@@ -27,49 +27,30 @@ def parse(request,result)
     hr = ''
     m = ''
 
-    doc = Hpricot(result.body.to_s)
-    doc.search('form').each do |f|
-      hr = f.attributes['action']
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('form').each do |f|
+      hr = f['action']
 
-      fname = f.attributes['name']
-      if fname.empty?
-        fname = "NONE"
-      end
-
-      m = "GET"
-      if !f.attributes['method'].empty?
-        m = f.attributes['method'].upcase
-      end
+      fname = f['name']
+      fname = "NONE" if fname.empty?
 
-      #puts "Parsing form name: #{fname} (#{m})"
+      m = f['method'].empty? ? 'GET' : f['method'].upcase
 
-      htmlform = Hpricot(f.inner_html)
+      htmlform = Nokogiri::HTML(f.inner_html)
 
       arrdata = []
 
-      htmlform.search('input').each do |p|
-        #puts p.attributes['name']
-        #puts p.attributes['type']
-        #puts p.attributes['value']
-
-        #raw_request has uri_encoding disabled as it encodes '='.
-        arrdata << (p.attributes['name'] + "=" + Rex::Text.uri_encode(p.attributes['value']))
+      htmlform.css('input').each do |p|
+        arrdata << "#{p['name']}=#{Rex::Text.uri_encode(p['value'])}"
       end
 
       data = arrdata.join("&").to_s
 
-
       begin
-        hreq = urltohash(m,hr,request['uri'],data)
-
+        hreq = urltohash(m, hr, request['uri'], data)
         hreq['ctype'] = 'application/x-www-form-urlencoded'
-
         insertnewpath(hreq)
-
-
       rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
       end
     end
   end

data/msfcrawler/frames.rb

@@ -9,33 +9,29 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerFrames < BaseParser
 
   def parse(request,result)
 
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('iframe').each do |ifra|
-
-    ir = ifra.attributes['src']
-
-    if ir and !ir.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',ir,request['uri'],nil)
+    return unless result['Content-Type'].include?('text/html')
 
-        insertnewpath(hreq)
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('iframe').each do |ifra|
+      ir = ifra['src']
 
-      rescue URI::InvalidURIError
-        #puts "Error"
+      if ir && !ir.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', ir, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
+
 end
 

data/msfcrawler/image.rb

@@ -10,33 +10,26 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerImage < BaseParser
 
   def parse(request,result)
 
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('img').each do |i|
-
-    im = i.attributes['src']
-
-    if im and !im.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',im,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{i[0]}"
+    return unless result['Content-Type'].include?('text/html')
+
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('img').each do |i|
+      im = i['src']
+      if im && !im.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', im, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
 end

data/msfcrawler/link.rb

@@ -10,33 +10,25 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerLink < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
-    doc = Hpricot(result.body.to_s)
-    doc.search('link').each do |link|
-
-    hr = link.attributes['href']
-
-    if hr and !hr.match(/^(\#|javascript\:)/)
-      begin
-        hreq = urltohash('GET',hr,request['uri'],nil)
-
-        insertnewpath(hreq)
-
-      rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
+    return unless result['Content-Type'].include?('text/html')
+
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.css('link').each do |link|
+      hr = link['href']
+      if hr && !hr.match(/^(\#|javascript\:)/)
+        begin
+          hreq = urltohash('GET', hr, request['uri'], nil)
+          insertnewpath(hreq)
+        rescue URI::InvalidURIError
+        end
       end
-    end
+
     end
   end
 end

data/msfcrawler/objects.rb

@@ -13,36 +13,25 @@
 
 require 'rubygems'
 require 'pathname'
-require 'hpricot'
+require 'nokogiri'
 require 'uri'
 
 class CrawlerObjects < BaseParser
 
   def parse(request,result)
-
-    if !result['Content-Type'].include? "text/html"
-      return
-    end
-
+    return unless result['Content-Type'].include?('text/html') # TOOD: use MIXIN
     hr = ''
     m = ''
-
-    doc = Hpricot(result.body.to_s)
-    doc.search("//object/embed").each do |obj|
-
+    doc = Nokogiri::HTML(result.body.to_s)
+    doc.xpath("//object/embed").each do |obj|
       s = obj['src']
-
       begin
-        hreq = urltohash('GET',s,request['uri'],nil)
-
+        hreq = urltohash('GET', s, request['uri'], nil)
         insertnewpath(hreq)
-
-
       rescue URI::InvalidURIError
-        #puts "Parse error"
-        #puts "Error: #{link[0]}"
       end
     end
   end
+
 end