Merge branch 'upstream/master' into group_tlv_refactors

Author: OJ

.gitignore

@@ -1,4 +1,6 @@
 .bundle
+Gemfile.local
+Gemfile.local.lock
 # Rubymine project directory
 .idea
 # Sublime Text project directory (not created by ST by default)
@@ -11,10 +13,10 @@
 .DS_Store
 # database config for testing
 config/database.yml
+# target config file for testing
+features/support/targets.yml
 # simplecov coverage data
 coverage
-data/meterpreter/ext_server_pivot.x86.dll
-data/meterpreter/ext_server_pivot.x64.dll
 doc/
 external/source/meterpreter/java/bin
 external/source/meterpreter/java/build
@@ -48,6 +50,30 @@ tags
 *.opensdf
 *.user
 
+# Rails log directory
+/log
+# Rails tmp directory
+/tmp
+
 # ignore release/debug folders for exploits
 external/source/exploits/**/Debug
 external/source/exploits/**/Release
+
+# Avoid checking in Meterpreter binaries. These are supplied upstream by
+# the meterpreter_bins gem.
+data/meterpreter/elevator.*.dll
+data/meterpreter/ext_server_espia.*.dll
+data/meterpreter/ext_server_extapi.*.dll
+data/meterpreter/ext_server_incognito.*.dll
+data/meterpreter/ext_server_kiwi.*.dll
+data/meterpreter/ext_server_lanattacks.*.dll
+data/meterpreter/ext_server_mimikatz.*.dll
+data/meterpreter/ext_server_priv.*.dll
+data/meterpreter/ext_server_stdapi.*.dll
+data/meterpreter/metsrv.*.dll
+data/meterpreter/screenshot.*.dll
+
+# Avoid checking in Meterpreter libs that are built from
+# private source. If you're interested in this functionality,
+# check out Metasploit Pro: http://metasploit.com/download
+data/meterpreter/ext_server_pivot.*.dll

.mailmap

@@ -18,6 +18,7 @@ todb-r7 <todb-r7@github>           Tod Beardsley <[email protected]>
 todb-r7 <todb-r7@github>           Tod Beardsley <[email protected]>
 todb-r7 <todb-r7@github>           Tod Beardsley <[email protected]>
 trosen-r7 <trosen-r7@github>       Trevor Rosen <[email protected]>
+trosen-r7 <trosen-r7@github>       Trevor Rosen <[email protected]>
 wchen-r7 <wchen-r7@github>         sinn3r <[email protected]> # aka sinn3r
 wchen-r7 <wchen-r7@github>         sinn3r <[email protected]>
 wchen-r7 <wchen-r7@github>         Wei Chen <[email protected]>

.rspec

@@ -1,2 +1,3 @@
 --color
 --format Fivemat
+--require spec_helper

.rubocop.yml

@@ -0,0 +1,79 @@
+# This list was intially created by analyzing the last three months (51
+# modules) committed to Metasploit Framework. Many, many older modules
+# will have offenses, but this should at least provide a baseline for
+# new modules.
+#
+# Updates to this file should include a 'Description' parameter for any
+# explaination needed.
+
+# inherit_from: .rubocop_todo.yml
+
+Style/ClassLength:
+  Description: 'Most Metasploit modules are quite large. This is ok.'
+  Enabled: true
+  Exclude:
+    - 'modules/**/*'
+
+Style/Documentation:
+  Enabled: true
+  Description: 'Most Metasploit modules do not have class documentation.'
+  Exclude:
+    - 'modules/**/*'
+
+Style/Encoding:
+  Enabled: true
+  Description: 'We prefer binary to UTF-8.'
+  EnforcedStyle: 'when_needed'
+
+Style/LineLength:
+  Description: >-
+                  Metasploit modules often pattern match against very
+                  long strings when identifying targets.
+  Enabled: true
+  Max: 180
+
+Style/MethodLength:
+  Enabled: true
+  Description: >-
+                  While the style guide suggests 10 lines, exploit definitions
+                  often exceed 200 lines.
+  Max: 300
+
+# Basically everything in metasploit needs binary encoding, not UTF-8.
+# Disable this here and enforce it through msftidy
+Style/Encoding:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+  Description: 'This often hurts readability for exploit-ish code.'
+
+Style/SpaceInsideBrackets:
+  Enabled: false
+  Description: 'Until module template are final, most modules will fail this.'
+
+Style/StringLiterals:
+  Enabled: false
+  Description: 'Single vs double quote fights are largely unproductive.'
+
+Style/WordArray:
+  Enabled: false
+  Description: 'Metasploit prefers consistent use of []'
+
+Style/RedundantBegin:
+  Exclude:
+    # this pattern is very common and somewhat unavoidable
+    # def run_host(ip)
+    #   begin
+    #     ...
+    #   rescue ...
+    #     ...
+    #   ensure
+    #     disconnect
+    #   end
+    # end
+    - 'modules/**/*'
+
+Documentation:
+  Exclude:
+    - 'modules/**/*'

.ruby-version

@@ -1 +1 @@
-1.9.3-p484
+1.9.3-p547

.simplecov

@@ -39,7 +39,6 @@ SimpleCov.configure do
 	# Other library groups
 	#
 
-	add_group 'Fastlib', 'lib/fastlib'
 	add_group 'Metasm', 'lib/metasm'
 	add_group 'PacketFu', 'lib/packetfu'
 	add_group 'Rex', 'lib/rex'

.travis.yml

@@ -1,15 +1,25 @@
+env:
+  - RAKE_TASK=cucumber
+  - RAKE_TASK=cucumber:boot
+  - RAKE_TASK=spec
+
 language: ruby
-env: MSF_SPOTCHECK_RECENT=1
 before_install:
   - rake --version
   - sudo apt-get update -qq
   - sudo apt-get install -qq libpcap-dev
+  # Uncomment when we have fewer shipping msftidy warnings.
+  # Merge committers will still be checking, just not autofailing.
+  # See https://dev.metasploit.com/redmine/issues/8498
+  # - ln -sf ../../tools/dev/pre-commit-hook.rb ./.git/hooks/post-merge
+  # - ls -la ./.git/hooks
+  # - ./.git/hooks/post-merge
 before_script:
-  - ./tools/msftidy.rb
   - cp config/database.yml.travis config/database.yml
   - bundle exec rake --version
   - bundle exec rake db:create
   - bundle exec rake db:migrate
+script: "bundle exec rake $RAKE_TASK"
 
 rvm:
   #- '1.8.7'

.yardopts

@@ -3,5 +3,8 @@
 --exclude \.ut\.rb/
 --exclude \.ts\.rb/
 --files CONTRIBUTING.md,COPYING,HACKING,LICENSE
+app/**/*.rb
 lib/msf/**/*.rb
+lib/metasploit/**/*.rb
 lib/rex/**/*.rb
+plugins/**/*.rb

CONTRIBUTING.md

@@ -1,14 +1,33 @@
-# Contributing to Metasploit
+# Hello, World!
 
 Thanks for your interest in making Metasploit -- and therefore, the
-world -- a better place! What you see here in CONTRIBUTING.md is a
-bullet-point list of the do's and don'ts of how to make sure *your*
-valuable contributions actually make it into Metasploit's master branch.
+world -- a better place!
+
+Are you about to report a bug? Sorry to hear it.
+
+Here's our [Issue tracker](https://github.com/rapid7/metasploit-framework/issues).
+Please try to be as specific as you can about your problem, include steps
+to reproduce (cut and paste from your console output if it's helpful), and
+what you were expecting to happen.
+
+Are you about to report a security vulnerability in Metasploit itself?
+How ironic! Please take a look at Rapid7's [Vulnerability
+Disclosure Policy](https://www.rapid7.com/disclosure.jsp), and send
+your report to [email protected] using [our PGP key](http://pgp.mit.edu:11371/pks/lookup?op=vindex&search=0x2380F85B8AD4DB8D).
+
+Are you about to contribute some new functionality, a bug fix, or a new
+Metasploit module? If so, read on...
+
+# Contributing to Metasploit
+
+What you see here in CONTRIBUTING.md is a bullet-point list of the do's
+and don'ts of how to make sure *your* valuable contributions actually
+make it into Metasploit's master branch.
 
 If you care not to follow these rules, your contribution **will** be
 closed (*Road House* style). Sorry!
 
-Incidentally, this is a **short** list. The
+This is intended to be a **short** list. The
 [wiki](https://github.com/rapid7/metasploit-framework/wiki) is much more
 exhaustive and reveals many mysteries. If you read nothing else, take a
 look at the standard [development environment setup
@@ -18,11 +37,13 @@ and Metasploit's [Common Coding Mistakes](https://github.com/rapid7/metasploit-f
 ## Code Contributions
 
 * **Do** stick to the [Ruby style guide](https://github.com/bbatsov/ruby-style-guide).
+* *Do* get [Rubocop](https://rubygems.org/search?query=rubocop) relatively quiet against the code you are adding or modifying.
 * **Do** follow the [50/72 rule](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) for Git commit messages.
 * **Do** create a [topic branch](http://git-scm.com/book/en/Git-Branching-Branching-Workflows#Topic-Branches) to work on instead of working directly on `master`.
 
 ### Pull Requests
 
+* **Do** target your pull request to the **master branch**. Not staging, not develop, not release.
 * **Do** specify a descriptive title to make searching for your pull request easier.
 * **Do** include [console output](https://help.github.com/articles/github-flavored-markdown#fenced-code-blocks), especially for witnessable effects in `msfconsole`.
 * **Do** list [verification steps](https://help.github.com/articles/writing-on-github#task-lists) so your code is testable.
@@ -34,30 +55,27 @@ Pull requests [#2940](https://github.com/rapid7/metasploit-framework/pull/2940)
 #### New Modules
 
 * **Do** run `tools/msftidy.rb` against your module and fix any errors or warnings that come up. Even better would be to set up `msftidy.rb` as a [pre-commit hook](https://github.com/rapid7/metasploit-framework/blob/master/tools/dev/pre-commit-hook.rb).
-* **Do** use the [API](https://dev.metasploit.com/documents/api/). Wheel improvements are welcome; wheel reinventions, not so much.
+* **Do** use the [many module mixin APIs](https://dev.metasploit.com/api/). Wheel improvements are welcome; wheel reinventions, not so much.
 * **Don't** include more than one module per pull request.
 
 #### Library Code
 
 * **Do** write [RSpec](http://rspec.info/) tests - even the smallest change in library land can thoroughly screw things up.
 * **Do** follow [Better Specs](http://betterspecs.org/) - it's like the style guide for specs.
 * **Do** write [YARD](http://yardoc.org/) documentation - this makes it easier for people to use your code.
+* **Don't** fix a lot of things in one pull request. Small fixes are easier to validate.
 
 #### Bug Fixes
 
 * **Do** include reproduction steps in the form of verification steps.
-* **Do** include a link to the corresponding [Redmine](https://dev.metasploit.com/redmine/projects/framework) issue in the format of `SeeRM #1234` in your commit description.
+* **Do** include a link to any corresponding [Issue](https://github.com/rapid7/metasploit-framework/issues) in the format of `See #1234` in your commit description.
 
 ## Bug Reports
 
-* **Do** report vulnerabilities in Rapid7 software to [email protected].
-* **Do** create a Redmine account and report your bug there.
+* **Do** report vulnerabilities in Rapid7 software directly to [email protected].
 * **Do** write a detailed description of your bug and use a descriptive title.
 * **Do** include reproduction steps, stack traces, and anything else that might help us verify and fix your bug.
 * **Don't** file duplicate reports - search for your bug before filing a new report.
-* **Don't** report a bug on GitHub. Use [Redmine](https://dev.metasploit.com/redmine/projects/framework) instead.
-
-Redmine issues [#8762](https://dev.metasploit.com/redmine/issues/8762) and [#8764](https://dev.metasploit.com/redmine/issues/8764) are a couple good examples to follow.
 
 If you need some more guidance, talk to the main body of open
 source contributors over on the [Freenode IRC channel](http://webchat.freenode.net/?channels=%23metasploit&uio=d4)

Gemfile

@@ -1,60 +1,58 @@
 source 'https://rubygems.org'
-
-# Need 3+ for ActiveSupport::Concern
-gem 'activesupport', '>= 3.0.0'
-# Needed for some admin modules (cfme_manageiq_evm_pass_reset.rb)
-gem 'bcrypt-ruby'
-# Needed for some admin modules (scrutinizer_add_user.rb)
-gem 'json'
-# Needed by msfgui and other rpc components
-gem 'msgpack'
-# Needed by anemone crawler
-gem 'nokogiri'
-# Needed by anemone crawler
-gem 'robots'
-# Needed by db.rb and Msf::Exploit::Capture
-gem 'packetfu', '1.1.9'
+# Add default group gems to `metasploit-framework.gemspec`:
+#   spec.add_runtime_dependency '<name>', [<version requirements>]
+gemspec
 
 group :db do
   # Needed for Msf::DbManager
-  gem 'activerecord'
+  gem 'activerecord', '>= 3.0.0', '< 4.0.0'
+
+  # Metasploit::Credential database models
+  gem 'metasploit-credential', '~> 0.12.0'
   # Database models shared between framework and Pro.
-  gem 'metasploit_data_models', '~> 0.17.0'
+  gem 'metasploit_data_models', '~> 0.21.1'
   # Needed for module caching in Mdm::ModuleDetails
   gem 'pg', '>= 0.11'
 end
 
-group :pcap do
-  gem 'network_interface', '~> 0.0.1'
-  # For sniffer and raw socket modules
-  gem 'pcaprub'
-end
-
 group :development do
   # Markdown formatting for yard
   gem 'redcarpet'
   # generating documentation
   gem 'yard'
+  # for development and testing purposes
+  gem 'pry'
 end
 
 group :development, :test do
   # supplies factories for producing model instance for specs
   # Version 4.1.0 or newer is needed to support generate calls without the
   # 'FactoryGirl.' in factory definitions syntax.
   gem 'factory_girl', '>= 4.1.0'
+  # automatically include factories from spec/factories
+  gem 'factory_girl_rails'
   # Make rspec output shorter and more useful
   gem 'fivemat', '1.2.1'
   # running documentation generation tasks and rspec tasks
   gem 'rake', '>= 10.0.0'
+  # testing framework
+  gem 'rspec', '>= 2.12', '< 3.0.0'
+  # Define `rake spec`.  Must be in development AND test so that its available by default as a rake test when the
+  # environment is development
+  gem 'rspec-rails' , '>= 2.12', '< 3.0.0'
+end
+
+group :pcap do
+  gem 'network_interface', '~> 0.0.1'
+  # For sniffer and raw socket modules
+  gem 'pcaprub'
 end
 
 group :test do
-  # Removes records from database created during tests.  Can't use rspec-rails'
-  # transactional fixtures because multiple connections are in use so
-  # transactions won't work.
-  gem 'database_cleaner'
-  # testing framework
-  gem 'rspec', '>= 2.12'
+  # cucumber extension for testing command line applications, like msfconsole
+  gem 'aruba'
+  # cucumber + automatic database cleaning with database_cleaner
+  gem 'cucumber-rails', :require => false
   gem 'shoulda-matchers'
   # code coverage for tests
   # any version newer than 0.5.4 gives an Encoding error when trying to read the source files.