Merge pull request rapid7#1813 from limhoff-r7/feature/mdm-module-namespace

Updates to MDM 0.11.2

Author: bturner-r7

Gemfile

@@ -12,43 +12,48 @@ gem 'nokogiri'
 gem 'robots'
 
 group :db do
-  # Needed for Msf::DbManager
-  gem 'activerecord'
-  # Database models shared between framework and Pro.
-  gem 'metasploit_data_models', '~> 0.6.16'
-  # Needed for module caching in Mdm::ModuleDetails
-  gem 'pg', '>= 0.11'
+	# Needed for Msf::DbManager
+	gem 'activerecord'
+	# Database models shared between framework and Pro.
+	gem 'metasploit_data_models', '~> 0.11.2'
+	# Needed for module caching in Mdm::ModuleDetails
+	gem 'pg', '>= 0.11'
 end
 
 group :pcap do
-  # For sniffer and raw socket modules
-  gem 'pcaprub'
+	# For sniffer and raw socket modules
+	gem 'pcaprub'
 end
 
 group :development do
-  # Markdown formatting for yard
-  gem 'redcarpet'
-  # generating documentation
-  gem 'yard'
+	# Markdown formatting for yard
+	gem 'redcarpet'
+	# generating documentation
+	gem 'yard'
 end
 
 group :development, :test do
 	# supplies factories for producing model instance for specs
-  # Version 4.1.0 or newer is needed to support generate calls without the
-  # 'FactoryGirl.' in factory definitions syntax.
-  gem 'factory_girl', '>= 4.1.0'
-  # running documentation generation tasks and rspec tasks
-  gem 'rake'
+	# Version 4.1.0 or newer is needed to support generate calls without the
+	# 'FactoryGirl.' in factory definitions syntax.
+	gem 'factory_girl', '>= 4.1.0'
+	# running documentation generation tasks and rspec tasks
+	gem 'rake'
 end
 
 group :test do
 	# Removes records from database created during tests.  Can't use rspec-rails'
 	# transactional fixtures because multiple connections are in use so
 	# transactions won't work.
 	gem 'database_cleaner'
-  # testing framework
-  gem 'rspec', '>= 2.12'
-  # code coverage for tests
-  # any version newer than 0.5.4 gives an Encoding error when trying to read the source files.
-  gem 'simplecov', '0.5.4', :require => false
+	# testing framework
+	gem 'rspec', '>= 2.12'
+	# add matchers from shoulda, such as query_the_database, which is useful for
+	# testing that the Msf::DBManager activation is respected.
+	gem 'shoulda-matchers'
+	# code coverage for tests
+	# any version newer than 0.5.4 gives an Encoding error when trying to read the source files.
+	gem 'simplecov', '0.5.4', :require => false
+	# Manipulate Time.now in specs
+	gem 'timecop'
 end

Gemfile.lock

@@ -13,22 +13,27 @@ GEM
       i18n (= 0.6.1)
       multi_json (~> 1.0)
     arel (3.0.2)
+    bourne (1.4.0)
+      mocha (~> 0.13.2)
     builder (3.0.4)
     database_cleaner (0.9.1)
     diff-lcs (1.2.2)
     factory_girl (4.2.0)
       activesupport (>= 3.0.0)
     i18n (0.6.1)
     json (1.7.7)
-    metasploit_data_models (0.6.16)
+    metaclass (0.0.1)
+    metasploit_data_models (0.11.2)
       activerecord (>= 3.2.13)
       activesupport
       pg
+    mocha (0.13.3)
+      metaclass (~> 0.0.1)
     msgpack (0.5.4)
     multi_json (1.0.4)
     nokogiri (1.5.9)
     pcaprub (0.11.3)
-    pg (0.15.0)
+    pg (0.15.1)
     rake (10.0.4)
     redcarpet (2.2.2)
     robots (0.10.1)
@@ -40,10 +45,14 @@ GEM
     rspec-expectations (2.13.0)
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.13.0)
+    shoulda-matchers (1.5.2)
+      activesupport (>= 3.0.0)
+      bourne (~> 1.3)
     simplecov (0.5.4)
       multi_json (~> 1.0.3)
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
+    timecop (0.6.1)
     tzinfo (0.3.37)
     yard (0.8.5.2)
 
@@ -56,7 +65,7 @@ DEPENDENCIES
   database_cleaner
   factory_girl (>= 4.1.0)
   json
-  metasploit_data_models (~> 0.6.16)
+  metasploit_data_models (~> 0.11.2)
   msgpack
   nokogiri
   pcaprub
@@ -65,5 +74,7 @@ DEPENDENCIES
   redcarpet
   robots
   rspec (>= 2.12)
+  shoulda-matchers
   simplecov (= 0.5.4)
+  timecop
   yard

Rakefile

@@ -36,6 +36,17 @@ else
 	task :default => :spec
 end
 
+# Require yard before loading metasploit_data_models rake tasks as the yard tasks won't be defined if
+# YARD is not defined when yard.rake is loaded.
+begin
+  require 'yard'
+rescue LoadError
+	puts "yard not in bundle, so can't set up yard tasks.  " \
+	     "To generate documentation ensure to install the development group."
+
+	print_without = true
+end
+
 begin
 	require 'metasploit_data_models'
 rescue LoadError
@@ -58,14 +69,6 @@ else
 	end
 end
 
-begin
-  require 'yard'
-rescue LoadError
-	puts "yard not in bundle, so can't set up yard tasks.  " \
-	     "To generate documentation ensure to install the development group."
-
-	print_without = true
-end
 
 
 if print_without

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20130228214900) do
+ActiveRecord::Schema.define(:version => 20130430162145) do
 
   create_table "api_keys", :force => true do |t|
     t.text     "token"
@@ -135,7 +135,7 @@
 
   create_table "hosts", :force => true do |t|
     t.datetime "created_at"
-    t.string   "address",               :limit => nil
+    t.string   "address",               :limit => nil,                  :null => false
     t.string   "mac"
     t.string   "comm"
     t.string   "name"
@@ -145,7 +145,7 @@
     t.string   "os_sp"
     t.string   "os_lang"
     t.string   "arch"
-    t.integer  "workspace_id"
+    t.integer  "workspace_id",                                          :null => false
     t.datetime "updated_at"
     t.text     "purpose"
     t.string   "info",                  :limit => 65536
@@ -157,14 +157,15 @@
     t.integer  "service_count",                          :default => 0
     t.integer  "host_detail_count",                      :default => 0
     t.integer  "exploit_attempt_count",                  :default => 0
+    t.integer  "cred_count",                             :default => 0
   end
 
-  add_index "hosts", ["address"], :name => "index_hosts_on_address"
   add_index "hosts", ["name"], :name => "index_hosts_on_name"
   add_index "hosts", ["os_flavor"], :name => "index_hosts_on_os_flavor"
   add_index "hosts", ["os_name"], :name => "index_hosts_on_os_name"
   add_index "hosts", ["purpose"], :name => "index_hosts_on_purpose"
   add_index "hosts", ["state"], :name => "index_hosts_on_state"
+  add_index "hosts", ["workspace_id", "address"], :name => "index_hosts_on_workspace_id_and_address", :unique => true
 
   create_table "hosts_tags", :id => false, :force => true do |t|
     t.integer "host_id"
@@ -223,26 +224,26 @@
   end
 
   create_table "module_actions", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
   end
 
-  add_index "module_actions", ["module_detail_id"], :name => "index_module_actions_on_module_detail_id"
+  add_index "module_actions", ["detail_id"], :name => "index_module_actions_on_module_detail_id"
 
   create_table "module_archs", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
   end
 
-  add_index "module_archs", ["module_detail_id"], :name => "index_module_archs_on_module_detail_id"
+  add_index "module_archs", ["detail_id"], :name => "index_module_archs_on_module_detail_id"
 
   create_table "module_authors", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
     t.text    "email"
   end
 
-  add_index "module_authors", ["module_detail_id"], :name => "index_module_authors_on_module_detail_id"
+  add_index "module_authors", ["detail_id"], :name => "index_module_authors_on_module_detail_id"
 
   create_table "module_details", :force => true do |t|
     t.datetime "mtime"
@@ -268,34 +269,34 @@
   add_index "module_details", ["refname"], :name => "index_module_details_on_refname"
 
   create_table "module_mixins", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
   end
 
-  add_index "module_mixins", ["module_detail_id"], :name => "index_module_mixins_on_module_detail_id"
+  add_index "module_mixins", ["detail_id"], :name => "index_module_mixins_on_module_detail_id"
 
   create_table "module_platforms", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
   end
 
-  add_index "module_platforms", ["module_detail_id"], :name => "index_module_platforms_on_module_detail_id"
+  add_index "module_platforms", ["detail_id"], :name => "index_module_platforms_on_module_detail_id"
 
   create_table "module_refs", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.text    "name"
   end
 
-  add_index "module_refs", ["module_detail_id"], :name => "index_module_refs_on_module_detail_id"
+  add_index "module_refs", ["detail_id"], :name => "index_module_refs_on_module_detail_id"
   add_index "module_refs", ["name"], :name => "index_module_refs_on_name"
 
   create_table "module_targets", :force => true do |t|
-    t.integer "module_detail_id"
+    t.integer "detail_id"
     t.integer "index"
     t.text    "name"
   end
 
-  add_index "module_targets", ["module_detail_id"], :name => "index_module_targets_on_module_detail_id"
+  add_index "module_targets", ["detail_id"], :name => "index_module_targets_on_module_detail_id"
 
   create_table "nexpose_consoles", :force => true do |t|
     t.datetime "created_at",                     :null => false

lib/msf/core/db.rb

@@ -645,12 +645,69 @@ def get_session(opts)
 	}
 	end
 
-	# Record a new session in the database
-	#
-	# opts MUST contain either
-	# +:session+:: the Msf::Session object we are reporting
-	# +:host+::    the Host object we are reporting a session on.
-	#
+	# @note The Mdm::Session#desc will be truncated to 255 characters.
+  # @todo https://www.pivotaltracker.com/story/show/48249739
+	#
+	# @overload report_session(opts)
+	#   Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the
+	#   +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the
+	#   session's host.  The Mdm::Host for the +session_host+ is created using
+	#   The session.session_host, +session.arch+ (if +session+ responds to arch),
+	#   and the workspace derived from opts or the +session+.  The Mdm::Session is
+	#   assumed to be +last_seen+ and +opened_at+ at the time report_session is
+	#   called.  +session.exploit_datastore['ParentModule']+ is used for the
+	#   Mdm::Session#via_exploit if +session.via_exploit+ is
+	#   'exploit/multi/handler'.
+	#
+	#   @param opts [Hash{Symbol => Object}] options
+	#   @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session
+	#     The in-memory session to persist to the database.
+	#   @option opts [Mdm::Workspace] :workspace The workspace for in which the
+	#     :session host is contained.  Also used as the workspace for the
+	#     Mdm::ExploitAttempt and Mdm::Vuln.  Defaults to Mdm::Worksapce with
+	#     Mdm::Workspace#name equal to +session.workspace+.
+	#   @return [nil] if {Msf::DBManager#active} is +false+.
+	#   @return [Mdm::Session] if session is saved
+	#   @raise [ArgumentError] if :session is not an {Msf::Session}.
+	#   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
+	#     saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be
+	#     created, but the Mdm::Host will have been.   (There is no transaction
+	#       to rollback the Mdm::Host creation.)
+	#   @see #find_or_create_host
+	#   @see #normalize_host
+	#   @see #report_exploit_success
+	#   @see #report_vuln
+	#
+	# @overload report_session(opts)
+	#   Creates an Mdm::Session from Mdm::Host.
+	#
+	#   @param opts [Hash{Symbol => Object}] options
+	#   @option opts [DateTime, Time] :closed_at The date and time the sesion was
+	#     closed.
+	#   @option opts [String] :close_reason Reason the session was closed.
+	#   @option opts [Hash] :datastore {Msf::DataStore#to_h}.
+	#   @option opts [String] :desc Session description.  Will be truncated to 255
+	#     characters.
+	#   @option opts [Mdm::Host] :host The host on which the session was opened.
+	#   @option opts [DateTime, Time] :last_seen The last date and time the
+	#     session was seen to be open.  Defaults to :closed_at's value.
+	#   @option opts [DateTime, Time] :opened_at The date and time that the
+	#     session was opened.
+	#   @option opts [String] :platform The platform of the host.
+	#   @option opts [Array] :routes ([]) The routes through the session for
+	#     pivoting.
+	#   @option opts [String] :stype Session type.
+	#   @option opts [String] :via_exploit The {Msf::Module#fullname} of the
+	#     exploit that was used to open the session.
+	#   @option option [String] :via_payload the {MSf::Module#fullname} of the
+	#     payload sent to the host when the exploit was successful.
+	#   @return [nil] if {Msf::DBManager#active} is +false+.
+	#   @return [Mdm::Session] if session is saved.
+	#   @raise [ArgumentError] if :host is not an Mdm::Host.
+	#   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
+	#     saved.
+	#
+	# @raise ArgumentError if :host and :session is +nil+
 	def report_session(opts)
 		return if not active
 	::ActiveRecord::Base.connection_pool.with_connection {
@@ -719,13 +776,11 @@ def report_session(opts)
 
 		# If this is a live session, we know the host is vulnerable to something.
 		if opts[:session] and session.via_exploit
-			return unless host
-
 			mod = framework.modules.create(session.via_exploit)
 
 			if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
 				mod_fullname = sess_data[:datastore]['ParentModule']
-				mod_name = ::Mdm::ModuleDetail.find_by_fullname(mod_fullname).name
+				mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name
 			else
 				mod_name = mod.name
 				mod_fullname = mod.fullname