Pass 1: externalize database

Author: clee-r7

Gemfile

@@ -3,6 +3,9 @@ source 'https://rubygems.org'
 #   spec.add_runtime_dependency '<name>', [<version requirements>]
 gemspec name: 'metasploit-framework'
 
+gem 'thin'
+gem 'sinatra'
+gem 'ruby-prof'
 gem 'bit-struct', git: 'https://github.com/busterb/bit-struct', branch: 'ruby-2.4'
 gem 'method_source', git: 'https://github.com/banister/method_source', branch: 'master'
 
@@ -24,6 +27,8 @@ group :development do
   # metasploit-aggregator as a framework only option for now
   # Metasploit::Aggregator external session proxy
   gem 'metasploit-aggregator'
+
+  #gem 'rex-core', path: '/home/chlee/rapid7/rex-core'
 end
 
 group :development, :test do

lib/metasploit/framework/command/base.rb

@@ -44,7 +44,11 @@ class Metasploit::Framework::Command::Base
   #
   # @return (see parsed_options)
   def self.require_environment!
+    # TODO: Look into removing Rails.application (save ~20mb)
+    # return self.parsed_options if ( self.parsed_options.options.database.remote_process)
+
     parsed_options = self.parsed_options
+
     # RAILS_ENV must be set before requiring 'config/application.rb'
     parsed_options.environment!
     ARGV.replace(parsed_options.positional)
@@ -79,7 +83,9 @@ def self.parsed_options_class_name
 
   def self.start
     parsed_options = require_environment!
-    new(application: Rails.application, parsed_options: parsed_options).start
+    is_db_remote = false # parsed_options.options.database.remote_process
+    application = is_db_remote ? nil : Rails.application
+    new(application: application, parsed_options: parsed_options).start
   end
 
   #

lib/metasploit/framework/command/console.rb

@@ -79,6 +79,7 @@ def driver_options
       driver_options['DatabaseEnv'] = options.environment
       driver_options['DatabaseMigrationPaths'] = options.database.migrations_paths
       driver_options['DatabaseYAML'] = options.database.config
+      driver_options['DatabaseRemoteProcess'] = options.database.remote_process
       driver_options['DeferModuleLoads'] = options.modules.defer_loads
       driver_options['DisableBanner'] = options.console.quiet
       driver_options['DisableDatabase'] = options.database.disable

lib/metasploit/framework/data_service.rb

@@ -0,0 +1,32 @@
+require 'metasploit/framework/data_service/stubs/host_data_service'
+require 'metasploit/framework/data_service/stubs/vuln_data_service'
+require 'metasploit/framework/data_service/stubs/event_data_service'
+require 'metasploit/framework/data_service/stubs/workspace_data_service'
+require 'metasploit/framework/data_service/stubs/note_data_service'
+require 'metasploit/framework/data_service/stubs/web_data_service'
+require 'metasploit/framework/data_service/stubs/service_data_service'
+
+#
+# All data service implementations should include this module to ensure proper implementation
+#
+module Metasploit
+module Framework
+module DataService
+  include HostDataService
+  include EventDataService
+  include VulnDataService
+  include WorkspaceDataService
+  include WebDataService
+  include NoteDataService
+  include ServiceDataService
+
+  def name
+    raise 'DataLService#name is not implemented';
+  end
+
+  def active
+    raise 'DataLService#active is not implemented';
+  end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/core.rb

@@ -0,0 +1,202 @@
+require 'singleton'
+require 'open3'
+require 'rex/ui'
+require 'rex/logging'
+require 'msf/core/db_manager'
+require 'metasploit/framework/data_service/remote/http/core'
+require 'metasploit/framework/data_service/remote/http/remote_service_endpoint'
+require 'metasploit/framework/data_service/proxy/data_proxy_auto_loader'
+
+#
+# Holds references to data services (@see Metasploit::Framework::DataService)
+# and forwards data to the implementation set as current.
+#
+module Metasploit
+module Framework
+module DataService
+class DataProxy
+  include Singleton
+  include DataProxyAutoLoader
+
+  attr_reader :usable
+
+  #
+  # Returns current error state
+  #
+  def error
+    return @error if (@error)
+    return @data_service.error if @data_service
+    return "none"
+  end
+
+  #
+  # Determines if the data service is active
+  #
+  def active
+    if (@data_service)
+      return @data_service.active
+    end
+
+    return false
+  end
+
+  #
+  # Initializes the data service to be used - primarily on startup
+  #
+  def init(framework, opts)
+    @mutex.synchronize {
+      if (@initialized)
+        return
+      end
+
+      begin
+        if (opts['DisableDatabase'])
+          @error = 'disabled'
+          return
+        elsif (opts['DatabaseRemoteProcess'])
+          run_remote_db_process(opts)
+        else
+          run_local_db_process(framework, opts)
+        end
+        @usable = true
+        @initialized = true
+      rescue Exception => e
+        puts "Unable to initialize a dataservice #{e.message}"
+        return
+      end
+    }
+
+  end
+
+  #
+  # Registers a data service with the proxy and immediately
+  # set as primary if online
+  #
+  def register_data_service(data_service, online=false)
+    validate(data_service)
+
+    puts "Registering data service: #{data_service.name}"
+    data_service_id = @data_service_id += 1
+    @data_services[data_service_id] = data_service
+    set_data_service(data_service_id, online)
+  end
+
+  #
+  # Set the data service to be used
+  #
+  def set_data_service(data_service_id, online=false)
+    data_service = @data_services[data_service_id.to_i]
+    if (data_service.nil?)
+      puts "Data service with id: #{data_service_id} does not exist"
+      return nil
+    end
+
+    if (!online && !data_service.active)
+      puts "Data service not online: #{data_service.name}, not setting as active"
+      return nil
+    end
+
+    puts "Setting active data service: #{data_service.name}"
+    @data_service = data_service
+  end
+
+  #
+  # Prints out a list of the current data services
+  #
+  def print_data_services()
+    @data_services.each_key {|key|
+      out = "id: #{key}, description: #{@data_services[key].name}"
+      if (!@data_service.nil? && @data_services[key].name == @data_service.name)
+        out += " [active]"
+      end
+      puts out  #hahaha
+    }
+  end
+
+  #
+  # Used to bridge the local db
+  #
+  def method_missing(method, *args, &block)
+    puts "Attempting to delegate method: #{method}"
+    unless @data_service.nil?
+      @data_service.send(method, *args, &block)
+    end
+  end
+
+  #
+  # Attempt to shutdown the local db process if it exists
+  #
+  def exit_called
+    if @pid
+      puts 'Killing db process'
+      begin
+        Process.kill("TERM", @pid)
+      rescue Exception => e
+        puts "Unable to kill db process: #{e.message}"
+      end
+    end
+  end
+
+  #########
+  protected
+  #########
+
+  def get_data_service
+    raise 'No registered data_service' unless @data_service
+    return @data_service
+  end
+
+  #######
+  private
+  #######
+
+  def initialize
+    @data_services = {}
+    @data_service_id = 0
+    @usable = false
+    @initialized = false
+    @mutex = Mutex.new()
+  end
+
+  def validate(data_service)
+    raise "Invalid data_service: #{data_service.class}, not of type Metasploit::Framework::DataService" unless data_service.is_a? (Metasploit::Framework::DataService)
+    raise 'Cannot register null data service data_service' unless data_service
+    raise 'Data Service already exists' if data_service_exist?(data_service)
+  end
+
+  def data_service_exist?(data_service)
+    @data_services.each_value{|value|
+      if (value.name == data_service.name)
+        return true
+      end
+    }
+
+    return false
+  end
+
+
+  def run_local_db_process(framework, opts)
+    puts 'Initializing local db process'
+    db_manager = Msf::DBManager.new(framework)
+    if (db_manager.usable and not opts['SkipDatabaseInit'])
+        register_data_service(db_manager, true)
+        db_manager.init_db(opts)
+    end
+  end
+
+  def run_remote_db_process(opts)
+    # started with no signal to prevent ctrl-c from taking out db
+    db_script = File.join( Msf::Config.install_root, "msfdb -ns")
+    wait_t = Open3.pipeline_start(db_script)
+    @pid = wait_t[0].pid
+    puts "Started process with pid #{@pid}"
+
+    endpoint = Metasploit::Framework::DataService::RemoteServiceEndpoint.new('localhost', 8080)
+    remote_host_data_service = Metasploit::Framework::DataService::RemoteHTTPDataService.new(endpoint)
+    register_data_service(remote_host_data_service, true)
+  end
+
+end
+end
+end
+end

lib/metasploit/framework/data_service/proxy/data_proxy_auto_loader.rb

@@ -0,0 +1,20 @@
+#
+# Autoloads specific data proxies
+#
+module DataProxyAutoLoader
+  autoload :HostDataProxy, 'metasploit/framework/data_service/proxy/host_data_proxy'
+  autoload :VulnDataProxy, 'metasploit/framework/data_service/proxy/vuln_data_proxy'
+  autoload :EventDataProxy, 'metasploit/framework/data_service/proxy/event_data_proxy'
+  autoload :WorkspaceDataProxy, 'metasploit/framework/data_service/proxy/workspace_data_proxy'
+  autoload :NoteDataProxy, 'metasploit/framework/data_service/proxy/note_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :WebDataProxy, 'metasploit/framework/data_service/proxy/web_data_proxy'
+  autoload :ServiceDataProxy, 'metasploit/framework/data_service/proxy/service_data_proxy'
+  include ServiceDataProxy
+  include HostDataProxy
+  include VulnDataProxy
+  include EventDataProxy
+  include WorkspaceDataProxy
+  include NoteDataProxy
+  include WebDataProxy
+end

lib/metasploit/framework/data_service/proxy/event_data_proxy.rb

@@ -0,0 +1,12 @@
+module EventDataProxy
+
+  def report_event(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_event(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_event threw exception: #{e.message}"
+    end
+  end
+
+end

lib/metasploit/framework/data_service/proxy/host_data_proxy.rb

@@ -0,0 +1,59 @@
+module HostDataProxy
+
+  def hosts(wspace, non_dead = false, addresses = nil)
+    begin
+      data_service = self.get_data_service()
+      opts = {}
+      opts[:wspace] = wspace
+      opts[:non_dead] = non_dead
+      opts[:addresses] = addresses
+      data_service.hosts(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#hosts threw exception: #{e.message}"
+    end
+  end
+
+  def find_or_create_host(opts)
+    puts 'Calling find host'
+    report_host(opts)
+  end
+
+  def report_host(opts)
+    return unless valid(opts)
+
+    begin
+      data_service = self.get_data_service()
+      data_service.report_host(opts)
+    rescue  Exception => e
+        puts"Call to  #{data_service.class}#report_host threw exception: #{e.message}"
+        opts.each do |key, value| puts "#{key} : #{value}" end
+    end
+  end
+
+  def report_hosts(hosts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_hosts(hosts)
+    rescue  Exception => e
+      puts "Call to  #{data_service.class}#report_hosts threw exception: #{e.message}"
+    end
+  end
+
+  private
+
+  def valid(opts)
+    unless opts[:host]
+      puts 'Invalid host hash passed, :host is missing'
+      return false
+    end
+
+    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
+    if opts[:host].eql? "Remote Pipe"
+      puts "Invalid host hash passed, address was of type 'Remote Pipe'"
+      return false
+    end
+
+    return true
+  end
+
+end

lib/metasploit/framework/data_service/proxy/note_data_proxy.rb

@@ -0,0 +1,10 @@
+module NoteDataProxy
+  def report_note(opts)
+    begin
+      data_service = self.get_data_service()
+      data_service.report_note(opts)
+    rescue  Exception => e
+      puts"Call to  #{data_service.class}#report_note threw exception: #{e.message}"
+    end
+  end
+end