Initial fix for non db cache

Author: clee-r7

lib/msf/core/framework.rb

@@ -61,6 +61,7 @@ module Offspring
   require 'msf/core/db_manager'
   require 'msf/core/event_dispatcher'
   require 'rex/json_hash_file'
+  require 'msf/core/modules/metadata/cache'
 
   #
   # Creates an instance of the framework context.
@@ -90,6 +91,8 @@ def initialize(options={})
     events.add_general_subscriber(subscriber)
     events.add_db_subscriber(subscriber)
     events.add_ui_subscriber(subscriber)
+
+    Msf::Modules::Metadata::Cache.instance.init(self)
   end
 
   def inspect

lib/msf/core/module/full_name.rb

@@ -21,7 +21,7 @@ module ClassMethods
     #
 
     def fullname
-      type + '/' + refname
+      type + '/' + (refname.nil? ? "" : refname)
     end
 
     def shortname

lib/msf/core/module_set.rb

@@ -205,6 +205,8 @@ def add_module(klass, reference_name, info = {})
 
     self[reference_name] = klass
 
+    Msf::Modules::Metadata::Cache.instance.cache_module_metadata(klass)
+
     klass
   end
 

lib/msf/core/modules/metadata.rb

@@ -0,0 +1,3 @@
+module Msf::Modules::Metadata
+
+end

lib/msf/core/modules/metadata/cache.rb

@@ -0,0 +1,160 @@
+require 'singleton'
+require 'msf/events'
+require 'rex/ui/text/output/stdio'
+require 'msf/core/constants'
+require 'msf/core/modules/metadata/obj'
+require 'msf/core/modules/metadata/search'
+
+#
+# Core service class that provides storage of module metadata as well as operations on the metadata.
+# Note that operations on this metadata are included as separate modules.
+#
+# To prevent excessive startup times module definitions are not parsed for metadata until startup
+# is complete. Once startup is complete to prevent CPU spin loading is then done gradually and
+# only when an operation using the cache is called is CPU use maximized.
+#
+module Msf
+module Modules
+module Metadata
+
+class Cache
+  include Singleton
+  include ::Msf::UiEventSubscriber
+  include Msf::Modules::Metadata::Search
+
+  #
+  # Init registers this class as a listener to be notified when the console is done loading,
+  # this acts a hint to this class to start trickle loading the metadata
+  #
+  def init(framework)
+    register_ui_listener(framework)
+    @framework = framework
+  end
+
+  #
+  # Parses module metadata into an in memory cache.
+  #
+  # @param klass_or_instance - An instantiated or class instance of a module.
+  #
+  def cache_module_metadata(klass_or_instance)
+    # Only use cache if db is not in use for now
+    return  if @framework.db.active
+
+    if klass_or_instance.is_a?(Class)
+      if @module_load_complete
+        add_module_metadata_from_class(klass_or_instance)
+      else
+        @module_definitions << klass_or_instance
+      end
+    else
+      add_module_metadata_from_instance(klass_or_instance)
+    end
+  end
+
+  #########
+  protected
+  #########
+
+  #
+  # Notify the thread responsible for loading metadata that it can start loading.
+  #
+  def on_ui_start(rev)
+    return if @module_load_complete
+    @startup_called = true
+  end
+
+  #
+  #  Returns the module data cache, but first ensures all the metadata is loaded
+  #
+  def get_module_metadata_cache
+    wait_for_load
+    return @module_metadata_cache
+  end
+
+  #######
+  private
+  #######
+
+  def register_ui_listener(framework)
+    begin
+      framework.events.add_ui_subscriber(self)
+    rescue Exception => e
+      elog('Unable to register metadata cache service as UI listener')
+    end
+  end
+
+  def wait_for_load
+    if (!@module_load_complete)
+      @trickle_load = false
+      @console.print_warning('Waiting to finish parsing module metadata')
+      @module_load_thread.join
+    end
+  end
+
+  def add_module_metadata_from_class(module_class_definition)
+    begin
+      instance = module_class_definition.new
+      add_module_metadata_from_instance(instance)
+    rescue Exception => e
+      elog("Error adding module metadata: #{e.message}")
+    end
+  end
+
+  def add_module_metadata_from_instance(module_instance)
+    module_metadata = Obj.new(module_instance)
+    @module_metadata_cache[get_cache_key(module_instance)] = module_metadata
+  end
+
+  def get_cache_key(module_instance)
+    key = ''
+    key << (module_instance.type.nil? ? '' : module_instance.type)
+    key << '_'
+    key << module_instance.name
+    return key
+  end
+
+  #
+  # This method is used by the @module_load_thread
+  #
+  def load_module_definitions
+    loop do
+      if @startup_called
+        break;
+      end
+
+      sleep 0.3
+    end
+
+    count = 0
+    @module_definitions.each {|module_definition|
+      add_module_metadata_from_class(module_definition)
+      count = count + 1
+      if (@trickle_load && count > @trickle_load_batch)
+        sleep @trickle_load_interval
+        count = 0
+      end
+    }
+
+    @module_load_complete = true
+    GC.start(full_mark: true, immediate_sweep: true)
+  end
+
+  def initialize
+    @module_load_complete = false
+    @startup_called = false;
+    @trickle_load = true
+    @trickle_load_batch = 200
+    @trickle_load_interval = 0.5
+    @module_metadata_cache = {}
+    @module_definitions = []
+    @module_load_thread = Thread.new {
+      load_module_definitions
+    }
+
+    @console = Rex::Ui::Text::Output::Stdio.new
+  end
+end
+
+end
+end
+end

lib/msf/core/modules/metadata/obj.rb

@@ -0,0 +1,46 @@
+#
+# Simple accessor object for storing module metadata.
+#
+module Msf
+module Modules
+module Metadata
+
+class Obj
+  attr_reader :name
+  attr_reader :full_name
+  attr_reader :rank
+  attr_reader :disclosure_date
+  attr_reader :type
+  attr_reader :author
+  attr_reader :description
+  attr_reader :references
+  attr_reader :is_server
+  attr_reader :is_client
+  attr_reader :platform
+  attr_reader :arch
+  attr_reader :rport
+  attr_reader :targets
+
+  def initialize(module_instance)
+    @name = module_instance.name
+    @full_name = module_instance.fullname
+    @disclosure_date = module_instance.disclosure_date
+    @rank = module_instance.rank
+    @type = module_instance.type
+    @description = module_instance.description
+    @author = module_instance.author.map{|x| x.to_s}
+    @references = module_instance.references.map{|x| [x.ctx_id, x.ctx_val].join("-") }
+    @is_server = (module_instance.respond_to?(:stance) and module_instance.stance == "aggressive")
+    @is_client = (module_instance.respond_to?(:stance) and module_instance.stance == "passive")
+    @platform = module_instance.platform_to_s
+    @arch = module_instance.arch_to_s
+    @rport = module_instance.datastore['RPORT'].to_s
+
+    if module_instance.respond_to?(:targets) and module_instance.targets
+      @targets = module_instance.targets.map{|x| x.name}
+    end
+  end
+end
+end
+end
+end

lib/msf/core/modules/metadata/search.rb

@@ -0,0 +1,117 @@
+#
+# Provides search operations on the module metadata cache.
+#
+module Msf::Modules::Metadata::Search
+  #
+  # Searches the module metadata using the passed search string.
+  #
+  def find(search_string)
+    search_results = []
+
+    get_module_metadata_cache.values.each { |module_metadata|
+      if is_match(search_string, module_metadata)
+        search_results << module_metadata
+      end
+    }
+
+    return search_results
+  end
+
+  #######
+  private
+  #######
+
+  def is_match(search_string, module_metadata)
+    return false if not search_string
+
+    search_string += ' '
+
+    # Split search terms by space, but allow quoted strings
+    terms = search_string.split(/\"/).collect{|t| t.strip==t ? t : t.split(' ')}.flatten
+    terms.delete('')
+
+    # All terms are either included or excluded
+    res = {}
+
+    terms.each do |t|
+      f,v = t.split(":", 2)
+      if not v
+        v = f
+        f = 'text'
+      end
+      next if v.length == 0
+      f.downcase!
+      v.downcase!
+      res[f] ||=[   [],    []   ]
+      if v[0,1] == "-"
+        next if v.length == 1
+        res[f][1] << v[1,v.length-1]
+      else
+        res[f][0] << v
+      end
+    end
+
+    k = res
+
+    [0,1].each do |mode|
+      match = false
+      k.keys.each do |t|
+        next if k[t][mode].length == 0
+
+        k[t][mode].each do |w|
+          # Reset the match flag for each keyword for inclusive search
+          match = false if mode == 0
+
+          # Convert into a case-insensitive regex
+          r = Regexp.new(Regexp.escape(w), true)
+
+          case t
+            when 'text'
+              terms = [module_metadata.name, module_metadata.full_name, module_metadata.description] + module_metadata.references + module_metadata.author
+
+              if module_metadata.targets
+                terms = terms + module_metadata.targets
+              end
+              match = [t,w] if terms.any? { |x| x =~ r }
+            when 'name'
+              match = [t,w] if module_metadata.name =~ r
+            when 'path'
+              match = [t,w] if module_metadata.full_name =~ r
+            when 'author'
+              match = [t,w] if module_metadata.author.any? { |a| a =~ r }
+            when 'os', 'platform'
+              match = [t,w] if module_metadata.platform  =~ r or module_metadata.arch  =~ r
+
+              if module_metadata.targets
+                match = [t,w] if module_metadata.targets.any? { |t| t =~ r }
+              end
+            when 'port'
+              match = [t,w] if module_metadata.rport =~ r
+            when 'type'
+              match = [t,w] if Msf::MODULE_TYPES.any? { |modt| w == modt and module_metadata.type == modt }
+            when 'app'
+              match = [t,w] if (w == "server" and module_metadata.is_server)
+              match = [t,w] if (w == "client" and module_metadata.is_client)
+            when 'cve'
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^cve\-/i and ref =~ r }
+            when 'bid'
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^bid\-/i and ref =~ r }
+            when 'edb'
+              match = [t,w] if module_metadata.references.any? { |ref| ref =~ /^edb\-/i and ref =~ r }
+          end
+          break if match
+        end
+        # Filter this module if no matches for a given keyword type
+        if mode == 0 and not match
+          return false
+        end
+      end
+      # Filter this module if we matched an exclusion keyword (-value)
+      if mode == 1 and match
+        return false
+      end
+    end
+
+    true
+  end
+end

lib/msf/core/payload_set.rb

@@ -197,6 +197,8 @@ def add_module(payload_module, reference_name, modinfo={})
     # our own evil purposes.
     instance = build_payload(payload_module).new
 
+    Msf::Modules::Metadata::Cache.instance.cache_module_metadata(instance)
+
     # Create an array of information about this payload module
     pinfo =
       [