Land #6 JSON to MDM

Deserialize JSON returned from a remote data service to an in-memory MDM object

Author: mkienow-r7

lib/metasploit/framework/data_service/proxy/host_data_proxy.rb

@@ -13,6 +13,8 @@ def hosts(wspace = workspace, non_dead = false, addresses = nil)
     end
   end
 
+  # TODO: Shouldn't this proxy to RemoteHostDataService#find_or_create_host ?
+  # It's currently skipping the "find" part
   def find_or_create_host(opts)
     puts 'Calling find host'
     report_host(opts)

lib/metasploit/framework/data_service/remote/http/remote_credential_data_service.rb

@@ -3,13 +3,22 @@
 module RemoteCredentialDataService
   include ResponseDataHelper
 
-  CREDENTIAL_PATH = '/api/1/msf/credential'
+  CREDENTIAL_API_PATH = '/api/1/msf/credential'
+  # "MDM_CLASS" is a little misleading since it is not in that repo but trying to keep naming consistent across DataServices
+  CREDENTIAL_MDM_CLASS = 'Metasploit::Credential::Core'
 
   def creds(opts = {})
-    json_to_open_struct_object(self.get_data(CREDENTIAL_PATH, opts), [])
+    data = self.get_data(CREDENTIAL_API_PATH, opts)
+    rv = json_to_mdm_object(data, CREDENTIAL_MDM_CLASS, [])
+    parsed_body = JSON.parse(data.response.body)
+    parsed_body.each do |cred|
+      private_object = to_ar(cred['private_class'].constantize, cred['private'])
+      rv[parsed_body.index(cred)].private = private_object
+    end
+    rv
   end
 
   def create_credential(opts)
-    self.post_data_async(CREDENTIAL_PATH, opts)
+    self.post_data_async(CREDENTIAL_API_PATH, opts)
   end
 end

lib/metasploit/framework/data_service/remote/http/remote_host_data_service.rb

@@ -3,27 +3,28 @@
 module RemoteHostDataService
   include ResponseDataHelper
 
-  HOST_PATH = '/api/1/msf/host'
-  HOST_SEARCH_PATH = HOST_PATH + "/search"
+  HOST_API_PATH = '/api/1/msf/host'
+  HOST_SEARCH_PATH = HOST_API_PATH + "/search"
+  HOST_MDM_CLASS = 'Mdm::Host'
 
   def hosts(opts)
-    json_to_open_struct_object(self.get_data(HOST_PATH, opts), [])
+    json_to_mdm_object(self.get_data(HOST_API_PATH, opts), HOST_MDM_CLASS, [])
   end
 
   def report_host(opts)
-    json_to_open_struct_object(self.post_data(HOST_PATH, opts))
+    json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
   end
 
   def find_or_create_host(opts)
-    json_to_open_struct_object(self.post_data(HOST_PATH, opts))
+    json_to_mdm_object(self.post_data(HOST_API_PATH, opts), HOST_MDM_CLASS, []).first
   end
 
   def report_hosts(hosts)
-    self.post_data(HOST_PATH, hosts)
+    self.post_data(HOST_API_PATH, hosts)
   end
 
   def delete_host(opts)
-    json_to_open_struct_object(self.delete_data(HOST_PATH, opts))
+    json_to_mdm_object(self.delete_data(HOST_API_PATH, opts), HOST_MDM_CLASS, [])
   end
 
   # TODO: Remove? What is the purpose of this method?

lib/metasploit/framework/data_service/remote/http/remote_loot_data_service.rb

@@ -3,11 +3,12 @@
 module RemoteLootDataService
   include ResponseDataHelper
 
-  LOOT_PATH = '/api/1/msf/loot'
+  LOOT_API_PATH = '/api/1/msf/loot'
+  LOOT_MDM_CLASS = 'Mdm::Loot'
 
   def loot(opts = {})
     # TODO: Add an option to toggle whether the file data is returned or not
-    loots = json_to_open_struct_object(self.get_data(LOOT_PATH, opts), [])
+    loots = json_to_mdm_object(self.get_data(LOOT_API_PATH, opts), LOOT_MDM_CLASS, [])
     # Save a local copy of the file
     loots.each do |loot|
       if loot.data
@@ -19,14 +20,14 @@ def loot(opts = {})
   end
 
   def report_loot(opts)
-    self.post_data_async(LOOT_PATH, opts)
+    self.post_data_async(LOOT_API_PATH, opts)
   end
 
   def find_or_create_loot(opts)
-    json_to_open_struct_object(self.post_data(LOOT_PATH, opts))
+    json_to_mdm_object(self.post_data(LOOT_API_PATH, opts), LOOT_MDM_CLASS, [])
   end
 
   def report_loots(loot)
-    self.post_data(LOOT_PATH, loot)
+    self.post_data(LOOT_API_PATH, loot)
   end
 end

lib/metasploit/framework/data_service/remote/http/remote_session_data_service.rb

@@ -1,6 +1,7 @@
 module RemoteSessionDataService
 
   SESSION_API_PATH = '/api/1/msf/session'
+  SESSION_MDM_CLASS = 'Mdm::Session'
 
   def report_session(opts)
     session = opts[:session]
@@ -12,7 +13,7 @@ def report_session(opts)
     end
 
     opts[:time_stamp] = Time.now.utc
-    sess_db = json_to_open_struct_object(self.post_data(SESSION_API_PATH, opts))
+    sess_db = json_to_mdm_object(self.post_data(SESSION_API_PATH, opts), SESSION_MDM_CLASS, []).first
     session.db_record = sess_db
   end
 

lib/metasploit/framework/data_service/remote/http/remote_session_event_data_service.rb

@@ -3,14 +3,15 @@
 module RemoteSessionEventDataService
   include ResponseDataHelper
 
-  SESSION_EVENT_PATH = '/api/1/msf/session_event'
+  SESSION_EVENT_API_PATH = '/api/1/msf/session_event'
+  SESSION_EVENT_MDM_CLASS = 'Mdm::SessionEvent'
 
   def session_events(opts = {})
-    json_to_open_struct_object(self.get_data(SESSION_EVENT_PATH, opts), [])
+    json_to_mdm_object(self.get_data(SESSION_EVENT_API_PATH, opts), SESSION_EVENT_MDM_CLASS, [])
   end
 
   def report_session_event(opts)
     opts[:session] = opts[:session].db_record
-    self.post_data_async(SESSION_EVENT_PATH, opts)
+    self.post_data_async(SESSION_EVENT_API_PATH, opts)
   end
 end

lib/metasploit/framework/data_service/remote/http/remote_workspace_data_service.rb

@@ -5,19 +5,20 @@ module RemoteWorkspaceDataService
 
   WORKSPACE_COUNTS_API_PATH = '/api/1/msf/workspace/counts'
   WORKSPACE_API_PATH = '/api/1/msf/workspace'
+  WORKSPACE_MDM_CLASS = 'Mdm::Workspace'
   DEFAULT_WORKSPACE_NAME = 'default'
 
   def find_workspace(workspace_name)
     workspace = workspace_cache[workspace_name]
     return workspace unless (workspace.nil?)
 
-    workspace = json_to_open_struct_object(self.get_data(WORKSPACE_API_PATH, {:workspace_name => workspace_name}))
+    workspace = json_to_mdm_object(self.get_data(WORKSPACE_API_PATH, {:workspace_name => workspace_name}), WORKSPACE_MDM_CLASS).first
     workspace_cache[workspace_name] = workspace
   end
 
   def add_workspace(workspace_name)
     response = self.post_data(WORKSPACE_API_PATH, {:workspace_name => workspace_name})
-    json_to_open_struct_object(response, nil)
+    json_to_mdm_object(response, WORKSPACE_MDM_CLASS, nil)
   end
 
   def default_workspace
@@ -33,11 +34,11 @@ def workspace=(workspace)
   end
 
   def workspaces
-    json_to_open_struct_object(self.get_data(WORKSPACE_API_PATH, {:all => true}), [])
+    json_to_mdm_object(self.get_data(WORKSPACE_API_PATH, {:all => true}), WORKSPACE_MDM_CLASS, [])
   end
 
   def workspace_associations_counts()
-    json_to_open_struct_object(self.get_data(WORKSPACE_COUNTS_API_PATH), [])
+    json_to_mdm_object(self.get_data(WORKSPACE_API_PATH, []), WORKSPACE_MDM_CLASS, [])
   end
 
   #########

lib/metasploit/framework/data_service/remote/http/response_data_helper.rb

@@ -10,10 +10,10 @@ module ResponseDataHelper
   # Converts an HTTP response to an OpenStruct object
   #
   def json_to_open_struct_object(response_wrapper, returns_on_error = nil)
-    if (response_wrapper.expected)
+    if response_wrapper.expected
       begin
         body = response_wrapper.response.body
-        if (not body.nil? and not body.empty?)
+        if not body.nil? and not body.empty?
           return JSON.parse(body, object_class: OpenStruct)
         end
       rescue Exception => e
@@ -24,6 +24,34 @@ def json_to_open_struct_object(response_wrapper, returns_on_error = nil)
     return returns_on_error
   end
 
+  #
+  # Converts an HTTP response to an Mdm Object
+  #
+  # @param [ResponseWrapper] A wrapped HTTP response containing a JSON body.
+  # @param [String] The Mdm class to convert the JSON to.
+  # @param [Anything] A failsafe response to return if no objects are found.
+  # @return [ActiveRecord::Base] An object of type mdm_class, which inherits from ActiveRecord::Base
+  def json_to_mdm_object(response_wrapper, mdm_class, returns_on_error = nil)
+    if response_wrapper.expected
+      begin
+        body = response_wrapper.response.body
+        if not body.nil? and not body.empty?
+          parsed_body = Array.wrap(JSON.parse(body))
+          rv = []
+          parsed_body.each do |json_object|
+            rv << to_ar(mdm_class.constantize, json_object)
+          end
+          return rv
+        end
+      rescue Exception => e
+        puts "Mdm Object conversion failed #{e.message}"
+        e.backtrace.each { |line| puts "#{line}\n" }
+      end
+    end
+
+    return returns_on_error
+  end
+
   # Processes a Base64 encoded file included in a JSON request.
   # Saves the file in the location specified in the parameter.
   #
@@ -45,6 +73,60 @@ def process_file(base64_file, save_path)
     save_path
   end
 
+  # Converts a Hash or JSON string to an ActiveRecord object.
+  # Importantly, this retains associated objects if they are in the JSON string.
+  #
+  # Modified from https://github.com/swdyh/toar/
+  # Credit to https://github.com/swdyh
+  #
+  # @param [String] klass The ActiveRecord class to convert the JSON/Hash to.
+  # @param [String] val The JSON string, or Hash, to convert.
+  # @param [Class] base_class The base class to build back to. Used for recursion.
+  # @return [ActiveRecord::Base] A klass object, which inherits from ActiveRecord::Base.
+  def to_ar(klass, val, base_object = nil)
+    data = val.class == Hash ? val.dup : JSON.parse(val)
+    obj = base_object || klass.new
+
+    obj_associations = klass.reflect_on_all_associations(:has_many).reduce({}) do |reflection, i|
+      reflection[i.options[:through]] = i if i.options[:through]
+      reflection
+    end
+
+    data.except(*obj.attributes.keys).each do |k, v|
+      association = klass.reflect_on_association(k)
+      next unless association
+
+      case association.macro
+        when :belongs_to
+          data.delete("#{k}_id")
+          to_ar(association.klass, v, obj.send("build_#{k}"))
+          obj.class_eval do
+            define_method("#{k}_id") { obj.send(k).id }
+          end
+        when :has_one
+          to_ar(association.klass, v, obj.send("build_#{k}"))
+        when :has_many
+          obj.send(k).proxy_association.target =
+              v.map { |i| to_ar(association.klass, i) }
+
+          as_th = obj_associations[k.to_sym]
+          if as_th
+            obj.send(as_th.name).proxy_association.target =
+                v.map { |i| to_ar(as_th.klass, i[as_th.source_reflection_name.to_s]) }
+          end
+      end
+    end
+    obj.assign_attributes(data.slice(*obj.attributes.keys))
+
+    obj.instance_eval do
+      # prevent save
+      def valid?(_context = nil)
+        false
+      end
+    end
+    obj
+  end
+
   #
   # Converts a hash to an open struct
   #

lib/msf/core/db_manager/host.rb

@@ -31,16 +31,14 @@ def delete_host(opts)
       deleted = []
       hosts.each do |host|
         begin
-          host.destroy
-          deleted << host.address.to_s
+          deleted << host.destroy
         rescue # refs suck
           elog("Forcibly deleting #{host.address}")
-          host.delete
-          deleted << host.address.to_s
+          deleted << host.delete
         end
       end
 
-      return { deleted: deleted }
+      return deleted
     }
   end
 

lib/msf/core/db_manager/http/servlet/credential_servlet.rb

@@ -18,12 +18,12 @@ def self.get_credentials
       begin
         opts = parse_json_request(request, false)
         data = get_db().creds(opts)
-        includes = [:logins, :public, :private, :origin, :realm]
+        includes = [:logins, :public, :private, :realm]
         # Need to append the human attribute into the private sub-object before converting to json
         # This is normally pulled from a class method from the MetasploitCredential class
         response = []
         data.each do |cred|
-          json = cred.as_json(include: includes).merge('human' => cred.private.class.model_name.human)
+          json = cred.as_json(include: includes).merge('private_class' => cred.private.class.to_s)
           response << json
         end
         set_json_response(response)