Merge branch 'goliath' into http_client_change

Author: jbarnett-r7

lib/metasploit/framework/data_service/remote/http/remote_loot_data_service.rb

@@ -6,7 +6,16 @@ module RemoteLootDataService
   LOOT_PATH = '/api/1/msf/loot'
 
   def loot(opts = {})
-    json_to_open_struct_object(self.get_data(LOOT_PATH, opts), [])
+    # TODO: Add an option to toggle whether the file data is returned or not
+    loots = json_to_open_struct_object(self.get_data(LOOT_PATH, opts), [])
+    # Save a local copy of the file
+    loots.each do |loot|
+      if loot.data
+        local_path = File.join(Msf::Config.loot_directory, File.basename(loot.path))
+        loot.path = process_file(loot.data, local_path)
+      end
+    end
+    loots
   end
 
   def report_loot(opts)

lib/metasploit/framework/data_service/remote/http/remote_nmap_data_service.rb

@@ -14,6 +14,6 @@ def import_nmap_xml_file(opts)
 
     opts[:data] = Base64.urlsafe_encode64(data)
 
-    self.post_data_async(NMAP_PATH, opts)
+    self.post_data(NMAP_PATH, opts)
   end
 end

lib/metasploit/framework/data_service/remote/http/response_data_helper.rb

@@ -1,4 +1,5 @@
 require 'ostruct'
+require 'digest'
 
 #
 # HTTP response helper class
@@ -23,6 +24,27 @@ def json_to_open_struct_object(response_wrapper, returns_on_error = nil)
     return returns_on_error
   end
 
+  # Processes a Base64 encoded file included in a JSON request.
+  # Saves the file in the location specified in the parameter.
+  #
+  # @param base64_file [String] The Base64 encoded file.
+  # @param save_path [String] The location to store the file. This should include the file's name.
+  # @return [String] The location where the file was successfully stored.
+  def process_file(base64_file, save_path)
+    decoded_file = Base64.urlsafe_decode64(base64_file)
+    begin
+      # If we are running the data service on the same box this will ensure we only write
+      # the file if it is somehow not there already.
+      unless File.exists?(save_path) && File.read(save_path) == decoded_file
+        File.open(save_path, 'w+') { |file| file.write(decoded_file) }
+      end
+    rescue Exception => e
+      puts "There was an error writing the file: #{e}"
+      e.backtrace.each { |line| puts "#{line}\n"}
+    end
+    save_path
+  end
+
   #
   # Converts a hash to an open struct
   #

lib/msf/core/db_manager/http/servlet/loot_servlet.rb

@@ -27,48 +27,11 @@ def self.get_loot
 
   def self.report_loot
     lambda {
-
       job = lambda { |opts|
-        # This regex does a best attempt to determine if opts[:data] is valid Base64
-        # See https://stackoverflow.com/questions/8571501/how-to-check-whether-the-string-is-base64-encoded-or-not
-        if opts[:data] =~ /^([A-Za-z0-9+\/]{4})*([A-Za-z0-9+\/]{4}|[A-Za-z0-9+\/]{3}=|[A-Za-z0-9+\/]{2}==)$/
-          opts[:data] = Base64.urlsafe_decode64(opts[:data]) if opts[:data]
-        end
-
-        # This code is all for writing out the file locally.
-        # It is copied from lib/msf/core/auxiliary/report.rb
-        # We shouldn't duplicate it so a better method should be used
-        if ! ::File.directory?(Msf::Config.loot_directory)
-          FileUtils.mkdir_p(Msf::Config.loot_directory)
-        end
-
-        ext = 'bin'
-        if opts[:name]
-          parts = opts[:name].to_s.split('.')
-          if parts.length > 1 and parts[-1].length < 4
-            ext = parts[-1]
-          end
-        end
-
-        case opts[:content_type]
-          when /^text\/[\w\.]+$/
-            ext = "txt"
-        end
-        # This method is available even if there is no database, don't bother checking
-        host = Msf::Util::Host.normalize_host(opts[:host])
-
-        ws = (opts[:workspace] ? opts[:workspace] : 'default')
-        name =
-            Time.now.strftime("%Y%m%d%H%M%S") + "_" + ws + "_" +
-                (host || 'unknown') + '_' + opts[:type][0,16] + '_' +
-                Rex::Text.rand_text_numeric(6) + '.' + ext
-
-        name.gsub!(/[^a-z0-9\.\_]+/i, '')
-
-        path = File.join(Msf::Config.loot_directory, name)
-        full_path = ::File.expand_path(path)
-        File.open(full_path, "wb") do |fd|
-          fd.write(opts[:data])
+        if opts[:data]
+          filename = File.basename(opts[:path])
+          local_path = File.join(Msf::Config.loot_directory, filename)
+          opts[:path] = process_file(opts[:data], local_path)
         end
 
         get_db().report_loot(opts)

lib/msf/core/db_manager/http/servlet/nmap_servlet.rb

@@ -16,12 +16,9 @@ def self.import_nmap_xml_file
     lambda {
 
       job = lambda { |opts|
-
-        nmap_file = opts[:filename].split('/').last
-        local_file = File.open(File.join(Msf::Config.local_directory, nmap_file), 'w')
-        local_file.write(Base64.urlsafe_decode64(opts[:data]))
-        local_file.close
-        opts[:filename] = File.expand_path(local_file)
+        nmap_file = File.basename(opts[:filename])
+        nmap_file_path = File.join(Msf::Config.local_directory, nmap_file)
+        opts[:filename] = process_file(opts[:data], nmap_file_path)
         get_db().import_nmap_xml_file(opts)
       }
       exec_report_job(request, &job)