Land rapid7#7086, Add 'continue' and 'tries' wget-like options to meterpreter 'download'

Author: Brent Cook

lib/rex/post/meterpreter/extensions/stdapi/fs/dir.rb

@@ -195,17 +195,53 @@ def Dir.unlink(path)
   # Downloads the contents of a remote directory a
   # local directory, optionally in a recursive fashion.
   #
-  def Dir.download(dst, src, recursive = false, force = true, glob = nil, timestamp = nil, &stat)
+  def Dir.download(dst, src, opts, force = true, glob = nil, &stat)
+    recursive = false
+    continue = false
+    tries = false
+    tries_no = 0
+    tries_cnt = 0
+    if opts
+      timestamp = opts["timestamp"]
+      recursive = true if opts["recursive"]     
+      continue = true if opts["continue"]
+      tries = true if opts["tries"]
+      tries_no = opts["tries_no"]
+    end
+    begin
+      dir_files = self.entries(src, glob)
+    rescue Rex::TimeoutError
+      if (tries && (tries_no == 0 || tries_cnt < tries_no))
+        tries_cnt += 1
+        stat.call('error listing  - retry #', tries_cnt, src) if (stat)
+        retry
+      else
+        stat.call('error listing directory - giving up', src, dst) if (stat)
+        raise
+      end
+    end
 
-    self.entries(src, glob).each { |src_sub|
+    dir_files.each { |src_sub|
       dst_item = dst + ::File::SEPARATOR + client.unicode_filter_encode(src_sub)
       src_item = src + client.fs.file.separator + client.unicode_filter_encode(src_sub)
 
       if (src_sub == '.' or src_sub == '..')
         next
       end
 
-      src_stat = client.fs.filestat.new(src_item)
+      tries_cnt = 0
+      begin
+        src_stat = client.fs.filestat.new(src_item)
+      rescue Rex::TimeoutError
+        if (tries && (tries_no == 0 || tries_cnt < tries_no))
+          tries_cnt += 1
+          stat.call('error opening file - retry #', tries_cnt, src_item) if (stat)
+          retry
+        else
+          stat.call('error opening file - giving up', tries_cnt, src_item) if (stat)
+          raise
+        end
+      end
 
       if (src_stat.file?)
         if timestamp
@@ -215,7 +251,11 @@ def Dir.download(dst, src, recursive = false, force = true, glob = nil, timestam
         stat.call('downloading', src_item, dst_item) if (stat)
 
         begin
-          result = client.fs.file.download_file(dst_item, src_item)
+          if (continue || tries)  # allow to file.download to log messages
+            result = client.fs.file.download_file(dst_item, src_item, opts, &stat)
+          else
+            result = client.fs.file.download_file(dst_item, src_item, opts)
+          end
           stat.call(result, src_item, dst_item) if (stat)
         rescue ::Rex::Post::Meterpreter::RequestError => e
           if force
@@ -236,10 +276,10 @@ def Dir.download(dst, src, recursive = false, force = true, glob = nil, timestam
         end
 
         stat.call('mirroring', src_item, dst_item) if (stat)
-        download(dst_item, src_item, recursive, force, glob, timestamp, &stat)
+        download(dst_item, src_item, opts, force, glob, &stat)
         stat.call('mirrored', src_item, dst_item) if (stat)
       end
-    }
+    } # entries
   end
 
   #

lib/rex/post/meterpreter/extensions/stdapi/fs/file.rb

@@ -280,7 +280,8 @@ def File.is_glob?(name)
   # If a block is given, it will be called before each file is downloaded and
   # again when each download is complete.
   #
-  def File.download(dest, src_files, timestamp = nil, &stat)
+  def File.download(dest, src_files, opts = nil, &stat)
+    timestamp = opts["timestamp"] if opts
     [*src_files].each { |src|
       if (::File.basename(dest) != File.basename(src))
         # The destination when downloading is a local file so use this
@@ -294,15 +295,23 @@ def File.download(dest, src_files, timestamp = nil, &stat)
       end
 
       stat.call('downloading', src, dest) if (stat)
-      result = download_file(dest, src)
+      result = download_file(dest, src, opts, &stat)
       stat.call(result, src, dest) if (stat)
     }
   end
 
   #
   # Download a single file.
   #
-  def File.download_file(dest_file, src_file)
+  def File.download_file(dest_file, src_file, opts = nil, &stat)
+    continue=false
+    tries=false
+    tries_no=0
+    if opts
+      continue = true if opts["continue"]
+      tries = true if opts["tries"]
+      tries_no = opts["tries_no"]
+    end
     src_fd = client.fs.file.new(src_file, "rb")
 
     # Check for changes
@@ -318,12 +327,61 @@ def File.download_file(dest_file, src_file)
     dir = ::File.dirname(dest_file)
     ::FileUtils.mkdir_p(dir) if dir and not ::File.directory?(dir)
 
-    dst_fd = ::File.new(dest_file, "wb")
+    if continue
+      # continue downloading the file - skip downloaded part in the source
+      dst_fd = ::File.new(dest_file, "ab")
+      begin
+        dst_fd.seek(0, ::IO::SEEK_END)
+        in_pos = dst_fd.pos
+        src_fd.seek(in_pos)
+        stat.call('continuing from ', in_pos, src_file) if (stat)
+      rescue
+        # if we can't seek, download again
+        stat.call('error continuing - downloading from scratch', src_file, dest_file) if (stat)
+        dst_fd.close
+        dst_fd = ::File.new(dest_file, "wb")
+      end
+    else
+      dst_fd = ::File.new(dest_file, "wb")
+    end
 
     # Keep transferring until EOF is reached...
     begin
-      while ((data = src_fd.read) != nil)
-        dst_fd.write(data)
+      if tries
+        # resume when timeouts encountered
+        seek_back = false
+        tries_cnt = 0
+        begin # while
+          begin # exception
+            if seek_back
+              in_pos = dst_fd.pos
+              src_fd.seek(in_pos)
+              seek_back = false
+              stat.call('resuming at ', in_pos, src_file) if (stat)
+            else
+              # succesfully read and wrote - reset the counter
+              tries_cnt = 0
+            end
+            data = src_fd.read
+          rescue Rex::TimeoutError
+            # timeout encountered - either seek back and retry or quit
+            if (tries && (tries_no == 0 || tries_cnt < tries_no))
+              tries_cnt += 1
+              seek_back = true
+              stat.call('error downloading - retry #', tries_cnt, src_file) if (stat)
+              retry
+            else
+              stat.call('error downloading - giving up', src_file, dest_file) if (stat)
+              raise
+            end
+          end
+          dst_fd.write(data) if (data != nil)
+        end while (data != nil)
+      else
+        # do the simple copying quiting on the first error
+        while ((data = src_fd.read) != nil)
+          dst_fd.write(data)
+        end
       end
     rescue EOFError
     ensure

lib/rex/post/meterpreter/ui/console/command_dispatcher/extapi/clipboard.rb

@@ -376,7 +376,8 @@ def download_file( dest_folder, source )
     dest = File.join( dest_folder, base )
 
     if stat.directory?
-      client.fs.dir.download( dest, source, true, true ) { |step, src, dst|
+      client.fs.dir.download( dest, source, {"recursive" => true}, true ) { |step, src, dst|
+#      client.fs.dir.download( dest, source, true, true ) { |step, src, dst|
         print_line( "#{step.ljust(11)} : #{src} -> #{dst}" )
         client.framework.events.on_session_download( client, src, dest ) if msf_loaded?
       }

lib/rex/post/meterpreter/ui/console/command_dispatcher/stdapi/fs.rb

@@ -24,6 +24,8 @@ class Console::CommandDispatcher::Stdapi::Fs
   #
   @@download_opts = Rex::Parser::Arguments.new(
     "-h" => [ false, "Help banner." ],
+    "-c" => [ false, "Resume getting a partially-downloaded file." ],
+    "-l" => [ true, "Set the limit of retries (0 unlimits)." ],
     "-r" => [ false, "Download recursively." ],
     "-t" => [ false, "Timestamp downloaded files." ])
   #
@@ -333,17 +335,29 @@ def cmd_download(*args)
     end
 
     recursive = false
-    timestamp = false
     src_items = []
     last      = nil
     dest      = nil
+    continue  = false
+    tries     = false
+    tries_no  = 0
+    opts = {}
 
     @@download_opts.parse(args) { |opt, idx, val|
       case opt
       when "-r"
         recursive = true
+        opts['recursive'] = true
+      when "-c"
+        continue = true
+        opts['continue'] = true
+      when "-l"
+        tries = true
+        tries_no = val.to_i
+        opts['tries'] = true
+        opts['tries_no'] = tries_no
       when "-t"
-        timestamp = true
+        opts['timestamp'] = '_' + Time.now.iso8601
       when nil
         src_items << last if (last)
         last = val
@@ -371,10 +385,6 @@ def cmd_download(*args)
       dest = ::File.dirname(dest)
     end
 
-    if timestamp
-      ts = '_' + Time.now.iso8601
-    end
-
     # Go through each source item and download them
     src_items.each { |src|
       glob = nil
@@ -397,7 +407,8 @@ def cmd_download(*args)
             src_path = file['path'] + client.fs.file.separator + file['name']
             dest_path = src_path.tr(src_separator, ::File::SEPARATOR)
 
-            client.fs.file.download(dest_path, src_path, ts) do |step, src, dst|
+            client.fs.file.download(dest_path, src_path, opts) do |step, src, dst|
+              puts step
               print_status("#{step.ljust(11)}: #{src} -> #{dst}")
               client.framework.events.on_session_download(client, src, dest) if msf_loaded?
             end
@@ -409,14 +420,27 @@ def cmd_download(*args)
 
       else
         # Perform direct matching
-        stat = client.fs.file.stat(src)
+        tries_cnt = 0
+        begin
+          stat = client.fs.file.stat(src)
+        rescue Rex::TimeoutError
+          if (tries && (tries_no == 0 || tries_cnt < tries_no))
+            tries_cnt += 1
+            print_error("Error opening: #{src} - retry (#{tries_cnt})")
+            retry
+          else
+            print_error("Error opening: #{src} - giving up")
+            raise
+          end
+        end
+
         if (stat.directory?)
-          client.fs.dir.download(dest, src, recursive, true, glob, ts) do |step, src, dst|
+          client.fs.dir.download(dest, src, opts, true, glob) do |step, src, dst|
             print_status("#{step.ljust(11)}: #{src} -> #{dst}")
             client.framework.events.on_session_download(client, src, dest) if msf_loaded?
           end
         elsif (stat.file?)
-          client.fs.file.download(dest, src, ts) do |step, src, dst|
+          client.fs.file.download(dest, src, opts) do |step, src, dst|
             print_status("#{step.ljust(11)}: #{src} -> #{dst}")
             client.framework.events.on_session_download(client, src, dest) if msf_loaded?
           end