adding a quick concurrency feature

Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     s3_asset_deploy (1.0.0)
       aws-sdk-s3 (~> 1.0)
+      concurrent-ruby (~> 1.1)
       mime-types (~> 3.0)
 
 GEM
@@ -26,6 +27,7 @@ GEM
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     coderay (1.1.3)
+    concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
     jmespath (1.4.0)
     method_source (1.0.0)

lib/s3_asset_deploy/manager.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "concurrent"
 require "logger"
 require "time"
 require "aws-sdk-s3"
@@ -53,12 +54,22 @@ def upload(dry_run: false)
     end
 
     uploaded_assets = []
+
+    pool = Concurrent::FixedThreadPool.new(10)
+    upload_errors = Concurrent::Array.new
     assets_to_upload.each do |asset|
       next unless File.file?(asset.full_path)
       log "Uploading #{asset.path}..."
-      upload_asset(asset) unless dry_run
+      pool.post do
+        begin
+          upload_asset(asset) unless dry_run
+        rescue StandardError => e
+          upload_errors << e
+        end
+      end
       uploaded_assets << asset.path
     end
+    wait_for_pool_to_finish(pool, upload_errors)
 
     removal_manifest.save unless dry_run
     remote_asset_collector.clear_cache
@@ -202,4 +213,19 @@ def delete_objects(keys = [])
   def log(msg)
     logger.info("#{self.class.name}: #{msg}")
   end
+
+  def wait_for_pool_to_finish(pool, errors)
+    pool.shutdown
+
+    until pool.shutdown?
+      if errors.any?
+        pool.kill
+        fail errors.first
+      end
+      sleep 1
+    end
+
+    pool.wait_for_termination
+  end
+
 end

lib/s3_asset_deploy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3AssetDeploy
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

s3_asset_deploy.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "aws-sdk-s3", "~> 1.0"
   spec.add_dependency "mime-types", "~> 3.0"
+  spec.add_dependency "concurrent-ruby", "~> 1.1"
 
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"