Use bucket based on placeholder

Signed-off-by: rmontenegroo <[email protected]>

Author: rmontenegroo

lib/fluent/plugin/out_s3.rb

@@ -252,11 +252,12 @@ def start
       s3_client = Aws::S3::Client.new(options)
       @s3 = Aws::S3::Resource.new(client: s3_client)
 
-      if not @s3_bucket =~ /\$\{.*\}/
-        @bucket = @s3.bucket(@s3_bucket)
-        check_apikeys(@bucket) if @check_apikey_on_start
-        ensure_bucket(@bucket) if @check_bucket
-        ensure_bucket_lifecycle(@bucket)
+      @tag_placeholders = get_placeholders_tag(@s3_bucket)
+      @keys_placeholders = get_placeholders_keys(@s3_bucket)
+      @time_placeholders = ext_get_placeholders_time(@s3_bucket)
+      
+      if @tag_placeholders.empty? and @keys_placeholders.empty? and @time_placeholders.empty?
+        @bucket = create_bucket(@s3_bucket)
       end
 
       super
@@ -267,6 +268,26 @@ def format(tag, time, record)
       @formatter.format(tag, time, r)
     end
 
+    def create_bucket(name)
+      bucket = @s3.bucket(name)
+      check_apikeys(bucket) if @check_apikey_on_start
+      ensure_bucket(bucket) if @check_bucket
+      ensure_bucket_lifecycle(bucket)
+      bucket
+    end
+
+    def use_fallback(placeholder)
+      if not @s3_bucket_fallback
+        raise "It was not possible to extract '#{placeholder}' placeholder from chunk and @s3_bucket_fallback is not set."
+      end
+      log.warn "Using @s3_bucket_fallback ('#{@s3_bucket_fallback}') as a fallback bucket name."
+      @s3_bucket_fallback
+    end
+
+    def ext_get_placeholders_time(str)
+      output = [ "%S", "%M", "%H", "%d", "%m", "%Y" ].select { |tp| str.include? tp }
+    end
+
     def write(chunk)
 
       i = 0
@@ -277,34 +298,30 @@ def write(chunk)
                    else
                      @time_slice_with_tz.call(metadata.timekey)
                    end
-      
-      bucket_name = nil
 
-      if @s3_bucket =~ /\$\{.*\}/
-        @s3_bucket.scan(/\$\{([^\$\{\}]+)\}/) do |placeholder|
-          placeholder = placeholder.join
+      bucket = @bucket ? @bucket : nil
+
+      if (not bucket) and (not @tag_placeholders.empty?)
+        if (not chunk.metadata.tag) or ((@tag_placeholders.max + 1) > chunk.metadata.tag.split('.').length)
+          bucket = create_bucket(use_fallback("tag"))
+        end
+      end
+
+      if not bucket
+        @keys_placeholders.each do |placeholder|
           if (not chunk.metadata.variables) or (not chunk.metadata.variables.keys.include?(placeholder.to_sym))
-            log.warn "There is no placeholder '#{placeholder}'" 
-            if @s3_bucket_fallback
-              bucket_name = @s3_bucket_fallback
-              log.warn "Using @s3_bucket_fallback ('#{@s3_bucket_fallback}') as a fallback bucket name."
-              break
-            else
-              raise "It was not possible to extract placeholder '#{placeholder}' from chunk and @s3_bucket_fallback is not set."
-            end
+            bucket = create_bucket(use_fallback(placeholder))
+            break
           end
         end
+      end
 
-        if not bucket_name
-          bucket_name = extract_placeholders(@s3_bucket, chunk)
-        end
+      if (not bucket) and (not chunk.metadata.timekey) and @time_placeholders
+        bucket = create_bucket(use_fallback("time"))
+      end
 
-        bucket = @s3.bucket(bucket_name)
-        check_apikeys(bucket) if @check_apikey_on_start
-        ensure_bucket(bucket) if @check_bucket
-        ensure_bucket_lifecycle(bucket)
-      else
-        bucket = @bucket
+      if not bucket
+        bucket = create_bucket(extract_placeholders(@s3_bucket, chunk))
       end
 
       if @check_object