s3_bucket_time_slice_substring suffix created as needed

Signed-off-by: rodmonte <[email protected]>

Author: rmontenegroo

ChangeLog

@@ -1,3 +1,7 @@
+Release 1.4.1 - 2020/09/30
+
+  * Added feature s3_bucket_timestamp_suffix
+
 Release 1.4.0 - 2020/08/02
 
   * Remove uuidtools dependency

Dockerfile

@@ -0,0 +1,11 @@
+FROM fluent/fluentd:edge
+
+ARG  VERSION=1.4.1
+
+USER root
+
+COPY fluent-plugin-s3-$VERSION.gem /
+
+RUN  gem install /fluent-plugin-s3-$VERSION.gem --no-document
+
+USER fluent

Makefile

@@ -0,0 +1,11 @@
+all: build
+
+gem: clean
+	gem build fluent-plugin-s3
+
+build: gem
+	docker build -t fluentd-custom:edge .
+
+clean:
+	gem clean;
+	rm -f fluent-plugin-s3*.gem

README.md

@@ -295,6 +295,16 @@ full list of regions are available here. >
 http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region. We
 recommend using `s3_region` instead of `s3_endpoint`.
 
+**s3_bucket_use_time_slice_sub**
+
+Use a time slice substring to create new buckets as time slice changes.  **auto_create_bucket** is required!
+For example: 
+bucket = MYBUCKET
+time_slice = %Y-%m-%d-%H
+s3_bucket_use_time_slice_sub = %Y-%m-%d
+
+Buckets will be created like: MYBUCKET-2020-09-25, MYBUCKET-2020-09-26 as days go by.
+
 **s3_endpoint**
 
 endpoint for S3 compatible services. For example, Riak CS based storage or

VERSION

@@ -1 +1 @@
-1.4.0
+1.4.1

lib/fluent/plugin/out_s3.rb

@@ -87,6 +87,8 @@ def initialize
     config_param :aws_iam_retries, :integer, default: nil, deprecated: "Use 'instance_profile_credentials' instead"
     desc "S3 bucket name"
     config_param :s3_bucket, :string
+    desc "Use time slice substring to rotate bucket name"
+    config_param :s3_bucket_use_time_slice_sub, :string, default: ""
     desc "S3 region name"
     config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
     desc "Use 's3_region' instead"
@@ -249,13 +251,29 @@ def start
 
       s3_client = Aws::S3::Client.new(options)
       @s3 = Aws::S3::Resource.new(client: s3_client)
+      @s3_bucket_prefix = @s3_bucket
+
+      if @s3_bucket_use_time_slice_sub != ""
+        if !@configured_time_slice_format.start_with?(@s3_bucket_use_time_slice_sub)
+          raise "s3_bucket_use_time_slice_sub (" + @s3_bucket_use_time_slice_sub + ") must be a substring of time_slice (" + @configured_time_slice_format + ")"
+        end
+      end
+
+      super
+    end
+
+    def set_bucket
+      if @s3_bucket_use_time_slice_sub != ""
+        @s3_bucket = @s3_bucket_prefix + "-" + Time.new.strftime(@s3_bucket_use_time_slice_sub)
+      else
+        @s3_bucket = @s3_bucket_prefix
+      end
+
       @bucket = @s3.bucket(@s3_bucket)
 
       check_apikeys if @check_apikey_on_start
       ensure_bucket if @check_bucket
       ensure_bucket_lifecycle
-
-      super
     end
 
     def format(tag, time, record)
@@ -272,6 +290,8 @@ def write(chunk)
                    else
                      @time_slice_with_tz.call(metadata.timekey)
                    end
+      
+      set_bucket
 
       if @check_object
         begin