From af7e86e30e90aa558e3afe515f567ced66569680 Mon Sep 17 00:00:00 2001 From: rodmonte Date: Fri, 25 Sep 2020 15:27:04 -0300 Subject: [PATCH] s3_bucket_time_slice_substring suffix created as needed Signed-off-by: rodmonte --- ChangeLog | 4 ++++ Dockerfile | 11 +++++++++++ Makefile | 11 +++++++++++ README.md | 10 ++++++++++ VERSION | 2 +- lib/fluent/plugin/out_s3.rb | 24 ++++++++++++++++++++++-- 6 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 Makefile diff --git a/ChangeLog b/ChangeLog index 0e15e1b7..cf25d727 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Release 1.4.1 - 2020/09/30 + + * Added feature s3_bucket_timestamp_suffix + Release 1.4.0 - 2020/08/02 * Remove uuidtools dependency diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..242db28d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM fluent/fluentd:edge + +ARG VERSION=1.4.1 + +USER root + +COPY fluent-plugin-s3-$VERSION.gem / + +RUN gem install /fluent-plugin-s3-$VERSION.gem --no-document + +USER fluent diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..9930fb00 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +all: build + +gem: clean + gem build fluent-plugin-s3 + +build: gem + docker build -t fluentd-custom:edge . + +clean: + gem clean; + rm -f fluent-plugin-s3*.gem diff --git a/README.md b/README.md index a10b0d9b..f55d70ef 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,16 @@ full list of regions are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region. We recommend using `s3_region` instead of `s3_endpoint`. +**s3_bucket_use_time_slice_sub** + +Use a time slice substring to create new buckets as time slice changes. **auto_create_bucket** is required! +For example: +bucket = MYBUCKET +time_slice = %Y-%m-%d-%H +s3_bucket_use_time_slice_sub = %Y-%m-%d + +Buckets will be created like: MYBUCKET-2020-09-25, MYBUCKET-2020-09-26 as days go by. + **s3_endpoint** endpoint for S3 compatible services. For example, Riak CS based storage or diff --git a/VERSION b/VERSION index e21e727f..347f5833 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.0 \ No newline at end of file +1.4.1 diff --git a/lib/fluent/plugin/out_s3.rb b/lib/fluent/plugin/out_s3.rb index 5fb653db..f38d1d5e 100644 --- a/lib/fluent/plugin/out_s3.rb +++ b/lib/fluent/plugin/out_s3.rb @@ -87,6 +87,8 @@ def initialize config_param :aws_iam_retries, :integer, default: nil, deprecated: "Use 'instance_profile_credentials' instead" desc "S3 bucket name" config_param :s3_bucket, :string + desc "Use time slice substring to rotate bucket name" + config_param :s3_bucket_use_time_slice_sub, :string, default: "" desc "S3 region name" config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1" desc "Use 's3_region' instead" @@ -249,13 +251,29 @@ def start s3_client = Aws::S3::Client.new(options) @s3 = Aws::S3::Resource.new(client: s3_client) + @s3_bucket_prefix = @s3_bucket + + if @s3_bucket_use_time_slice_sub != "" + if !@configured_time_slice_format.start_with?(@s3_bucket_use_time_slice_sub) + raise "s3_bucket_use_time_slice_sub (" + @s3_bucket_use_time_slice_sub + ") must be a substring of time_slice (" + @configured_time_slice_format + ")" + end + end + + super + end + + def set_bucket + if @s3_bucket_use_time_slice_sub != "" + @s3_bucket = @s3_bucket_prefix + "-" + Time.new.strftime(@s3_bucket_use_time_slice_sub) + else + @s3_bucket = @s3_bucket_prefix + end + @bucket = @s3.bucket(@s3_bucket) check_apikeys if @check_apikey_on_start ensure_bucket if @check_bucket ensure_bucket_lifecycle - - super end def format(tag, time, record) @@ -272,6 +290,8 @@ def write(chunk) else @time_slice_with_tz.call(metadata.timekey) end + + set_bucket if @check_object begin