Add 'Workers' parameter for S3 Output plugin

Signed-off-by: Steve Scaffidi <[email protected]>

Author: hercynium

apis/fluentbit/v1alpha2/plugins/output/s3_types.go

@@ -65,7 +65,9 @@ type S3 struct {
 	// Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID.
 	ExternalId string `json:"ExternalId,omitempty"`
 	// Option to specify an AWS Profile for credentials.
-	Profile      string `json:"Profile,omitempty"`
+	Profile string `json:"Profile,omitempty"`
+	// Specify number of worker threads to use to output to S3
+	Workers      *int32 `json:"Workers,omitempty"`
 	*plugins.TLS `json:"tls,omitempty"`
 }
 
@@ -159,6 +161,9 @@ func (o *S3) Params(sl plugins.SecretLoader) (*params.KVs, error) {
 	if o.Profile != "" {
 		kvs.Insert("profile", o.Profile)
 	}
+	if o.Workers != nil {
+		kvs.Insert("workers", fmt.Sprint(*o.Workers))
+	}
 	if o.TLS != nil {
 		tls, err := o.TLS.Params(sl)
 		if err != nil {

apis/fluentbit/v1alpha2/plugins/output/s3_types_test.go

@@ -41,6 +41,7 @@ func TestOutput_S3_Params(t *testing.T) {
 		RetryLimit:               ptrAny(int32(1)),
 		ExternalId:               "external_id",
 		Profile:                  "my-profile",
+		Workers:                  ptrAny(int32(1)),
 	}
 
 	expected := params.NewKVs()
@@ -71,6 +72,7 @@ func TestOutput_S3_Params(t *testing.T) {
 	expected.Insert("retry_limit", "1")
 	expected.Insert("external_id", "external_id")
 	expected.Insert("profile", "my-profile")
+	expected.Insert("workers", "1")
 
 	kvs, err := s3.Params(sl)
 	g.Expect(err).NotTo(HaveOccurred())

docs/plugins/fluentbit/output/s3.md

@@ -32,4 +32,5 @@ The S3 output plugin, allows to flush your records into a S3 time series databas
 | RetryLimit | Integer value to set the maximum number of retries allowed. | *int32 |
 | ExternalId | Specify an external ID for the STS API, can be used with the role_arn parameter if your role requires an external ID. | string |
 | Profile | Option to specify an AWS Profile for credentials. | string |
+| Workers | Specify number of worker threads to use to output to S3 | *int32 |
 | tls |  | *[plugins.TLS](../tls.md) |