S3 Plugin: Implement SSE option

- Introduces a server_side_encryption parameter for the S3 [OUTPUT]
  plugin. Possible values are AES256 and aws:kms, as per AWS API
  documentation:
  https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#API_PutObject_ResponseSyntax
- If either value is provided, the x-amz-server-side-encryption header
  will be included with S3 requests with the corresponding value set.

Signed-off-by: Mark Solters <[email protected]>

Author: Mark Solters

include/fluent-bit/aws/flb_aws_sse.h

@@ -0,0 +1,37 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef FLB_AWS_SSE
+#define FLB_AWS_SSE
+
+// Per https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html#API_PutObject_ResponseSyntax
+#include <sys/types.h>
+#define FLB_AWS_SSE_NONE  0
+#define FLB_AWS_SSE_AWSKMS  1
+#define FLB_AWS_SSE_AES256 2
+
+/*
+ * Get sse type from sse keyword. The return value is used to identify
+ * what sse option to utilize.
+ *
+ * Returns int sse type id - FLB_AWS_SSE_<sse-type>
+ */
+int flb_aws_sse_get_type(const char *sse_keyword);
+
+#endif

plugins/out_s3/s3.c

@@ -25,6 +25,7 @@
 #include <fluent-bit/flb_config_map.h>
 #include <fluent-bit/flb_aws_util.h>
 #include <fluent-bit/aws/flb_aws_compress.h>
+#include <fluent-bit/aws/flb_aws_sse.h>
 #include <fluent-bit/flb_hash.h>
 #include <fluent-bit/flb_crypto.h>
 #include <fluent-bit/flb_signv4.h>
@@ -98,6 +99,13 @@ static struct flb_aws_header storage_class_header = {
     .val_len = 0,
 };
 
+static struct flb_aws_header server_side_encryption_header = {
+    .key = "x-amz-server-side-encryption",
+    .key_len = 28,
+    .val = "",
+    .val_len = 0,
+};
+
 static char *mock_error_response(char *error_env_var)
 {
     char *err_val = NULL;
@@ -150,6 +158,9 @@ int create_headers(struct flb_s3 *ctx, char *body_md5,
     if (body_md5 != NULL && strlen(body_md5) && multipart_upload == FLB_FALSE) {
         headers_len++;
     }
+    if (strlen(ctx->sse)) {
+        headers_len++;
+    }
     if (ctx->storage_class != NULL) {
         headers_len++;
     }
@@ -187,6 +198,12 @@ int create_headers(struct flb_s3 *ctx, char *body_md5,
         s3_headers[n].val_len = strlen(body_md5);
         n++;
     }
+    if (strlen(ctx->sse)) {
+        s3_headers[n] = server_side_encryption_header;
+        s3_headers[n].val = ctx->sse;
+        s3_headers[n].val_len = strlen(ctx->sse);
+        n++;
+    }
     if (ctx->storage_class != NULL) {
         s3_headers[n] = storage_class_header;
         s3_headers[n].val = ctx->storage_class;
@@ -756,6 +773,16 @@ static int cb_s3_init(struct flb_output_instance *ins,
         }
     }
 
+    tmp = flb_output_get_property("server_side_encryption", ins);
+    if (tmp) {
+        ret = flb_aws_sse_get_type(tmp);
+        if (ret == -1) {
+            flb_plg_error(ctx->ins, "unknown server-side encryption type: %s", tmp);
+            return -1;
+        }
+        ctx->sse = tmp;
+    }
+
     tmp = flb_output_get_property("sts_endpoint", ins);
     if (tmp) {
         ctx->sts_endpoint = (char *) tmp;
@@ -2376,7 +2403,12 @@ static struct flb_config_map config_map[] = {
     "A standard MIME type for the S3 object; this will be set "
     "as the Content-Type HTTP header."
     },
-
+    {
+     FLB_CONFIG_MAP_STR, "server_side_encryption", NULL,
+     0, FLB_FALSE, 0,
+    "Optional serve-side encryption type to use"
+    "Defaults to no encryption header. "
+    },
     {
      FLB_CONFIG_MAP_STR, "store_dir", "/tmp/fluent-bit/s3",
      0, FLB_TRUE, offsetof(struct flb_s3, store_dir),

plugins/out_s3/s3.h

@@ -113,6 +113,7 @@ struct flb_s3 {
     char *log_key;
     char *external_id;
     char *profile;
+    char *sse;
     int free_endpoint;
     int retry_requests;
     int use_put_object;

src/aws/CMakeLists.txt

@@ -15,6 +15,7 @@ set(src
   "flb_aws_imds.c"
   "flb_aws_credentials_http.c"
   "flb_aws_credentials_profile.c"
+  "flb_aws_sse.c"
   )
 
 if(FLB_HAVE_AWS_CREDENTIAL_PROCESS)

src/aws/flb_aws_sse.c

@@ -0,0 +1,66 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2019-2021 The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include <fluent-bit/flb_log.h>
+
+#include <fluent-bit/aws/flb_aws_sse.h>
+
+#include <stdint.h>
+
+struct sse_option {
+    int sse_type;
+    char *sse_keyword;
+};
+
+/*
+ * Library of sse options
+ * AWS plugins that support sse will have these options.
+ * Referenced function should return -1 on error and 0 on success.
+ */
+static const struct sse_option sse_options[] = {
+    /* FLB_AWS_SSE_NONE which is 0 is reserved for array footer */
+    {
+        FLB_AWS_SSE_AWSKMS,
+        "aws:kms"
+    },
+    {
+        FLB_AWS_SSE_AES256,
+        "AES256"
+    },
+    { 0 }
+};
+
+int flb_aws_sse_get_type(const char *sse_keyword)
+{
+    int ret;
+    const struct sse_option *o;
+
+    o = sse_options;
+
+    while (o->sse_type != 0) {
+        ret = strcmp(o->sse_keyword, sse_keyword);
+        if (ret == 0) {
+            return o->sse_type;
+        }
+        ++o;
+    }
+
+    flb_error("[aws_compress] unknown sse type: %s", sse_keyword);
+    return -1;
+}