Merge branch 'fluent:master' into master

Author: runderwo

cmake/plugins_options.cmake

@@ -45,6 +45,7 @@ DEFINE_OPTION(FLB_IN_PROCESS_EXPORTER_METRICS "Enable process exporter metrics i
 DEFINE_OPTION(FLB_IN_PROC                     "Enable Process input plugin"                  ON)
 DEFINE_OPTION(FLB_IN_PROMETHEUS_REMOTE_WRITE  "Enable prometheus remote write input plugin"  ON)
 DEFINE_OPTION(FLB_IN_PROMETHEUS_SCRAPE        "Enable Prometheus Scrape input plugin"        ON)
+DEFINE_OPTION(FLB_IN_PROMETHEUS_TEXTFILE      "Enable Prometheus textfile input plugin"      ON)
 DEFINE_OPTION(FLB_IN_RANDOM                   "Enable random input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SERIAL                   "Enable Serial input plugin"                   ON)
 DEFINE_OPTION(FLB_IN_SPLUNK                   "Enable Splunk HTTP HEC input plugin"          ON)

lib/cmetrics/CMakeLists.txt

@@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 # CMetrics Version
 set(CMT_VERSION_MAJOR  1)
 set(CMT_VERSION_MINOR  0)
-set(CMT_VERSION_PATCH  4)
+set(CMT_VERSION_PATCH  5)
 set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}")
 
 # Include helpers

lib/cmetrics/src/cmt_cat.c

@@ -108,7 +108,7 @@ static inline int cat_histogram_values(struct cmt_metric *metric_dst, struct cmt
         }
     }
 
-    for (i = 0; i < histogram->buckets->count; i++) {
+    for (i = 0; i <= histogram->buckets->count; i++) {
         /* histogram buckets are always integers, no need to convert them */
         metric_dst->hist_buckets[i] += metric_src->hist_buckets[i];
     }

lib/cmetrics/src/cmt_histogram.c

@@ -36,7 +36,7 @@ struct cmt_histogram_buckets *cmt_histogram_buckets_create_size(double *bkts, si
     }
 
     /* besides buckets set by the user, we add an implicit bucket for +inf */
-    upper_bounds = calloc(1, sizeof(double) * count + 1);
+    upper_bounds = calloc(1, sizeof(double) * (count + 1));
     if (!upper_bounds) {
         cmt_errno();
         return NULL;

plugins/CMakeLists.txt

@@ -276,6 +276,7 @@ endif()
 
 REGISTER_IN_PLUGIN("in_fluentbit_metrics")
 REGISTER_IN_PLUGIN("in_prometheus_scrape")
+REGISTER_IN_PLUGIN("in_prometheus_textfile")
 REGISTER_IN_PLUGIN("in_emitter")
 REGISTER_IN_PLUGIN("in_tail")
 REGISTER_IN_PLUGIN("in_dummy")

plugins/filter_aws/aws.c

@@ -926,6 +926,30 @@ static int get_ec2_metadata(struct flb_filter_aws *ctx)
         ctx->metadata_retrieved = FLB_TRUE;
     }
 
+    if (ctx->enable_entity) {
+        if (!ctx->account_id) {
+            ret = flb_aws_imds_request_by_key(ctx->client_imds, FLB_AWS_IMDS_ACCOUNT_ID_PATH,
+                                  &ctx->account_id, &ctx->account_id_len,
+                                  "accountId");
+
+            if (ret < 0) {
+                flb_plg_error(ctx->ins, "Failed to get Account ID");
+                return -1;
+            }
+        }
+
+        if (!ctx->instance_id) {
+            ret = flb_aws_imds_request(ctx->client_imds, FLB_AWS_IMDS_INSTANCE_ID_PATH,
+                                       &ctx->instance_id,
+                                       &ctx->instance_id_len);
+            if (ret < 0) {
+                flb_plg_error(ctx->ins, "Failed to get instance ID");
+                return -1;
+            }
+        }
+    }
+
+    ctx->metadata_retrieved = FLB_TRUE;
     return 0;
 }
 
@@ -1104,6 +1128,23 @@ static int cb_aws_filter(const void *data, size_t bytes,
             }
         }
 
+        if (ctx->enable_entity &&
+            ctx->instance_id &&
+            ctx->account_id &&
+            ret == FLB_EVENT_ENCODER_SUCCESS) {
+            ret = flb_log_event_encoder_append_body_values(
+                    &log_encoder,
+                    FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY),
+                    FLB_LOG_EVENT_STRING_VALUE(ctx->instance_id,
+                                               ctx->instance_id_len));
+            ret = flb_log_event_encoder_append_body_values(
+                    &log_encoder,
+                    FLB_LOG_EVENT_CSTRING_VALUE(FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY),
+                    FLB_LOG_EVENT_STRING_VALUE(ctx->account_id,
+                                               ctx->account_id_len));
+        }
+
+
         if (ret == FLB_EVENT_ENCODER_SUCCESS) {
             ret = flb_log_event_encoder_commit_record(&log_encoder);
         }
@@ -1273,6 +1314,12 @@ static struct flb_config_map config_map[] = {
      0, FLB_TRUE, offsetof(struct flb_filter_aws, retry_required_interval),
      "Defines minimum duration between retries for fetching metadata groups"
     },
+    {
+    FLB_CONFIG_MAP_BOOL, "enable_entity", "false",
+    0, FLB_TRUE, offsetof(struct flb_filter_aws, enable_entity),
+    "Enable entity prefix for fields used for constructing entity."
+    "This currently only affects instance ID"
+    },
     {0}
 };
 

plugins/filter_aws/aws.h

@@ -27,6 +27,8 @@
 #define FLB_FILTER_AWS_AVAILABILITY_ZONE_KEY_LEN          2
 #define FLB_FILTER_AWS_INSTANCE_ID_KEY                    "ec2_instance_id"
 #define FLB_FILTER_AWS_INSTANCE_ID_KEY_LEN                15
+#define FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY             "aws_entity_ec2_instance_id"
+#define FLB_FILTER_AWS_ENTITY_INSTANCE_ID_KEY_LEN         26
 #define FLB_FILTER_AWS_INSTANCE_TYPE_KEY                  "ec2_instance_type"
 #define FLB_FILTER_AWS_INSTANCE_TYPE_KEY_LEN              17
 #define FLB_FILTER_AWS_PRIVATE_IP_KEY                     "private_ip"
@@ -37,6 +39,8 @@
 #define FLB_FILTER_AWS_AMI_ID_KEY_LEN                     6
 #define FLB_FILTER_AWS_ACCOUNT_ID_KEY                     "account_id"
 #define FLB_FILTER_AWS_ACCOUNT_ID_KEY_LEN                 10
+#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY              "aws_entity_account_id"
+#define FLB_FILTER_AWS_ENTITY_ACCOUNT_ID_KEY_LEN          21
 #define FLB_FILTER_AWS_HOSTNAME_KEY                       "hostname"
 #define FLB_FILTER_AWS_HOSTNAME_KEY_LEN                   8
 
@@ -110,6 +114,11 @@ struct flb_filter_aws {
     /* tags_* fields are related to exposing EC2 tags in log labels
      * tags_enabled defines if EC2 tags functionality is enabled */
     int tags_enabled;
+    /*
+    * Enable entity prefix appending. This appends
+    * 'aws_entity' to relevant keys
+    */
+    int enable_entity;
 
     /* tags_fetched defines if tag keys and values were fetched successfully
      * and might be used to inject into msgpack */

plugins/in_prometheus_textfile/CMakeLists.txt

@@ -0,0 +1,4 @@
+set(src
+  prometheus_textfile.c)
+
+FLB_PLUGIN(in_prometheus_textfile "${src}" "")