Fluentbit improvements (#189)

* Bump chart version

* Take only log message

* Add multiline parsing

* Add variable config for refresh interval

Author: bonclay7

modules/eks-monitoring/add-ons/aws-for-fluentbit/README.md

@@ -42,6 +42,7 @@ See this [Helm Chart](https://github.com/aws/eks-charts/tree/master/stable/aws-f
 | <a name="input_helm_config"></a> [helm\_config](#input\_helm\_config) | Helm provider config aws\_for\_fluent\_bit. | `any` | `{}` | no |
 | <a name="input_irsa_policies"></a> [irsa\_policies](#input\_irsa\_policies) | Additional IAM policies for a IAM role for service accounts | `list(string)` | `[]` | no |
 | <a name="input_manage_via_gitops"></a> [manage\_via\_gitops](#input\_manage\_via\_gitops) | Determines if the add-on should be managed via GitOps. | `bool` | `false` | no |
+| <a name="input_refresh_interval"></a> [refresh\_interval](#input\_refresh\_interval) | FluentBit input refresh interval | `number` | `60` | no |
 
 ## Outputs
 

modules/eks-monitoring/add-ons/aws-for-fluentbit/locals.tf

@@ -18,7 +18,7 @@ locals {
     name        = local.name
     chart       = local.name
     repository  = "https://aws.github.io/eks-charts"
-    version     = "0.1.24"
+    version     = "0.1.27"
     namespace   = local.name
     values      = local.default_helm_values
     description = "aws-for-fluentbit Helm Chart deployment configuration"
@@ -33,6 +33,7 @@ locals {
     aws_region         = var.addon_context.aws_region_name
     cluster_name       = var.addon_context.eks_cluster_id
     log_retention_days = var.cw_log_retention_days
+    refresh_interval   = var.refresh_interval
     service_account    = local.service_account
   })]
 

modules/eks-monitoring/add-ons/aws-for-fluentbit/values.yaml

@@ -12,5 +12,19 @@ cloudWatchLogs:
   logGroupName: /aws/eks/observability-accelerator/workloads
   logGroupTemplate: /aws/eks/observability-accelerator/${cluster_name}/$kubernetes['namespace_name']
   logStreamTemplate: $kubernetes['container_name'].$kubernetes['pod_name']
-  log_key: log
+  logKey: log
   logRetentionDays: ${log_retention_days}
+
+input:
+  enabled: false
+
+additionalInputs: |
+  [INPUT]
+      Name              tail
+      Tag               kube.*
+      Path              /var/log/containers/*.log
+      DB                /var/log/flb_kube.db
+      Mem_Buf_Limit     5MB
+      Skip_Long_Lines   On
+      Refresh_Interval  ${refresh_interval}
+      multiline.parser  cri, docker, go, java, python

modules/eks-monitoring/add-ons/aws-for-fluentbit/variables.tf

@@ -10,6 +10,13 @@ variable "cw_log_retention_days" {
   default     = 90
 }
 
+variable "refresh_interval" {
+  description = "FluentBit input refresh interval"
+  type        = number
+  default     = 60
+}
+
+
 variable "manage_via_gitops" {
   type        = bool
   description = "Determines if the add-on should be managed via GitOps."