Fluentbit kubernetes - Indexing based on namespace

[meektechie]

Hi Folks,

We want to create the index based on namespace, for eg: if we have a, b, c namespaces the index should be test-a-, test-b-, test-c-* (here test is the prefix). Currently we have only one index for the whole cluster but we want it as requested.

config:
service: |
[SERVICE]
Daemon Off
Flush 5
Log_Level info
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
Health_Check On
## https://docs.fluentbit.io/manual/pipeline/inputs
inputs: |
[INPUT]
Name tail
Path /var/log/containers/.log
multiline.parser docker, cri
Tag kube.${$.kubernetes.namespace_name}.
Mem_Buf_Limit 50MB
Skip_Long_Lines On
parser docker
[INPUT]
Name systemd
Tag host.*
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Read_From_Tail On
## https://docs.fluentbit.io/manual/pipeline/filters
filters: |
[FILTER]
Name kubernetes
Match kube.*
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
Merge_Log_Trim On
Labels On
Annotations Off

  [FILTER]
      name                  multiline
      match                 *
      multiline.key_content log
      multiline.parser      go, java, python
      emitter_mem_buf_limit 1G


      
  [FILTER]
      Name modify
      Match kube.*
      Rename log message
      Remove kubernetes_host
      Remove kubernetes_container_hash
  

## https://docs.fluentbit.io/manual/pipeline/outputs
outputs: |
  [OUTPUT]
      Name es
      Match *
      Host ${es_host}
      Port 9243
      Cloud_ID ${Cloud_ID}  
      Cloud_Auth ${Cloud_Auth}
      Index test-${$.kubernetes.namespace_name}
      Logstash_Format Off
      Logstash_Prefix test-${$.kubernetes.namespace_name}
      Time_Key @timestamp
      Time_Key_Format %Y-%m-%dT%H:%M:%S 
      Retry_Limit 10
      tls On
      tls.verify Off
      Suppress_Type_Name On
      Trace_Error On
      Replace_Dots On
      

## https://docs.fluentbit.io/manual/pipeline/parsers
customParsers: |
  [PARSER]
      Name docker_no_time
      Format json
      Time_Keep Off
      Time_Key time
      Time_Format %Y-%m-%dT%H:%M:%S.%L

parsers.conf: |-
  [PARSER]
      Name        docker
      Format      json
      Time_Key    time
      Time_Format %Y-%m-%dT%H:%M:%S.%