Skip to content

Commit 564a98b

Browse files
treyrichbourgtreyrichbourg
committed
Refactor: Use marker-based templating for fluent-bit configs
1 parent 060cc50 commit 564a98b

File tree

2 files changed

+21
-177
lines changed

2 files changed

+21
-177
lines changed

‎charts/amazon-cloudwatch-observability/templates/linux/fluent-bit-configmap.yaml‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,17 @@ data:
1414
{{- end }}
1515
parsers.conf: |
1616
{{- .Values.containerLogs.fluentBit.config.customParsers | nindent 4 }}
17-
{{- if hasKey .Values.adcEndpointOverrides .Values.region }}
18-
{{- range $key, $val := .Values.containerLogs.fluentBit.config.adcRegionExtraFiles }}
17+
{{- range $key, $val := .Values.containerLogs.fluentBit.config.extraFiles }}
1918
{{ $key }}: |
20-
{{- include "fluent-bit.add-dualstack-endpoints" (dict "Values" $.Values "config" (tpl $val $)) | nindent 4 }}
21-
{{- end -}}
22-
{{- else }}
23-
{{- range $key, $val := .Values.containerLogs.fluentBit.config.extraFiles }}
24-
{{ $key }}: |
25-
{{- include "fluent-bit.add-dualstack-endpoints" (dict "Values" $.Values "config" (tpl $val $)) | nindent 4 }}
26-
{{- end -}}
19+
{{- $config := tpl $val $ }}
20+
{{- if hasKey $.Values.adcEndpointOverrides $.Values.region }}
21+
{{- $endpoint := index $.Values.adcEndpointOverrides $.Values.region }}
22+
{{- $config = regexReplaceAll "(?s)# ISOLATED_REGION_REMOVE_START.*?# ISOLATED_REGION_REMOVE_END\n?" $config "" }}
23+
{{- $config = $config | replace "# ISOLATED_ENDPOINT_OVERRIDE" (printf "endpoint logs.${AWS_REGION}.%s" $endpoint) }}
24+
{{- else }}
25+
{{- $config = regexReplaceAll "(?m)^.*# ISOLATED_REGION_REMOVE_(START|END).*$\n?" $config "" }}
26+
{{- $config = regexReplaceAll "(?m)^.*# ISOLATED_ENDPOINT_OVERRIDE.*$\n?" $config "" }}
27+
{{- end }}
28+
{{- include "fluent-bit.add-dualstack-endpoints" (dict "Values" $.Values "config" $config) | nindent 4 }}
29+
{{- end -}}
2730
{{- end -}}
28-
{{- end -}}

‎charts/amazon-cloudwatch-observability/values.yaml‎

Lines changed: 9 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,14 @@ containerLogs:
124124
Refresh_Interval 10
125125
Read_from_Head ${READ_FROM_HEAD}
126126
127+
# ISOLATED_REGION_REMOVE_START
127128
[FILTER]
128129
Name aws
129130
Match application.*
130131
az false
131132
ec2_instance_id false
132133
Enable_Entity true
134+
# ISOLATED_REGION_REMOVE_END
133135
134136
[FILTER]
135137
Name kubernetes
@@ -145,7 +147,9 @@ containerLogs:
145147
Use_Kubelet On
146148
Kubelet_Port 10250
147149
Buffer_Size 0
150+
# ISOLATED_REGION_REMOVE_START
148151
Use_Pod_Association On
152+
# ISOLATED_REGION_REMOVE_END
149153
150154
[OUTPUT]
151155
Name cloudwatch_logs
@@ -155,7 +159,10 @@ containerLogs:
155159
log_stream_prefix ${HOST_NAME}-
156160
auto_create_group true
157161
extra_user_agent container-insights
162+
# ISOLATED_ENDPOINT_OVERRIDE
163+
# ISOLATED_REGION_REMOVE_START
158164
add_entity true
165+
# ISOLATED_REGION_REMOVE_END
159166
dataplane-log.conf: |
160167
[INPUT]
161168
Name systemd
@@ -201,6 +208,7 @@ containerLogs:
201208
log_stream_prefix ${HOST_NAME}-
202209
auto_create_group true
203210
extra_user_agent container-insights
211+
# ISOLATED_ENDPOINT_OVERRIDE
204212
host-log.conf: |
205213
[INPUT]
206214
Name systemd
@@ -259,173 +267,7 @@ containerLogs:
259267
log_stream_prefix ${HOST_NAME}.
260268
auto_create_group true
261269
extra_user_agent container-insights
262-
adcRegionExtraFiles:
263-
application-log.conf: |
264-
[INPUT]
265-
Name tail
266-
Tag application.*
267-
Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
268-
Path /var/log/containers/*.log
269-
multiline.parser docker, cri
270-
DB /var/fluent-bit/state/flb_container.db
271-
Mem_Buf_Limit 50MB
272-
Skip_Long_Lines On
273-
Refresh_Interval 10
274-
Rotate_Wait 30
275-
storage.type filesystem
276-
Read_from_Head ${READ_FROM_HEAD}
277-
278-
[INPUT]
279-
Name tail
280-
Tag application.*
281-
Path /var/log/containers/fluent-bit*
282-
multiline.parser docker, cri
283-
DB /var/fluent-bit/state/flb_log.db
284-
Mem_Buf_Limit 5MB
285-
Skip_Long_Lines On
286-
Refresh_Interval 10
287-
Read_from_Head ${READ_FROM_HEAD}
288-
289-
[INPUT]
290-
Name tail
291-
Tag application.*
292-
Path /var/log/containers/cloudwatch-agent*
293-
multiline.parser docker, cri
294-
DB /var/fluent-bit/state/flb_cwagent.db
295-
Mem_Buf_Limit 5MB
296-
Skip_Long_Lines On
297-
Refresh_Interval 10
298-
Read_from_Head ${READ_FROM_HEAD}
299-
300-
[FILTER]
301-
Name kubernetes
302-
Match application.*
303-
Kube_URL https://kubernetes.default.svc:443
304-
Kube_Tag_Prefix application.var.log.containers.
305-
Merge_Log On
306-
Merge_Log_Key log_processed
307-
K8S-Logging.Parser On
308-
K8S-Logging.Exclude Off
309-
Labels Off
310-
Annotations Off
311-
Use_Kubelet On
312-
Kubelet_Port 10250
313-
Buffer_Size 0
314-
315-
[OUTPUT]
316-
Name cloudwatch_logs
317-
Match application.*
318-
region ${AWS_REGION}
319-
log_group_name /aws/containerinsights/${CLUSTER_NAME}/application
320-
log_stream_prefix ${HOST_NAME}-
321-
auto_create_group true
322-
endpoint logs.${AWS_REGION}.${ADC_REGION_ENDPOINT}
323-
extra_user_agent container-insights
324-
dataplane-log.conf: |
325-
[INPUT]
326-
Name systemd
327-
Tag dataplane.systemd.*
328-
Systemd_Filter _SYSTEMD_UNIT=docker.service
329-
Systemd_Filter _SYSTEMD_UNIT=containerd.service
330-
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
331-
DB /var/fluent-bit/state/systemd.db
332-
Path /var/log/journal
333-
Read_From_Tail ${READ_FROM_TAIL}
334-
335-
[INPUT]
336-
Name tail
337-
Tag dataplane.tail.*
338-
Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy*
339-
multiline.parser docker, cri
340-
DB /var/fluent-bit/state/flb_dataplane_tail.db
341-
Mem_Buf_Limit 50MB
342-
Skip_Long_Lines On
343-
Refresh_Interval 10
344-
Rotate_Wait 30
345-
storage.type filesystem
346-
Read_from_Head ${READ_FROM_HEAD}
347-
348-
[FILTER]
349-
Name modify
350-
Match dataplane.systemd.*
351-
Rename _HOSTNAME hostname
352-
Rename _SYSTEMD_UNIT systemd_unit
353-
Rename MESSAGE message
354-
Remove_regex ^((?!hostname|systemd_unit|message).)*$
355-
356-
[FILTER]
357-
Name aws
358-
Match dataplane.*
359-
imds_version v2
360-
361-
[OUTPUT]
362-
Name cloudwatch_logs
363-
Match dataplane.*
364-
region ${AWS_REGION}
365-
log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane
366-
log_stream_prefix ${HOST_NAME}-
367-
auto_create_group true
368-
endpoint logs.${AWS_REGION}.${ADC_REGION_ENDPOINT}
369-
extra_user_agent container-insights
370-
host-log.conf: |
371-
[INPUT]
372-
Name systemd
373-
Tag host.dmesg
374-
Systemd_Filter _TRANSPORT=kernel
375-
DB /var/fluent-bit/state/flb_dmesg.db
376-
Path /var/log/journal
377-
Read_From_Tail ${READ_FROM_TAIL}
378-
379-
[INPUT]
380-
Name systemd
381-
Tag host.messages
382-
Systemd_Filter PRIORITY=0
383-
Systemd_Filter PRIORITY=1
384-
Systemd_Filter PRIORITY=2
385-
Systemd_Filter PRIORITY=3
386-
Systemd_Filter PRIORITY=4
387-
Systemd_Filter PRIORITY=5
388-
Systemd_Filter PRIORITY=6
389-
DB /var/fluent-bit/state/flb_messages.db
390-
Path /var/log/journal
391-
Read_From_Tail ${READ_FROM_TAIL}
392-
393-
[INPUT]
394-
Name systemd
395-
Tag host.secure
396-
Systemd_Filter SYSLOG_FACILITY=10
397-
DB /var/fluent-bit/state/flb_secure.db
398-
Path /var/log/journal
399-
Read_From_Tail ${READ_FROM_TAIL}
400-
401-
[FILTER]
402-
Name aws
403-
Match host.*
404-
imds_version v2
405-
406-
[FILTER]
407-
Name grep
408-
Match host.messages
409-
Exclude SYSLOG_FACILITY /^(2|9|10)$/
410-
411-
[FILTER]
412-
Name modify
413-
Match host.*
414-
Rename _HOSTNAME host
415-
Rename MESSAGE message
416-
Rename SYSLOG_IDENTIFIER ident
417-
Rename SYSLOG_PID pid
418-
Remove_regex [A-Z]
419-
420-
[OUTPUT]
421-
Name cloudwatch_logs
422-
Match host.*
423-
region ${AWS_REGION}
424-
log_group_name /aws/containerinsights/${CLUSTER_NAME}/host
425-
log_stream_prefix ${HOST_NAME}.
426-
auto_create_group true
427-
endpoint logs.${AWS_REGION}.${ADC_REGION_ENDPOINT}
428-
extra_user_agent container-insights
270+
# ISOLATED_ENDPOINT_OVERRIDE
429271
configWindows:
430272
service: |
431273
[ SERVICE ]

0 commit comments

Comments
 (0)