Exclude AMACoreAgent connect logs when OTLP is disabled (#1542)

* Exclude AMACoreAgent connect issue when OTLP is disabled

* Exclude logs from RS

* Additional filter

Author: suyadav1

build/linux/installer/conf/fluent-bit-common.conf

@@ -15,7 +15,7 @@
 [INPUT]
     Name tail
     Alias ama-logs_mdsd_err_tail
-    Tag oms.container.log.flbplugin.mdsd.*
+    Tag oms.container.log.flbplugin.mdsd.err.*
     Path /var/opt/microsoft/linuxmonagent/log/mdsd.err
     Read_from_Head true
     DB /var/opt/microsoft/docker-cimprov/state/mdsd-ai.db
@@ -91,6 +91,18 @@
     Match oms.container.log.flbplugin.mdsd.qos.*
     regex log ^MaODSRequest,https.*_LogManagement_LINUX_SYSLOGS_BLOB
 
+[FILTER]
+    Name grep
+    Alias ama-logs_mdsd_err_filter
+    Match oms.container.log.flbplugin.mdsd.err.*
+    # Exclude noisy AMACoreAgent startup tenant not started messages in RS
+    # Consolidated exclusion of specific transient AMACoreAgent startup/connection retry messages:
+    #  - OtlpTokenFetcher AMACoreAgent tenant not started
+    #  - CreateSocket / ConnectToPipelineAgent socket connection failures to AMACoreAgent
+    #  - PAConnector::StartTenant / SendAmcsData failures
+    #  - "Will retry one more time in 5 seconds." retry notice (does not include AMACoreAgent keyword)
+    Exclude log (OtlpTokenFetcher.*AMACoreAgent tenant not started|CreateSocket.*AMACoreAgent|ConnectToPipelineAgent.*AMACoreAgent|PAConnector::StartTenant.*AMACoreAgent|PAConnector::SendAmcsData.*AMACoreAgent|PAConnector::AmcsStopTenant.*AMACoreAgent|Will retry one more time in 5 seconds\.)
+
 [FILTER]
     Name grep
     Alias ama-logs_amaca_log_tail

build/linux/installer/conf/fluent-bit-rs.conf

@@ -25,7 +25,7 @@
 
 [INPUT]
     Name tail
-    Tag oms.container.log.flbplugin.mdsd.*
+    Tag oms.container.log.flbplugin.mdsd.err.*
     Path /var/opt/microsoft/linuxmonagent/log/mdsd.err
     Read_from_Head true
     DB /var/opt/microsoft/docker-cimprov/state/mdsd-ai.db
@@ -45,6 +45,18 @@
     Buffer_Size ${AZMON_FBIT_BUFFER_SIZE}
     Mem_Buf_Limit ${AZMON_FBIT_MEM_BUF_LIMIT}
 
+[FILTER]
+    Name grep
+    Alias ama-logs_mdsd_err_filter
+    Match oms.container.log.flbplugin.mdsd.err.*
+    # Exclude noisy AMACoreAgent startup tenant not started messages in RS
+    # Consolidated exclusion of specific transient AMACoreAgent startup/connection retry messages:
+    #  - OtlpTokenFetcher AMACoreAgent tenant not started
+    #  - CreateSocket / ConnectToPipelineAgent socket connection failures to AMACoreAgent
+    #  - PAConnector::StartTenant / SendAmcsData failures
+    #  - "Will retry one more time in 5 seconds." retry notice (does not include AMACoreAgent keyword)
+    Exclude log (OtlpTokenFetcher.*AMACoreAgent tenant not started|CreateSocket.*AMACoreAgent|ConnectToPipelineAgent.*AMACoreAgent|PAConnector::StartTenant.*AMACoreAgent|PAConnector::SendAmcsData.*AMACoreAgent|PAConnector::AmcsStopTenant.*AMACoreAgent|Will retry one more time in 5 seconds\.)
+
 [OUTPUT]
     Name                            oms
     EnableTelemetry                 true