Enabled OTel logs and traces support (#1527)

* OTel logs support

* Telemetry changes

* Resolving comments

* Add auto instrument telemetry

* Removed additional env vars

Author: suyadav1

build/linux/installer/scripts/livenessprobe.sh

@@ -70,13 +70,13 @@ then
  exit 1
 fi
 
-if [[ "${IS_HIGH_LOG_SCALE_MODE}" == "true" || "${AZMON_MULTI_TENANCY_LOGS_SERVICE_MODE}" == "true" ]]; then
-      (ps -ef | grep "amacoreagent" | grep -v "grep")
-      if [ $? -ne 0 ]
-      then
-        echo "amacoreagent is not running" > /dev/termination-log
-        exit 1
-      fi
+if [[ "${IS_HIGH_LOG_SCALE_MODE}" == "true" || "${AZMON_MULTI_TENANCY_LOGS_SERVICE_MODE}" == "true" || "${APPMONITORING_OPENTELEMETRYLOGS_ENABLED}" == "true" ]]; then
+  (ps -ef | grep "amacoreagent" | grep -v "grep")
+  if [ $? -ne 0 ]
+  then
+   echo "amacoreagent is not running when high log scale or OpenTelemetry logs is enabled" > /dev/termination-log
+   exit 1
+  fi
 fi
 
 

kubernetes/ama-logs.yaml

@@ -454,6 +454,8 @@ spec:
               value: "false"
             - name: APPMONITORING_OPENTELEMETRYLOGS_PORT
               value: "28331"
+            - name: AZMON_OPENTELEMETRYLOGS_CONTAINER_PORT
+              value: "4319"
             - name: AZMON_RETINA_FLOW_LOGS_ENABLED
               value: "false"
             - name: CLUSTER_CLOUD_ENVIRONMENT
@@ -474,6 +476,10 @@ spec:
               containerPort: 28330
               hostPort: 28330
               protocol: TCP
+            - name: otlp-logs
+              containerPort: 4319
+              hostPort: 28331
+              protocol: TCP
           volumeMounts:
             - name: kube-api-access
               mountPath: /var/run/secrets/kubernetes.io/serviceaccount
@@ -1093,10 +1099,6 @@ spec:
             #   value: "true"
             - name: APPMONITORING_AUTOINSTRUMENTATION_ENABLED
               value: "false"
-            - name: APPMONITORING_OPENTELEMETRYLOGS_ENABLED
-              value: "false"
-            - name: APPMONITORING_OPENTELEMETRYLOGS_PORT
-              value: "28331"
             - name: AZMON_TELEGRAF_LIVENESSPROBE_ENABLED
               value: "false"
             - name: AZMON_WINDOWS_FLUENT_BIT_ENABLED

kubernetes/linux/main.sh

@@ -60,7 +60,7 @@ getClusterCloudEnvironment() {
 }
 
 startAMACoreAgent() {
-      echo "AMACoreAgent: Starting AMA Core Agent since High Log scale mode is enabled"
+      echo "AMACoreAgent: Starting AMA Core Agent since High Log scale or OpenTelemetry logs is enabled"
 
       export AMACALogFileDir="/var/opt/microsoft/linuxmonagent/amaca/log"
       export AMACALogFilePath="$AMACALogFileDir"/amaca.log
@@ -79,7 +79,26 @@ startAMACoreAgent() {
          echo "export PA_CONFIG_PORT=$PA_CONFIG_PORT"
          echo "export CounterDataReportFrequencyInMinutes=$CounterDataReportFrequencyInMinutes"
          echo "export AMACALogFilePath=$AMACALogFilePath"
-      } >> ~/.bashrc
+      } >> ~/.bashrc  
+
+      if isOpenTelemetryLogsEnabled; then
+            export PA_AMCS_PROTOCOL="HttpProtobuf"
+            export PA_AMCS_HOST="0.0.0.0"
+            export PA_AMCS_PORT=4319
+            if [ -n "${AZMON_OPENTELEMETRYLOGS_CONTAINER_PORT}" ]; then
+                  # Convert AZMON_OPENTELEMETRYLOGS_CONTAINER_PORT from string to int
+                  port_int=$((AZMON_OPENTELEMETRYLOGS_CONTAINER_PORT + 0))
+                  if [ "$port_int" -gt 0 ] 2>/dev/null; then
+                        export PA_AMCS_PORT=$port_int
+                  fi
+            fi
+            
+            {
+                  echo "export PA_AMCS_PROTOCOL=$PA_AMCS_PROTOCOL"
+                  echo "export PA_AMCS_PORT=$PA_AMCS_PORT"
+                  echo "export PA_AMCS_HOST=$PA_AMCS_HOST"
+            } >> ~/.bashrc
+      fi
 
       source ~/.bashrc
       /opt/microsoft/azure-mdsd/bin/amacoreagent --configport $PA_CONFIG_PORT --amacalog $AMACALogFilePath --giglaport $PA_DATA_PORT > /dev/null 2>&1 &
@@ -230,6 +249,14 @@ isHighLogScaleMode() {
      fi
 }
 
+isOpenTelemetryLogsEnabled() {
+      if [[ "${APPMONITORING_OPENTELEMETRYLOGS_ENABLED}" == "true" ]]; then
+          true
+      else
+          false
+      fi
+}
+
 checkAgentOnboardingStatus() {
       local sleepdurationsecs=1
       local totalsleptsecs=0
@@ -494,10 +521,10 @@ source common_agent_config_env_var
 
 # check if high log scale mode enabled
 if isHighLogScaleMode; then
-    echo "Enabled High Log Scale Mode"
-    export IS_HIGH_LOG_SCALE_MODE=true
-    echo "export IS_HIGH_LOG_SCALE_MODE=$IS_HIGH_LOG_SCALE_MODE" >>~/.bashrc
-    source ~/.bashrc
+      echo "Enabled High Log Scale Mode"
+      export IS_HIGH_LOG_SCALE_MODE=true
+      echo "export IS_HIGH_LOG_SCALE_MODE=$IS_HIGH_LOG_SCALE_MODE" >>~/.bashrc
+      source ~/.bashrc
 fi
 
 #Parse the configmap to set the right environment variables for agent config.
@@ -1088,7 +1115,7 @@ if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
     fi
 else
       echo "starting mdsd in main container..."
-      if isHighLogScaleMode; then
+      if isHighLogScaleMode || isOpenTelemetryLogsEnabled; then
             startAMACoreAgent
       fi
       export MDSD_ROLE_PREFIX=/var/run/mdsd-ci/default

kubernetes/linux/setup.sh

@@ -37,7 +37,7 @@ rm -rf /usr/lib/ruby/gems/3.3.0/gems/rdoc-6.6.3.1
 # remove net-imap gem as it has a known CVE (CVE-2025-43857) and is not used by the agent
 gem uninstall net-imap --force
 
-sudo tdnf install -y azure-mdsd-1.35.7
+sudo tdnf install -y azure-mdsd-1.37.0
 cp -f $TMPDIR/mdsd.xml /etc/mdsd.d
 cp -f $TMPDIR/envmdsd /etc/mdsd.d
 rm /usr/sbin/telegraf

source/plugins/go/src/telemetry.go

@@ -292,6 +292,23 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) {
 					telemetryDimensions["isHighLogScaleMode"] = isHighLogScaleMode
 				}
 
+				isOpenTelemetryLogsEnabled := os.Getenv("APPMONITORING_OPENTELEMETRYLOGS_ENABLED")
+				if isOpenTelemetryLogsEnabled != "" {
+					telemetryDimensions["OtelLogs"] = isOpenTelemetryLogsEnabled
+					if strings.EqualFold(isOpenTelemetryLogsEnabled, "true") {
+						otelLogsPort := os.Getenv("APPMONITORING_OPENTELEMETRYLOGS_PORT")
+						// Otel node port 28331 is the default port. We will send the port only if it is configured to a different port
+						if otelLogsPort != "28331" {
+							telemetryDimensions["OtelLogsPort"] = otelLogsPort
+						}
+						isAppMonitoringAutoInstrumentationEnabled := os.Getenv("APPMONITORING_AUTOINSTRUMENTATION_ENABLED")
+						// By default otel is enabled with App monitoring auto instrumentation so only send the value if it is set to false
+						if isAppMonitoringAutoInstrumentationEnabled == "false" {
+							telemetryDimensions["AppMonAutoInstrument"] = "false"
+						}
+					}
+				}
+
 				isAzMonMultitenancyEnabled := os.Getenv("AZMON_MULTI_TENANCY_LOG_COLLECTION")
 				isAzMonMultitenancyAdvancedMode := os.Getenv("AZMON_MULTI_TENANCY_LOG_COLLECTION_ADVANCED_MODE")
 				isAzMonMultitenancyDefaultFallbackIngestionDisabled := os.Getenv("AZMON_MULTI_TENANCY_FALLBACK_INGESTION_DISABLED")