attempt to ensure otel collector logs go to stdout (#1228)

Author: brandon-pereira

.changeset/quick-otters-camp.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+Add new logging pararmeter for otel collector

docker-compose.dev.yml

@@ -33,7 +33,7 @@ services:
       OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
       CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
       # Uncomment to enable stdout logging for the OTel collector
-      OTEL_SUPERVISOR_PASSTHROUGH_LOGS: 'false'
+      # OTEL_SUPERVISOR_LOGS: 'true'
       # Uncomment to enable JSON schema in ClickHouse
       # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
       # OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'

docker/otel-collector/Dockerfile

@@ -26,6 +26,7 @@ COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol
 # Copy entrypoint and log rotation scripts
 COPY --chmod=755 ./entrypoint.sh /entrypoint.sh
 COPY --chmod=755 ./log-rotator.sh /log-rotator.sh
+COPY --chmod=755 ./log-tailer.sh /log-tailer.sh
 
 ## dev ##############################################################################################
 FROM base AS dev

docker/otel-collector/entrypoint.sh

@@ -5,6 +5,12 @@ set -e
 # Arguments: log_file_path [max_size_mb] [max_archives] [check_interval_seconds]
 /log-rotator.sh /etc/otel/supervisor-data/agent.log 16 1 60 &
 
+# Start log tailer script in background for agent.log
+# Arguments: log_file_path [check_interval_seconds]
+if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then
+    /log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
+fi
+
 # Render the supervisor config template using gomplate
 # Write to supervisor-data directory which has proper permissions for otel user
 gomplate -f /etc/otel/supervisor.yaml.tmpl -o /etc/otel/supervisor-data/supervisor-runtime.yaml

docker/otel-collector/log-tailer.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# Generic log tailer script that follows a log file and echoes new lines to stdout
+# Handles file rotation and truncation (e.g., when used with log-rotator.sh)
+# Usage: log-tailer.sh <log_file_path> [sleep_interval]
+
+# Parse arguments
+LOG_FILE="${1}"
+SLEEP_INTERVAL="${2:-1}"
+
+# Validate required argument
+if [ -z "$LOG_FILE" ]; then
+    echo "Error: Log file path is required" >&2
+    echo "Usage: $0 <log_file_path> [sleep_interval]" >&2
+    exit 1
+fi
+
+# Wait for file to exist if it doesn't yet
+while [ ! -f "$LOG_FILE" ]; do
+    echo "Waiting for log file to be created: $LOG_FILE" >&2
+    sleep "$SLEEP_INTERVAL"
+done
+
+echo "Starting to tail: $LOG_FILE" >&2
+
+# Use tail -F to follow the file by name, not by descriptor
+# This handles rotation and truncation gracefully
+# -n 0: Start from the end (don't output existing content)
+# -F: Follow by name and retry if file is inaccessible
+# -s: Sleep interval between checks
+tail -n 0 -F -s "$SLEEP_INTERVAL" "$LOG_FILE"

docker/otel-collector/supervisor_docker.yaml.tmpl

@@ -27,7 +27,6 @@ agent:
 {{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
     - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
 {{- end }}
-  passthrough_logs: {{ getenv "OTEL_SUPERVISOR_PASSTHROUGH_LOGS" | default "false" }}
 
 storage:
   directory: /etc/otel/supervisor-data/