fix: support otel collector logging with minimal storage (#1509)

This commit sets up a FIFO named pipe with the same name/path that the otel collector and supervisor are expecting. By starting to tail that pipe before starting the collector, we can send log files to stdio without the memory required by the `passthrough_logs` feature or and storage on the volume.

---
Running locally in orbstack, we're still seeing logs on stdout

<img width="2316" height="1452" alt="image" src="https://github.com/user-attachments/assets/f86961cf-6ea4-4faa-82f8-54f9596b5f16" />

but the file size for `agent.log` remains at 0

<img width="1606" height="838" alt="image" src="https://github.com/user-attachments/assets/6feb9470-e220-4d8a-b5b3-10a221926158" />

disk usage stats also remain stable

<img width="1728" height="860" alt="image" src="https://github.com/user-attachments/assets/ff5fe593-4936-446e-9396-606fb495d60d" />

Author: dhable

docker/otel-collector/Dockerfile

@@ -23,9 +23,8 @@ USER ${USER_UID}:${USER_GID}
 COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
 COPY --from=col --chmod=755 /otelcol-contrib /otelcontribcol
 
-# Copy entrypoint and log rotation scripts
+# Copy entrypoint and log tail wrapper scripts
 COPY --chmod=755 ./entrypoint.sh /entrypoint.sh
-COPY --chmod=755 ./log-rotator.sh /log-rotator.sh
 COPY --chmod=755 ./log-tailer.sh /log-tailer.sh
 
 ## dev ##############################################################################################

docker/otel-collector/entrypoint.sh

@@ -1,14 +1,19 @@
 #!/bin/sh
 set -e
 
-# Start log rotation script in background for agent.log
-# Arguments: log_file_path [max_size_mb] [max_archives] [check_interval_seconds]
-/log-rotator.sh /etc/otel/supervisor-data/agent.log 16 1 60 &
-
-# Start log tailer script in background for agent.log
-# Arguments: log_file_path [check_interval_seconds]
 if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then
-    /log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
+ # Start log tailer process in background for agent.log
+  # Arguments: log_file_path [check_interval_seconds]
+  /log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
+
+  # Create a agent log file for the supervisor and collector child process. Normally
+  # this file would be created as a standard file but we just want a FIFO pipe that
+  # will pass data over to the tail process in the entrypoint script. This avoids
+  # the need to the supervisor to store and forward the logs in its memory while also
+  # eliminating the need for volume based storage.
+  if [ ! -e /etc/otel/supervisor-data/agent.log ]; then
+    mkfifo /etc/otel/supervisor-data/agent.log || echo "Failed to create FIFO" >&2
+  fi
 fi
 
 # Render the supervisor config template using gomplate

docker/otel-collector/log-rotator.sh

@@ -14,17 +14,11 @@ if [ -z "$LOG_FILE" ]; then
     exit 1
 fi
 
-# Wait for file to exist if it doesn't yet
-while [ ! -f "$LOG_FILE" ]; do
-    echo "Waiting for log file to be created: $LOG_FILE" >&2
-    sleep "$SLEEP_INTERVAL"
+while true; do
+  # Use tail -F to follow the file by name, not by descriptor
+  # This handles rotation and truncation gracefully
+  # -n 0: Start from the end (don't output existing content)
+  # -F: Follow by name and retry if file is inaccessible
+  # -s: Sleep interval between checks
+  tail -n 0 -F -s "$SLEEP_INTERVAL" "$LOG_FILE" || true
 done
-
-echo "Starting to tail: $LOG_FILE" >&2
-
-# Use tail -F to follow the file by name, not by descriptor
-# This handles rotation and truncation gracefully
-# -n 0: Start from the end (don't output existing content)
-# -F: Follow by name and retry if file is inaccessible
-# -s: Sleep interval between checks
-tail -n 0 -F -s "$SLEEP_INTERVAL" "$LOG_FILE"