Skip to content

Commit 43de467

Browse files
wrn14897wrn14897
authored
feat: allow otel-collector to run without OpAMP server (#1672)
Today, users have to set up an OpAMP server to run with our clickstack OTel collector. Instead, we should allow users to disable OpAMP when they're using ClickHouse Cloud with the clickstack integration. This can be determined by `OPAMP_SERVER_URL` not being defined by the user. The end result is that a user can do ``` docker run \ -e CLICKHOUSE_ENDPOINT=${CLICKHOUSE_ENDPOINT} \ -e CLICKHOUSE_USER=default \ -e CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD} \ -p 8080:8080 -p 4317:4317 -p 4318:4318 \ clickhouse/clickstack-otel-collector:latest ``` Ref: HDX-3300
1 parent 7ce9485 commit 43de467

File tree

9 files changed

+153
-215
lines changed

9 files changed

+153
-215
lines changed

‎.changeset/config.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"$schema": "https://unpkg.com/@changesets/config@2.3.1/schema.json",
33
"changelog": "@changesets/cli/changelog",
44
"commit": false,
5-
"fixed": [["@hyperdx/api", "@hyperdx/app"]],
5+
"fixed": [["@hyperdx/api", "@hyperdx/app", "@hyperdx/otel-collector"]],
66
"linked": [],
77
"access": "restricted",
88
"baseBranch": "main",

‎.changeset/smooth-bananas-hammer.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/otel-collector": patch
3+
---
4+
5+
feat: allow otel-collector to run without OpAMP server

‎docker/otel-collector/Dockerfile‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ ARG USER_GID=10001
1313
RUN apk add --no-cache ca-certificates && \
1414
addgroup -S -g ${USER_GID} otel && \
1515
adduser -S -u ${USER_UID} -G otel otel && \
16-
install -d -m 0777 -o ${USER_UID} -g ${USER_GID} /etc/otel/supervisor-data
16+
install -d -m 0777 -o ${USER_UID} -g ${USER_GID} /etc/otel/supervisor-data && \
17+
install -d -m 0755 -o ${USER_UID} -g ${USER_GID} /etc/otelcol-contrib
1718

1819
# Copy gomplate binary from the gomplate image
1920
COPY --from=gomplate /bin/gomplate /usr/local/bin/gomplate
@@ -30,8 +31,9 @@ COPY --chmod=755 ./log-tailer.sh /log-tailer.sh
3031
## dev ##############################################################################################
3132
FROM base AS dev
3233

33-
COPY ./config.yaml /etc/otelcol-contrib/config.yaml
34-
COPY ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
34+
COPY --chown=10001:10001 ./config.yaml /etc/otelcol-contrib/config.yaml
35+
COPY --chown=10001:10001 ./config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
36+
COPY --chown=10001:10001 ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
3537

3638
EXPOSE 4317 4318 13133
3739

@@ -40,8 +42,9 @@ ENTRYPOINT ["/entrypoint.sh", "/opampsupervisor"]
4042
## prod #############################################################################################
4143
FROM base AS prod
4244

43-
COPY ./config.yaml /etc/otelcol-contrib/config.yaml
44-
COPY ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
45+
COPY --chown=10001:10001 ./config.yaml /etc/otelcol-contrib/config.yaml
46+
COPY --chown=10001:10001 ./config.standalone.yaml /etc/otelcol-contrib/standalone-config.yaml
47+
COPY --chown=10001:10001 ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
4548

4649
EXPOSE 4317 4318 13133
4750

‎docker/otel-collector/config.deprecated.yaml‎

Lines changed: 0 additions & 174 deletions
This file was deleted.

‎docker/otel-collector/config.standalone.yaml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# This configuration is derived from packages/api/src/opamp/controllers/opampController.ts
2+
# When updating this file, ensure it stays in sync with buildOtelCollectorConfig()
3+
4+
receivers:
5+
otlp/hyperdx:
6+
protocols:
7+
grpc:
8+
include_metadata: true
9+
endpoint: "0.0.0.0:4317"
10+
http:
11+
cors:
12+
allowed_origins: ["*"]
13+
allowed_headers: ["*"]
14+
include_metadata: true
15+
endpoint: "0.0.0.0:4318"
16+
17+
connectors:
18+
routing/logs:
19+
default_pipelines: [logs/out-default]
20+
error_mode: ignore
21+
table:
22+
- context: log
23+
statement: route() where IsMatch(attributes["rr-web.event"], ".*")
24+
pipelines: [logs/out-rrweb]
25+
26+
exporters:
27+
clickhouse/rrweb:
28+
database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
29+
endpoint: ${env:CLICKHOUSE_ENDPOINT}
30+
password: ${env:CLICKHOUSE_PASSWORD}
31+
username: ${env:CLICKHOUSE_USER}
32+
ttl: 720h
33+
logs_table_name: hyperdx_sessions
34+
timeout: 5s
35+
retry_on_failure:
36+
enabled: true
37+
initial_interval: 5s
38+
max_interval: 30s
39+
max_elapsed_time: 300s
40+
clickhouse:
41+
database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
42+
endpoint: ${env:CLICKHOUSE_ENDPOINT}
43+
password: ${env:CLICKHOUSE_PASSWORD}
44+
username: ${env:CLICKHOUSE_USER}
45+
ttl: 720h
46+
timeout: 5s
47+
retry_on_failure:
48+
enabled: true
49+
initial_interval: 5s
50+
max_interval: 30s
51+
max_elapsed_time: 300s
52+
53+
service:
54+
pipelines:
55+
traces:
56+
receivers: [otlp/hyperdx]
57+
processors: [memory_limiter, batch]
58+
exporters: [clickhouse]
59+
metrics:
60+
receivers: [otlp/hyperdx]
61+
processors: [memory_limiter, batch]
62+
exporters: [clickhouse]
63+
logs/in:
64+
receivers: [otlp/hyperdx]
65+
exporters: [routing/logs]
66+
logs/out-default:
67+
receivers: [routing/logs]
68+
processors: [memory_limiter, transform, batch]
69+
exporters: [clickhouse]
70+
logs/out-rrweb:
71+
receivers: [routing/logs]
72+
processors: [memory_limiter, batch]
73+
exporters: [clickhouse/rrweb]

‎docker/otel-collector/entrypoint.sh‎

Lines changed: 47 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,55 @@
11
#!/bin/sh
22
set -e
33

4-
if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then
5-
# Start log tailer process in background for agent.log
6-
# Arguments: log_file_path [check_interval_seconds]
7-
/log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
8-
9-
# Create a agent log file for the supervisor and collector child process. Normally
10-
# this file would be created as a standard file but we just want a FIFO pipe that
11-
# will pass data over to the tail process in the entrypoint script. This avoids
12-
# the need to the supervisor to store and forward the logs in its memory while also
13-
# eliminating the need for volume based storage.
14-
if [ ! -e /etc/otel/supervisor-data/agent.log ]; then
15-
mkfifo /etc/otel/supervisor-data/agent.log || echo "Failed to create FIFO" >&2
16-
fi
17-
fi
4+
# Check if OPAMP_SERVER_URL is defined to determine mode
5+
if [ -z "$OPAMP_SERVER_URL" ]; then
6+
# Standalone mode - run collector directly without supervisor
7+
echo "Running in standalone mode (OPAMP_SERVER_URL not set)"
8+
9+
# Build collector arguments with multiple config files
10+
COLLECTOR_ARGS="--config /etc/otelcol-contrib/config.yaml --config /etc/otelcol-contrib/standalone-config.yaml"
1811

19-
# Render the supervisor config template using gomplate
20-
# Write to supervisor-data directory which has proper permissions for otel user
21-
gomplate -f /etc/otel/supervisor.yaml.tmpl -o /etc/otel/supervisor-data/supervisor-runtime.yaml
12+
# Add custom config file if specified
13+
if [ -n "$CUSTOM_OTELCOL_CONFIG_FILE" ]; then
14+
echo "Including custom config: $CUSTOM_OTELCOL_CONFIG_FILE"
15+
COLLECTOR_ARGS="$COLLECTOR_ARGS --config $CUSTOM_OTELCOL_CONFIG_FILE"
16+
fi
2217

23-
# Log the configuration being used
24-
if [ -n "$CUSTOM_OTELCOL_CONFIG_FILE" ]; then
25-
echo "Using custom OTEL config file: $CUSTOM_OTELCOL_CONFIG_FILE"
18+
# Execute collector directly
19+
exec /otelcontribcol $COLLECTOR_ARGS
2620
else
27-
echo "CUSTOM_OTELCOL_CONFIG_FILE not set, using default configuration"
28-
fi
21+
# Supervisor mode - run with OpAMP supervisor
22+
echo "Running in supervisor mode (OPAMP_SERVER_URL: $OPAMP_SERVER_URL)"
23+
24+
if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then
25+
# Start log tailer process in background for agent.log
26+
# Arguments: log_file_path [check_interval_seconds]
27+
/log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
2928

30-
# Update the command arguments to use the rendered config file
31-
set -- "$1" --config /etc/otel/supervisor-data/supervisor-runtime.yaml
29+
# Create a agent log file for the supervisor and collector child process. Normally
30+
# this file would be created as a standard file but we just want a FIFO pipe that
31+
# will pass data over to the tail process in the entrypoint script. This avoids
32+
# the need to the supervisor to store and forward the logs in its memory while also
33+
# eliminating the need for volume based storage.
34+
if [ ! -e /etc/otel/supervisor-data/agent.log ]; then
35+
mkfifo /etc/otel/supervisor-data/agent.log || echo "Failed to create FIFO" >&2
36+
fi
37+
fi
38+
39+
# Render the supervisor config template using gomplate
40+
# Write to supervisor-data directory which has proper permissions for otel user
41+
gomplate -f /etc/otel/supervisor.yaml.tmpl -o /etc/otel/supervisor-data/supervisor-runtime.yaml
42+
43+
# Log the configuration being used
44+
if [ -n "$CUSTOM_OTELCOL_CONFIG_FILE" ]; then
45+
echo "Using custom OTEL config file: $CUSTOM_OTELCOL_CONFIG_FILE"
46+
else
47+
echo "CUSTOM_OTELCOL_CONFIG_FILE not set, using default configuration"
48+
fi
49+
50+
# Update the command arguments to use the rendered config file
51+
set -- "$1" --config /etc/otel/supervisor-data/supervisor-runtime.yaml
3252

33-
# Execute the supervisor with all passed arguments
34-
exec "$@"
53+
# Execute the supervisor with all passed arguments
54+
exec "$@"
55+
fi

‎packages/otel-collector/package.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "@hyperdx/otel-collector",
3+
"description": "HyperDX OpenTelemetry Collector configuration and Docker image",
4+
"version": "2.13.0",
5+
"license": "MIT",
6+
"private": true
7+
}

0 commit comments

Comments
 (0)