chore: split json otel collector to enable both during dev (#1247)

knudtty · wrn14897 · web-flow · commit 19c5085cdedf · 2025-11-04T21:16:41.000Z
Gets us closer to a staging instance of json <img width="216" height="174" alt="image" src="https://github.com/user-attachments/assets/b5cc3cf8-aef0-4ba4-9e9a-8c1d4fad5451" /> Co-authored-by: Warren <5959690+wrn14897@users.noreply.github.com>
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -33,10 +33,7 @@ services:
       OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
       CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
       # Uncomment to enable stdout logging for the OTel collector
-      # OTEL_SUPERVISOR_LOGS: 'true'
-      # Uncomment to enable JSON schema in ClickHouse
-      # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
-      # OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'
+      OTEL_SUPERVISOR_LOGS: 'true'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
@@ -54,6 +51,37 @@ services:
     depends_on:
       ch-server:
         condition: service_healthy
+  otel-collector-json:
+    # image: otel/opentelemetry-collector-contrib:0.120.0
+    build:
+      context: ./docker/otel-collector
+      target: dev
+    environment:
+      CLICKHOUSE_ENDPOINT: 'tcp://ch-server:9000?dial_timeout=10s'
+      CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT: 'ch-server:9363'
+      HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: 'otel_json'
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+      OPAMP_SERVER_URL: 'http://host.docker.internal:${HYPERDX_OPAMP_PORT}'
+      CUSTOM_OTELCOL_CONFIG_FILE: '/etc/otelcol-contrib/custom.config.yaml'
+      # Uncomment to enable stdout logging for the OTel collector
+      OTEL_SUPERVISOR_LOGS: 'true'
+      # Uncomment to enable JSON schema in ClickHouse
+      # Be sure to also set BETA_CH_OTEL_JSON_SCHEMA_ENABLED to 'true' in ch-server
+      OTEL_AGENT_FEATURE_GATE_ARG: '--feature-gates=clickhouse.json'
+    volumes:
+      - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
+      - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
+      # Add a custom config file
+      - ./docker/otel-collector/dev.json.config.yaml:/etc/otelcol-contrib/custom.config.yaml
+    ports:
+      - '14318:4318' # OTLP http receiver
+    restart: always
+    networks:
+      - internal
+    depends_on:
+      ch-server:
+        condition: service_healthy
   ch-server:
     image: clickhouse/clickhouse-server:25.7-alpine
     ports:
diff --git a/docker/otel-collector/custom.config.yaml b/docker/otel-collector/custom.config.yaml
@@ -15,11 +15,45 @@ receivers:
 #     timeout: 10s
 #   memory_limiter:
 #     limit_mib: 2000
+exporters:
+  # fork data to otel-collector-json service (DEV ONLY)
+  otlphttp/json:
+    endpoint: http://otel-collector-json:4318
+    headers:
+      authorization: "super-secure-ingestion-api-key"
+    compression: gzip
 service:
   pipelines:
     metrics/hostmetrics:
-      receivers: [hostmetrics]
-      # attach existing processors
-      processors: [memory_limiter, batch]
-      # attach existing exporters
-      exporters: [clickhouse]
+      exporters:
+        - clickhouse
+      processors:
+        - memory_limiter
+        - batch
+      receivers:
+        - hostmetrics
+    traces/json:
+      exporters:
+        - otlphttp/json
+      processors:
+        - memory_limiter
+        - batch
+      receivers:
+        - otlp/hyperdx
+    # Metrics hasn't used JSON attributes yet
+    # metrics/json:
+    #   exporters:
+    #     - otlphttp/json
+    #   processors:
+    #     - memory_limiter
+    #     - batch
+    #   receivers:
+    #     - otlp/hyperdx
+    logs/json:
+      exporters:
+        - otlphttp/json
+      processors:
+        - memory_limiter
+        - batch
+      receivers:
+        - otlp/hyperdx
diff --git a/docker/otel-collector/dev.json.config.yaml b/docker/otel-collector/dev.json.config.yaml
@@ -0,0 +1,11 @@
+service:
+  pipelines:
+    # ignore rrweb events
+    logs/out-rrweb:
+      exporters:
+        - nop
+      processors:
+        - memory_limiter
+        - batch
+      receivers:
+        - routing/logs
diff --git a/packages/api/.env.development b/packages/api/.env.development
@@ -22,7 +22,7 @@ USAGE_STATS_ENABLED=false
 NODE_OPTIONS="--max-http-header-size=131072"
 ENABLE_SWAGGER=true
 DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]
-DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]
+DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}]
 INGESTION_API_KEY="super-secure-ingestion-api-key"
 HYPERDX_API_KEY=$INGESTION_API_KEY
 ANTHROPIC_API_KEY="your-anthropic-api-key-here"
diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts
@@ -67,6 +67,7 @@ type CollectorConfig = {
     };
   };
   exporters?: {
+    nop?: null;
     debug?: {
       verbosity: string;
       sampling_initial: number;
@@ -184,6 +185,7 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
       },
     },
     exporters: {
+      nop: null,
       debug: {
         verbosity: 'detailed',
         sampling_initial: 5,
