Skip to content

Commit 867f83a

Browse files
adambenhassenGemini
adambenhassen
authored and
Gemini
committed
feat: add redpanda buffering layer with split ingress/egress otel collectors
1 parent 08f22ec commit 867f83a

File tree

5 files changed

+169
-121
lines changed

5 files changed

+169
-121
lines changed

‎docker/configs/otel-collector/builder-config.yaml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ dist:
66

77
receivers:
88
- gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.122.0
9+
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.122.0
910

1011
processors:
1112
- gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.122.0
@@ -20,18 +21,15 @@ exporters:
2021
- gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.122.0
2122
- gomod:
2223
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v0.122.0
24+
- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.122.0
2325

2426
extensions:
2527
- gomod:
2628
github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension
2729
v0.122.0
2830
- gomod:
29-
github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension
30-
v0.122.0
31+
github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.122.0
3132
- gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.122.0
32-
- gomod:
33-
github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage
34-
v0.122.0
3533
- gomod: github.com/graphql-hive/console/docker/configs/otel-collector/extension-hiveauth v0.0.0
3634
path: ./extension-hiveauth
3735
name: hiveauthextension # when using local extensions, package name is required, otherwise you get "missing import path"

‎docker/configs/otel-collector/config-egress.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
extensions:
2+
health_check:
3+
endpoint: '0.0.0.0:13133'
4+
receivers:
5+
kafka:
6+
brokers:
7+
- broker:29092
8+
topic: otel-traces
9+
encoding: otlp_proto
10+
group_id: otel-collector-egress
11+
session_timeout: 30s
12+
fetch:
13+
min: 1
14+
default: 524288 # 512 KB - fetch conservatively
15+
processors:
16+
batch:
17+
timeout: 5s
18+
send_batch_size: 8000
19+
exporters:
20+
debug:
21+
verbosity: basic
22+
clickhouse:
23+
endpoint: ${CLICKHOUSE_PROTOCOL}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}?dial_timeout=10s&compress=lz4&async_insert=1&wait_for_async_insert=0
24+
database: default
25+
async_insert: true
26+
username: ${CLICKHOUSE_USERNAME}
27+
password: ${CLICKHOUSE_PASSWORD}
28+
create_schema: false
29+
ttl: 720h
30+
compress: lz4
31+
logs_table_name: otel_logs
32+
traces_table_name: otel_traces
33+
metrics_table_name: otel_metrics
34+
timeout: 5s
35+
retry_on_failure:
36+
enabled: true
37+
initial_interval: 5s
38+
max_interval: 30s
39+
max_elapsed_time: 300s
40+
service:
41+
extensions:
42+
- health_check
43+
telemetry:
44+
logs:
45+
level: DEBUG
46+
encoding: json
47+
output_paths: ['stdout']
48+
error_output_paths: ['stderr']
49+
metrics:
50+
address: '0.0.0.0:10254'
51+
level: detailed
52+
pipelines:
53+
traces:
54+
receivers: [kafka]
55+
processors:
56+
- batch
57+
exporters:
58+
- clickhouse
59+
# - debug

‎docker/configs/otel-collector/config-ingress.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
extensions:
2+
hiveauth:
3+
endpoint: ${HIVE_OTEL_AUTH_ENDPOINT}
4+
health_check:
5+
endpoint: '0.0.0.0:13133'
6+
receivers:
7+
otlp:
8+
protocols:
9+
grpc:
10+
include_metadata: true
11+
endpoint: '0.0.0.0:4317'
12+
auth:
13+
authenticator: hiveauth
14+
http:
15+
cors:
16+
allowed_origins: ['*']
17+
allowed_headers: ['*']
18+
include_metadata: true
19+
endpoint: '0.0.0.0:4318'
20+
auth:
21+
authenticator: hiveauth
22+
processors:
23+
attributes:
24+
actions:
25+
- key: hive.target_id
26+
from_context: auth.targetId
27+
action: insert
28+
memory_limiter:
29+
check_interval: 1s
30+
limit_mib: 512
31+
spike_limit_mib: 128
32+
exporters:
33+
debug:
34+
verbosity: basic
35+
kafka:
36+
brokers:
37+
- broker:29092
38+
topic: otel-traces
39+
encoding: otlp_proto
40+
producer:
41+
compression: snappy
42+
max_message_bytes: 1000000
43+
timeout: 10s
44+
retry_on_failure:
45+
enabled: true
46+
initial_interval: 5s
47+
max_interval: 30s
48+
max_elapsed_time: 300s
49+
service:
50+
extensions:
51+
- hiveauth
52+
- health_check
53+
telemetry:
54+
logs:
55+
level: DEBUG
56+
encoding: json
57+
output_paths: ['stdout']
58+
error_output_paths: ['stderr']
59+
metrics:
60+
address: '0.0.0.0:10254'
61+
level: detailed
62+
pipelines:
63+
traces:
64+
receivers: [otlp]
65+
processors:
66+
- memory_limiter
67+
- attributes
68+
exporters:
69+
- kafka
70+
# - debug

‎docker/configs/otel-collector/config.yaml‎

Lines changed: 0 additions & 105 deletions
This file was deleted.

‎docker/docker-compose.dev.yml‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ services:
9393
- 'stack'
9494

9595
broker:
96-
image: redpandadata/redpanda:v23.3.21
96+
image: redpandadata/redpanda:v25.2.11
9797
mem_limit: 300m
9898
mem_reservation: 100m
9999
hostname: broker
@@ -123,6 +123,16 @@ services:
123123
volumes:
124124
- ./.hive-dev/broker/db:/var/lib/kafka/data
125125

126+
broker_provision_topics:
127+
image: redpandadata/redpanda:v25.2.11
128+
depends_on:
129+
- broker
130+
restart: 'no'
131+
networks:
132+
- 'stack'
133+
entrypoint: >
134+
/bin/sh -c " sleep 5; rpk topic create otel-traces --brokers broker:29092 || true; exit 0"
135+
126136
supertokens:
127137
image: registry.supertokens.io/supertokens/supertokens-postgresql:9.3
128138
mem_limit: 300m
@@ -176,31 +186,47 @@ services:
176186
networks:
177187
- 'stack'
178188

179-
otel-collector:
189+
otel-collector-ingress:
190+
depends_on:
191+
broker_provision_topics:
192+
condition: service_completed_successfully
193+
build:
194+
context: ./configs/otel-collector
195+
dockerfile: ./../../otel-collector.dockerfile
196+
mem_limit: 512m
197+
environment:
198+
HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth'
199+
volumes:
200+
- ./configs/otel-collector/builder-config.yaml:/builder-config.yaml
201+
- ./configs/otel-collector/config-ingress.yaml:/etc/otel-config.yaml
202+
ports:
203+
- '4317:4317'
204+
- '4318:4318'
205+
- '10254:10254'
206+
networks:
207+
- 'stack'
208+
209+
otel-collector-egress:
180210
depends_on:
181211
clickhouse:
182212
condition: service_healthy
213+
broker_provision_topics:
214+
condition: service_completed_successfully
183215
build:
184216
context: ./configs/otel-collector
185217
dockerfile: ./../../otel-collector.dockerfile
186-
mem_limit: 1000m
218+
mem_limit: 512m
187219
environment:
188-
HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth'
189220
CLICKHOUSE_PROTOCOL: 'http'
190221
CLICKHOUSE_HOST: clickhouse
191222
CLICKHOUSE_PORT: 8123
192223
CLICKHOUSE_USERNAME: test
193224
CLICKHOUSE_PASSWORD: test
194225
volumes:
195226
- ./configs/otel-collector/builder-config.yaml:/builder-config.yaml
196-
- ./configs/otel-collector/config.yaml:/etc/otel-config.yaml
227+
- ./configs/otel-collector/config-egress.yaml:/etc/otel-config.yaml
197228
ports:
198-
- '4317:4317'
199-
- '4318:4318'
200-
- '10254:10254'
201-
- '1777:1777'
202-
- '8081:8081'
203-
- '55679:55679'
229+
- '10255:10254'
204230
networks:
205231
- 'stack'
206232

0 commit comments

Comments
 (0)