Skip to content

Commit 98c13bb

Browse files
adambenhassenadambenhassen
committed
feat: add redpanda buffering layer with split ingress/egress otel collectors
1 parent 08f22ec commit 98c13bb

File tree

5 files changed

+173
-119
lines changed

5 files changed

+173
-119
lines changed

‎docker/configs/otel-collector/builder-config.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ dist:
66

77
receivers:
88
- gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.122.0
9+
- gomod:
10+
github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.122.0
911

1012
processors:
1113
- gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.122.0
@@ -20,6 +22,8 @@ exporters:
2022
- gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.122.0
2123
- gomod:
2224
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v0.122.0
25+
- gomod:
26+
github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.122.0
2327

2428
extensions:
2529
- gomod:
@@ -29,9 +33,6 @@ extensions:
2933
github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension
3034
v0.122.0
3135
- gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.122.0
32-
- gomod:
33-
github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage
34-
v0.122.0
3536
- gomod: github.com/graphql-hive/console/docker/configs/otel-collector/extension-hiveauth v0.0.0
3637
path: ./extension-hiveauth
3738
name: hiveauthextension # when using local extensions, package name is required, otherwise you get "missing import path"

‎docker/configs/otel-collector/config-egress.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
extensions:
2+
health_check:
3+
endpoint: '0.0.0.0:13133'
4+
receivers:
5+
kafka:
6+
brokers:
7+
- broker:29092
8+
topic: otel-traces
9+
encoding: otlp_proto
10+
group_id: otel-collector-egress
11+
session_timeout: 30s
12+
fetch:
13+
min: 1
14+
default: 524288 # 512 KB - fetch conservatively
15+
processors:
16+
batch:
17+
timeout: 5s
18+
send_batch_size: 8000
19+
exporters:
20+
debug:
21+
verbosity: basic
22+
clickhouse:
23+
endpoint: ${CLICKHOUSE_PROTOCOL}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}?dial_timeout=10s&compress=lz4&async_insert=1&wait_for_async_insert=0
24+
database: default
25+
async_insert: true
26+
username: ${CLICKHOUSE_USERNAME}
27+
password: ${CLICKHOUSE_PASSWORD}
28+
create_schema: false
29+
ttl: 720h
30+
compress: lz4
31+
logs_table_name: otel_logs
32+
traces_table_name: otel_traces
33+
metrics_table_name: otel_metrics
34+
timeout: 5s
35+
retry_on_failure:
36+
enabled: true
37+
initial_interval: 5s
38+
max_interval: 30s
39+
max_elapsed_time: 300s
40+
service:
41+
extensions:
42+
- health_check
43+
telemetry:
44+
logs:
45+
level: DEBUG
46+
encoding: json
47+
output_paths: ['stdout']
48+
error_output_paths: ['stderr']
49+
metrics:
50+
address: '0.0.0.0:10254'
51+
level: detailed
52+
pipelines:
53+
traces:
54+
receivers: [kafka]
55+
processors:
56+
- batch
57+
exporters:
58+
- clickhouse
59+
# - debug

‎docker/configs/otel-collector/config-ingress.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
extensions:
2+
hiveauth:
3+
endpoint: ${HIVE_OTEL_AUTH_ENDPOINT}
4+
health_check:
5+
endpoint: '0.0.0.0:13133'
6+
receivers:
7+
otlp:
8+
protocols:
9+
grpc:
10+
include_metadata: true
11+
endpoint: '0.0.0.0:4317'
12+
auth:
13+
authenticator: hiveauth
14+
http:
15+
cors:
16+
allowed_origins: ['*']
17+
allowed_headers: ['*']
18+
include_metadata: true
19+
endpoint: '0.0.0.0:4318'
20+
auth:
21+
authenticator: hiveauth
22+
processors:
23+
attributes:
24+
actions:
25+
- key: hive.target_id
26+
from_context: auth.targetId
27+
action: insert
28+
memory_limiter:
29+
check_interval: 1s
30+
limit_mib: 512
31+
spike_limit_mib: 128
32+
exporters:
33+
debug:
34+
verbosity: basic
35+
kafka:
36+
brokers:
37+
- broker:29092
38+
topic: otel-traces
39+
encoding: otlp_proto
40+
producer:
41+
compression: snappy
42+
max_message_bytes: 1000000
43+
timeout: 10s
44+
retry_on_failure:
45+
enabled: true
46+
initial_interval: 5s
47+
max_interval: 30s
48+
max_elapsed_time: 300s
49+
service:
50+
extensions:
51+
- hiveauth
52+
- health_check
53+
telemetry:
54+
logs:
55+
level: DEBUG
56+
encoding: json
57+
output_paths: ['stdout']
58+
error_output_paths: ['stderr']
59+
metrics:
60+
address: '0.0.0.0:10254'
61+
level: detailed
62+
pipelines:
63+
traces:
64+
receivers: [otlp]
65+
processors:
66+
- memory_limiter
67+
- attributes
68+
exporters:
69+
- kafka
70+
# - debug

‎docker/configs/otel-collector/config.yaml‎

Lines changed: 0 additions & 105 deletions
This file was deleted.

‎docker/docker-compose.dev.yml‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ services:
9393
- 'stack'
9494

9595
broker:
96-
image: redpandadata/redpanda:v23.3.21
96+
image: redpandadata/redpanda:v25.2.11
9797
mem_limit: 300m
9898
mem_reservation: 100m
9999
hostname: broker
@@ -123,6 +123,19 @@ services:
123123
volumes:
124124
- ./.hive-dev/broker/db:/var/lib/kafka/data
125125

126+
broker_provision_topics:
127+
image: redpandadata/redpanda:v25.2.11
128+
depends_on:
129+
- broker
130+
restart: 'no'
131+
networks:
132+
- 'stack'
133+
entrypoint: >
134+
/bin/sh -c "
135+
sleep 5;
136+
rpk topic create otel-traces --brokers broker:29092 || true;
137+
exit 0"
138+
126139
supertokens:
127140
image: registry.supertokens.io/supertokens/supertokens-postgresql:9.3
128141
mem_limit: 300m
@@ -176,31 +189,47 @@ services:
176189
networks:
177190
- 'stack'
178191

179-
otel-collector:
192+
otel-collector-ingress:
193+
depends_on:
194+
broker_provision_topics:
195+
condition: service_completed_successfully
196+
build:
197+
context: ./configs/otel-collector
198+
dockerfile: ./../../otel-collector.dockerfile
199+
mem_limit: 512m
200+
environment:
201+
HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth'
202+
volumes:
203+
- ./configs/otel-collector/builder-config.yaml:/builder-config.yaml
204+
- ./configs/otel-collector/config-ingress.yaml:/etc/otel-config.yaml
205+
ports:
206+
- '4317:4317'
207+
- '4318:4318'
208+
- '10254:10254'
209+
networks:
210+
- 'stack'
211+
212+
otel-collector-egress:
180213
depends_on:
181214
clickhouse:
182215
condition: service_healthy
216+
broker_provision_topics:
217+
condition: service_completed_successfully
183218
build:
184219
context: ./configs/otel-collector
185220
dockerfile: ./../../otel-collector.dockerfile
186-
mem_limit: 1000m
221+
mem_limit: 512m
187222
environment:
188-
HIVE_OTEL_AUTH_ENDPOINT: 'http://host.docker.internal:3001/otel-auth'
189223
CLICKHOUSE_PROTOCOL: 'http'
190224
CLICKHOUSE_HOST: clickhouse
191225
CLICKHOUSE_PORT: 8123
192226
CLICKHOUSE_USERNAME: test
193227
CLICKHOUSE_PASSWORD: test
194228
volumes:
195229
- ./configs/otel-collector/builder-config.yaml:/builder-config.yaml
196-
- ./configs/otel-collector/config.yaml:/etc/otel-config.yaml
230+
- ./configs/otel-collector/config-egress.yaml:/etc/otel-config.yaml
197231
ports:
198-
- '4317:4317'
199-
- '4318:4318'
200-
- '10254:10254'
201-
- '1777:1777'
202-
- '8081:8081'
203-
- '55679:55679'
232+
- '10255:10254'
204233
networks:
205234
- 'stack'
206235

0 commit comments

Comments
 (0)