Skip to content

Commit 31e22dc

Browse files
wrn14897wrn14897
authored
feat: introduce clickhouse db init script (#843)
Ref: HDX-1777 This shouldn't have any impact on users
1 parent 06332dc commit 31e22dc

File tree

5 files changed

+139
-2
lines changed

5 files changed

+139
-2
lines changed

‎.changeset/quiet-singers-fry.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@hyperdx/api": patch
3+
"@hyperdx/app": patch
4+
"@hyperdx/common-utils": patch
5+
---
6+
7+
feat: introduce clickhouse db init script

‎docker-compose.ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ services:
2929
volumes:
3030
- ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
3131
- ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
32+
- ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
3233
restart: on-failure
3334
ports:
3435
- 8123:8123 # http api

‎docker-compose.dev.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,11 @@ services:
5353
environment:
5454
# default settings
5555
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
56+
HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}
5657
volumes:
5758
- ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml
5859
- ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml
60+
- ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
5961
- .volumes/ch_data_dev:/var/lib/clickhouse
6062
- .volumes/ch_logs_dev:/var/log/clickhouse-server
6163
restart: on-failure

‎docker/clickhouse/local/init-db.sh‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/bin/bash
2+
set -e
3+
4+
DATABASE=${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default}
5+
6+
clickhouse client -n <<EOFSQL
7+
CREATE DATABASE IF NOT EXISTS ${DATABASE};
8+
9+
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
10+
(
11+
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
12+
\`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp),
13+
\`TraceId\` String CODEC(ZSTD(1)),
14+
\`SpanId\` String CODEC(ZSTD(1)),
15+
\`TraceFlags\` UInt8,
16+
\`SeverityText\` LowCardinality(String) CODEC(ZSTD(1)),
17+
\`SeverityNumber\` UInt8,
18+
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
19+
\`Body\` String CODEC(ZSTD(1)),
20+
\`ResourceSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
21+
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
22+
\`ScopeSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
23+
\`ScopeName\` String CODEC(ZSTD(1)),
24+
\`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)),
25+
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
26+
\`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
27+
\`__hdx_materialized_k8s.cluster.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)),
28+
\`__hdx_materialized_k8s.container.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)),
29+
\`__hdx_materialized_k8s.deployment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)),
30+
\`__hdx_materialized_k8s.namespace.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)),
31+
\`__hdx_materialized_k8s.node.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)),
32+
\`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)),
33+
\`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)),
34+
\`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)),
35+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
36+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
37+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
38+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
39+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
40+
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
41+
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
42+
INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
43+
)
44+
ENGINE = MergeTree
45+
PARTITION BY toDate(TimestampTime)
46+
PRIMARY KEY (ServiceName, TimestampTime)
47+
ORDER BY (ServiceName, TimestampTime, Timestamp)
48+
TTL TimestampTime + toIntervalDay(30)
49+
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
50+
51+
CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces
52+
(
53+
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
54+
\`TraceId\` String CODEC(ZSTD(1)),
55+
\`SpanId\` String CODEC(ZSTD(1)),
56+
\`ParentSpanId\` String CODEC(ZSTD(1)),
57+
\`TraceState\` String CODEC(ZSTD(1)),
58+
\`SpanName\` LowCardinality(String) CODEC(ZSTD(1)),
59+
\`SpanKind\` LowCardinality(String) CODEC(ZSTD(1)),
60+
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
61+
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
62+
\`ScopeName\` String CODEC(ZSTD(1)),
63+
\`ScopeVersion\` String CODEC(ZSTD(1)),
64+
\`SpanAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
65+
\`Duration\` UInt64 CODEC(ZSTD(1)),
66+
\`StatusCode\` LowCardinality(String) CODEC(ZSTD(1)),
67+
\`StatusMessage\` String CODEC(ZSTD(1)),
68+
\`Events.Timestamp\` Array(DateTime64(9)) CODEC(ZSTD(1)),
69+
\`Events.Name\` Array(LowCardinality(String)) CODEC(ZSTD(1)),
70+
\`Events.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
71+
\`Links.TraceId\` Array(String) CODEC(ZSTD(1)),
72+
\`Links.SpanId\` Array(String) CODEC(ZSTD(1)),
73+
\`Links.TraceState\` Array(String) CODEC(ZSTD(1)),
74+
\`Links.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
75+
\`__hdx_materialized_rum.sessionId\` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)),
76+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
77+
INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
78+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
79+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
80+
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
81+
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
82+
INDEX idx_duration Duration TYPE minmax GRANULARITY 1,
83+
INDEX idx_lower_span_name lower(SpanName) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
84+
)
85+
ENGINE = MergeTree
86+
PARTITION BY toDate(Timestamp)
87+
ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
88+
TTL toDate(Timestamp) + toIntervalDay(30)
89+
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1;
90+
91+
CREATE TABLE ${DATABASE}.hyperdx_sessions
92+
(
93+
\`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)),
94+
\`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp),
95+
\`TraceId\` String CODEC(ZSTD(1)),
96+
\`SpanId\` String CODEC(ZSTD(1)),
97+
\`TraceFlags\` UInt8,
98+
\`SeverityText\` LowCardinality(String) CODEC(ZSTD(1)),
99+
\`SeverityNumber\` UInt8,
100+
\`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)),
101+
\`Body\` String CODEC(ZSTD(1)),
102+
\`ResourceSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
103+
\`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
104+
\`ScopeSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)),
105+
\`ScopeName\` String CODEC(ZSTD(1)),
106+
\`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)),
107+
\`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
108+
\`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)),
109+
\`__hdx_materialized_rum.sessionId\` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)),
110+
\`__hdx_materialized_type\` LowCardinality(String) MATERIALIZED toString(simpleJSONExtractInt(Body, 'type')) CODEC(ZSTD(1)),
111+
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
112+
INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1,
113+
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
114+
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
115+
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
116+
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
117+
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
118+
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
119+
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8
120+
)
121+
ENGINE = MergeTree
122+
PARTITION BY toDate(TimestampTime)
123+
PRIMARY KEY (ServiceName, TimestampTime)
124+
ORDER BY (ServiceName, TimestampTime, Timestamp)
125+
TTL TimestampTime + toIntervalDay(30)
126+
SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1
127+
EOFSQL

‎docker/otel-collector/config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ exporters:
116116
endpoint: ${env:CLICKHOUSE_ENDPOINT}
117117
password: ${env:CLICKHOUSE_PASSWORD}
118118
username: ${env:CLICKHOUSE_USER}
119-
ttl: 72h
119+
ttl: 720h
120120
logs_table_name: hyperdx_sessions
121121
timeout: 5s
122122
retry_on_failure:
@@ -129,7 +129,7 @@ exporters:
129129
endpoint: ${env:CLICKHOUSE_ENDPOINT}
130130
password: ${env:CLICKHOUSE_PASSWORD}
131131
username: ${env:CLICKHOUSE_USER}
132-
ttl: 72h
132+
ttl: 720h
133133
timeout: 5s
134134
retry_on_failure:
135135
enabled: true

0 commit comments

Comments
 (0)