Skip to content

Commit 70f66f3

Browse files
laxmanchekkakotharironakjcchavezs
authored
Kafka streams config fix/tuning: hypertrace-ingester (#91)
Co-authored-by: Ronak <[email protected]> Co-authored-by: José Carlos Chávez <[email protected]>
1 parent 6ad02f4 commit 70f66f3

File tree

16 files changed

+229
-139
lines changed

16 files changed

+229
-139
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@ test-output
2020
# Local config to handle using Java 8 vs java 11.
2121
.java-version
2222
*.tgz
23-
23+
# helm
24+
charts/
25+
Chart.lock

hypertrace-ingester/build.gradle.kts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,13 @@ hypertraceDocker {
2525
}
2626

2727
dependencies {
28-
implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-framework:0.1.9")
29-
implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.9")
30-
implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.8")
28+
implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-framework:0.1.15")
29+
implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.18")
30+
implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.18")
3131
implementation("org.hypertrace.core.datamodel:data-model:0.1.12")
3232
implementation("org.hypertrace.core.viewgenerator:view-generator-framework:0.1.21")
3333
implementation("com.typesafe:config:1.4.0")
34+
implementation("org.apache.commons:commons-lang3:3.11")
3435

3536
implementation(project(":span-normalizer:span-normalizer"))
3637
implementation(project(":raw-spans-grouper:raw-spans-grouper"))
@@ -40,7 +41,7 @@ dependencies {
4041
testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
4142
testImplementation("org.mockito:mockito-core:3.6.0")
4243
testImplementation("org.junit-pioneer:junit-pioneer:1.0.0")
43-
testImplementation("org.apache.kafka:kafka-streams-test-utils:5.5.1-ccs")
44+
testImplementation("org.apache.kafka:kafka-streams-test-utils:6.0.1-ccs")
4445
testImplementation(project(":hypertrace-view-generator:hypertrace-view-generator-api"))
4546
testImplementation(project(":span-normalizer:span-normalizer-api"))
4647
}

hypertrace-trace-enricher/enriched-span-constants/build.gradle.kts

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,6 @@ dependencies {
6565
implementation(project(":span-normalizer:span-normalizer-constants"))
6666
implementation("org.hypertrace.entity.service:entity-service-api:0.1.23")
6767

68-
constraints {
69-
implementation("com.google.guava:guava:30.0-jre") {
70-
because("https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415")
71-
}
72-
}
73-
7468
testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
7569
testImplementation("org.mockito:mockito-core:3.6.28")
7670
}

hypertrace-trace-enricher/helm/templates/trace-enricher-config.yaml

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,32 @@ data:
88
application.conf: |-
99
kafka.streams.config {
1010
application.id = structured-traces-enrichment-job
11-
metrics.recording.level = "{{ .Values.traceEnricherConfig.kafka.streams.config.metricsRecordingLevel }}"
12-
num.stream.threads = "{{ .Values.traceEnricherConfig.kafka.streams.config.numStreamThreads }}"
13-
bootstrap.servers = "{{ .Values.traceEnricherConfig.kafka.streams.config.bootstrapServers }}"
14-
schema.registry.url = "{{ .Values.traceEnricherConfig.kafka.streams.config.schemaRegistryUrl }}"
11+
bootstrap.servers = "{{ .Values.traceEnricherConfig.kafkaStreamsConfig.bootstrapServers }}"
12+
schema.registry.url = "{{ .Values.traceEnricherConfig.kafkaStreamsConfig.schemaRegistryUrl }}"
13+
# kafka streams config
14+
num.stream.threads = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.numStreamThreads }}"
15+
commit.interval.ms = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.commitIntervalMs }}"
16+
# Common client (prodcuer, consumer, admin) configs
17+
receive.buffer.bytes = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.receiveBufferBytes }}"
18+
send.buffer.bytes = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.sendBufferBytes }}"
19+
# Producer configs
20+
producer.acks = "{{ .Values.traceEnricherConfig.kafkaStreamsConfig.producerAcks }}"
21+
producer.batch.size = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.producerBatchSize }}"
22+
producer.linger.ms = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.producerLingerMs }}"
23+
producer.compression.type = "{{ .Values.traceEnricherConfig.kafkaStreamsConfig.producerCompressionType }}"
24+
producer.max.request.size = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.producerMaxRequestSize }}"
25+
producer.buffer.memory = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.producerBufferMemory }}"
26+
# Consumer configs
27+
consumer.max.partition.fetch.bytes = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.consumerMaxPartitionFetchBytes }}"
28+
consumer.max.poll.records = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.consumerMaxPollRecords }}"
29+
consumer.session.timeout.ms = "{{ int .Values.traceEnricherConfig.kafkaStreamsConfig.consumerSessionTimeoutMs }}"
30+
# Others
31+
metrics.recording.level = "{{ .Values.traceEnricherConfig.kafkaStreamsConfig.metricsRecordingLevel }}"
32+
{{- if .Values.traceEnricherConfig.extraKafkaStreamsConfig }}
33+
{{- range $key,$value := .Values.traceEnricherConfig.extraKafkaStreamsConfig }}
34+
{{ $key }} = {{ $value }}
35+
{{- end }}
36+
{{- end }}
1537
}
1638
1739
enricher {

hypertrace-trace-enricher/helm/values.yaml

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ nodeLabels: {}
2525
# This is defined in resources/configs/trace-enricher/application.conf as service.admin.port
2626
containerAdminPort: 8099
2727

28-
javaOpts: "-Xms512M -Xmx1024M"
28+
javaOpts: "-XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0 -XX:MaxDirectMemorySize=128M"
2929

3030
livenessProbe:
3131
initialDelaySeconds: 10
@@ -45,7 +45,7 @@ resources:
4545
memory: 1024Mi
4646
limits:
4747
cpu: 1.0
48-
memory: 1536Mi
48+
memory: 1024Mi
4949

5050
podLabels:
5151
app: hypertrace-trace-enricher
@@ -65,13 +65,32 @@ deploymentSelectorMatchLabels:
6565
###########
6666
traceEnricherConfig:
6767
name: hypertrace-trace-enricher-config
68-
kafka:
69-
streams:
70-
config:
71-
metricsRecordingLevel: INFO
72-
numStreamThreads: 2
73-
bootstrapServers: "bootstrap:9092"
74-
schemaRegistryUrl: "http://schema-registry-service:8081"
68+
# Important kafka streams configurations which are used in config template goes here.
69+
kafkaStreamsConfig:
70+
bootstrapServers: "bootstrap:9092"
71+
schemaRegistryUrl: "http://schema-registry-service:8081"
72+
# Core config
73+
numStreamThreads: 2 # default = 1
74+
commitIntervalMs: 30000 # default = 30000
75+
# Common client (producer, consumer, admin) configs
76+
receiveBufferBytes: 4194304 # default = 32768 (kafka streams default)
77+
sendBufferBytes: 4194304 # default = 131072 (kafka streams default)
78+
# Producer configs
79+
producerAcks: all # default: 1
80+
producerBatchSize: 524288 # default = 16384
81+
producerLingerMs: 1000 # default = 100 (kafka streams default)
82+
producerCompressionType: "gzip" # default = none
83+
producerMaxRequestSize: 10485760 # default = 1048576
84+
producerBufferMemory: 134217728 # default = 33554432
85+
# Consumer configs
86+
consumerMaxPartitionFetchBytes: 4194304 # default = 1048576
87+
consumerMaxPollRecords: 1000 # default = 1000 (kafka streams default)
88+
consumerSessionTimeoutMs: 10000 # default = 10000
89+
# Others
90+
metricsRecordingLevel: INFO # default = INFO
91+
# All other streams config goes here.
92+
# Remove the flower braces and add key: value pair here.
93+
extraKafkaStreamsConfig: {}
7594

7695
logConfig:
7796
name: hypertrace-trace-enricher-log-config
@@ -88,11 +107,12 @@ kafka-topic-creator:
88107
kafka:
89108
topics:
90109
- name: enriched-structured-traces
91-
replicationFactor: 1
110+
replicationFactor: 3
92111
partitions: 8
93112
configs:
94-
- retention.bytes=4294967296
95-
- retention.ms=259200000
113+
- retention.bytes=8589934592 # default = -1
114+
- retention.ms=86400000 # default = 604800000 (7 days)
115+
- max.message.bytes=10485760 # Allow larger messages for traces
96116
zookeeper:
97117
address: zookeeper:2181
98118
imagePullSecrets: []

hypertrace-trace-enricher/hypertrace-trace-enricher-api/build.gradle.kts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,9 @@ dependencies {
1414
implementation("org.slf4j:slf4j-api:1.7.30")
1515
implementation("org.apache.commons:commons-lang3:3.11")
1616
constraints {
17-
implementation("com.google.guava:guava:30.0-jre") {
18-
because("https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415")
17+
implementation("com.google.guava:guava:30.1-jre") {
18+
because("Information Disclosure [Medium Severity][https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415] in com.google.guava:[email protected]")
1919
}
2020
}
21-
2221
testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
2322
}

hypertrace-trace-enricher/hypertrace-trace-enricher/build.gradle.kts

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,28 +34,22 @@ tasks.test {
3434
}
3535

3636
dependencies {
37-
constraints {
38-
implementation("org.hibernate.validator:hibernate-validator:6.1.5.Final") {
39-
because("Cross-site Scripting (XSS) [Medium Severity][https://snyk.io/vuln/SNYK-JAVA-ORGHIBERNATEVALIDATOR-541187] in org.hibernate.validator:[email protected]\n" +
40-
" introduced by io.confluent:[email protected] > io.confluent:[email protected] > org.glassfish.jersey.ext:[email protected] > org.hibernate.validator:[email protected]")
41-
}
42-
implementation("org.yaml:snakeyaml:1.26") {
43-
because("Denial of Service (DoS) [Medium Severity][https://snyk.io/vuln/SNYK-JAVA-ORGYAML-537645] in org.yaml:[email protected]\n" +
44-
" introduced by io.confluent:[email protected] > io.confluent:[email protected] > io.swagger:[email protected] > com.fasterxml.jackson.dataformat:[email protected] > org.yaml:[email protected]")
45-
}
46-
}
47-
4837
implementation(project(":hypertrace-trace-enricher:hypertrace-trace-enricher-impl"))
4938
implementation("org.hypertrace.core.datamodel:data-model:0.1.12")
50-
implementation("org.hypertrace.core.flinkutils:flink-utils:0.1.6")
5139
implementation("org.hypertrace.core.serviceframework:platform-service-framework:0.1.18")
5240
implementation("org.hypertrace.entity.service:entity-service-client:0.1.23")
5341

5442
implementation("com.typesafe:config:1.4.1")
55-
implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-framework:0.1.13")
43+
implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-framework:0.1.15")
44+
constraints {
45+
implementation("com.google.guava:guava:30.1-jre") {
46+
because("Information Disclosure [Medium Severity][https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415] in com.google.guava:[email protected]")
47+
}
48+
}
5649

5750
// Required for the GRPC clients.
5851
runtimeOnly("io.grpc:grpc-netty-shaded:1.33.1")
52+
5953
// Logging
6054
implementation("org.slf4j:slf4j-api:1.7.30")
6155
runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.14.0")
@@ -64,5 +58,5 @@ dependencies {
6458
testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
6559
testImplementation("org.mockito:mockito-core:3.6.28")
6660
testImplementation("org.junit-pioneer:junit-pioneer:1.1.0")
67-
testImplementation("org.apache.kafka:kafka-streams-test-utils:5.5.1-ccs")
61+
testImplementation("org.apache.kafka:kafka-streams-test-utils:6.0.1-ccs")
6862
}

hypertrace-trace-enricher/hypertrace-trace-visualizer/build.gradle.kts

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ dependencies {
77

88
implementation("org.json:json:20201115")
99
implementation("org.apache.commons:commons-lang3:3.11")
10-
constraints {
11-
implementation("com.google.guava:guava:30.0-jre") {
12-
because("https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415")
13-
}
14-
}
1510
}
1611

1712
description = "Trace Visualizer to help visualize a structured trace."

hypertrace-trace-enricher/trace-reader/build.gradle.kts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ dependencies {
1515
implementation("org.hypertrace.core.grpcutils:grpc-client-rx-utils:0.3.2")
1616
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.3.2")
1717
implementation("io.reactivex.rxjava3:rxjava:3.0.7")
18+
constraints {
19+
implementation("com.google.guava:guava:30.1-jre") {
20+
because("Information Disclosure [Medium Severity][https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415] in com.google.guava:[email protected]")
21+
}
22+
}
1823

1924
testImplementation("org.junit.jupiter:junit-jupiter:5.7.0")
2025
testImplementation("org.mockito:mockito-core:3.6.28")

hypertrace-view-generator/hypertrace-view-creator/build.gradle.kts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,6 @@ dependencies {
1919
implementation(project(":hypertrace-view-generator:hypertrace-view-generator-api"))
2020
implementation("org.hypertrace.core.viewcreator:view-creator-framework:0.1.21")
2121
constraints {
22-
implementation("com.google.guava:guava:30.0-jre") {
23-
because("https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415")
24-
}
2522
// to have calcite libs on the same version
2623
implementation("org.apache.calcite:calcite-babel:1.26.0") {
2724
because("https://snyk.io/vuln/SNYK-JAVA-ORGAPACHECALCITE-1038296")

0 commit comments

Comments
 (0)