Skip to content

Commit 27a31f3

Browse files
tmacrotmacro
committed
add templates
1 parent f3d878b commit 27a31f3

File tree

11 files changed

+181
-4
lines changed

11 files changed

+181
-4
lines changed

‎cmd/configure.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func configureEnv(cfg Config, envDir string) error {
4747
generateScubaConfig,
4848
generateS3MetadataConfig,
4949
generateScubaMetadataConfig,
50+
generateKafkaConfig,
5051
}
5152

5253
configDir := filepath.Join(envDir, "config")
@@ -118,3 +119,16 @@ func generateScubaMetadataConfig(cfg Config, path string) error {
118119
cfgPath := filepath.Join(path, "metadata-scuba")
119120
return generateMetadataConfig(cfg.ScubaMetadata, cfgPath)
120121
}
122+
123+
func generateKafkaConfig(cfg Config, path string) error {
124+
templates := []string{
125+
"Dockerfile",
126+
"setup.sh",
127+
"server.backbeat.properties",
128+
"server.destination.properties",
129+
"config.properties",
130+
"zookeeper.properties",
131+
}
132+
133+
return renderTemplates(cfg, "templates/kafka", filepath.Join(path, "kafka"), templates)
134+
}

‎templates/backbeat/config.notification.json‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,16 @@
3535
{
3636
"resource": "destination1",
3737
"type": "kafka",
38-
"host": "localhost:9092",
39-
"topic": "destination-topic",
38+
"host": "127.0.0.1:9094",
39+
"topic": "notifications",
40+
{{ if eq .Features.BucketNotifications.DestinationAuth.Type "basic" }}
41+
"auth": {
42+
"type": "basic",
43+
"credentialsFile": "notificationCredentials.json"
44+
}
45+
{{ else if eq .Features.BucketNotifications.DestinationAuth.Type "none" }}
4046
"auth": {}
47+
{{ end }}
4148
}
4249
]
4350
}

‎templates/backbeat/notificationCredentials.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"username": "{{ .Features.BucketNotifications.DestinationAuth.Username }}",
3+
"password": "{{ .Features.BucketNotifications.DestinationAuth.Password }}"
4+
}

‎templates/cloudserver/config.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,8 @@
8989
"resource": "destination1",
9090
"type": "kafka",
9191
"host": "127.0.0.1",
92-
"port": 9093,
93-
"topic": "destination-topic",
92+
"port": 9094,
93+
"topic": "notifications",
9494
"auth": {}
9595
}
9696
],

‎templates/kafka/Dockerfile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM alpine:latest
2+
3+
RUN apk add --no-cache \
4+
openjdk17-jre \
5+
bash
6+
7+
RUN wget https://archive.apache.org/dist/kafka/3.4.0/kafka_2.13-3.4.0.tgz \
8+
&& tar -xzf kafka_2.13-3.4.0.tgz \
9+
&& mv kafka_2.13-3.4.0 /opt/kafka \
10+
&& rm kafka_2.13-3.4.0.tgz
11+
12+
COPY --chmod=755 setup.sh /usr/local/bin/setup-kafka.sh
13+
14+
ENV PATH="/opt/kafka/bin:$PATH"
15+
16+
WORKDIR /opt/kafka

‎templates/kafka/auth.conf‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
2+
# List of enabled mechanisms, can be more than one
3+
sasl.enabled.mechanisms=SASL_PLAINTEXT
4+
5+
# Specify one of of the SASL mechanisms
6+
sasl.mechanism.inter.broker.protocol=SASL_PLAINTEXT
7+
8+
listeners=SASL_PLAINTEXT://:9093
9+
advertised.listeners=SASL_PLAINTEXT://127.0.0.1:9093
10+
11+
listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
12+
username="admin" \
13+
password="admin-secret" \
14+
user_admin="admin-secret" \
15+
user_notify="kafka-password";

‎templates/kafka/config.properties‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{{ if eq .Features.BucketNotifications.DestinationAuth.Type "basic" }}
2+
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="{{.Features.BucketNotifications.DestinationAuth.Username}}" password="{{.Features.BucketNotifications.DestinationAuth.Password}}";
3+
security.protocol=SASL_PLAINTEXT
4+
sasl.mechanism=PLAIN
5+
{{ end }}

‎templates/kafka/server.backbeat.properties‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
broker.id=1
2+
num.network.threads=3
3+
num.io.threads=8
4+
socket.send.buffer.bytes=102400
5+
socket.receive.buffer.bytes=102400
6+
socket.request.max.bytes=104857600
7+
log.dir=/data
8+
num.partitions=1
9+
num.recovery.threads.per.data.dir=1
10+
offsets.topic.replication.factor=1
11+
transaction.state.log.replication.factor=1
12+
transaction.state.log.min.isr=1
13+
log.retention.hours=168
14+
log.retention.check.interval.ms=300000
15+
zookeeper.connect=127.0.0.1:2181/backbeat
16+
zookeeper.connection.timeout.ms=18000
17+
group.initial.rebalance.delay.ms=0

‎templates/kafka/server.destination.properties‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
broker.id=1
2+
num.network.threads=3
3+
num.io.threads=8
4+
socket.send.buffer.bytes=102400
5+
socket.receive.buffer.bytes=102400
6+
socket.request.max.bytes=104857600
7+
log.dir=/data
8+
num.partitions=1
9+
num.recovery.threads.per.data.dir=1
10+
offsets.topic.replication.factor=1
11+
transaction.state.log.replication.factor=1
12+
transaction.state.log.min.isr=1
13+
log.retention.hours=168
14+
log.retention.check.interval.ms=300000
15+
zookeeper.connect=127.0.0.1:2181/destination
16+
zookeeper.connection.timeout.ms=18000
17+
group.initial.rebalance.delay.ms=0
18+
19+
{{ if eq .Features.BucketNotifications.DestinationAuth.Type "none" }}
20+
listeners=PLAINTEXT://:9094
21+
{{ else if eq .Features.BucketNotifications.DestinationAuth.Type "basic" }}
22+
listeners=SASL_PLAINTEXT://:9094
23+
security.inter.broker.protocol=SASL_PLAINTEXT
24+
sasl.mechanism.inter.broker.protocol=PLAIN
25+
sasl.enabled.mechanisms=PLAIN
26+
27+
listener.name.sasl_plaintext.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
28+
username="broker" \
29+
password="broker_pass" \
30+
user_broker="broker_pass" \
31+
user_{{.Features.BucketNotifications.DestinationAuth.Username}}="{{.Features.BucketNotifications.DestinationAuth.Password}}";
32+
{{ end }}

‎templates/kafka/setup.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/usr/bin/env bash
2+
set -ex
3+
4+
if [[ -z "$KAFKA_HOST" ]]; then
5+
KAFKA_HOST=127.0.0.1
6+
fi
7+
8+
if [[ -z "$KAFKA_PORT" ]]; then
9+
KAFKA_PORT=9092
10+
fi
11+
12+
KAFKA_BROKER="${KAFKA_HOST}:${KAFKA_PORT}"
13+
14+
echo "[setup] Kafka broker: $KAFKA_BROKER"
15+
16+
echo "[setup] Waiting for Kafka broker..."
17+
until nc -z "$KAFKA_HOST" "$KAFKA_PORT"; do
18+
echo "[setup] Kafka not ready, retrying in 2s..."
19+
sleep 2
20+
done
21+
echo "[setup] Kafka is ready!"
22+
echo
23+
24+
25+
if [[ -n "$TOPICS_TO_CREATE" ]]; then
26+
echo "[setup] Topics to create: $topic"
27+
echo
28+
common_opts="--create --if-not-exists --bootstrap-server $KAFKA_BROKER --partitions 1 --replication-factor 1"
29+
if [[ -n "$JAAS_CONFIG" ]]; then
30+
common_opts="$common_opts --command-config $JAAS_CONFIG"
31+
fi
32+
for topic in $TOPICS_TO_CREATE; do
33+
echo "[setup] Creating topic: $topic"
34+
kafka-topics.sh \
35+
$common_opts \
36+
--topic "$topic" \
37+
echo "[setup] Topic '$topic' created (or already exists)."
38+
echo
39+
done
40+
fi
41+
42+
if [[ "$CREATE_NOTIFICATION_PATHS" == "true" ]]; then
43+
if [[ -z "$ZOOKEEPER_ENDPOINT" ]]; then
44+
echo "[setup] Zookeeper endpoint not set"
45+
exit 1
46+
fi
47+
48+
echo "[setup] Creating Zookeeper paths..."
49+
zookeeper-shell.sh localhost:2181/backbeat <<EOF
50+
create /
51+
create /bucket-notification
52+
create /bucket-notification/raft-id-dispatcher
53+
create /bucket-notification/raft-id-dispatcher/owners
54+
create /bucket-notification/raft-id-dispatcher/leaders
55+
create /bucket-notification/raft-id-dispatcher/provisions
56+
create /bucket-notification/raft-id-dispatcher/provisions/1
57+
create /bucket-notification/raft-id-dispatcher/provisions/2
58+
create /bucket-notification/raft-id-dispatcher/provisions/3
59+
quit
60+
EOF
61+
echo "[setup] Zookeeper paths created."
62+
echo
63+
fi

0 commit comments

Comments
 (0)