Skip to content

Commit b81c7bc

Browse files
izadoesdevizadoesdev
committed
fix: redpanda console auth
1 parent ae65c82 commit b81c7bc

File tree

7 files changed

+275
-31
lines changed

7 files changed

+275
-31
lines changed

‎infra/env.example‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎infra/ingest/console-config.yml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,21 @@
11
kafka:
22
brokers: ["redpanda:9092"]
3+
sasl:
4+
enabled: true
5+
impersonateUser: false
36
redpanda:
47
adminApi:
58
enabled: true
69
urls: ["http://redpanda:9644"]
7-
10+
authentication:
11+
impersonateUser: false
12+
authentication:
13+
useSecureCookies: false
14+
basic:
15+
enabled: true
16+
authorization:
17+
roleBindings:
18+
- roleName: admin
19+
users:
20+
- loginType: basic
21+
name: "admin"

‎infra/ingest/docker-compose.yml‎

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,27 @@ services:
88
- "9644:9644"
99
volumes:
1010
- redpanda_data:/var/lib/redpanda/data
11+
- ./init-redpanda.sh:/init-redpanda.sh:ro
12+
- ./redpanda.yaml:/etc/redpanda/redpanda.yaml.template:ro
1113
environment:
12-
REDPANDA_LOG_LEVEL: ${REDPANDA_LOG_LEVEL:-info}
13-
REDPANDA_ADVERTISED_HOST: ${REDPANDA_ADVERTISED_HOST:-localhost}
14-
entrypoint: [ "/bin/bash", "-c" ]
14+
REDPANDA_LOG_LEVEL: ${REDPANDA_LOG_LEVEL}
15+
REDPANDA_ADVERTISED_HOST: ${REDPANDA_ADVERTISED_HOST}
16+
REDPANDA_USER: ${REDPANDA_USER}
17+
REDPANDA_PASSWORD: ${REDPANDA_PASSWORD}
18+
VECTOR_KAFKA_USER: ${VECTOR_KAFKA_USER}
19+
VECTOR_KAFKA_PASSWORD: ${VECTOR_KAFKA_PASSWORD}
20+
entrypoint: ["/bin/bash", "-c"]
1521
command:
1622
- |
1723
if [ -z "$$REDPANDA_ADVERTISED_HOST" ]; then
1824
REDPANDA_ADVERTISED_HOST=$$(ip route get 1.1.1.1 2>/dev/null | grep -oP 'src \K\S+' || hostname -i)
1925
fi
26+
echo "Cleaning up crash loop state if present..."
27+
rm -f /var/lib/redpanda/data/startup_log
28+
echo "Copying config file to writable location..."
29+
cp /etc/redpanda/redpanda.yaml.template /etc/redpanda/redpanda.yaml
2030
echo "Advertising Kafka at: $$REDPANDA_ADVERTISED_HOST:19092"
21-
exec /entrypoint.sh redpanda start \
31+
/entrypoint.sh redpanda start \
2232
--kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 \
2333
--advertise-kafka-addr internal://redpanda:9092,external://$$REDPANDA_ADVERTISED_HOST:19092 \
2434
--pandaproxy-addr internal://0.0.0.0:8082 \
@@ -29,11 +39,42 @@ services:
2939
--smp 2 \
3040
--memory 2G \
3141
--reserve-memory 0M \
32-
--default-log-level=info
42+
--default-log-level=$${REDPANDA_LOG_LEVEL:-warn} &
43+
REDPANDA_PID=$$!
44+
echo "Waiting for Redpanda to be ready..."
45+
MAX_RETRIES=30
46+
RETRY_COUNT=0
47+
while [ $$RETRY_COUNT -lt $$MAX_RETRIES ]; do
48+
# Try without SASL first (SASL may not be enabled yet)
49+
if rpk cluster health >/dev/null 2>&1; then
50+
echo "Redpanda is ready!"
51+
break
52+
fi
53+
# If that fails, try with SASL (in case it was already enabled)
54+
if rpk cluster health -X user="$$REDPANDA_USER" -X pass="$$REDPANDA_PASSWORD" -X sasl.mechanism=SCRAM-SHA-256 >/dev/null 2>&1; then
55+
echo "Redpanda is ready!"
56+
break
57+
fi
58+
RETRY_COUNT=$$((RETRY_COUNT + 1))
59+
echo "Waiting for Redpanda... ($$RETRY_COUNT/$$MAX_RETRIES)"
60+
sleep 2
61+
done
62+
if [ $$RETRY_COUNT -eq $$MAX_RETRIES ]; then
63+
echo "ERROR: Redpanda did not become ready in time"
64+
kill $$REDPANDA_PID 2>/dev/null || true
65+
exit 1
66+
fi
67+
echo "Initializing Redpanda users and ACLs..."
68+
/bin/bash /init-redpanda.sh || echo "Warning: Initialization script failed, but continuing..."
69+
wait $$REDPANDA_PID
3370
networks:
3471
- databuddy
3572
healthcheck:
36-
test: [ "CMD-SHELL", "rpk cluster health | grep -q 'Healthy:.*true' && rpk topic list" ]
73+
test:
74+
[
75+
"CMD-SHELL",
76+
'rpk cluster health -X user="$$REDPANDA_USER" -X pass="$$REDPANDA_PASSWORD" -X sasl.mechanism=SCRAM-SHA-256 | grep -q ''Healthy:.*true'' && rpk topic list -X user="$$REDPANDA_USER" -X pass="$$REDPANDA_PASSWORD" -X sasl.mechanism=SCRAM-SHA-256',
77+
]
3778
interval: 10s
3879
timeout: 10s
3980
retries: 10
@@ -49,6 +90,13 @@ services:
4990
- ./console-config.yml:/tmp/console-config.yml:ro
5091
environment:
5192
CONFIG_FILEPATH: /tmp/console-config.yml
93+
KAFKA_BROKERS: ${REDPANDA_BROKER}
94+
KAFKA_SASL_USERNAME: ${REDPANDA_USER}
95+
KAFKA_SASL_PASSWORD: ${REDPANDA_PASSWORD}
96+
KAFKA_SASL_MECHANISM: SCRAM-SHA-256
97+
REDPANDA_ADMINAPI_AUTHENTICATION_BASIC_USERNAME: ${REDPANDA_USER}
98+
REDPANDA_ADMINAPI_AUTHENTICATION_BASIC_PASSWORD: ${REDPANDA_PASSWORD}
99+
AUTHENTICATION_JWTSIGNINGKEY: ${CONSOLE_JWT_SIGNING_KEY}
52100
networks:
53101
- databuddy
54102
depends_on:
@@ -62,21 +110,22 @@ services:
62110
volumes:
63111
- ./vector.yaml:/etc/vector/vector.yaml:ro
64112
environment:
65-
CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default}
66-
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-defaultpass}
67-
CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
68-
VECTOR_LOG: ${VECTOR_LOG:-info}
69-
VECTOR_KAFKA_USER: ${VECTOR_KAFKA_USER:-vector-agent}
70-
VECTOR_KAFKA_PASSWORD: ${VECTOR_KAFKA_PASSWORD:-super_secret_password}
71-
REDPANDA_BROKER: ${REDPANDA_BROKER:-redpanda:9092}
113+
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
114+
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
115+
CLICKHOUSE_URL: ${CLICKHOUSE_URL}
116+
VECTOR_LOG: ${VECTOR_LOG}
117+
VECTOR_KAFKA_USER: ${VECTOR_KAFKA_USER}
118+
VECTOR_KAFKA_PASSWORD: ${VECTOR_KAFKA_PASSWORD}
119+
REDPANDA_BROKER: ${REDPANDA_BROKER}
72120
networks:
73121
- databuddy
122+
- databuddy-network
74123
depends_on:
75124
redpanda:
76125
condition: service_healthy
77126
restart: true
78127
healthcheck:
79-
test: [ "CMD", "vector", "validate", "/etc/vector/vector.yaml" ]
128+
test: ["CMD", "vector", "validate", "/etc/vector/vector.yaml"]
80129
interval: 30s
81130
timeout: 10s
82131
retries: 3
@@ -85,8 +134,10 @@ services:
85134
volumes:
86135
redpanda_data:
87136

88-
89137
networks:
90138
databuddy:
91139
name: databuddy
92140
driver: bridge
141+
databuddy-network:
142+
name: databuddy-network
143+
driver: bridge

‎infra/ingest/env.example‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# ClickHouse Configuration
2+
CLICKHOUSE_USER=default
3+
CLICKHOUSE_PASSWORD=defaultpass
4+
CLICKHOUSE_URL=http://clickhouse:8123
5+
6+
# Redpanda Configuration
7+
REDPANDA_LOG_LEVEL=warn
8+
REDPANDA_ADVERTISED_HOST=localhost
9+
REDPANDA_BROKER=redpanda:9092
10+
REDPANDA_USER=admin
11+
REDPANDA_PASSWORD=super_secret_password
12+
13+
# Vector Configuration
14+
VECTOR_LOG=warn
15+
VECTOR_KAFKA_USER=vector-agent
16+
VECTOR_KAFKA_PASSWORD=super_secret_password
17+
18+
# Console Configuration
19+
CONSOLE_JWT_SIGNING_KEY=databuddy-console-jwt-secret-key-change-in-production-min-32-chars

‎infra/ingest/init-redpanda.sh‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/bin/bash
2+
set -e
3+
4+
VECTOR_KAFKA_USER="${VECTOR_KAFKA_USER}"
5+
VECTOR_KAFKA_PASSWORD="${VECTOR_KAFKA_PASSWORD}"
6+
REDPANDA_USER="${REDPANDA_USER}"
7+
REDPANDA_PASSWORD="${REDPANDA_PASSWORD}"
8+
9+
echo "Setting up Redpanda SASL users and ACLs..."
10+
11+
echo "Step 1: Creating admin superuser (without SASL)..."
12+
OUTPUT=$(rpk security user create "${REDPANDA_USER}" -p "${REDPANDA_PASSWORD}" --mechanism SCRAM-SHA-256 2>&1 || true)
13+
if echo "${OUTPUT}" | grep -q "already exists"; then
14+
echo "Admin user already exists, skipping..."
15+
elif echo "${OUTPUT}" | grep -q "Created user"; then
16+
echo "Admin user created successfully"
17+
else
18+
echo "Warning: Failed to create admin user"
19+
echo "${OUTPUT}"
20+
fi
21+
22+
echo "Step 2: Setting admin as superuser..."
23+
OUTPUT=$(rpk cluster config set superusers "[\"${REDPANDA_USER}\"]" 2>&1 || true)
24+
if echo "${OUTPUT}" | grep -q "Set\|Successfully"; then
25+
echo "Admin set as superuser successfully"
26+
elif echo "${OUTPUT}" | grep -q "already"; then
27+
echo "Admin is already configured as superuser"
28+
else
29+
echo "Warning: Failed to set superuser"
30+
echo "${OUTPUT}"
31+
fi
32+
33+
echo "Step 3: Ensuring enterprise features are disabled (before enabling SASL)..."
34+
# Disable audit logging (enterprise feature)
35+
OUTPUT=$(rpk cluster config set audit_enabled false 2>&1 || true)
36+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
37+
echo "Audit logging disabled"
38+
fi
39+
40+
# Disable continuous data balancing (enterprise feature)
41+
OUTPUT=$(rpk cluster config set partition_autobalancing_mode node_add 2>&1 || true)
42+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
43+
echo "Continuous data balancing disabled"
44+
fi
45+
46+
# Disable continuous intra-broker partition balancing (enterprise feature)
47+
OUTPUT=$(rpk cluster config set core_balancing_continuous false 2>&1 || true)
48+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
49+
echo "Continuous intra-broker balancing disabled"
50+
fi
51+
52+
# Disable tiered storage (enterprise feature)
53+
OUTPUT=$(rpk cluster config set cloud_storage_enabled false 2>&1 || true)
54+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
55+
echo "Tiered storage disabled"
56+
fi
57+
58+
# Disable remote read replicas (enterprise feature)
59+
OUTPUT=$(rpk cluster config set cloud_storage_enable_remote_read false 2>&1 || true)
60+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
61+
echo "Remote read replicas disabled"
62+
fi
63+
64+
# Disable leader pinning (enterprise feature)
65+
OUTPUT=$(rpk cluster config set default_leaders_preference none 2>&1 || true)
66+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
67+
echo "Leader pinning disabled"
68+
fi
69+
70+
# Disable server-side schema ID validation (enterprise feature)
71+
OUTPUT=$(rpk cluster config set enable_schema_id_validation none 2>&1 || true)
72+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
73+
echo "Server-side schema ID validation disabled"
74+
fi
75+
76+
# Ensure we're only using SCRAM (not OIDC/OAUTHBEARER which are enterprise)
77+
OUTPUT=$(rpk cluster config set sasl_mechanisms '["SCRAM"]' 2>&1 || true)
78+
if echo "${OUTPUT}" | grep -q "Set\|already\|Successfully"; then
79+
echo "SASL mechanisms set to SCRAM only"
80+
fi
81+
82+
echo "Step 4: Enabling auto-create topics..."
83+
OUTPUT=$(rpk cluster config set auto_create_topics_enabled true 2>&1 || true)
84+
if echo "${OUTPUT}" | grep -q "Set\|Successfully"; then
85+
echo "Auto-create topics enabled successfully"
86+
elif echo "${OUTPUT}" | grep -q "already"; then
87+
echo "Auto-create topics is already enabled"
88+
else
89+
echo "Warning: Failed to enable auto-create topics"
90+
echo "${OUTPUT}"
91+
fi
92+
93+
echo "Step 5: Enabling SASL authentication..."
94+
OUTPUT=$(rpk cluster config set enable_sasl true 2>&1 || true)
95+
if echo "${OUTPUT}" | grep -q "Set\|Successfully"; then
96+
echo "SASL enabled successfully"
97+
elif echo "${OUTPUT}" | grep -q "already"; then
98+
echo "SASL is already enabled"
99+
else
100+
echo "Warning: Failed to enable SASL (might already be enabled via config)"
101+
echo "${OUTPUT}"
102+
fi
103+
104+
echo "Step 6: Waiting for SASL to be fully ready..."
105+
sleep 3
106+
107+
SASL_CONFIG="-X user=${REDPANDA_USER} -X pass=${REDPANDA_PASSWORD} -X sasl.mechanism=SCRAM-SHA-256"
108+
109+
echo "Step 7: Creating vector-agent user (with SASL)..."
110+
OUTPUT=$(rpk security user create "${VECTOR_KAFKA_USER}" -p "${VECTOR_KAFKA_PASSWORD}" --mechanism SCRAM-SHA-256 ${SASL_CONFIG} 2>&1 || true)
111+
if echo "${OUTPUT}" | grep -q "already exists"; then
112+
echo "vector-agent user already exists, skipping..."
113+
elif echo "${OUTPUT}" | grep -q "Created user"; then
114+
echo "vector-agent user created successfully"
115+
else
116+
echo "Error: Failed to create vector-agent user"
117+
echo "${OUTPUT}"
118+
exit 1
119+
fi
120+
121+
echo "Step 8: Setting ACL permissions for vector-agent..."
122+
OUTPUT=$(rpk security acl create \
123+
--allow-principal "User:${VECTOR_KAFKA_USER}" \
124+
--operation read,write,create \
125+
--topic '*' \
126+
--group '*' \
127+
${SASL_CONFIG} 2>&1 || true)
128+
if echo "${OUTPUT}" | grep -q "already exists"; then
129+
echo "ACL permissions already exist, skipping..."
130+
elif echo "${OUTPUT}" | grep -q "Created ACL"; then
131+
echo "ACL permissions created successfully"
132+
else
133+
echo "Warning: Failed to create ACLs (might already exist)"
134+
echo "${OUTPUT}"
135+
fi
136+
137+
echo "Redpanda initialization completed successfully!"
138+

‎infra/ingest/redpanda.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
redpanda:
2+
data_directory: /var/lib/redpanda/data
3+
node_id: 0
4+
rpc_server:
5+
address: 0.0.0.0
6+
port: 33145
7+
kafka_api:
8+
- address: 0.0.0.0
9+
port: 9092
10+
name: internal
11+
authentication_method: sasl
12+
- address: 0.0.0.0
13+
port: 19092
14+
name: external
15+
authentication_method: sasl
16+
admin:
17+
- address: 0.0.0.0
18+
port: 9644
19+
admin_api_doc_dir: /usr/share/redpanda/admin-api-doc
20+
21+
rpk:
22+
enable_usage_stats: false
23+
24+
pandaproxy:
25+
pandaproxy_api:
26+
- address: 0.0.0.0
27+
port: 8082
28+
29+
schema_registry:
30+
schema_registry_api:
31+
- address: 0.0.0.0
32+
port: 8081

‎infra/ingest/vector.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ sources:
1212
- analytics-email-events
1313
- analytics-blocked-traffic
1414
auto_offset_reset: earliest
15+
sasl:
16+
enabled: true
17+
mechanism: SCRAM-SHA-256
18+
username: "${VECTOR_KAFKA_USER}"
19+
password: "${VECTOR_KAFKA_PASSWORD}"
1520
decoding:
1621
codec: json
1722

@@ -177,6 +182,3 @@ sinks:
177182
path: /var/log/vector/dead_letters.log
178183
encoding:
179184
codec: json
180-
181-
182-

0 commit comments

Comments
 (0)