|
| 1 | +import * as k8s from '@pulumi/kubernetes'; |
| 2 | +import * as pulumi from '@pulumi/pulumi'; |
| 3 | + |
| 4 | +export type Redpanda = ReturnType<typeof deployRedpanda>; |
| 5 | + |
| 6 | +export function deployRedpanda() { |
| 7 | + const redpandaConfig = new pulumi.Config('redpanda'); |
| 8 | + const replicas = redpandaConfig.getNumber('replicas') || 3; |
| 9 | + const storageSize = redpandaConfig.get('storageSize') || '20Gi'; |
| 10 | + const memoryLimit = redpandaConfig.get('memoryLimit') || '2Gi'; |
| 11 | + const cpuLimit = redpandaConfig.get('cpuLimit') || '1000m'; |
| 12 | + |
| 13 | + const labels = { app: 'redpanda' }; |
| 14 | + |
| 15 | + // StatefulSet for Redpanda |
| 16 | + const statefulSet = new k8s.apps.v1.StatefulSet('redpanda', { |
| 17 | + metadata: { |
| 18 | + name: 'redpanda', |
| 19 | + }, |
| 20 | + spec: { |
| 21 | + serviceName: 'redpanda', |
| 22 | + replicas, |
| 23 | + selector: { |
| 24 | + matchLabels: labels, |
| 25 | + }, |
| 26 | + template: { |
| 27 | + metadata: { |
| 28 | + labels, |
| 29 | + }, |
| 30 | + spec: { |
| 31 | + containers: [ |
| 32 | + { |
| 33 | + name: 'redpanda', |
| 34 | + image: 'redpandadata/redpanda:v25.2.11', |
| 35 | + resources: { |
| 36 | + limits: { |
| 37 | + cpu: cpuLimit, |
| 38 | + memory: memoryLimit, |
| 39 | + }, |
| 40 | + requests: { |
| 41 | + cpu: '500m', |
| 42 | + memory: '1Gi', |
| 43 | + }, |
| 44 | + }, |
| 45 | + command: [ |
| 46 | + '/bin/bash', |
| 47 | + '-c', |
| 48 | + ` |
| 49 | + set -e |
| 50 | + POD_ORDINAL=\${HOSTNAME##*-} |
| 51 | + INTERNAL_ADVERTISE="redpanda-\${POD_ORDINAL}.redpanda.default.svc.cluster.local" |
| 52 | +
|
| 53 | + # Build seeds list for cluster formation |
| 54 | + SEEDS="" |
| 55 | + for i in $(seq 0 $((${replicas} - 1))); do |
| 56 | + if [ $i -ne 0 ]; then |
| 57 | + SEEDS="$SEEDS," |
| 58 | + fi |
| 59 | + SEEDS="$SEEDS{\\\"node_id\\\":$i,\\\"host\\\":\\\"redpanda-$i.redpanda.default.svc.cluster.local\\\",\\\"port\\\":33145}" |
| 60 | + done |
| 61 | +
|
| 62 | + /usr/bin/redpanda start \\ |
| 63 | + --node-id \${POD_ORDINAL} \\ |
| 64 | + --smp 1 \\ |
| 65 | + --memory 1G \\ |
| 66 | + --reserve-memory 0M \\ |
| 67 | + --overprovisioned \\ |
| 68 | + --kafka-addr PLAINTEXT://0.0.0.0:29092 \\ |
| 69 | + --advertise-kafka-addr PLAINTEXT://\${INTERNAL_ADVERTISE}:29092 \\ |
| 70 | + --pandaproxy-addr 0.0.0.0:8082 \\ |
| 71 | + --advertise-pandaproxy-addr \${INTERNAL_ADVERTISE}:8082 \\ |
| 72 | + --rpc-addr 0.0.0.0:33145 \\ |
| 73 | + --advertise-rpc-addr \${INTERNAL_ADVERTISE}:33145 \\ |
| 74 | + --seeds "[$SEEDS]" \\ |
| 75 | + --set redpanda.enable_idempotence=true \\ |
| 76 | + --set redpanda.enable_transactions=true \\ |
| 77 | + --set redpanda.default_topic_replications=1 \\ |
| 78 | + --set redpanda.auto_create_topics_enabled=false |
| 79 | + `, |
| 80 | + ], |
| 81 | + ports: [ |
| 82 | + { containerPort: 29092, name: 'kafka' }, |
| 83 | + { containerPort: 8082, name: 'http' }, |
| 84 | + { containerPort: 33145, name: 'rpc' }, |
| 85 | + { containerPort: 9644, name: 'admin' }, |
| 86 | + ], |
| 87 | + volumeMounts: [ |
| 88 | + { |
| 89 | + name: 'datadir', |
| 90 | + mountPath: '/var/lib/redpanda/data', |
| 91 | + }, |
| 92 | + ], |
| 93 | + livenessProbe: { |
| 94 | + httpGet: { |
| 95 | + path: '/v1/status/ready', |
| 96 | + port: 9644 as any, |
| 97 | + }, |
| 98 | + initialDelaySeconds: 30, |
| 99 | + periodSeconds: 10, |
| 100 | + }, |
| 101 | + readinessProbe: { |
| 102 | + httpGet: { |
| 103 | + path: '/v1/status/ready', |
| 104 | + port: 9644 as any, |
| 105 | + }, |
| 106 | + initialDelaySeconds: 10, |
| 107 | + periodSeconds: 5, |
| 108 | + }, |
| 109 | + }, |
| 110 | + ], |
| 111 | + }, |
| 112 | + }, |
| 113 | + volumeClaimTemplates: [ |
| 114 | + { |
| 115 | + metadata: { |
| 116 | + name: 'datadir', |
| 117 | + }, |
| 118 | + spec: { |
| 119 | + accessModes: ['ReadWriteOnce'], |
| 120 | + resources: { |
| 121 | + requests: { |
| 122 | + storage: storageSize, |
| 123 | + }, |
| 124 | + }, |
| 125 | + }, |
| 126 | + }, |
| 127 | + ], |
| 128 | + }, |
| 129 | + }); |
| 130 | + |
| 131 | + // Headless Service for StatefulSet (used for internal cluster communication) |
| 132 | + const headlessService = new k8s.core.v1.Service('redpanda-headless', { |
| 133 | + metadata: { |
| 134 | + name: 'redpanda', |
| 135 | + }, |
| 136 | + spec: { |
| 137 | + clusterIP: 'None', |
| 138 | + selector: labels, |
| 139 | + ports: [ |
| 140 | + { name: 'kafka', port: 29092, targetPort: 29092 as any }, |
| 141 | + { name: 'http', port: 8082, targetPort: 8082 as any }, |
| 142 | + { name: 'rpc', port: 33145, targetPort: 33145 as any }, |
| 143 | + { name: 'admin', port: 9644, targetPort: 9644 as any }, |
| 144 | + ], |
| 145 | + }, |
| 146 | + }); |
| 147 | + |
| 148 | + // ClusterIP Service for clients (load balances across all pods) |
| 149 | + const clientService = new k8s.core.v1.Service('redpanda-client-service', { |
| 150 | + metadata: { |
| 151 | + name: 'redpanda-client', |
| 152 | + }, |
| 153 | + spec: { |
| 154 | + type: 'ClusterIP', |
| 155 | + selector: labels, |
| 156 | + ports: [ |
| 157 | + { name: 'kafka', port: 29092, targetPort: 29092 as any }, |
| 158 | + { name: 'http', port: 8082, targetPort: 8082 as any }, |
| 159 | + ], |
| 160 | + }, |
| 161 | + }); |
| 162 | + |
| 163 | + // Create otel-traces topic with proper replication |
| 164 | + const topicCreationJob = new k8s.batch.v1.Job('redpanda-topic-creation', { |
| 165 | + metadata: { |
| 166 | + name: 'redpanda-topic-creation', |
| 167 | + }, |
| 168 | + spec: { |
| 169 | + template: { |
| 170 | + spec: { |
| 171 | + restartPolicy: 'OnFailure', |
| 172 | + containers: [ |
| 173 | + { |
| 174 | + name: 'rpk', |
| 175 | + image: 'redpandadata/redpanda:v25.2.11', |
| 176 | + command: [ |
| 177 | + '/bin/bash', |
| 178 | + '-c', |
| 179 | + ` |
| 180 | + # Wait for Redpanda to be ready |
| 181 | + until rpk cluster health --brokers redpanda-0.redpanda:29092 2>/dev/null | grep -q 'Healthy'; do |
| 182 | + echo "Waiting for Redpanda cluster..." |
| 183 | + sleep 5 |
| 184 | + done |
| 185 | +
|
| 186 | + # Create topic with partitioning only (no replication) |
| 187 | + rpk topic create otel-traces \\ |
| 188 | + --brokers redpanda-0.redpanda:29092 \\ |
| 189 | + --replicas 1 \\ |
| 190 | + --partitions 6 \\ |
| 191 | + --config retention.ms=86400000 \\ |
| 192 | + --config compression.type=snappy \\ |
| 193 | + || echo "Topic may already exist" |
| 194 | + `, |
| 195 | + ], |
| 196 | + }, |
| 197 | + ], |
| 198 | + }, |
| 199 | + }, |
| 200 | + }, |
| 201 | + }, { dependsOn: [statefulSet, headlessService] }); |
| 202 | + |
| 203 | + return { |
| 204 | + statefulSet, |
| 205 | + headlessService, |
| 206 | + clientService, |
| 207 | + topicCreationJob, |
| 208 | + // Client service endpoint - auto-discovers all brokers |
| 209 | + brokerEndpoint: 'redpanda-client:29092', |
| 210 | + }; |
| 211 | +} |
0 commit comments