update kafka replica connections

Author: eloiferrer

charts/change-propagation/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: change-propagation
 description: A Helm chart for the Mediawiki service change propagation
 type: application
-version: 0.1.2
+version: 0.1.3
 appVersion: "1.0.0"

charts/change-propagation/templates/configmap.yaml

@@ -29,7 +29,7 @@ data:
                 x-modules:
                   - path: sys/kafka.js
                     options:
-                      metadata_broker_list: 'kafka-0.kafka.{{ .Release.Namespace }}.svc.cluster.local:9092'
+                      metadata_broker_list: '{{- $ns := .Release.Namespace -}}{{- $replicas := .Values.kafka.replicas | int -}}{{- range $i, $e := until $replicas -}}{{- if $i }},{{ end -}}kafka-{{ $i }}.kafka.{{ $ns }}.svc.cluster.local:9092{{- end -}}'
                       dc_name: {{ .Values.kafka.dcName }}
                       startup_delay: {{ .Values.jobqueue.startupDelay }}
                       consumer:

charts/change-propagation/values.yaml

@@ -10,6 +10,7 @@ service:
 
 kafka:
   dcName: mardi
+  replicas: 3
 
 mediawiki:
   host: http://wikibase-apache

charts/eventgate/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: eventgate
 description: Eventgate chart to push events into kafka
 type: application
-version: 0.1.3
+version: 0.1.4
 appVersion: "1.0.0"

charts/eventgate/templates/config.yaml

@@ -22,7 +22,7 @@ data:
           test_mode: {{ .Values.testMode | default false }}
           kafka:
             conf:
-              metadata.broker.list: 'kafka-0.kafka.{{ .Release.Namespace }}.svc.cluster.local:9092'
+              metadata.broker.list: '{{- $ns := .Release.Namespace -}}{{- $replicas := .Values.kafka.replicas | int -}}{{- range $i, $e := until $replicas -}}{{- if $i }},{{ end -}}kafka-{{ $i }}.kafka.{{ $ns }}.svc.cluster.local:9092{{- end -}}'
               topic.metadata.refresh.interval.ms: 30000
               compression.codec: snappy
               api.version.request: true

charts/eventgate/values.yaml

@@ -1,7 +1,6 @@
 image:
   repository: docker-registry.wikimedia.org/repos/data-engineering/eventgate-wikimedia
   tag: v1.27.0
-numWorkers: 1
 loggingLevel: "info"
 testMode: false
 resources:
@@ -10,4 +9,6 @@ resources:
     memory: "256Mi"
   limits:
     cpu: "500m"
-    memory: "512Mi"
+    memory: "512Mi"
+kafka:
+  replicas: 3

charts/eventstreams/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: eventstreams
 description: Eventstreams chart (stream mediawiki events)
 type: application
-version: 0.1.3
+version: 0.1.4
 appVersion: "1.0.0"

charts/eventstreams/templates/config.yaml

@@ -18,7 +18,7 @@ data:
           stream_config_ttl: 0
           stream_config_defaults: {}          
           kafka:
-            metadata.broker.list: 'kafka-0.kafka.{{ .Release.Namespace }}.svc.cluster.local:9092'
+            metadata.broker.list: '{{- $ns := .Release.Namespace -}}{{- $replicas := .Values.kafka.replicas | int -}}{{- range $i, $e := until $replicas -}}{{- if $i }},{{ end -}}kafka-{{ $i }}.kafka.{{ $ns }}.svc.cluster.local:9092{{- end -}}'
             group.id: eventstreams-consumer
             api.version.request: true
           allowed_streams:

charts/eventstreams/values.yaml

@@ -9,4 +9,6 @@ resources:
     memory: "256Mi"
   limits:
     cpu: "500m"
-    memory: "512Mi"
+    memory: "512Mi"
+kafka:
+  replicas: 3

charts/rdf-streaming-updater/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
 name: rdf-streaming-updater
 description: A Helm chart for the Flink-based RDF streaming updater
-version: 0.1.21
+version: 0.1.22
 appVersion: "1.0.0"