Merge pull request #897 from RedHatInsights/psav/fix_msk

Fix MSK replicas

Author: psav

apis/cloud.redhat.com/v1alpha1/clowdenvironment_types.go

@@ -275,6 +275,9 @@ type KafkaConfig struct {
 
 	// (Deprecated) (Unused)
 	Suffix string `json:"suffix,omitempty"`
+
+	// Sets the replica count for ephem-msk mode for kafka connect
+	KafkaConnectReplicaCount int `json:"kafkaConnectReplicaCount,omitempty"`
 }
 
 // DatabaseMode details the mode of operation of the Clowder Database Provider

config/crd/bases/cloud.redhat.com_clowdenvironments.yaml

@@ -316,6 +316,10 @@ spec:
                       enableLegacyStrimzi:
                         description: EnableLegacyStrimzi disables TLS + user auth
                         type: boolean
+                      kafkaConnectReplicaCount:
+                        description: Sets the replica count for ephem-msk mode for
+                          kafka connect
+                        type: integer
                       managedPrefix:
                         description: Managed topic prefix for the managed cluster.
                           Only used in (*_managed_*) mode.

controllers/cloud.redhat.com/providers/kafka/msk.go

@@ -3,6 +3,7 @@ package kafka
 import (
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 
 	crd "github.com/RedHatInsights/clowder/apis/cloud.redhat.com/v1alpha1"
@@ -199,14 +200,19 @@ type genericConfig map[string]string
 
 func (s mskProvider) connectConfig(config *apiextensions.JSON) error {
 
+	replicas := 3
+	if s.Env.Spec.Providers.Kafka.KafkaConnectReplicaCount != 0 {
+		replicas = s.Env.Spec.Providers.Kafka.KafkaConnectReplicaCount
+	}
+
 	connectConfig := genericConfig{
-		"config.storage.replication.factor":       "1",
+		"config.storage.replication.factor":       strconv.Itoa(replicas),
 		"config.storage.topic":                    fmt.Sprintf("%v-connect-cluster-configs", s.Env.Name),
 		"connector.client.config.override.policy": "All",
 		"group.id":                          "connect-cluster",
-		"offset.storage.replication.factor": "1",
+		"offset.storage.replication.factor": strconv.Itoa(replicas),
 		"offset.storage.topic":              fmt.Sprintf("%v-connect-cluster-offsets", s.Env.Name),
-		"status.storage.replication.factor": "1",
+		"status.storage.replication.factor": strconv.Itoa(replicas),
 		"status.storage.topic":              fmt.Sprintf("%v-connect-cluster-status", s.Env.Name),
 	}
 

deploy-mutate.yml

@@ -6524,6 +6524,10 @@ objects:
                         enableLegacyStrimzi:
                           description: EnableLegacyStrimzi disables TLS + user auth
                           type: boolean
+                        kafkaConnectReplicaCount:
+                          description: Sets the replica count for ephem-msk mode for
+                            kafka connect
+                          type: integer
                         managedPrefix:
                           description: Managed topic prefix for the managed cluster.
                             Only used in (*_managed_*) mode.

deploy.yml

@@ -6524,6 +6524,10 @@ objects:
                         enableLegacyStrimzi:
                           description: EnableLegacyStrimzi disables TLS + user auth
                           type: boolean
+                        kafkaConnectReplicaCount:
+                          description: Sets the replica count for ephem-msk mode for
+                            kafka connect
+                          type: integer
                         managedPrefix:
                           description: Managed topic prefix for the managed cluster.
                             Only used in (*_managed_*) mode.

docs/antora/modules/ROOT/pages/api_reference.adoc

@@ -1009,6 +1009,7 @@ KafkaConfig configures the Clowder provider controlling the creation of Kafka in
 | *`connectNamespace`* __string__ | (Deprecated) The namespace that the Kafka Connect cluster is expected to reside in. This is only used in (*_app-interface_*) and (*_operator_*) modes.
 | *`connectClusterName`* __string__ | (Deprecated) Defines the kafka connect cluster name that is used in this environment.
 | *`suffix`* __string__ | (Deprecated) (Unused)
+| *`kafkaConnectReplicaCount`* __integer__ | Sets the replica count for ephem-msk mode for kafka connect
 |===
 
 

tests/kuttl/test-kafka-msk/03-pods.yaml

@@ -20,6 +20,7 @@ spec:
         namespace: test-kafka-msk-sec-source
       clusterAnnotation: test-kafka-msk
       topicNamespace: test-kafka-msk-sec-source
+      kafkaConnectReplicaCount: 1
     db:
       mode: none
     logging: