Use adobe's kafka-operator and zookeeper-operator

Issue: ZENKO-5188

Author: francoisferrand

.github/scripts/end2end/install-kind-dependencies.sh

@@ -10,9 +10,11 @@ VERSION_FILE="${REPOSITORY_DIR}/VERSION"
 
 source "${VERSION_FILE}"
 
-ZK_OPERATOR_VERSION=0.2.15
+ZK_OPERATOR_VERSION=0.2.15-adobe-20250923
+ZK_OPERATOR_CHART=oci://ghcr.io/adobe/helm-charts/zookeeper-operator
 CERT_MANAGER_VERSION=v1.13.3
-KAFKA_OPERATOR_VERSION=0.25.1
+KAFKA_OPERATOR_VERSION=0.28.0-adobe-20251203
+KAFKA_OPERATOR_CHART=oci://ghcr.io/adobe/helm-charts/kafka-operator
 INGRESS_NGINX_VERSION=controller-v1.10.3
 PROMETHEUS_VERSION=v0.52.1
 KEYCLOAK_VERSION=${KEYCLOAK_VERSION:-'18.4.4'}
@@ -35,29 +37,13 @@ MONGODB_SHARD_COUNT=${MONGODB_SHARD_COUNT:-1}
 
 ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'}
 
-KAFKA_CHART=banzaicloud-stable/kafka-operator
-
 if [ $ENABLE_KEYCLOAK_HTTPS == 'true' ]; then
     KEYCLOAK_INGRESS_OPTIONS="$DIR/configs/keycloak_ingress_https.yaml"
 else
     KEYCLOAK_INGRESS_OPTIONS="$DIR/configs/keycloak_ingress_http.yaml"
 fi
 
-helm repo add --force-update bitnami https://charts.bitnami.com/bitnami
-helm repo add --force-update pravega https://charts.pravega.io
 helm repo add --force-update codecentric https://codecentric.github.io/helm-charts/
-# BanzaiCloud repo may not work, c.f. https://scality.atlassian.net/browse/AN-225
-helm repo add --force-update banzaicloud-stable https://kubernetes-charts.banzaicloud.com || {
-		echo -n "::notice file=$(basename $0),line=$LINENO,title=Banzaicloud Charts not available::"
-		echo "Failed to add banzaicloud-stable repo, using local checkout"
-
-		kafka_operator="$(mktemp -d)"
-		git -c advice.detachedHead=false clone -q --depth 1 -b "v${KAFKA_OPERATOR_VERSION}" \
-            https://github.com/banzaicloud/koperator "${kafka_operator}"
-
-		KAFKA_CHART="${kafka_operator}/charts/kafka-operator"
-	}
-helm repo update
 
 # nginx-controller
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/${INGRESS_NGINX_VERSION}/deploy/static/provider/kind/deploy.yaml
@@ -127,12 +113,16 @@ kubectl wait --for=condition=established --timeout=10m crd/alertmanagers.monitor
 envsubst < configs/prometheus.yaml | kubectl apply -f -
 
 # zookeeper
-helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator pravega/zookeeper-operator --set "watchNamespace=default"
-
-# kafka
-kafka_crd_url=https://github.com/banzaicloud/koperator/releases/download/v${KAFKA_OPERATOR_VERSION}/kafka-operator.crds.yaml
-kubectl create -f $kafka_crd_url || kubectl replace -f $kafka_crd_url
-helm upgrade --install --version ${KAFKA_OPERATOR_VERSION} -n default kafka-operator ${KAFKA_CHART}
+# Use Adobe's OCI Helm chart from GitHub Container Registry
+helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator ${ZK_OPERATOR_CHART} --set "watchNamespace=default"
+
+# kafka (also requires Contour CRDs)
+kubectl apply -f https://raw.githubusercontent.com/projectcontour/contour/refs/heads/main/examples/contour/01-crds.yaml
+for crd in cruisecontroloperations kafkaclusters kafkatopics kafkausers ; do
+    kafka_crd_url=https://github.com/adobe/koperator/raw/refs/tags/${KAFKA_OPERATOR_VERSION}/master/config/base/crds/kafka.banzaicloud.io_${crd}.yaml
+    kubectl create -f $kafka_crd_url || kubectl replace -f $kafka_crd_url
+done
+helm upgrade --install --version ${KAFKA_OPERATOR_VERSION} -n default kafka-operator ${KAFKA_OPERATOR_CHART}
 
 # keycloak
 envsubst < $DIR/configs/keycloak_config.json > $DIR/configs/keycloak-realm.json

solution/deps.yaml

@@ -39,9 +39,9 @@ haproxy:
   tag: 3.0.12-alpine
   envsubst: HAPROXY_TAG
 jmx-javaagent:
-  sourceRegistry: ghcr.io/banzaicloud
+  sourceRegistry:  ghcr.io/adobe/koperator
   image: jmx-javaagent
-  tag: 0.16.1
+  tag: 1.4.0
   envsubst: JMX_JAVAAGENT_TAG
 kaf: # CLI Kafka client all-in-one
   tag: 0.2.8
@@ -51,7 +51,7 @@ kaf: # CLI Kafka client all-in-one
 kafka:
   sourceRegistry: ghcr.io/scality/zenko
   image: kafka
-  tag: 2.13-3.1.2
+  tag: 2.13-3.9.1
   envsubst: KAFKA_TAG
 kafka-cleaner:
   sourceRegistry: ghcr.io/scality
@@ -62,7 +62,7 @@ kafka-cleaner:
 kafka-connect:
   sourceRegistry: ghcr.io/scality/zenko
   image: kafka-connect
-  tag: 2.13-3.1.2-1.13.0
+  tag: 2.13-3.9.1-1.13.0
   envsubst: KAFKA_CONNECT_TAG
 kafka-cruise-control:
   sourceRegistry: ghcr.io/banzaicloud

solution/kafka/Dockerfile

@@ -1,44 +1,7 @@
-FROM alpine:3.17 AS kafka_dist
-
-ARG scala_version=2.13
-ARG kafka_version=3.1.0
-ARG kafka_distro_base_url=https://archive.apache.org/dist/kafka
-
-ENV kafka_distro=kafka_$scala_version-$kafka_version.tgz
-ENV kafka_distro_asc=$kafka_distro.asc
-
-RUN apk add --no-cache gnupg
-
-WORKDIR /var/tmp
-
-RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro
-RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro_asc
-RUN wget -q $kafka_distro_base_url/KEYS
-
-RUN gpg --import KEYS
-RUN gpg --verify $kafka_distro_asc $kafka_distro
-
-RUN tar -xzf $kafka_distro
-RUN rm -r kafka_$scala_version-$kafka_version/bin/windows
-
-####################################################################################################
-FROM eclipse-temurin:17.0.18_8-jre
-
 ARG scala_version=2.13
-ARG kafka_version=3.1.0
-
-ENV KAFKA_VERSION=$kafka_version \
-    SCALA_VERSION=$scala_version \
-    KAFKA_HOME=/opt/kafka \
-    KAFKA_GC_LOG_OPTS=-Dnogclog
+ARG kafka_version=3.9.1
 
-ENV PATH=${PATH}:${KAFKA_HOME}/bin
+FROM ghcr.io/adobe/koperator/kafka:${scala_version}-${kafka_version}
 
-RUN mkdir ${KAFKA_HOME} && apt-get update && apt-get install curl -y && apt-get clean
-
-COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$kafka_version ${KAFKA_HOME}
+ENV KAFKA_GC_LOG_OPTS=-Dnogclog
 COPY log4j.properties ${KAFKA_HOME}/config/
-
-RUN chmod a+x ${KAFKA_HOME}/bin/*.sh
-
-CMD ["kafka-server-start.sh"]