Merge branch 'improvement/ZENKO-5196/use-generated-kafka-image-ci' into w/2.14/improvement/ZENKO-5196/use-generated-kafka-image-ci

Author: francoisferrand

.github/scripts/end2end/configs/notification_destinations.yaml

@@ -29,7 +29,7 @@ spec:
 apiVersion: zenko.io/v1alpha2
 kind: ZenkoNotificationTarget
 metadata:
-  name: ${NOTIF_AUTH_DEST_NAME}
+  name: ${NOTIF_PLAIN_DEST_NAME}
   labels:
     app.kubernetes.io/instance: ${ZENKO_NAME}
 spec:
@@ -41,3 +41,22 @@ spec:
   plain:
     username: ${NOTIF_AUTH_DEST_USERNAME}
     password: ${NOTIF_AUTH_DEST_PASSWORD}
+
+---
+
+apiVersion: zenko.io/v1alpha2
+kind: ZenkoNotificationTarget
+metadata:
+  name: ${NOTIF_SCRAM_DEST_NAME}
+  labels:
+    app.kubernetes.io/instance: ${ZENKO_NAME}
+spec:
+  type: kafka
+  host: ${NOTIF_KAFKA_AUTH_HOST}
+  port: ${NOTIF_KAFKA_SCRAM_PORT}
+  destinationTopic: ${NOTIF_SCRAM_DEST_TOPIC}
+  auth: scram
+  scram:
+    username: ${NOTIF_SCRAM_DEST_USERNAME}
+    password: ${NOTIF_SCRAM_DEST_PASSWORD}
+    mechanism: SHA-512

.github/scripts/end2end/configure-e2e-ctst.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
 set -exu
 
+# Get kafka image name and tag
+KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
+KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
+KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
+KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
+BUILD_TREE_HASH=$(git rev-parse HEAD:solution/kafka)
+
 # Setup test environment variables
 export ZENKO_NAME=${1:-"end2end"}
 # Getting kafka host from backbeat's config
@@ -14,8 +21,9 @@ export NOTIF_KAFKA_PORT=${KAFKA_HOST_PORT#*:}
 export NOTIF_KAFKA_AUTH_HOST="${ZENKO_NAME}-base-queue-auth-0"
 export NOTIF_KAFKA_AUTH_HOST_PORT="$NOTIF_KAFKA_AUTH_HOST:$NOTIF_KAFKA_PORT"
 export NOTIF_KAFKA_AUTH_PORT=9094
+export NOTIF_KAFKA_SCRAM_PORT=9095
 
-# Add an extra SASL_PLAIN Kafka listener, to support testing authenticated Kafka for bucket notifications
+# Add extra SASL_PLAIN & SASL_SCRAM Kafka listeners, to support testing authenticated Kafka for bucket notifications
 kubectl get zookeepercluster "${ZENKO_NAME}-base-quorum" -o json | jq '.
 | .metadata |= {namespace, name: "\(.name)-auth" }
 | del(.spec.labels)
@@ -30,10 +38,16 @@ kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 --timeout 10m zookeeperc
 kubectl get kafkacluster "${ZENKO_NAME}-base-queue" -o json | jq '.
 | .metadata |= {namespace, name: "\(.name)-auth" }
 | del(.status)
-| .spec.listenersConfig.internalListeners |= . + [{containerPort: 9094, name: "auth", type: "sasl_plaintext", usedForInnerBrokerCommunication: false}]
+| .spec.listenersConfig.internalListeners |= . + [
+    {containerPort: 9094, name: "auth", type: "sasl_plaintext", usedForInnerBrokerCommunication: false},
+    {containerPort: 9095, name: "scram", type: "sasl_plaintext", usedForInnerBrokerCommunication: false}
+  ]
 | .spec.readOnlyConfig |= (. + "
-sasl.enabled.mechanisms=PLAIN
+sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-512
+listener.name.auth.sasl.enabled.mechanisms=PLAIN
 listener.name.auth.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"'"$NOTIF_AUTH_DEST_USERNAME"'\" password=\"'"$NOTIF_AUTH_DEST_PASSWORD"'\" user_'"$NOTIF_AUTH_DEST_USERNAME"'=\"'"$NOTIF_AUTH_DEST_PASSWORD"'\";
+listener.name.scram.sasl.enabled.mechanisms=SCRAM-SHA-512
+listener.name.scram.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username=\"'"$NOTIF_SCRAM_DEST_USERNAME"'\" password=\"'"$NOTIF_SCRAM_DEST_PASSWORD"'\" user_'"$NOTIF_SCRAM_DEST_USERNAME"'=\"'"$NOTIF_SCRAM_DEST_PASSWORD"'\";
 ")
 | del(.spec.brokerConfigGroups.default.storageConfigs[].pvcSpec)
 | .spec.brokerConfigGroups.default.storageConfigs[].emptyDir |= {medium: "Memory"}
@@ -42,6 +56,19 @@ listener.name.auth.plain.sasl.jaas.config=org.apache.kafka.common.security.plain
 ' | kubectl apply -f -
 kubectl wait --for=jsonpath='{.status.state}'=ClusterRunning --timeout 10m kafkacluster "${ZENKO_NAME}-base-queue-auth"
 
+# Create SCRAM credentials for the SCRAM listener
+kubectl run kafka-config \
+    --image=$KAFKA_IMAGE-$BUILD_TREE_HASH \
+    --pod-running-timeout=5m \
+    --rm \
+    --restart=Never \
+    --attach=True \
+    --command -- bash -c \
+    "kafka-configs.sh --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT \
+        --alter --add-config 'SCRAM-SHA-512=[password=$NOTIF_SCRAM_DEST_PASSWORD]' \
+        --entity-type users \
+        --entity-name $NOTIF_SCRAM_DEST_USERNAME"
+
 UUID=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .extensions.replication.topic)
 UUID=${UUID%.*}
@@ -57,12 +84,6 @@ kubectl wait --for condition=DeploymentInProgress=true --timeout 10m zenko/${ZEN
 kubectl wait --for condition=DeploymentFailure=false --timeout 10m zenko/${ZENKO_NAME}
 kubectl wait --for condition=DeploymentInProgress=false --timeout 10m zenko/${ZENKO_NAME}
 
-# Get kafka image name and tag
-KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
-KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
-KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
-KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
-
 # Cold location topic
 AZURE_ARCHIVE_STATUS_TOPIC="${UUID}.cold-status-e2e-azure-archive"
 AZURE_ARCHIVE_STATUS_TOPIC_2_NV="${UUID}.cold-status-e2e-azure-archive-2-non-versioned"
@@ -71,7 +92,7 @@ AZURE_ARCHIVE_STATUS_TOPIC_2_S="${UUID}.cold-status-e2e-azure-archive-2-suspende
 
 # Creating bucket notification topic in kafka
 kubectl run kafka-topics \
-    --image=$KAFKA_IMAGE \
+    --image=$KAFKA_IMAGE-$BUILD_TREE_HASH \
     --pod-running-timeout=5m \
     --rm \
     --restart=Never \
@@ -80,6 +101,7 @@ kubectl run kafka-topics \
     "kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
         kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
         kafka-topics.sh --create --topic $NOTIF_AUTH_DEST_TOPIC --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $NOTIF_SCRAM_DEST_TOPIC --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT --if-not-exists ; \
         kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
         kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
         kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \

.github/scripts/end2end/configure-e2e.sh

@@ -55,10 +55,11 @@ KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
 KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
 KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
+BUILD_TREE_HASH=$(git rev-parse HEAD:solution/kafka)
 
 # Creating replication/transition and notification topics in kafka
 kubectl run kafka-topics \
-    --image=$KAFKA_IMAGE \
+    --image=$KAFKA_IMAGE-$BUILD_TREE_HASH \
     --pod-running-timeout=5m \
     --rm \
     --restart=Never \

.github/scripts/end2end/deploy-zenko.sh

@@ -89,6 +89,7 @@ function dependencies_env()
     echo $(dependencies_policy_env)
     echo $(dependencies_config_env)
     echo "ZENKO_VERSION_NAME=${ZENKO_VERSION_NAME}"
+    echo "BUILD_TREE_HASH=$(git rev-parse HEAD:solution/kafka)"
 }
 
 create_encryption_secret()

.github/scripts/end2end/run-e2e-ctst.sh

@@ -62,6 +62,11 @@ KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,a
 KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
 KAFKA_PORT=${KAFKA_HOST_PORT#*:}
 
+# Subtle: we push to the authenticated Kafka through SASL/PLAIN and SASL/SCRAM,
+# as defined in notification_destinations.yaml, but we check the resulting
+# notification in the tests through the unauthenticated listener.
+# This is why we reuse the base Kafka port here, rather than 9094/9095.
+# This variable is used for checking the notifications only.
 KAFKA_AUTH_HOST="end2end-base-queue-auth-0"
 KAFKA_AUTH_HOST_PORT="$KAFKA_AUTH_HOST:$KAFKA_PORT"
 
@@ -98,10 +103,10 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "NotificationDestinationTopic":"${NOTIF_DEST_TOPIC}",
   "NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
   "NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
-  "NotificationDestinationAuth":"${NOTIF_AUTH_DEST_NAME}",
-  "NotificationDestinationTopicAuth":"${NOTIF_AUTH_DEST_TOPIC}",
-  "NotificationDestinationAuthUsername":"${NOTIF_AUTH_DEST_USERNAME}",
-  "NotificationDestinationAuthPassword":"${NOTIF_AUTH_DEST_PASSWORD}",
+  "NotificationDestinationPlain":"${NOTIF_PLAIN_DEST_NAME}",
+  "NotificationDestinationTopicPlain":"${NOTIF_AUTH_DEST_TOPIC}",
+  "NotificationDestinationScram":"${NOTIF_SCRAM_DEST_NAME}",
+  "NotificationDestinationTopicScram":"${NOTIF_SCRAM_DEST_TOPIC}",
   "KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
   "PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
   "KafkaHosts":"${KAFKA_HOST_PORT}",

.github/workflows/end2end.yaml

@@ -95,10 +95,14 @@ env:
   NOTIF_DEST_TOPIC: "destination-topic-1"
   NOTIF_ALT_DEST_NAME: "destination2"
   NOTIF_ALT_DEST_TOPIC: "destination-topic-2"
-  NOTIF_AUTH_DEST_NAME: "destination3"
+  NOTIF_PLAIN_DEST_NAME: "destination3"
   NOTIF_AUTH_DEST_TOPIC: "destination-topic-3"
   NOTIF_AUTH_DEST_USERNAME: "admin"
   NOTIF_AUTH_DEST_PASSWORD: "admin-secret"
+  NOTIF_SCRAM_DEST_NAME: "destination4"
+  NOTIF_SCRAM_DEST_TOPIC: "destination-topic-4"
+  NOTIF_SCRAM_DEST_USERNAME: "admin"
+  NOTIF_SCRAM_DEST_PASSWORD: "admin-secret"
   SUBDOMAIN: "zenko.local"
   DR_SUBDOMAIN: "dr.zenko.local"
   SKOPEO_PATH: "/tmp"

solution/deps.yaml

@@ -130,10 +130,10 @@ vault:
 zenko-operator:
   sourceRegistry: ghcr.io/scality
   image: zenko-operator
-  tag: v1.8.2
+  tag: v1.8.3
   envsubst: ZENKO_OPERATOR_TAG
 zookeeper:
-  sourceRegistry: pravega
+  sourceRegistry: ghcr.io/adobe/zookeeper-operator
   image: zookeeper
-  tag: 0.2.15
+  tag: 3.8.4-0.2.15-adobe-20250923
   envsubst: ZOOKEEPER_TAG

solution/kafka/Dockerfile

@@ -22,7 +22,7 @@ RUN tar -xzf $kafka_distro
 RUN rm -r kafka_$scala_version-$kafka_version/bin/windows
 
 ####################################################################################################
-FROM eclipse-temurin:17.0.3_7-jre
+FROM eclipse-temurin:17.0.18_8-jre
 
 ARG scala_version=2.13
 ARG kafka_version=3.1.0

solution/zenkoversion.yaml

@@ -96,10 +96,10 @@ spec:
         tag: ${KAFKA_CLEANER_TAG}
       cluster:
         image: ${KAFKA_IMAGE}
-        tag: ${KAFKA_TAG}
+        tag: ${KAFKA_TAG}-${BUILD_TREE_HASH}
       connect:
         image: ${KAFKA_CONNECT_IMAGE}
-        tag: ${KAFKA_CONNECT_TAG}
+        tag: ${KAFKA_CONNECT_TAG}-${BUILD_TREE_HASH}
       cruiseControl:
         image: ${KAFKA_CRUISECONTROL_IMAGE}
         tag: ${KAFKA_CRUISECONTROL_TAG}

tests/ctst/features/bucket-notifications/notifications.feature

@@ -100,7 +100,24 @@ Feature: Bucket notifications
   @BucketNotification
   Scenario Outline: Receive notification for configured events in authenticated notification destinations
     Given a "<versioningConfiguration>" bucket
-    And one authenticated notification destination
+    And one PLAIN authenticated notification destination
+    When i subscribe to "<subscribedNotificationType>" notifications for destination <destination>
+    And a "<notificationType>" event is triggered "<enable>" "<filterType>"
+    Then i should "<shouldReceive>" a notification for "<notificationType>" event in destination <destination>
+
+    Examples:
+      | versioningConfiguration |               subscribedNotificationType |                         notificationType |  enable | filterType  | shouldReceive | destination |
+      |           Non versioned |                       s3:ObjectCreated:* |                     s3:ObjectCreated:Put | without |      filter |       receive |           0 |
+      |               Versioned |                       s3:ObjectCreated:* |                    s3:ObjectCreated:Copy | without |      filter |       receive |           0 |
+      |   Versioning suspended  |                       s3:ObjectCreated:* |                     s3:ObjectCreated:Put | without |      filter |       receive |           0 |
+
+  @2.6.0
+  @PreMerge
+  @Flaky
+  @BucketNotification
+  Scenario Outline: Receive notification for configured events in SCRAM authenticated notification destinations
+    Given a "<versioningConfiguration>" bucket
+    And one SCRAM authenticated notification destination
     When i subscribe to "<subscribedNotificationType>" notifications for destination <destination>
     And a "<notificationType>" event is triggered "<enable>" "<filterType>"
     Then i should "<shouldReceive>" a notification for "<notificationType>" event in destination <destination>