Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion .github/scripts/end2end/configs/notification_destinations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ spec:
apiVersion: zenko.io/v1alpha2
kind: ZenkoNotificationTarget
metadata:
name: ${NOTIF_AUTH_DEST_NAME}
name: ${NOTIF_PLAIN_DEST_NAME}
labels:
app.kubernetes.io/instance: ${ZENKO_NAME}
spec:
Expand All @@ -41,3 +41,22 @@ spec:
plain:
username: ${NOTIF_AUTH_DEST_USERNAME}
password: ${NOTIF_AUTH_DEST_PASSWORD}

---

apiVersion: zenko.io/v1alpha2
kind: ZenkoNotificationTarget
metadata:
name: ${NOTIF_SCRAM_DEST_NAME}
labels:
app.kubernetes.io/instance: ${ZENKO_NAME}
spec:
type: kafka
host: ${NOTIF_KAFKA_AUTH_HOST}
port: ${NOTIF_KAFKA_SCRAM_PORT}
destinationTopic: ${NOTIF_SCRAM_DEST_TOPIC}
auth: scram
scram:
username: ${NOTIF_SCRAM_DEST_USERNAME}
password: ${NOTIF_SCRAM_DEST_PASSWORD}
mechanism: SHA-512
42 changes: 33 additions & 9 deletions .github/scripts/end2end/configure-e2e-ctst.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
#!/bin/bash
set -exu

DIR=$(dirname "$0")

# Get kafka image name and tag
kafka_image() {
source <( "$DIR"/../../../solution/kafka_build_vars.sh )
echo "$KAFKA_IMAGE:$KAFKA_TAG-$BUILD_TREE_HASH"
}
KAFKA_IMAGE=$(kafka_image)

# Setup test environment variables
export ZENKO_NAME=${1:-"end2end"}
# Getting kafka host from backbeat's config
Expand All @@ -14,8 +23,9 @@ export NOTIF_KAFKA_PORT=${KAFKA_HOST_PORT#*:}
export NOTIF_KAFKA_AUTH_HOST="${ZENKO_NAME}-base-queue-auth-0"
export NOTIF_KAFKA_AUTH_HOST_PORT="$NOTIF_KAFKA_AUTH_HOST:$NOTIF_KAFKA_PORT"
export NOTIF_KAFKA_AUTH_PORT=9094
export NOTIF_KAFKA_SCRAM_PORT=9095

# Add an extra SASL_PLAIN Kafka listener, to support testing authenticated Kafka for bucket notifications
# Add extra SASL_PLAIN & SASL_SCRAM Kafka listeners, to support testing authenticated Kafka for bucket notifications
kubectl get zookeepercluster "${ZENKO_NAME}-base-quorum" -o json | jq '.
| .metadata |= {namespace, name: "\(.name)-auth" }
| del(.spec.labels)
Expand All @@ -30,10 +40,16 @@ kubectl wait --for=jsonpath='{.status.readyReplicas}'=1 --timeout 10m zookeeperc
kubectl get kafkacluster "${ZENKO_NAME}-base-queue" -o json | jq '.
| .metadata |= {namespace, name: "\(.name)-auth" }
| del(.status)
| .spec.listenersConfig.internalListeners |= . + [{containerPort: 9094, name: "auth", type: "sasl_plaintext", usedForInnerBrokerCommunication: false}]
| .spec.listenersConfig.internalListeners |= . + [
{containerPort: 9094, name: "auth", type: "sasl_plaintext", usedForInnerBrokerCommunication: false},
{containerPort: 9095, name: "scram", type: "sasl_plaintext", usedForInnerBrokerCommunication: false}
]
| .spec.readOnlyConfig |= (. + "
sasl.enabled.mechanisms=PLAIN
sasl.enabled.mechanisms=PLAIN,SCRAM-SHA-512
listener.name.auth.sasl.enabled.mechanisms=PLAIN
listener.name.auth.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username=\"'"$NOTIF_AUTH_DEST_USERNAME"'\" password=\"'"$NOTIF_AUTH_DEST_PASSWORD"'\" user_'"$NOTIF_AUTH_DEST_USERNAME"'=\"'"$NOTIF_AUTH_DEST_PASSWORD"'\";
listener.name.scram.sasl.enabled.mechanisms=SCRAM-SHA-512
listener.name.scram.scram-sha-512.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username=\"'"$NOTIF_SCRAM_DEST_USERNAME"'\" password=\"'"$NOTIF_SCRAM_DEST_PASSWORD"'\" user_'"$NOTIF_SCRAM_DEST_USERNAME"'=\"'"$NOTIF_SCRAM_DEST_PASSWORD"'\";
")
| del(.spec.brokerConfigGroups.default.storageConfigs[].pvcSpec)
| .spec.brokerConfigGroups.default.storageConfigs[].emptyDir |= {medium: "Memory"}
Expand All @@ -42,6 +58,19 @@ listener.name.auth.plain.sasl.jaas.config=org.apache.kafka.common.security.plain
' | kubectl apply -f -
kubectl wait --for=jsonpath='{.status.state}'=ClusterRunning --timeout 10m kafkacluster "${ZENKO_NAME}-base-queue-auth"

# Create SCRAM credentials for the SCRAM listener
kubectl run kafka-config \
--image=$KAFKA_IMAGE \
--pod-running-timeout=5m \
--rm \
--restart=Never \
--attach=True \
--command -- bash -c \
"kafka-configs.sh --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT \
--alter --add-config 'SCRAM-SHA-512=[password=$NOTIF_SCRAM_DEST_PASSWORD]' \
--entity-type users \
--entity-name $NOTIF_SCRAM_DEST_USERNAME"

UUID=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .extensions.replication.topic)
UUID=${UUID%.*}
Expand All @@ -57,12 +86,6 @@ kubectl wait --for condition=DeploymentInProgress=true --timeout 10m zenko/${ZEN
kubectl wait --for condition=DeploymentFailure=false --timeout 10m zenko/${ZENKO_NAME}
kubectl wait --for condition=DeploymentInProgress=false --timeout 10m zenko/${ZENKO_NAME}

# Get kafka image name and tag
KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG

# Cold location topic
AZURE_ARCHIVE_STATUS_TOPIC="${UUID}.cold-status-e2e-azure-archive"
AZURE_ARCHIVE_STATUS_TOPIC_2_NV="${UUID}.cold-status-e2e-azure-archive-2-non-versioned"
Expand All @@ -80,6 +103,7 @@ kubectl run kafka-topics \
"kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $NOTIF_AUTH_DEST_TOPIC --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $NOTIF_SCRAM_DEST_TOPIC --bootstrap-server $NOTIF_KAFKA_AUTH_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
Expand Down
13 changes: 8 additions & 5 deletions .github/scripts/end2end/configure-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@

set -exu

. "$(dirname $0)/common.sh"
DIR=$(dirname "${0}")
. "$DIR"/common.sh

ZENKO_NAME=${1:-end2end}
E2E_IMAGE=${2:-ghcr.io/scality/zenko/zenko-e2e:latest}
Expand Down Expand Up @@ -48,10 +49,12 @@ roleRef:
apiGroup: rbac.authorization.k8s.io
EOF

KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
kafka_image() {
source <( "$DIR"/../../../solution/kafka_build_vars.sh )
echo "$KAFKA_IMAGE:$KAFKA_TAG-$BUILD_TREE_HASH"
}

KAFKA_IMAGE=$(kafka_image)
KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
-o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
Expand Down
10 changes: 7 additions & 3 deletions .github/scripts/end2end/deploy-zenko.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@

set -exu

DIR=$(dirname "$0")
DIR="$(dirname "$0")"
REPOSITORY_DIR="${DIR}/../../.."

export ZENKO_NAME=${1:-end2end}
export NAMESPACE=${2:-default}
export ZENKO_CR_PATH=${3:-'./configs/zenko.yaml'}
export ZENKOVERSION_PATH=${4:-'../../../solution/zenkoversion.yaml'}
export DEPS_PATH=${5:-'../../../solution/deps.yaml'}
export ZENKOVERSION_PATH=${4:-"${REPOSITORY_DIR}/solution/zenkoversion.yaml"}
export DEPS_PATH=${5:-"${REPOSITORY_DIR}/solution/deps.yaml"}
export ZENKO_VERSION_NAME="${ZENKO_NAME}-version"
export ZENKO_ANNOTATIONS=""
export ZENKO_MONGODB_SECRET_NAME=${ZENKO_MONGODB_SECRET_NAME:-'mongodb-db-creds'}
Expand Down Expand Up @@ -79,6 +80,9 @@ function dependencies_config_env()
function dependencies_versions_env()
{
yq eval '.[] | .envsubst + "=" + .tag' ${DEPS_PATH}

source <( "${REPOSITORY_DIR}/solution/kafka_build_vars.sh" )
echo "KAFKA_BUILD_TREE_HASH=${BUILD_TREE_HASH}"
}

function dependencies_env()
Expand Down
13 changes: 9 additions & 4 deletions .github/scripts/end2end/run-e2e-ctst.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,a
KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
KAFKA_PORT=${KAFKA_HOST_PORT#*:}

# Subtle: we push to the authenticated Kafka through SASL/PLAIN and SASL/SCRAM,
# as defined in notification_destinations.yaml, but we check the resulting
# notification in the tests through the unauthenticated listener.
# This is why we reuse the base Kafka port here, rather than 9094/9095.
# This variable is used for checking the notifications only.
KAFKA_AUTH_HOST="end2end-base-queue-auth-0"
KAFKA_AUTH_HOST_PORT="$KAFKA_AUTH_HOST:$KAFKA_PORT"

Expand Down Expand Up @@ -97,10 +102,10 @@ WORLD_PARAMETERS="$(jq -c <<EOF
"NotificationDestinationTopic":"${NOTIF_DEST_TOPIC}",
"NotificationDestinationAlt":"${NOTIF_ALT_DEST_NAME}",
"NotificationDestinationTopicAlt":"${NOTIF_ALT_DEST_TOPIC}",
"NotificationDestinationAuth":"${NOTIF_AUTH_DEST_NAME}",
"NotificationDestinationTopicAuth":"${NOTIF_AUTH_DEST_TOPIC}",
"NotificationDestinationAuthUsername":"${NOTIF_AUTH_DEST_USERNAME}",
"NotificationDestinationAuthPassword":"${NOTIF_AUTH_DEST_PASSWORD}",
"NotificationDestinationPlain":"${NOTIF_PLAIN_DEST_NAME}",
"NotificationDestinationTopicPlain":"${NOTIF_AUTH_DEST_TOPIC}",
"NotificationDestinationScram":"${NOTIF_SCRAM_DEST_NAME}",
"NotificationDestinationTopicScram":"${NOTIF_SCRAM_DEST_TOPIC}",
"KafkaExternalIps": "${KAFKA_EXTERNAL_IP:-}",
"PrometheusService":"${PROMETHEUS_NAME}-operated.default.svc.cluster.local",
"KafkaHosts":"${KAFKA_HOST_PORT}",
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/end2end.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,14 @@ env:
NOTIF_DEST_TOPIC: "destination-topic-1"
NOTIF_ALT_DEST_NAME: "destination2"
NOTIF_ALT_DEST_TOPIC: "destination-topic-2"
NOTIF_AUTH_DEST_NAME: "destination3"
NOTIF_PLAIN_DEST_NAME: "destination3"
NOTIF_AUTH_DEST_TOPIC: "destination-topic-3"
NOTIF_AUTH_DEST_USERNAME: "admin"
NOTIF_AUTH_DEST_PASSWORD: "admin-secret"
NOTIF_SCRAM_DEST_NAME: "destination4"
NOTIF_SCRAM_DEST_TOPIC: "destination-topic-4"
NOTIF_SCRAM_DEST_USERNAME: "admin"
NOTIF_SCRAM_DEST_PASSWORD: "admin-secret"
SUBDOMAIN: "zenko.local"
DR_SUBDOMAIN: "dr.zenko.local"
SKOPEO_PATH: "/tmp"
Expand Down
27 changes: 7 additions & 20 deletions solution/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,10 @@ EOF

function flatten_source_images()
{
yq eval '.* | (.sourceRegistry // "docker.io") + "/" + .image + ":" + .tag' deps.yaml
source <( ${REPOSITORY_DIR}/solution/kafka_build_vars.sh )

yq eval '.* | (.sourceRegistry // "docker.io") + "/" + .image + ":" + .tag' deps.yaml |
sed '/ghcr.io\/scality\/zenko\/kafka/ s/$/-'"${BUILD_TREE_HASH}"'/'
}

function zenko_operator_tag()
Expand All @@ -86,6 +89,9 @@ function dependencies_versions_env()
done
yq eval '.[] | select(.tag) | .envsubst + "=" + .tag' deps.yaml
echo ZENKO_VERSION_NAME=${VERSION_FULL}

source <( "${REPOSITORY_DIR}/solution/kafka_build_vars.sh" )
echo "KAFKA_BUILD_TREE_HASH=${BUILD_TREE_HASH}"
}

function copy_yamls()
Expand Down Expand Up @@ -319,31 +325,12 @@ function download_tools()
done
}

function retag()
{
local image=$1
local tag=$2
local suffix=$3
${DOCKER} image inspect "${image}:${tag}-${suffix}" > /dev/null 2>&1 || \
${DOCKER} ${DOCKER_OPTS} pull "${image}:${tag}-${suffix}"
${DOCKER} tag "${image}:${tag}-${suffix}" "${image}:${tag}"
}

function prepare_kafka_images()
(
source <( ${REPOSITORY_DIR}/solution/kafka_build_vars.sh )

retag "$KAFKA_IMAGE" "$KAFKA_TAG" "$BUILD_TREE_HASH"
retag "$KAFKA_CONNECT_IMAGE" "$KAFKA_CONNECT_TAG" "$BUILD_TREE_HASH"
)

# run everything in order
clean
mkdirs
download_tools
gen_manifest_yaml
copy_yamls
prepare_kafka_images
flatten_source_images | while read img ; do
# only pull if the image isnt already local
${DOCKER} image inspect ${img} > /dev/null 2>&1 || ${DOCKER} ${DOCKER_OPTS} pull ${img}
Expand Down
8 changes: 4 additions & 4 deletions solution/deps.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ backbeat:
dashboard: backbeat/backbeat-dashboards
image: backbeat
policy: backbeat/backbeat-policies
tag: 9.1.5
tag: 9.1.7
envsubst: BACKBEAT_TAG
busybox:
image: busybox
Expand Down Expand Up @@ -130,10 +130,10 @@ vault:
zenko-operator:
sourceRegistry: ghcr.io/scality
image: zenko-operator
tag: v1.8.2
tag: v1.8.3
envsubst: ZENKO_OPERATOR_TAG
zookeeper:
sourceRegistry: pravega
sourceRegistry: ghcr.io/adobe/zookeeper-operator
image: zookeeper
tag: 0.2.15
tag: 3.8.4-0.2.15-adobe-20250923
envsubst: ZOOKEEPER_TAG
2 changes: 1 addition & 1 deletion solution/kafka/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN tar -xzf $kafka_distro
RUN rm -r kafka_$scala_version-$kafka_version/bin/windows

####################################################################################################
FROM eclipse-temurin:17.0.3_7-jre
FROM eclipse-temurin:17.0.18_8-jre

ARG scala_version=2.13
ARG kafka_version=3.1.0
Expand Down
4 changes: 2 additions & 2 deletions solution/zenkoversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,10 @@ spec:
tag: ${KAFKA_CLEANER_TAG}
cluster:
image: ${KAFKA_IMAGE}
tag: ${KAFKA_TAG}
tag: ${KAFKA_TAG}-${KAFKA_BUILD_TREE_HASH}
connect:
image: ${KAFKA_CONNECT_IMAGE}
tag: ${KAFKA_CONNECT_TAG}
tag: ${KAFKA_CONNECT_TAG}-${KAFKA_BUILD_TREE_HASH}
cruiseControl:
image: ${KAFKA_CRUISECONTROL_IMAGE}
tag: ${KAFKA_CRUISECONTROL_TAG}
Expand Down
19 changes: 18 additions & 1 deletion tests/ctst/features/bucket-notifications/notifications.feature
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,24 @@ Feature: Bucket notifications
@BucketNotification
Scenario Outline: Receive notification for configured events in authenticated notification destinations
Given a "<versioningConfiguration>" bucket
And one authenticated notification destination
And one PLAIN authenticated notification destination
When i subscribe to "<subscribedNotificationType>" notifications for destination <destination>
And a "<notificationType>" event is triggered "<enable>" "<filterType>"
Then i should "<shouldReceive>" a notification for "<notificationType>" event in destination <destination>

Examples:
| versioningConfiguration | subscribedNotificationType | notificationType | enable | filterType | shouldReceive | destination |
| Non versioned | s3:ObjectCreated:* | s3:ObjectCreated:Put | without | filter | receive | 0 |
| Versioned | s3:ObjectCreated:* | s3:ObjectCreated:Copy | without | filter | receive | 0 |
| Versioning suspended | s3:ObjectCreated:* | s3:ObjectCreated:Put | without | filter | receive | 0 |

@2.6.0
@PreMerge
@Flaky
@BucketNotification
Scenario Outline: Receive notification for configured events in SCRAM authenticated notification destinations
Given a "<versioningConfiguration>" bucket
And one SCRAM authenticated notification destination
When i subscribe to "<subscribedNotificationType>" notifications for destination <destination>
And a "<notificationType>" event is triggered "<enable>" "<filterType>"
Then i should "<shouldReceive>" a notification for "<notificationType>" event in destination <destination>
Expand Down
15 changes: 12 additions & 3 deletions tests/ctst/steps/notifications.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,20 @@ Given('one notification destination', function (this: Zenko) {
);
});

Given('one authenticated notification destination', function (this: Zenko) {
Given('one PLAIN authenticated notification destination', function (this: Zenko) {
setNotificationDestination(
this,
this.parameters.NotificationDestinationAuth,
this.parameters.NotificationDestinationTopicAuth,
this.parameters.NotificationDestinationPlain,
this.parameters.NotificationDestinationTopicPlain,
this.parameters.KafkaAuthHosts,
);
});

Given('one SCRAM authenticated notification destination', function (this: Zenko) {
setNotificationDestination(
this,
this.parameters.NotificationDestinationScram,
this.parameters.NotificationDestinationTopicScram,
this.parameters.KafkaAuthHosts,
);
});
Expand Down
8 changes: 4 additions & 4 deletions tests/ctst/world/Zenko.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ export interface ZenkoWorldParameters extends ClientOptions {
NotificationDestinationTopic: string;
NotificationDestinationAlt: string;
NotificationDestinationTopicAlt: string;
NotificationDestinationAuth: string;
NotificationDestinationTopicAuth: string;
NotificationDestinationAuthUsername: string;
NotificationDestinationAuthPassword: string;
NotificationDestinationPlain: string;
NotificationDestinationTopicPlain: string;
NotificationDestinationScram: string;
NotificationDestinationTopicScram: string;
KafkaExternalIps: string;
KafkaHosts: string;
KafkaAuthHosts: string;
Expand Down
Loading