Bump Cli-testing to 1.3.0, update kafka libraries

.devcontainer/devcontainer.json

@@ -96,7 +96,17 @@
         "vscode": {
             "extensions": [
                 "ms-kubernetes-tools.vscode-kubernetes-tools",
-            ]
+                "cucumberopen.cucumber-official"
+            ],
+            "settings": {
+                "cucumber.glue": [
+                    "tests/ctst/steps/**/*.ts",
+                    "tests/ctst/common/**/*.ts"
+                ],
+                "cucumber.features": [
+                    "tests/ctst/features/**/*.feature"
+                ]
+            }
         }
     },
     "containerEnv": {

.github/scripts/end2end/run-e2e-ctst.sh

@@ -156,13 +156,6 @@ E2E_IMAGE=$E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG
 POD_NAME="${ZENKO_NAME}-ctst-tests"
 CTST_VERSION=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ../../../tests/ctst/package.json)
 
-# Configure keycloak
-docker run \
-    --rm \
-    --network=host \
-    "${E2E_IMAGE}" /bin/bash \
-    -c "SUBDOMAIN=${SUBDOMAIN} CONTROL_PLANE_INGRESS_ENDPOINT=${OIDC_ENDPOINT} ACCOUNT=${ZENKO_ACCOUNT_NAME} KEYCLOAK_REALM=${KEYCLOAK_TEST_REALM_NAME} STORAGE_MANAGER=${STORAGE_MANAGER_USER_NAME} STORAGE_ACCOUNT_OWNER=${STORAGE_ACCOUNT_OWNER_USER_NAME} DATA_CONSUMER=${DATA_CONSUMER_USER_NAME} DATA_ACCESSOR=${DATA_ACCESSOR_USER_NAME} /ctst/node_modules/cli-testing/bin/seedKeycloak.sh"; [[ $? -eq 1 ]] && exit 1 || echo 'Keycloak Configured!'
-
 # Grant access to Kube API (insecure, only for testing)
 kubectl create clusterrolebinding serviceaccounts-cluster-admin \
   --clusterrole=cluster-admin \
@@ -178,6 +171,14 @@ kubectl run $POD_NAME \
         --attach=True \
         --image-pull-policy=IfNotPresent \
         --env=TARGET_VERSION=$VERSION  \
+        --env=ACCOUNT=${ZENKO_ACCOUNT_NAME} \
+        --env=STORAGE_MANAGER=${STORAGE_MANAGER_USER_NAME} \
+        --env=STORAGE_ACCOUNT_OWNER=${STORAGE_ACCOUNT_OWNER_USER_NAME} \
+        --env=DATA_CONSUMER=${DATA_CONSUMER_USER_NAME} \
+        --env=DATA_ACCESSOR=${DATA_ACCESSOR_USER_NAME} \
+        --env=SEED_KEYCLOAK_DEFAULT_ROLES=true \
+        --env=KEYCLOAK_HOST=${KEYCLOAK_TEST_HOST} \
+        --env=KEYCLOAK_REALM=${KEYCLOAK_TEST_REALM_NAME} \
         --env=AZURE_BLOB_URL=$AZURE_BACKEND_ENDPOINT  \
         --env=AZURE_QUEUE_URL=$AZURE_BACKEND_QUEUE_ENDPOINT \
         --env=VERBOSE=1 \
@@ -226,5 +227,6 @@ kubectl run $POD_NAME \
     --parallel $PARALLEL_RUNS \
     --retry 3 \
     --retry-tag-filter @Flaky \
+    --format pretty \
     --format junit:/reports/ctst-junit.xml \
     --format html:/reports/report.html

.github/workflows/end2end.yaml

@@ -404,7 +404,7 @@ jobs:
           GIT_ACCESS_TOKEN: ${{ steps.app-token.outputs.token }}
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '24'
           cache: yarn
           cache-dependency-path: tests/ctst/yarn.lock
       - name: Install ctst test dependencies

tests/ctst/Dockerfile

@@ -4,7 +4,7 @@ ARG DRCTL_TAG=latest
 FROM ghcr.io/scality/sorbet:$SORBET_TAG AS sorbet
 FROM ghcr.io/scality/zenko-drctl:$DRCTL_TAG AS drctl
 
-FROM node:22.19.0-bookworm-slim
+FROM node:24.14.0-bookworm-slim
 
 ARG AWSCLI_VERSION=2.17.39
 
@@ -18,19 +18,12 @@ WORKDIR /ctst
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
     apt-utils \
-    python3 \
-    build-essential \
     ssh \
     git \
     curl \
     unzip \
     jq \
     ca-certificates \
-    librdkafka-dev \
-    zlib1g-dev \
-    libssl-dev \
-    libffi-dev \
-    libzstd-dev \
     && rm -rf /var/lib/apt/lists/* \
     && apt-get clean
 

tests/ctst/common/common.ts

@@ -4,7 +4,7 @@ import { CacheHelper, Constants, Identity, IdentityEnum, S3, Utils } from 'cli-t
 import Zenko from 'world/Zenko';
 import { safeJsonParse } from './utils';
 import assert from 'assert';
-import { Admin, Kafka } from 'kafkajs';
+import { Admin } from '@platformatic/kafka';
 import {
     createBucketWithConfiguration,
     putMpuObject,
@@ -99,8 +99,24 @@ async function addUserMetadataToObject(this: Zenko, objectName: string | undefin
 async function getTopicsOffsets(topics: string[], kafkaAdmin: Admin) {
     const offsets = [];
     for (const topic of topics) {
-        const partitions: ({ high: string; low: string; })[] =
-            await kafkaAdmin.fetchTopicOffsets(topic);
+        const metadata = await kafkaAdmin.metadata({ topics: [topic] });
+        const partitionCount = metadata.topics.get(topic)?.partitionsCount ?? 0;
+        const partitionIndexes = Array.from({ length: partitionCount }, (_, i) => ({
+            partitionIndex: i,
+            timestamp: BigInt(-2),
+        }));
+        const earliestResult = await kafkaAdmin.listOffsets({
+            topics: [{ name: topic, partitions: partitionIndexes }],
+        });
+        const latestResult = await kafkaAdmin.listOffsets({
+            topics: [{ name: topic, partitions: partitionIndexes.map(p => ({ ...p, timestamp: BigInt(-1) })) }],
+        });
+        const partitions = [];
+        for (let i = 0; i < partitionCount; i++) {
+            const low = earliestResult[0]?.partitions.find(p => p.partitionIndex === i)?.offset ?? BigInt(0);
+            const high = latestResult[0]?.partitions.find(p => p.partitionIndex === i)?.offset ?? BigInt(0);
+            partitions.push({ low: String(low), high: String(high) });
+        }
         offsets.push({ topic, partitions });
     }
     return offsets;
@@ -305,10 +321,15 @@ Then('kafka consumed messages should not take too much place on disk', { timeout
             assert.fail('Kafka cleaner did not clean the topics within the expected time');
         }, checkInterval * 10); // Timeout after 10 Kafka cleaner intervals
 
+        const kafkaAdmin = new Admin({
+            clientId: 'ctst-kafka-cleaner-check',
+            bootstrapBrokers: [this.parameters.KafkaHosts],
+        });
+
         try {
             const ignoredTopics = ['dead-letter'];
-            const kafkaAdmin = new Kafka({ brokers: [this.parameters.KafkaHosts] }).admin();
-            const topics: string[] = (await kafkaAdmin.listTopics())
+            const allTopics = await kafkaAdmin.listTopics();
+            const topics: string[] = allTopics
                 .filter(t => (t.includes(this.parameters.InstanceID) &&
                     !ignoredTopics.some(e => t.includes(e))));
 
@@ -368,6 +389,7 @@ Then('kafka consumed messages should not take too much place on disk', { timeout
             assert(topics.length === 0, `Topics ${topics.join(', ')} still have not been cleaned`);
         } finally {
             clearTimeout(timeoutID);
+            await kafkaAdmin.close();
         }
     });
 

tests/ctst/common/hooks.ts

@@ -16,6 +16,10 @@ import {
     cleanupAccount,
 } from './utils';
 
+import 'cli-testing/hooks/KeycloakSetup';
+import 'cli-testing/hooks/Logger';
+import 'cli-testing/hooks/versionTags';
+
 // HTTPS should not cause any error for CTST
 process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
 

tests/ctst/cucumber.config.cjs

@@ -5,8 +5,7 @@ module.exports = {
         require: ['steps/**/*.ts', 'common/**/*.ts', 'world/**/*.ts'],
         paths: ['features/**/*.feature'],
         format: [
-            'progress-bar',
-            '@cucumber/pretty-formatter',
+            'pretty',
             'json:reports/cucumber-report.json',
             'html:reports/report.html',
         ],