Fork/pruivo/t mix cluster

quarkus/tests/junit5/src/main/java/org/keycloak/it/utils/DockerKeycloakDistribution.java

@@ -37,9 +37,16 @@ private static class BackupConsumer implements Consumer<OutputFrame> {
 
         final ToStringConsumer stdOut = new ToStringConsumer();
         final ToStringConsumer stdErr = new ToStringConsumer();
+        final Consumer<OutputFrame> customLogConsumer;
+        public BackupConsumer(Consumer<OutputFrame> customLogConsumer) {
+            this.customLogConsumer = customLogConsumer;
+        }
 
         @Override
         public void accept(OutputFrame t) {
+            if (customLogConsumer != null) {
+                customLogConsumer.accept(t);
+            }
             if (t.getType() == OutputType.STDERR) {
                 stdErr.accept(t);
             } else if (t.getType() == OutputType.STDOUT) {
@@ -59,9 +66,8 @@ public void accept(OutputFrame t) {
 
     private String stdout = "";
     private String stderr = "";
-    private BackupConsumer backupConsumer = new BackupConsumer();
-
-
+    private BackupConsumer backupConsumer;
+    private Consumer<OutputFrame> customLogConsumer;
     private GenericContainer<?> keycloakContainer = null;
     private String containerId = null;
 
@@ -88,6 +94,10 @@ public void setEnvVar(String name, String value) {
         this.envVars.put(name, value);
     }
 
+    public void setCustomLogConsumer(Consumer<OutputFrame> customLogConsumer) {
+        this.customLogConsumer = customLogConsumer;
+    }
+
     private GenericContainer<?> getKeycloakContainer() {
         return new GenericContainer<>(image)
                 .withEnv(envVars)
@@ -137,7 +147,7 @@ public CLIResult run(List<String> arguments) {
             this.stdout = "";
             this.stderr = "";
             this.containerId = null;
-            this.backupConsumer = new BackupConsumer();
+            this.backupConsumer = new BackupConsumer(customLogConsumer);
 
             keycloakContainer = getKeycloakContainer();
 

test-framework/README.md

@@ -6,34 +6,6 @@ framework handles the lifecycle of Keycloak, the database, and any injected reso
 Tests simply declare what they want, including specific configuration, and the framework takes care of the rest. 
 
 
-TODO, remove or organize at the bottom of the document.
-
-Mixed cluster
-
-KC_TEST_SERVER_IMAGES   -> if empty, uses the built distribution archive from quarkus/dist directory in all containers
-                        -> if single value, uses that value in all the containers
-                        -> if comma separated value ("imageA,imageB"), each container will use the image specified from the list. The number of items must match the cluster size.
-                        -> "-" special keyword to use the built distribution archive
-KC_TEST_SERVER=cluster  -> enables cluster mode
-KC_TEST_DATABASE_INTERNAL=true -> configure keycloak with the internal database container IP (instead of localhost)
-
-Example, 2 node cluster, the first using the distribution archive and the second the nightly image
-KC_TEST_DATABASE=postgres KC_TEST_DATABASE_INTERNAL=true KC_TEST_SERVER=cluster KC_TEST_SERVER_IMAGES=-,quay.io/keycloak/keycloak:nightly mvn verify -pl tests/base/  -Dtest=MixedVersionClusterTest
-
-Using a mixed cluster with 26.2.3 and 26.2.4
-KC_TEST_DATABASE=postgres KC_TEST_DATABASE_INTERNAL=true KC_TEST_SERVER=cluster KC_TEST_SERVER_IMAGES=quay.io/keycloak/keycloak:26.2.3,quay.io/keycloak/keycloak:26.2.4 mvn verify -pl tests/base/  -Dtest=MixedVersionClusterTest
-
-The test has some println to check the state. Example:
-
-```
-2025-05-26 15:37:09,055 INFO  [org.infinispan.CLUSTER] (main) ISPN000094: Received new cluster view for channel ISPN: [f258a293828f-16122|0] (1) [f258a293828f-16122]
-2025-05-26 15:37:10,452 INFO  [io.quarkus] (main) Keycloak 26.2.3 on JVM (powered by Quarkus 3.20.0) started in 7.848s. Listening on: http://0.0.0.0:8080
-2025-05-26 15:37:18,608 INFO  [org.infinispan.CLUSTER] (main) ISPN000094: Received new cluster view for channel ISPN: [f258a293828f-16122|1] (2) [f258a293828f-16122, 64864a76fcfa-14422]
-2025-05-26 15:37:19,289 INFO  [io.quarkus] (main) Keycloak 26.2.4 on JVM (powered by Quarkus 3.20.0) started in 4.124s. Listening on: http://0.0.0.0:8080
-url0->http://localhost:32889
-url1->http://localhost:32891
-```
-
 # Writing tests
 
 An example is better than a lot of words, so here is a very basic test:

test-framework/bom/pom.xml

@@ -105,6 +105,12 @@
                 <version>${project.version}</version>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>org.keycloak.testframework</groupId>
+                <artifactId>keycloak-test-framework-clustering</artifactId>
+                <version>${project.version}</version>
+                <scope>test</scope>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 </project>

test-framework/clustering/pom.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <groupId>org.keycloak.testframework</groupId>
+        <artifactId>keycloak-test-framework-parent</artifactId>
+        <version>999.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>keycloak-test-framework-clustering</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.keycloak.testframework</groupId>
+            <artifactId>keycloak-test-framework-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
+</project>

test-framework/clustering/src/main/java/org/keycloak/testframework/ClusteringTestFrameworkExtension.java

@@ -0,0 +1,15 @@
+package org.keycloak.testframework;
+
+import org.keycloak.testframework.clustering.LoadBalancerSupplier;
+import org.keycloak.testframework.injection.Supplier;
+import org.keycloak.testframework.server.ClusteredKeycloakServerSupplier;
+
+import java.util.List;
+
+public class ClusteringTestFrameworkExtension implements TestFrameworkExtension {
+
+    @Override
+    public List<Supplier<?, ?>> suppliers() {
+        return List.of(new ClusteredKeycloakServerSupplier(), new LoadBalancerSupplier());
+    }
+}

test-framework/clustering/src/main/java/org/keycloak/testframework/annotations/InjectLoadBalancer.java

@@ -0,0 +1,11 @@
+package org.keycloak.testframework.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface InjectLoadBalancer {
+}

test-framework/clustering/src/main/java/org/keycloak/testframework/clustering/LoadBalancer.java

@@ -0,0 +1,26 @@
+package org.keycloak.testframework.clustering;
+
+import org.keycloak.testframework.server.ClusteredKeycloakServer;
+import org.keycloak.testframework.server.KeycloakUrls;
+
+import java.util.HashMap;
+
+public class LoadBalancer {
+    private final ClusteredKeycloakServer server;
+    private final HashMap<Integer, KeycloakUrls> urls = new HashMap<>();
+
+    public LoadBalancer(ClusteredKeycloakServer server) {
+        this.server = server;
+    }
+
+    public KeycloakUrls node(int nodeIndex) {
+        if (nodeIndex >= server.clusterSize()) {
+            throw new IllegalArgumentException("Node index out of bounds. Requested nodeIndex: %d, cluster size: %d".formatted(server.clusterSize(), nodeIndex));
+        }
+        return urls.computeIfAbsent(nodeIndex, i -> new KeycloakUrls(server.getBaseUrl(i), server.getManagementBaseUrl(i)));
+    }
+
+    public int clusterSize() {
+        return server.clusterSize();
+    }
+}

test-framework/clustering/src/main/java/org/keycloak/testframework/clustering/LoadBalancerSupplier.java

@@ -0,0 +1,27 @@
+package org.keycloak.testframework.clustering;
+
+import org.keycloak.testframework.annotations.InjectLoadBalancer;
+import org.keycloak.testframework.injection.InstanceContext;
+import org.keycloak.testframework.injection.RequestedInstance;
+import org.keycloak.testframework.injection.Supplier;
+import org.keycloak.testframework.server.ClusteredKeycloakServer;
+import org.keycloak.testframework.server.KeycloakServer;
+
+public class LoadBalancerSupplier implements Supplier<LoadBalancer, InjectLoadBalancer> {
+
+    @Override
+    public LoadBalancer getValue(InstanceContext<LoadBalancer, InjectLoadBalancer> instanceContext) {
+        KeycloakServer server = instanceContext.getDependency(KeycloakServer.class);
+
+        if (!(server instanceof ClusteredKeycloakServer)) {
+            throw new IllegalStateException("Load balancer can only be used with ClusteredKeycloakServer");
+        }
+
+        return new LoadBalancer((ClusteredKeycloakServer) server);
+    }
+
+    @Override
+    public boolean compatible(InstanceContext<LoadBalancer, InjectLoadBalancer> a, RequestedInstance<LoadBalancer, InjectLoadBalancer> b) {
+        return true;
+    }
+}

test-framework/core/src/main/java/org/keycloak/testframework/server/ContainerKeycloakCluster.java → test-framework/clustering/src/main/java/org/keycloak/testframework/server/ClusteredKeycloakServer.java

@@ -24,12 +24,14 @@
 import java.util.Objects;
 
 import io.quarkus.bootstrap.utils.BuildToolHelper;
+import org.jboss.logging.Logger;
 import org.keycloak.it.utils.DockerKeycloakDistribution;
+import org.keycloak.testframework.database.JBossLogConsumer;
 import org.testcontainers.images.RemoteDockerImage;
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.LazyFuture;
 
-public class ContainerKeycloakCluster implements KeycloakServer {
+public class ClusteredKeycloakServer implements KeycloakServer {
 
     private static final boolean MANUAL_STOP = true;
     private static final int REQUEST_PORT = 8080;
@@ -38,7 +40,6 @@ public class ContainerKeycloakCluster implements KeycloakServer {
 
     private final DockerKeycloakDistribution[] containers;
     private final String images;
-    private final boolean debug;
 
     private static LazyFuture<String> defaultImage() {
         Path classPathDir;
@@ -51,10 +52,9 @@ private static LazyFuture<String> defaultImage() {
         return DockerKeycloakDistribution.createImage(quarkusModule,true);
     }
 
-    public ContainerKeycloakCluster(int mumServers, String images, boolean debug) {
+    public ClusteredKeycloakServer(int mumServers, String images) {
         containers = new DockerKeycloakDistribution[mumServers];
         this.images = images;
-        this.debug = debug;
     }
 
     @Override
@@ -84,11 +84,12 @@ private void startContainersWithMixedImage(KeycloakServerConfigBuilder configBui
             } else {
                 resolvedImage = new RemoteDockerImage(DockerImageName.parse(imagePeServer[i]));
             }
-            var container = new DockerKeycloakDistribution(debug, MANUAL_STOP, REQUEST_PORT, exposedPorts, resolvedImage);
+            var container = new DockerKeycloakDistribution(false, MANUAL_STOP, REQUEST_PORT, exposedPorts, resolvedImage);
             containers[i] = container;
 
             copyProvidersAndConfigs(container, configBuilder);
 
+            container.setCustomLogConsumer(new JBossLogConsumer(Logger.getLogger("managed.keycloak." + i)));
             container.run(configBuilder.toArgs());
         }
     }
@@ -99,11 +100,12 @@ private void startContainersWithSameImage(KeycloakServerConfigBuilder configBuil
                 defaultImage() :
                 new RemoteDockerImage(DockerImageName.parse(image));
         for (int i = 0; i < containers.length; ++i) {
-            var container = new DockerKeycloakDistribution(debug, MANUAL_STOP, REQUEST_PORT, exposedPorts, imageFuture);
+            var container = new DockerKeycloakDistribution(false, MANUAL_STOP, REQUEST_PORT, exposedPorts, imageFuture);
             containers[i] = container;
 
             copyProvidersAndConfigs(container, configBuilder);
 
+            container.setCustomLogConsumer(new JBossLogConsumer(Logger.getLogger("managed.keycloak." + i)));
             container.run(configBuilder.toArgs());
         }
     }

test-framework/core/src/main/java/org/keycloak/testframework/server/ContainerKeycloakClusterSupplier.java → test-framework/clustering/src/main/java/org/keycloak/testframework/server/ClusteredKeycloakServerSupplier.java

@@ -3,22 +3,19 @@
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.jboss.logging.Logger;
 
-public class ContainerKeycloakClusterSupplier extends AbstractKeycloakServerSupplier {
+public class ClusteredKeycloakServerSupplier extends AbstractKeycloakServerSupplier {
 
-    private static final Logger LOGGER = Logger.getLogger(ContainerKeycloakClusterSupplier.class);
-
-    @ConfigProperty(name = "debug", defaultValue = "false")
-    boolean debug = false;
+    private static final Logger LOGGER = Logger.getLogger(ClusteredKeycloakServerSupplier.class);
 
     @ConfigProperty(name = "numContainer", defaultValue = "2")
     int numContainers = 2;
 
-    @ConfigProperty(name = "images", defaultValue = ContainerKeycloakCluster.SNAPSHOT_IMAGE)
-    String images = ContainerKeycloakCluster.SNAPSHOT_IMAGE;
+    @ConfigProperty(name = "images", defaultValue = ClusteredKeycloakServer.SNAPSHOT_IMAGE)
+    String images = ClusteredKeycloakServer.SNAPSHOT_IMAGE;
 
     @Override
     public KeycloakServer getServer() {
-        return new ContainerKeycloakCluster(numContainers, images, debug);
+        return new ClusteredKeycloakServer(numContainers, images);
     }
 
     @Override

test-framework/clustering/src/main/resources/META-INF/services/org.keycloak.testframework.TestFrameworkExtension

@@ -0,0 +1 @@
+org.keycloak.testframework.ClusteringTestFrameworkExtension

test-framework/core/src/main/java/org/keycloak/testframework/CoreTestFrameworkExtension.java

@@ -14,7 +14,6 @@
 import org.keycloak.testframework.realm.ClientSupplier;
 import org.keycloak.testframework.realm.RealmSupplier;
 import org.keycloak.testframework.realm.UserSupplier;
-import org.keycloak.testframework.server.ContainerKeycloakClusterSupplier;
 import org.keycloak.testframework.server.DistributionKeycloakServerSupplier;
 import org.keycloak.testframework.server.EmbeddedKeycloakServerSupplier;
 import org.keycloak.testframework.server.KeycloakServer;
@@ -37,7 +36,6 @@ public class CoreTestFrameworkExtension implements TestFrameworkExtension {
                 new DistributionKeycloakServerSupplier(),
                 new EmbeddedKeycloakServerSupplier(),
                 new RemoteKeycloakServerSupplier(),
-                new ContainerKeycloakClusterSupplier(),
                 new KeycloakUrlsSupplier(),
                 new DevMemDatabaseSupplier(),
                 new DevFileDatabaseSupplier(),

test-framework/core/src/main/java/org/keycloak/testframework/annotations/InjectKeycloakUrls.java

@@ -9,6 +9,4 @@
 @Target(ElementType.FIELD)
 public @interface InjectKeycloakUrls {
 
-    int nodeIndex() default  0;
-
 }

test-framework/core/src/main/java/org/keycloak/testframework/server/KeycloakUrls.java

@@ -67,8 +67,4 @@ private KeycloakUriBuilder toBuilder(String url) {
     public String getToken(String realm) {
         return baseUrl + "/realms/" + realm + "/protocol/" + OIDCLoginProtocol.LOGIN_PROTOCOL + "/token";
     }
-
-    public boolean isEnabled() {
-        return baseUrl != null && managementBaseUrl != null;
-    }
 }

test-framework/core/src/main/java/org/keycloak/testframework/server/KeycloakUrlsSupplier.java

@@ -9,28 +9,12 @@ public class KeycloakUrlsSupplier implements Supplier<KeycloakUrls, InjectKeyclo
 
     @Override
     public KeycloakUrls getValue(InstanceContext<KeycloakUrls, InjectKeycloakUrls> instanceContext) {
-        var server = instanceContext.getDependency(KeycloakServer.class);
-        var index = instanceContext.getAnnotation().nodeIndex();
-        if (index < 0) {
-            throw new IllegalArgumentException("@InjectKeycloakUrls nodeIndex must be zero or positive");
-        }
-        if (index == 0) {
-            return new KeycloakUrls(server.getBaseUrl(), server.getManagementBaseUrl());
-        }
-        if (server instanceof ContainerKeycloakCluster cluster && index < cluster.clusterSize()) {
-            return new KeycloakUrls(cluster.getBaseUrl(index), cluster.getManagementBaseUrl(index));
-        }
-        return new KeycloakUrls(null, null);
+        KeycloakServer server = instanceContext.getDependency(KeycloakServer.class);
+        return new KeycloakUrls(server.getBaseUrl(), server.getManagementBaseUrl());
     }
 
     @Override
     public boolean compatible(InstanceContext<KeycloakUrls, InjectKeycloakUrls> a, RequestedInstance<KeycloakUrls, InjectKeycloakUrls> b) {
-        return a.getAnnotation().nodeIndex() == b.getAnnotation().nodeIndex();
-    }
-
-    @Override
-    public String getRef(InjectKeycloakUrls annotation) {
-        //TODO: ref to identify the instances, is this correct?
-        return "node-" + annotation.nodeIndex();
+        return true;
     }
 }

test-framework/pom.xml

@@ -47,6 +47,7 @@
         <module>remote</module>
         <module>remote-providers</module>
         <module>ui</module>
+        <module>clustering</module>
     </modules>
 
 </project>

tests/base/pom.xml

@@ -88,6 +88,11 @@
             <groupId>org.keycloak.testframework</groupId>
             <artifactId>keycloak-test-framework-remote</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.keycloak.testframework</groupId>
+            <artifactId>keycloak-test-framework-clustering</artifactId>
+            <version>999.0.0-SNAPSHOT</version>
+        </dependency>
         <dependency>
             <groupId>org.keycloak.tests</groupId>
             <artifactId>keycloak-tests-utils</artifactId>