diff --git a/.dev/dev_arm64.yaml b/.dev/dev_arm64.yaml index 220140d3d..dc1a8726e 100644 --- a/.dev/dev_arm64.yaml +++ b/.dev/dev_arm64.yaml @@ -32,7 +32,7 @@ services: KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true' kafka0: - image: confluentinc/cp-kafka:7.6.0.arm64 + image: confluentinc/cp-kafka:7.8.0.arm64 user: "0:0" hostname: kafka0 container_name: kafka0 @@ -60,7 +60,7 @@ services: CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' schema-registry0: - image: confluentinc/cp-schema-registry:7.6.0.arm64 + image: confluentinc/cp-schema-registry:7.8.0.arm64 ports: - 8085:8085 depends_on: @@ -76,7 +76,7 @@ services: SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas kafka-connect0: - image: confluentinc/cp-kafka-connect:7.6.0.arm64 + image: confluentinc/cp-kafka-connect:7.8.0.arm64 ports: - 8083:8083 depends_on: @@ -101,7 +101,7 @@ services: CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/usr/local/share/kafka/plugins,/usr/share/filestream-connectors" ksqldb0: - image: confluentinc/cp-ksqldb-server:7.6.0.arm64 + image: confluentinc/cp-ksqldb-server:7.8.0.arm64 depends_on: - kafka0 - kafka-connect0 @@ -119,7 +119,7 @@ services: KSQL_CACHE_MAX_BYTES_BUFFERING: 0 kafka-init-topics: - image: confluentinc/cp-kafka:7.6.0.arm64 + image: confluentinc/cp-kafka:7.8.0.arm64 volumes: - ../documentation/compose/data/message.json:/data/message.json depends_on: diff --git a/.gitignore b/.gitignore index efd6a9749..51efbef39 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ build/ *.tgz /docker/*.override.yaml +/e2e-tests/allure-results/ diff --git a/api/pom.xml b/api/pom.xml index 3f7c044d0..dc774c09a 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -50,7 +50,10 @@ org.apache.kafka kafka-clients - ${kafka-clients.version} + + ${confluent.version}-ccs org.apache.commons diff --git a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java index 72ab7386a..474a0c159 100644 --- a/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java +++ b/api/src/main/java/io/kafbat/ui/client/RetryingKafkaConnectClient.java @@ -22,6 +22,7 @@ import java.util.Objects; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.http.ResponseEntity; import org.springframework.util.unit.DataSize; import org.springframework.web.client.RestClientException; @@ -51,14 +52,36 @@ private static Retry conflictCodeRetry() { (WebClientResponseException.Conflict) signal.failure())); } - private static Mono withRetryOnConflict(Mono publisher) { - return publisher.retryWhen(conflictCodeRetry()); + private static @NotNull Retry retryOnRebalance() { + return Retry.fixedDelay(MAX_RETRIES, RETRIES_DELAY).filter(e -> { + + if (e instanceof WebClientResponseException.InternalServerError exception) { + final var errorMessage = getMessage(exception); + return StringUtils.equals(errorMessage, + // From https://github.com/apache/kafka/blob/dfc07e0e0c6e737a56a5402644265f634402b864/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java#L2340 + "Request cannot be completed because a rebalance is expected"); + } + return false; + }); + } + + private static Mono withRetryOnConflictOrRebalance(Mono publisher) { + return publisher + .retryWhen(retryOnRebalance()) + .retryWhen(conflictCodeRetry()); + } + + private static Flux withRetryOnConflictOrRebalance(Flux publisher) { + return publisher + .retryWhen(retryOnRebalance()) + .retryWhen(conflictCodeRetry()); } - private static Flux withRetryOnConflict(Flux publisher) { - return publisher.retryWhen(conflictCodeRetry()); + private static Mono withRetryOnRebalance(Mono publisher) { + return publisher.retryWhen(retryOnRebalance()); } + private static Mono withBadRequestErrorHandling(Mono publisher) { return publisher .onErrorResume(WebClientResponseException.BadRequest.class, @@ -73,18 +96,21 @@ private record ErrorMessage(@NotNull @JsonProperty("message") String message) { } private static @NotNull Mono parseConnectErrorMessage(WebClientResponseException parseException) { + return Mono.error(new ValidationException(getMessage(parseException))); + } + + private static String getMessage(WebClientResponseException parseException) { final var errorMessage = parseException.getResponseBodyAs(ErrorMessage.class); - return Mono.error(new ValidationException( - Objects.requireNonNull(errorMessage, - // see https://github.com/apache/kafka/blob/a0a501952b6d61f6f273bdb8f842346b51e9dfce/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/errors/ConnectExceptionMapper.java - "This should not happen according to the ConnectExceptionMapper") - .message())); + return Objects.requireNonNull(errorMessage, + // see https://github.com/apache/kafka/blob/a0a501952b6d61f6f273bdb8f842346b51e9dfce/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/errors/ConnectExceptionMapper.java + "This should not happen according to the ConnectExceptionMapper") + .message(); } @Override public Mono createConnector(NewConnector newConnector) throws RestClientException { return withBadRequestErrorHandling( - super.createConnector(newConnector) + withRetryOnRebalance(super.createConnector(newConnector)) ); } @@ -92,178 +118,178 @@ public Mono createConnector(NewConnector newConnector) throws RestCli public Mono setConnectorConfig(String connectorName, Map requestBody) throws RestClientException { return withBadRequestErrorHandling( - super.setConnectorConfig(connectorName, requestBody) + withRetryOnRebalance(super.setConnectorConfig(connectorName, requestBody)) ); } @Override public Mono> createConnectorWithHttpInfo(NewConnector newConnector) throws WebClientResponseException { - return withRetryOnConflict(super.createConnectorWithHttpInfo(newConnector)); + return withRetryOnConflictOrRebalance(super.createConnectorWithHttpInfo(newConnector)); } @Override public Mono deleteConnector(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.deleteConnector(connectorName)); + return withRetryOnConflictOrRebalance(super.deleteConnector(connectorName)); } @Override public Mono> deleteConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.deleteConnectorWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.deleteConnectorWithHttpInfo(connectorName)); } @Override public Mono getConnector(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnector(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnector(connectorName)); } @Override public Mono> getConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorWithHttpInfo(connectorName)); } @Override public Mono> getConnectorConfig(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorConfig(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorConfig(connectorName)); } @Override public Mono>> getConnectorConfigWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorConfigWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorConfigWithHttpInfo(connectorName)); } @Override public Flux getConnectorPlugins() throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorPlugins()); + return withRetryOnConflictOrRebalance(super.getConnectorPlugins()); } @Override public Mono>> getConnectorPluginsWithHttpInfo() throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorPluginsWithHttpInfo()); + return withRetryOnConflictOrRebalance(super.getConnectorPluginsWithHttpInfo()); } @Override public Mono getConnectorStatus(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorStatus(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorStatus(connectorName)); } @Override public Mono> getConnectorStatusWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorStatusWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorStatusWithHttpInfo(connectorName)); } @Override public Mono getConnectorTaskStatus(String connectorName, Integer taskId) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTaskStatus(connectorName, taskId)); + return withRetryOnConflictOrRebalance(super.getConnectorTaskStatus(connectorName, taskId)); } @Override public Mono> getConnectorTaskStatusWithHttpInfo(String connectorName, Integer taskId) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTaskStatusWithHttpInfo(connectorName, taskId)); + return withRetryOnConflictOrRebalance(super.getConnectorTaskStatusWithHttpInfo(connectorName, taskId)); } @Override public Flux getConnectorTasks(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTasks(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorTasks(connectorName)); } @Override public Mono>> getConnectorTasksWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTasksWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorTasksWithHttpInfo(connectorName)); } @Override public Mono> getConnectorTopics(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTopics(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorTopics(connectorName)); } @Override public Mono>> getConnectorTopicsWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorTopicsWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.getConnectorTopicsWithHttpInfo(connectorName)); } @Override public Mono> getConnectors(String search) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectors(search)); + return withRetryOnConflictOrRebalance(super.getConnectors(search)); } @Override public Mono>> getConnectorsWithHttpInfo(String search) throws WebClientResponseException { - return withRetryOnConflict(super.getConnectorsWithHttpInfo(search)); + return withRetryOnConflictOrRebalance(super.getConnectorsWithHttpInfo(search)); } @Override public Mono pauseConnector(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.pauseConnector(connectorName)); + return withRetryOnConflictOrRebalance(super.pauseConnector(connectorName)); } @Override public Mono> pauseConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.pauseConnectorWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.pauseConnectorWithHttpInfo(connectorName)); } @Override public Mono restartConnector(String connectorName, Boolean includeTasks, Boolean onlyFailed) throws WebClientResponseException { - return withRetryOnConflict(super.restartConnector(connectorName, includeTasks, onlyFailed)); + return withRetryOnConflictOrRebalance(super.restartConnector(connectorName, includeTasks, onlyFailed)); } @Override public Mono> restartConnectorWithHttpInfo(String connectorName, Boolean includeTasks, Boolean onlyFailed) throws WebClientResponseException { - return withRetryOnConflict(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); + return withRetryOnConflictOrRebalance(super.restartConnectorWithHttpInfo(connectorName, includeTasks, onlyFailed)); } @Override public Mono restartConnectorTask(String connectorName, Integer taskId) throws WebClientResponseException { - return withRetryOnConflict(super.restartConnectorTask(connectorName, taskId)); + return withRetryOnConflictOrRebalance(super.restartConnectorTask(connectorName, taskId)); } @Override public Mono> restartConnectorTaskWithHttpInfo(String connectorName, Integer taskId) throws WebClientResponseException { - return withRetryOnConflict(super.restartConnectorTaskWithHttpInfo(connectorName, taskId)); + return withRetryOnConflictOrRebalance(super.restartConnectorTaskWithHttpInfo(connectorName, taskId)); } @Override public Mono resumeConnector(String connectorName) throws WebClientResponseException { - return super.resumeConnector(connectorName); + return withRetryOnRebalance(super.resumeConnector(connectorName)); } @Override public Mono> resumeConnectorWithHttpInfo(String connectorName) throws WebClientResponseException { - return withRetryOnConflict(super.resumeConnectorWithHttpInfo(connectorName)); + return withRetryOnConflictOrRebalance(super.resumeConnectorWithHttpInfo(connectorName)); } @Override public Mono> setConnectorConfigWithHttpInfo(String connectorName, Map requestBody) throws WebClientResponseException { - return withRetryOnConflict(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); + return withRetryOnConflictOrRebalance(super.setConnectorConfigWithHttpInfo(connectorName, requestBody)); } @Override public Mono validateConnectorPluginConfig(String pluginName, Map requestBody) throws WebClientResponseException { - return withRetryOnConflict(super.validateConnectorPluginConfig(pluginName, requestBody)); + return withRetryOnConflictOrRebalance(super.validateConnectorPluginConfig(pluginName, requestBody)); } @Override public Mono> validateConnectorPluginConfigWithHttpInfo( String pluginName, Map requestBody) throws WebClientResponseException { - return withRetryOnConflict(super.validateConnectorPluginConfigWithHttpInfo(pluginName, requestBody)); + return withRetryOnConflictOrRebalance(super.validateConnectorPluginConfigWithHttpInfo(pluginName, requestBody)); } private static class RetryingApiClient extends ApiClient { diff --git a/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java b/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java index 9722f2c19..7224649b1 100644 --- a/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java +++ b/api/src/test/java/io/kafbat/ui/AbstractIntegrationTest.java @@ -40,7 +40,7 @@ public abstract class AbstractIntegrationTest { private static final boolean IS_ARM = System.getProperty("os.arch").contains("arm") || System.getProperty("os.arch").contains("aarch64"); - private static final String CONFLUENT_PLATFORM_VERSION = IS_ARM ? "7.2.1.arm64" : "7.2.1"; + private static final String CONFLUENT_PLATFORM_VERSION = IS_ARM ? "7.8.0.arm64" : "7.8.0"; public static final KafkaContainer kafka = new KafkaContainer( DockerImageName.parse("confluentinc/cp-kafka").withTag(CONFLUENT_PLATFORM_VERSION)) diff --git a/documentation/compose/e2e-tests.yaml b/documentation/compose/e2e-tests.yaml index da986cc39..e18eb7a16 100644 --- a/documentation/compose/e2e-tests.yaml +++ b/documentation/compose/e2e-tests.yaml @@ -29,7 +29,7 @@ services: KAFKA_CLUSTERS_0_KSQLDBSERVER: http://ksqldb:8088 kafka0: - image: confluentinc/cp-kafka:7.6.0 + image: confluentinc/cp-kafka:7.8.0 user: "0:0" hostname: kafka0 container_name: kafka0 @@ -62,7 +62,7 @@ services: CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' schemaregistry0: - image: confluentinc/cp-schema-registry:7.6.0 + image: confluentinc/cp-schema-registry:7.8.0 ports: - 8085:8085 depends_on: @@ -87,7 +87,7 @@ services: build: context: ./kafka-connect args: - image: confluentinc/cp-kafka-connect:7.6.0 + image: confluentinc/cp-kafka-connect:7.8.0 ports: - 8083:8083 depends_on: @@ -121,7 +121,7 @@ services: # AWS_SECRET_ACCESS_KEY: "" kafka-init-topics: - image: confluentinc/cp-kafka:7.6.0 + image: confluentinc/cp-kafka:7.8.0 volumes: - ./data/message.json:/data/message.json depends_on: @@ -161,7 +161,7 @@ services: command: sh -c '/connectors/start.sh' ksqldb: - image: confluentinc/cp-ksqldb-server:7.6.0 + image: confluentinc/cp-ksqldb-server:7.8.0 healthcheck: test: [ "CMD", "timeout", "1", "curl", "--silent", "--fail", "http://localhost:8088/info" ] interval: 30s diff --git a/e2e-tests/pom.xml b/e2e-tests/pom.xml index 8131dd11d..33db9404d 100644 --- a/e2e-tests/pom.xml +++ b/e2e-tests/pom.xml @@ -12,7 +12,6 @@ e2e-tests - 3.8.0 ${project.version} 21 3.5.1 @@ -25,7 +24,7 @@ org.apache.kafka kafka_2.13 - ${kafka.version} + ${confluent.version}-ccs io.kafbat.ui diff --git a/e2e-tests/src/main/java/io/kafbat/ui/screens/panels/NaviSideBar.java b/e2e-tests/src/main/java/io/kafbat/ui/screens/panels/NaviSideBar.java index 6972f379d..b5dc2be8d 100644 --- a/e2e-tests/src/main/java/io/kafbat/ui/screens/panels/NaviSideBar.java +++ b/e2e-tests/src/main/java/io/kafbat/ui/screens/panels/NaviSideBar.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.openqa.selenium.By; public class NaviSideBar extends BasePage { @@ -45,7 +46,7 @@ public String getPagePath(MenuItem menuItem) { @Step public NaviSideBar openSideMenu(String clusterName, MenuItem menuItem) { WebUtil.clickByActions(expandCluster(clusterName).parent() - .$x(String.format(sideMenuOptionElementLocator, menuItem.getNaviTitle()))); + .find(By.linkText(menuItem.getNaviTitle()))); return this; } diff --git a/e2e-tests/src/main/java/io/kafbat/ui/screens/topics/TopicSettingsTab.java b/e2e-tests/src/main/java/io/kafbat/ui/screens/topics/TopicSettingsTab.java index ef32fe352..75e3fe8e7 100644 --- a/e2e-tests/src/main/java/io/kafbat/ui/screens/topics/TopicSettingsTab.java +++ b/e2e-tests/src/main/java/io/kafbat/ui/screens/topics/TopicSettingsTab.java @@ -9,10 +9,13 @@ import io.qameta.allure.Step; import java.util.ArrayList; import java.util.List; +import java.util.NoSuchElementException; public class TopicSettingsTab extends BasePage { protected SelenideElement defaultValueColumnHeaderLocator = $x("//div[text() = 'Default Value']"); + protected SelenideElement nextButton = $x("//button[contains(text(), 'Next')]"); + protected SelenideElement previousButton = $x("//button[contains(text(), 'Previous')]"); @Step public TopicSettingsTab waitUntilScreenReady() { @@ -36,7 +39,25 @@ private TopicSettingsTab.SettingsGridItem getItemByKey(String key) { @Step public String getValueByKey(String key) { - return getItemByKey(key).getValue(); + while (true) { + try { + String value = getItemByKey(key).getValue(); + resetPageNavigation(); + return value; + } catch (NoSuchElementException e) { + if (nextButton.isEnabled()) { + nextButton.click(); + } else { + throw e; + } + } + } + } + + private void resetPageNavigation() { + while (previousButton.isEnabled()) { + previousButton.click(); + } } public static class SettingsGridItem extends BasePage { diff --git a/pom.xml b/pom.xml index d631e7751..43427b3f1 100644 --- a/pom.xml +++ b/pom.xml @@ -35,11 +35,10 @@ 3.25.3 1.11.4 1.14.19 - 7.4.4 + 7.8.0 3.1.0 3.0.13 2.14.0 - 3.8.0 1.6.2 1.18.34 3.25.5