chore: Fix flaky tests

.github/workflows/integration-tests.yml

@@ -516,7 +516,7 @@ jobs:
 
             -   name: Run CI Tests With SAF Provider
                 run: >
-                    ./gradlew :integration-tests:runContainerSAFProviderTests --info -Denvironment.config=-ha
+                    ENV_CONFIG=ha ./gradlew :integration-tests:runContainerSAFProviderTests --info -Denvironment.config=-ha
                     -Partifactory_user=${{ secrets.ARTIFACTORY_USERNAME }} -Partifactory_password=${{ secrets.ARTIFACTORY_PASSWORD }}
             -   uses: ./.github/actions/dump-jacoco
                 if: always()

api-catalog-ui/frontend/cypress/e2e/graphql/graphql-apiml.cy.js

@@ -163,10 +163,6 @@ describe('>>> GraphiQL Playground page test', () => {
         cy.get('.graphiql-dialog-header h2').should('be.visible').should('contain', 'Settings');
     });
 
-    // Skip flaky tests in the microservice setup
-    if (Cypress.env('microservices')) {
-        return;
-    }
     it('Variable usage', () => {
         login();
         cy.contains('Discoverable client with GraphQL').click();
@@ -177,7 +173,9 @@ describe('>>> GraphiQL Playground page test', () => {
 
         const variable = '{"id" :"book-1"}';
 
-        cy.get('.graphiql-editor-tool').first()
+        cy.get('.graphiql-editor-tool .cm-s-graphiql').first().as('variablesInput').click();
+
+        cy.get('@variablesInput').first()
             .type(variable, {parseSpecialCharSequences: false});
 
         cy.get('.graphiql-editor-tool').then(($container) => {
@@ -196,7 +194,9 @@ describe('>>> GraphiQL Playground page test', () => {
 
         const header = '{"X-Custom-Header": "CustomValue"}';
 
-        cy.get('.graphiql-editor-tool').first()
+        cy.get('.graphiql-editor-tool .cm-s-graphiql').last().as('headersInput').click();
+
+        cy.get('@headersInput').first()
             .type(header, {parseSpecialCharSequences: false});
 
         cy.get('.graphiql-editor-tool').then(($container) => {

config/local-multi/discovery-service-1.yml

@@ -19,7 +19,7 @@ spring:
             enabled: always
 
 ---
-spring.profiles: https
+spring.config.activate.on-profile: https
 eureka:
     client:
         instanceInfoReplicationIntervalSeconds: 30

config/local-multi/discovery-service-2.yml

@@ -19,7 +19,7 @@ spring:
             enabled: always
 
 ---
-spring.profiles: https
+spring.config.activate.on-profile: https
 eureka:
     client:
         instanceInfoReplicationIntervalSeconds: 30

integration-tests/build.gradle

@@ -248,6 +248,7 @@ task runAllIntegrationTestsForZoweHaTestingOnZos(type: Test) {
             'GatewayCentralRegistry',
             'SafIdTokenTest',
             'ChaoticHATest',
+            'RateLimitTest',
             'GraphQLTest'
         )
     }

integration-tests/src/test/java/org/zowe/apiml/functional/gateway/CentralRegistryTest.java

@@ -27,29 +27,22 @@
 import org.zowe.apiml.product.constants.CoreService;
 import org.zowe.apiml.util.TestWithStartedInstances;
 import org.zowe.apiml.util.categories.DiscoverableClientDependentTest;
-import org.zowe.apiml.util.config.ConfigReader;
-import org.zowe.apiml.util.config.DiscoveryServiceConfiguration;
-import org.zowe.apiml.util.config.ServiceConfiguration;
-import org.zowe.apiml.util.config.SslContext;
-import org.zowe.apiml.util.config.SslContextConfigurer;
-import org.zowe.apiml.util.config.TlsConfiguration;
-
-import java.net.MalformedURLException;
+import org.zowe.apiml.util.config.*;
+
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static io.restassured.RestAssured.given;
 import static io.restassured.RestAssured.with;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.hamcrest.Matchers.is;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.springframework.http.HttpHeaders.ACCEPT;
 import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
 import static org.zowe.apiml.util.SecurityUtils.GATEWAY_TOKEN_COOKIE_NAME;
@@ -170,33 +163,68 @@ private ValidatableResponse listEurekaApps() {
             .contentType(ContentType.JSON);
     }
 
-    @Test
-    void shouldContainCorrectBasePaths() throws MalformedURLException, URISyntaxException {
-        URI containers = new URL(conf.getScheme(), conf.getHost(), conf.getPort(), APIML_CONTAINER_PATH)
-            .toURI();
-
-        final String jwt = gatewayToken();
-        String responseBody = with().given()
-            .header(ACCEPT, APPLICATION_JSON_VALUE)
-            .cookie(GATEWAY_TOKEN_COOKIE_NAME, jwt)
-            .get(containers)
-            .then()
-            .statusCode(200)
-            .contentType("application/json")
-            .extract()
-            .body()
-            .asString();
-
-        DocumentContext jsonContext = JsonPath.parse(responseBody);
+    private String callContainers() {
+        try {
+            URI containers = new URL(
+                conf.getScheme(),
+                conf.getHost(),
+                conf.getPort(),
+                APIML_CONTAINER_PATH
+            ).toURI();
+
+            final String jwt = gatewayToken();
+
+            return with().given()
+                .header(ACCEPT, APPLICATION_JSON_VALUE)
+                .cookie(GATEWAY_TOKEN_COOKIE_NAME, jwt)
+                .get(containers)
+                .then()
+                .statusCode(200)
+                .contentType("application/json")
+                .extract()
+                .body()
+                .asString();
+
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to call containers endpoint", e);
+        }
+    }
 
-        JSONArray gatewayBasePath = jsonContext.read("$[0].services[?(@.serviceId == 'gateway')].basePath");
-        assertNotNull(gatewayBasePath, String.format("BasePath for central gw should not be null but it was '%s'", gatewayBasePath));
-        assertFalse(gatewayBasePath.isEmpty(), String.format("BasePath for central gw should not be empty but it was '%s'", gatewayBasePath));
+    @Test
+    void shouldContainCorrectBasePaths() {
+
+        await()
+            .atMost(60, TimeUnit.SECONDS)
+            .pollInterval(1, TimeUnit.SECONDS)
+            .untilAsserted(() -> {
+                String body = callContainers();
+                assertThat(body)
+                    .as("Domain gateway must be present in containers")
+                    .contains("\"serviceId\":\"domain-apiml\"");
+            });
+
+        String body = callContainers();
+        DocumentContext jsonContext = JsonPath.parse(body);
+
+        JSONArray gatewayBasePath =
+            jsonContext.read(
+                "$[0].services[?(@.serviceId == 'gateway')].basePath"
+            );
+
+
+        assertThat(gatewayBasePath)
+            .withFailMessage("Central gateway basePath not ready yet. Payload:\n%s", body)
+            .isNotEmpty();
         assertEquals("/", gatewayBasePath.get(0));
-        JSONArray domainGatewayBasePath = jsonContext.read("$[0].services[?(@.serviceId == 'domain-apiml')].basePath");
-        assertNotNull(domainGatewayBasePath, String.format("BasePath for domain gw should not be null but it was '%s'", domainGatewayBasePath));
-        assertFalse(domainGatewayBasePath.isEmpty(), String.format("BasePath for domain gw should not be empty but it was '%s'", domainGatewayBasePath));
-        assertEquals("/" + DOMAIN_APIML, domainGatewayBasePath.get(0));
+
+        JSONArray domainGatewayBasePath =
+            jsonContext.read(
+                "$[0].services[?(@.serviceId == 'domain-apiml')].basePath"
+            );
+        assertThat(domainGatewayBasePath)
+            .withFailMessage("Domain gateway basePath not ready yet. Payload:\n%s", body)
+            .isNotEmpty();
+        assertEquals("/domain-apiml", domainGatewayBasePath.get(0));
     }
 
     @SneakyThrows

integration-tests/src/test/java/org/zowe/apiml/integration/ha/AuthenticationHaTest.java

@@ -19,25 +19,36 @@
 import org.zowe.apiml.util.categories.HATest;
 import org.zowe.apiml.util.config.ConfigReader;
 import org.zowe.apiml.util.config.GatewayServiceConfiguration;
+import org.zowe.apiml.util.config.ZaasConfiguration;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.zowe.apiml.util.SecurityUtils.assertIfLogged;
 import static org.zowe.apiml.util.SecurityUtils.getConfiguredSslConfig;
+import static org.zowe.apiml.util.config.ConfigReader.IS_MODULITH_ENABLED;
+import static org.zowe.apiml.util.http.HttpRequestUtils.getUriFromService;
 import static org.zowe.apiml.util.requests.Endpoints.ROUTED_LOGOUT;
 
 /**
  * In initial version, basic logout test to verify token invalidation in HA scenarios
- *
  */
 @HATest
 @Tag("SAFProviderTest")
 class AuthenticationHaTest {
 
+    private static final String ZAAS_QUERY = "/zaas/api/v1/auth/query";
     private static final GatewayServiceConfiguration GATEWAY_CONF = ConfigReader.environmentConfiguration().getGatewayServiceConfiguration();
+    private static final ZaasConfiguration ZAAS_CONF = ConfigReader.environmentConfiguration().getZaasConfiguration();
+    private List<Throwable> errors;
 
     @BeforeEach
     void setUp() {
         RestAssured.useRelaxedHTTPSValidation();
         RestAssured.config = RestAssured.config().sslConfig(getConfiguredSslConfig());
+        errors = new ArrayList<>();
     }
 
     @Nested
@@ -49,22 +60,51 @@ class WhenUserLogOut {
             @Test
             void thenTokenIsInvalidatedInBoth() {
                 var jwt = SecurityUtils.gatewayToken();
+                var gatewayHosts = getGatewayHosts();
 
                 assertIfLogged(jwt, true);
 
                 // Logout on any instance
-                SecurityUtils.logoutOnGateway(SecurityUtils.getGatewayUrl(getGatewayHosts()[0], ROUTED_LOGOUT), jwt);
-
-                // Verify token is invalid in one or more Gateway instances
-                var gatewayHosts = getGatewayHosts();
-                for (String host : gatewayHosts) {
-                    SecurityUtils.assertIfLogged(jwt, false, host);
+                SecurityUtils.logoutOnGateway(SecurityUtils.getGatewayUrl(gatewayHosts[0], ROUTED_LOGOUT), jwt);
+
+                // Verify token is invalid in one or more Gateway and ZAAS instances. Do this twice
+                for (int i = 0; i < 2; i++) {
+                    assertIfGatewayLogged(jwt, false, gatewayHosts[0]);
+                    if (!(IS_MODULITH_ENABLED || ZAAS_CONF == null)) {
+                        assertIfZaasLogged(jwt, false, ZAAS_CONF);
+                    }
+
+                    assertIfGatewayLogged(jwt, false, gatewayHosts[1]);
+                    if (!(IS_MODULITH_ENABLED || ZAAS_CONF == null)) {
+                        // Since we have only one ZAAS instance in the configuration, manually add the second one
+                        assertIfZaasLogged(jwt, false, new ZaasConfiguration(ZAAS_CONF.getScheme(), ZAAS_CONF.getHost() + "-2", ZAAS_CONF.getPort(), 2));
+                    }
                 }
 
+                assertTrue(
+                    errors.isEmpty(),
+                    () -> "Errors:\n" + errors.stream()
+                        .map(Throwable::getMessage)
+                        .collect(Collectors.joining("\n"))
+                );
             }
+        }
+    }
 
+    private void assertIfGatewayLogged(String jwt, boolean logged, String gatewayHost) {
+        try {
+            SecurityUtils.assertIfLogged(jwt, logged, gatewayHost);
+        } catch (Throwable error) {
+            errors.add(new Throwable(gatewayHost, error));
         }
+    }
 
+    private void assertIfZaasLogged(String jwt, boolean logged, ZaasConfiguration zaasConfiguration) {
+        try {
+            SecurityUtils.assertIfLogged(jwt, logged, getUriFromService(zaasConfiguration, ZAAS_QUERY));
+        } catch (Throwable error) {
+            errors.add(new Throwable(zaasConfiguration.getHost(), error));
+        }
     }
 
     // assume only two gateway (or apiml) instances

integration-tests/src/test/java/org/zowe/apiml/integration/proxy/MultipartPutIntegrationTest.java

@@ -11,6 +11,8 @@
 package org.zowe.apiml.integration.proxy;
 
 import io.restassured.RestAssured;
+import io.restassured.config.HttpClientConfig;
+import io.restassured.config.RestAssuredConfig;
 import io.restassured.parsing.Parser;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Nested;
@@ -23,6 +25,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
+import java.util.Objects;
 import java.util.Random;
 
 import static io.restassured.RestAssured.given;
@@ -33,7 +36,7 @@
 class MultipartPutIntegrationTest implements TestWithStartedInstances {
     private final String configFileName = "example.txt";
     private final ClassLoader classLoader = ClassLoader.getSystemClassLoader();
-    private URI url = HttpRequestUtils.getUriFromGateway(DISCOVERABLE_MULTIPART);
+    private final URI url = HttpRequestUtils.getUriFromGateway(DISCOVERABLE_MULTIPART);
 
     @BeforeAll
     static void beforeClass() {
@@ -51,8 +54,8 @@ void givenPutRequest() {
 
                 given().
                     contentType("multipart/form-data").
-                    multiPart(new File(classLoader.getResource(configFileName).getFile())).
-                expect().
+                    multiPart(new File(Objects.requireNonNull(classLoader.getResource(configFileName)).getFile())).
+                    expect().
                     statusCode(200).
                     body("fileName", equalTo("example.txt")).
                     body("fileType", equalTo("application/octet-stream")).
@@ -66,8 +69,8 @@ void givenPostRequest() {
 
                 given().
                     contentType("multipart/form-data").
-                    multiPart(new File(classLoader.getResource(configFileName).getFile())).
-                expect().
+                    multiPart(new File(Objects.requireNonNull(classLoader.getResource(configFileName)).getFile())).
+                    expect().
                     statusCode(200).
                     body("fileName", equalTo("example.txt")).
                     body("fileType", equalTo("application/octet-stream")).
@@ -79,8 +82,16 @@ void givenPostRequest() {
         @Test
         void givenLargeFileUpload() {
             int payloadSize = 750 * 1024 * 1024; //750MB
+            //disable the retry to avoid NonRepeatableRequestException and increase the timeout to fix the flakiness
+            RestAssuredConfig config = RestAssured.config()
+                .httpClient(HttpClientConfig.httpClientConfig()
+                    .setParam("http.connection.timeout", 300000)
+                    .setParam("http.socket.timeout", 300000)
+                    .setParam("http.method.retry-handler", (org.apache.http.client.HttpRequestRetryHandler)
+                        (exception, executionCount, context) -> false));
 
             given()
+                .config(config)
                 .multiPart(
                     "file",
                     "largefile.dat",
@@ -113,6 +124,21 @@ public int read() throws IOException {
                 count++;
                 return random.nextInt(256);
             }
+
+            @Override
+            public int read(byte[] b) throws IOException {
+                return read(b, 0, b.length);
+            }
+
+            @Override
+            public int read(byte[] b, int off, int len) {
+                if (count >= targetSize) return -1;
+                long remaining = targetSize - count;
+                int toRead = (int) Math.min(len, remaining);
+                random.nextBytes(b);
+                count += toRead;
+                return toRead;
+            }
         }
     }
 }

integration-tests/src/test/java/org/zowe/apiml/integration/proxy/StompProxyTest.java

@@ -60,7 +60,7 @@ void stompOverWebsocketLargeMessageExchange() throws Exception {
 
         StompSession stompSession = stompClient.connectAsync(
             discoverableClientGatewayUrl(DISCOVERABLE_STOMP), VALID_AUTH_HEADERS, new StompSessionHandlerAdapter() {
-        }).get(5, SECONDS); // lower connection timeout fails on z/os test system
+        }).get(10, SECONDS); // lower connection timeout fails on z/os test system
         stompSession.subscribe(SUBSCRIBE_ENDPOINT + uuid, new StringStompFrameHandler());
 
         char c = 'A';