[wko-nightly/3.4] ItMultiDomainModelsWithLoadBalancer test class failing intermittently (#3491)

* ItMultiDomainModelsWithLoadBalancer test class failing intermittently

Author: xiancao

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItMultiDomainModelsWithLoadBalancer.java

@@ -35,16 +35,13 @@
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
 import oracle.weblogic.kubernetes.utils.DomainUtils;
 import oracle.weblogic.kubernetes.utils.ExecResult;
-import org.awaitility.core.ConditionFactory;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import static java.nio.file.Paths.get;
-import static java.util.concurrent.TimeUnit.MINUTES;
-import static java.util.concurrent.TimeUnit.SECONDS;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
@@ -109,8 +106,6 @@
 import static oracle.weblogic.kubernetes.utils.PodUtils.setPodAntiAffinity;
 import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.awaitility.Awaitility.with;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -145,11 +140,11 @@ class ItMultiDomainModelsWithLoadBalancer {
   private static final String WLDF_OPENSESSION_APP_CONTEXT_ROOT = "opensession";
   private static final String wlSecretName = "weblogic-credentials";
   private static final String DATA_HOME_OVERRIDE = "/u01/mydata";
-  private static final String miiImageName = "mii-image";
+  private static final String miiImageName = "mdlb-mii-image";
   private static final String wdtModelFileForMiiDomain = "model-multiclusterdomain-sampleapp-wls.yaml";
-  private static final String miiDomainUid = "miidomain";
-  private static final String dimDomainUid = "domaininimage";
-  private static final String domainOnPVUid = "domainonpv";
+  private static final String miiDomainUid = "mdlb-miidomain";
+  private static final String dimDomainUid = "mdlb-domaininimage";
+  private static final String domainOnPVUid = "mdlb-domainonpv";
   private static final String wdtModelFileForDomainInImage = "wdt-singlecluster-multiapps-usingprop-wls.yaml";
 
   private static String opNamespace = null;
@@ -167,11 +162,6 @@ class ItMultiDomainModelsWithLoadBalancer {
   private static String encryptionSecretName = "encryptionsecret";
   private String curlCmd = null;
 
-  private ConditionFactory withStandardRetryPolicy =
-      with().pollDelay(1, SECONDS)
-          .and().with().pollInterval(5, SECONDS)
-          .atMost(1, MINUTES).await();
-
   /**
    * Install operator and NGINX.
    * Create three different type of domains: model in image, domain in PV and domain in image.
@@ -418,7 +408,6 @@ void testLivenessProbe() {
     String domainNamespace = domain.getMetadata().getNamespace();
     int numClusters = domain.getSpec().getClusters().size();
 
-    String serverName;
     String serverNamePrefix;
     if (numClusters > 1) {
       serverNamePrefix = domainUid + "-" + clusterName + "-" + MANAGED_SERVER_NAME_BASE;
@@ -503,10 +492,10 @@ void testLivenessProbe() {
     logger.info("Accessing the sample app through NGINX load balancer");
     String curlCmd = generateCurlCmd(domainUid, domainNamespace, clusterName, SAMPLE_APP_CONTEXT_ROOT);
     List<String> managedServers = listManagedServersBeforeScale(numClusters, clusterName, replicaCount);
-    assertThat(callWebAppAndCheckForServerNameInResponse(curlCmd, managedServers, 20))
-        .as("Verify NGINX can access the test web app from all managed servers in the domain")
-        .withFailMessage("NGINX can not access the test web app from one or more of the managed servers")
-        .isTrue();
+    testUntil(() -> callWebAppAndCheckForServerNameInResponse(curlCmd, managedServers, 20),
+        logger,
+        "NGINX can access the test web app from all managed servers {0} in the domain",
+        managedServers);
 
     // shutdown domain and verify the domain is shutdown
     shutdownDomainAndVerify(domainNamespace, domainUid, replicaCount);
@@ -879,6 +868,21 @@ private static Domain createDomainOnPvUsingWdt(String domainUid, String domainNa
           domainNamespace);
     }
 
+    // check that admin server pod is ready and service exists in domain namespace
+    logger.info("Checking that admin server pod {0} is ready and service exists in namespace {1}",
+        adminServerPodName, domainNamespace);
+    checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace);
+
+    // check the readiness for the managed servers in each cluster
+    for (int j = 1; j <= replicaCount; j++) {
+      String managedServerPodName = domainUid + "-" + MANAGED_SERVER_NAME_BASE + j;
+
+      // check managed server pod is ready and service exists in the namespace
+      logger.info("Checking that managed server pod {0} is ready and service exists in namespace {1}",
+          managedServerPodName, domainNamespace);
+      checkPodReadyAndServiceExists(managedServerPodName, domainUid, domainNamespace);
+    }
+
     return domain;
   }
 
@@ -1245,7 +1249,12 @@ private void verifyAdminConsoleLoginUsingIngressController(String domainUid, Str
           + "/console/login/LoginForm.jsp --write-out %{http_code} -o /dev/null";
 
       logger.info("Executing curl command {0}", curlCmd);
-      assertTrue(callWebAppAndWaitTillReady(curlCmd, 60));
+      testUntil(() -> callWebAppAndWaitTillReady(curlCmd, 5),
+          logger,
+          "WebLogic console on domain {0} in namespace {1} is accessible",
+          domainUid,
+          domainNamespace);
+
       logger.info("WebLogic console on domain1 is accessible");
     } else {
       logger.info("Skipping the admin console login test using ingress controller in OKD environment");

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/DomainUtils.java

@@ -57,6 +57,7 @@
 import static java.nio.file.Files.createDirectories;
 import static java.nio.file.Files.readString;
 import static java.nio.file.Paths.get;
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_PASSWORD_DEFAULT;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
 import static oracle.weblogic.kubernetes.TestConstants.ADMIN_USERNAME_DEFAULT;
@@ -499,7 +500,7 @@ private static void createConfigMapForDomainCreation(String configMapName,
       data.put(file.getFileName().toString(), readString(file));
       getLogger().info("Making a copy of file {0} to {1} for diagnostic purposes", file,
           domainScriptsDir.resolve(file.getFileName()));
-      copy(file, domainScriptsDir.resolve(file.getFileName()));
+      copy(file, domainScriptsDir.resolve(file.getFileName()), REPLACE_EXISTING);
     }
     V1ObjectMeta meta = new V1ObjectMeta()
         .name(configMapName)

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/LoadBalancerUtils.java

@@ -34,6 +34,7 @@
 import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
 
+import static oracle.weblogic.kubernetes.TestConstants.ADMIN_SERVER_NAME_BASE;
 import static oracle.weblogic.kubernetes.TestConstants.APACHE_RELEASE_NAME;
 import static oracle.weblogic.kubernetes.TestConstants.APACHE_SAMPLE_CHART_DIR;
 import static oracle.weblogic.kubernetes.TestConstants.APPSCODE_REPO_NAME;
@@ -72,7 +73,9 @@
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.isVoyagerReady;
 import static oracle.weblogic.kubernetes.assertions.TestAssertions.secretExists;
 import static oracle.weblogic.kubernetes.utils.ApplicationUtils.callWebAppAndWaitTillReady;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkServiceExists;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
 import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -647,6 +650,12 @@ public static List<String> createIngressForDomainAndVerify(String domainUid,
     logger.info("ingress {0} for domain {1} was created in namespace {2}",
         ingressName, domainUid, domainNamespace);
 
+    // check that admin server pod is ready and service exists in domain namespace
+    String adminServerPodName = domainUid + "-" + ADMIN_SERVER_NAME_BASE;
+    logger.info("Checking that admin server pod {0} is ready and service exists in namespace {1}",
+        adminServerPodName, domainNamespace);
+    checkPodReadyAndServiceExists(adminServerPodName, domainUid, domainNamespace);
+
     // check the ingress is ready to route the app to the server pod
     if (nodeport != 0) {
       for (String ingressHost : ingressHostList) {
@@ -655,7 +664,10 @@ public static List<String> createIngressForDomainAndVerify(String domainUid,
             + "/weblogic/ready --write-out %{http_code} -o /dev/null";
 
         logger.info("Executing curl command {0}", curlCmd);
-        assertTrue(callWebAppAndWaitTillReady(curlCmd, 60));
+        testUntil(() -> callWebAppAndWaitTillReady(curlCmd, 5),
+            logger,
+            "ingress is ready to ping the ready app on server {0}",
+            K8S_NODEPORT_HOST);
       }
     }
 

kindtest.sh

@@ -305,8 +305,8 @@ else
     echo "Running mvn -Dwdt.download.url=${wdt_download_url} -Dwit.download.url=${wit_download_url} -Dwle.download.url=${wle_download_url} -DPARALLEL_CLASSES=${parallel_run} -DNUMBER_OF_THREADS=${threads} -pl integration-tests -P ${maven_profile_name} verify"
     time mvn -Dwdt.download.url="${wdt_download_url}" -Dwit.download.url="${wit_download_url}" -Dwle.download.url="${wle_download_url}" -DPARALLEL_CLASSES="${parallel_run}" -DNUMBER_OF_THREADS="${threads}" -pl integration-tests -P ${maven_profile_name} verify 2>&1 | tee "${RESULT_ROOT}/kindtest.log" || captureLogs
   else
-    echo "Running mvn -Dit.test=!ItOperatorWlsUpgrade, !ItDedicatedMode, !ItT3Channel, !ItOperatorFmwUpgrade, !ItOCILoadBalancer, !ItMiiSampleFmwMain, !ItIstioCrossClusters*, !ItMultiDomainModels, !ItOpenshiftIstioMiiDomain -Dwdt.download.url=${wdt_download_url} -Dwit.download.url=${wit_download_url} -Dwle.download.url=${wle_download_url} -DPARALLEL_CLASSES=${parallel_run} -DNUMBER_OF_THREADS=${threads}  -pl integration-tests -P integration-tests  verify"
-    time mvn -Dit.test="!ItOperatorWlsUpgrade, !ItFmwDomainInPVUsingWDT, !ItFmwDynamicDomainInPV, !ItDedicatedMode, !ItT3Channel, !ItOperatorFmwUpgrade, !ItOCILoadBalancer, !ItMiiSampleFmwMain, !ItIstioCrossClusters*, !ItResilience, !ItMultiDomainModels, !ItOpenshiftIstioMiiDomain" -Dwdt.download.url="${wdt_download_url}" -Dwit.download.url="${wit_download_url}" -Dwle.download.url="${wle_download_url}" -DPARALLEL_CLASSES="${parallel_run}" -DNUMBER_OF_THREADS="${threads}" -pl integration-tests -P integration-tests verify 2>&1 | tee "${RESULT_ROOT}/kindtest.log" || captureLogs
+    echo "Running mvn -Dit.test=!ItLargeCapacityDomainsClusters, !ItOperatorWlsUpgrade, !ItDedicatedMode, !ItT3Channel, !ItOperatorFmwUpgrade, !ItOCILoadBalancer, !ItMiiSampleFmwMain, !ItIstioCrossClusters*, !ItMultiDomainModels, !ItOpenshiftIstioMiiDomain -Dwdt.download.url=${wdt_download_url} -Dwit.download.url=${wit_download_url} -Dwle.download.url=${wle_download_url} -DPARALLEL_CLASSES=${parallel_run} -DNUMBER_OF_THREADS=${threads}  -pl integration-tests -P integration-tests  verify"
+    time mvn -Dit.test="!ItLargeCapacityDomainsClusters, !ItOperatorWlsUpgrade, !ItFmwDomainInPVUsingWDT, !ItFmwDynamicDomainInPV, !ItDedicatedMode, !ItT3Channel, !ItOperatorFmwUpgrade, !ItOCILoadBalancer, !ItMiiSampleFmwMain, !ItIstioCrossClusters*, !ItResilience, !ItMultiDomainModels, !ItOpenshiftIstioMiiDomain" -Dwdt.download.url="${wdt_download_url}" -Dwit.download.url="${wit_download_url}" -Dwle.download.url="${wle_download_url}" -DPARALLEL_CLASSES="${parallel_run}" -DNUMBER_OF_THREADS="${threads}" -pl integration-tests -P integration-tests verify 2>&1 | tee "${RESULT_ROOT}/kindtest.log" || captureLogs
   fi
 fi