Fix for ItLBTwoDomainsNginx, ItEvictedPodsCycling and ItLBTwoDomainsTraefik nightly failures

Author: sankarpn

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItEvictedPodsCycling.java

@@ -1,14 +1,19 @@
-// Copyright (c) 2022, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2022, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.weblogic.kubernetes;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Callable;
 
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.models.CoreV1Event;
 import io.kubernetes.client.openapi.models.V1ResourceRequirements;
+import io.kubernetes.client.util.Yaml;
 import oracle.weblogic.domain.DomainResource;
+import oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes;
 import oracle.weblogic.kubernetes.annotations.IntegrationTest;
 import oracle.weblogic.kubernetes.annotations.Namespaces;
 import oracle.weblogic.kubernetes.logging.LoggingFacade;
@@ -25,8 +30,9 @@
 import static oracle.weblogic.kubernetes.TestConstants.MII_BASIC_IMAGE_TAG;
 import static oracle.weblogic.kubernetes.TestConstants.TEST_IMAGES_REPO_SECRET_NAME;
 import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource;
-import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodEvictedStatus;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
 import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
 import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
@@ -38,11 +44,11 @@
 
 /**
  * Use Operator log to test WLS server pods were evicted due to Pod ephemeral local
- * storage usage exceeds the total limit of containers 50M and replaced.
+ * storage usage exceeds the total limit of containers 25M and replaced.
  * 1. Install and start Operators
- * 2. Create and start the WebLogic domain with configuration of resource limit "ephemeral-storage=50M"
+ * 2. Create and start the WebLogic domain with configuration of resource limit "ephemeral-storage=25M"
  * 3. Verify that WLS server pods were evicted due to Pod ephemeral local
- *    storage usage exceeds the total limit of containers 50M and replaced.
+ *    storage usage exceeds the total limit of containers 25M and replaced.
  */
 @DisplayName("Test WLS server pods were evicted due to Pod ephemeral storage usage exceeds the total limit")
 @IntegrationTest
@@ -63,13 +69,13 @@ class ItEvictedPodsCycling {
 
   private static Map<String, String> resourceRequest = new HashMap<>();
   private static Map<String, String> resourceLimit = new HashMap<>();
-  private static final String ephemeralStorage = "50M";
+  private static final String ephemeralStorage = "25M";
 
   private static LoggingFacade logger = null;
 
   /**
    * Install Operator.
-   * Config resource limit and set ephemeral-storage=50M
+   * Config resource limit and set ephemeral-storage=25M
    * Create domain.
    *
    * @param namespaces list of namespaces created by the IntegrationTestWatcher by the
@@ -97,22 +103,29 @@ public static void init(@Namespaces(2) List<String> namespaces) {
   }
 
   /**
-   * Set domain resources limits tp 50M and then use Operator log to verify
+   * Set domain resources limits to 25M and then use Operator log to verify
    * that WLS server pods were evicted due to Pod ephemeral local
-   * storage usage exceeds the total limit of containers 50M and replaced. 
+   * storage usage exceeds the total limit of containers 25M and replaced. 
    */
   @Test
   @DisplayName("Use Operator log to verify that WLS server pods were evicted and replaced")
-  void testEvictedPodReplaced() {
+  void testEvictedPodReplaced() throws ApiException {
     resourceLimit.put("ephemeral-storage", ephemeralStorage);
     resourceRequest.put("cpu", "250m");
     resourceRequest.put("memory", "768Mi");
 
-    // patch domain with ephemeral-storage = 50M
+    // patch domain with ephemeral-storage = 25M
+    String reason = "Pod ephemeral local storage usage exceeds the total limit of containers";
+    addServerPodResources(domainUid, domainNamespace, resourceLimit, resourceRequest);
+    // verify that admin server pod evicted event is logged
+    testUntil(withLongRetryPolicy,
+        checkEvictionEvent(adminServerPodName, "Evicted", reason, "Warning"),
+        logger,
+        "domain event {0} to be logged in namespace {1}",
+        reason,
+        domainNamespace);
+    resourceLimit.replace("ephemeral-storage", "250M");
     addServerPodResources(domainUid, domainNamespace, resourceLimit, resourceRequest);
-
-    // verify that admin server pod evicted status exists in Operator log
-    checkPodEvictedStatus(opNamespace, adminServerPodName, ephemeralStorage);
 
     // verify that evicted pods are replaced and started
     checkServerPodsAndServiceReady();
@@ -184,4 +197,22 @@ private static void checkServerPodsAndServiceReady() {
       checkPodReadyAndServiceExists(managedServerPodPrefix + i, domainUid, domainNamespace);
     }
   }
+  
+  private Callable<Boolean> checkEvictionEvent(String adminServerpodName,
+      String reason, String message, String type) throws ApiException {
+    return (() -> {
+      boolean gotEvent = false;
+      List<CoreV1Event> events = Kubernetes.listNamespacedEvents(domainNamespace);
+      for (CoreV1Event event : events) {
+        if (event.getType().equals(type)
+            && event.getInvolvedObject().getName().equals(adminServerpodName)
+            && event.getReason().equals(reason)
+            && event.getMessage().contains(message)) {
+          logger.info(Yaml.dump(event));
+          gotEvent = true;
+        }
+      }
+      return gotEvent;
+    });
+  }
 }

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItLBTwoDomainsNginx.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2022, 2023, Oracle and/or its affiliates.
+// Copyright (c) 2022, 2024, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.weblogic.kubernetes;
@@ -46,6 +46,7 @@
 import static oracle.weblogic.kubernetes.utils.CommonLBTestUtils.verifyAdminServerAccess;
 import static oracle.weblogic.kubernetes.utils.CommonLBTestUtils.verifyClusterLoadbalancing;
 import static oracle.weblogic.kubernetes.utils.CommonLBTestUtils.verifyHeadersInAdminServerLog;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.formatIPv6Host;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.getServiceExtIPAddrtOke;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.createBaseRepoSecret;
 import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx;
@@ -138,7 +139,7 @@ public static void initAll(@Namespaces(3) List<String> namespaces) {
 
     String ingressServiceName = nginxHelmParams.getHelmParams().getReleaseName() + "-ingress-nginx-controller";
     ingressIP = getServiceExtIPAddrtOke(ingressServiceName, nginxNamespace) != null
-        ? getServiceExtIPAddrtOke(ingressServiceName, nginxNamespace) : K8S_NODEPORT_HOST;
+        ? getServiceExtIPAddrtOke(ingressServiceName, nginxNamespace) : formatIPv6Host(K8S_NODEPORT_HOST);
   }
 
   /**

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonLBTestUtils.java

@@ -842,24 +842,18 @@ public static void verifyClusterLoadbalancing(String domainUid,
     // access application in managed servers through load balancer
     getLogger().info("Accessing the clusterview app through load balancer to verify all servers in cluster");
     String curlRequest;
+    String uri = "clusterview/ClusterViewServlet" + "\"?user=" + ADMIN_USERNAME_DEFAULT
+        + "&password=" + ADMIN_PASSWORD_DEFAULT + (host.contains(":") ? "&ipv6=true" : "&ipv6=false") + "\"";
     if (hostRouting) {
       curlRequest = OKE_CLUSTER_PRIVATEIP ? String.format("curl -g --show-error -ks --noproxy '*' "
-          + "-H 'host: %s' %s://%s/clusterview/ClusterViewServlet"
-          + "\"?user=" + ADMIN_USERNAME_DEFAULT
-          + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", ingressHostName, protocol, host)
-        : String.format("curl --show-error -ks --noproxy '*' "
-          + "-H 'host: %s' %s://%s:%s/clusterview/ClusterViewServlet"
-          + "\"?user=" + ADMIN_USERNAME_DEFAULT
-          + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", ingressHostName, protocol, host, lbPort);
+          + "-H 'host: %s' %s://%s/" + uri, ingressHostName, protocol, host)
+        : String.format("curl -g --show-error -ks --noproxy '*' "
+          + "-H 'host: %s' %s://%s:%s/" + uri, ingressHostName, protocol, host, lbPort);
     } else {
       curlRequest = OKE_CLUSTER_PRIVATEIP ? String.format("curl -g --show-error -ks --noproxy '*' "
-          + "%s://%s" + locationString + "/clusterview/ClusterViewServlet"
-          + "\"?user=" + ADMIN_USERNAME_DEFAULT
-          + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", protocol, host)
-        : String.format("curl --show-error -ks --noproxy '*' "
-          + "%s://%s:%s" + locationString + "/clusterview/ClusterViewServlet"
-          + "\"?user=" + ADMIN_USERNAME_DEFAULT
-          + "&password=" + ADMIN_PASSWORD_DEFAULT + "\"", protocol, host, lbPort);
+          + "%s://%s" + locationString + "/" + uri, protocol, host)
+        : String.format("curl -g --show-error -ks --noproxy '*' "
+          + "%s://%s:%s" + locationString + "/" + uri, protocol, host, lbPort);
     }
 
     List<String> managedServers = new ArrayList<>();

integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java

@@ -1093,6 +1093,7 @@ public static void verifyServerCommunication(String curlRequest, List<String> ma
             // check the response contains managed server name
             ExecResult result = null;
             try {
+              logger.info("Sending request {0}", curlRequest);
               result = ExecCommand.exec(curlRequest, true);
             } catch (IOException | InterruptedException ex) {
               logger.severe(ex.getMessage());