more changes for Coherence rolling restart test

Author: pfmackin

integration-tests/src/test/java/oracle/kubernetes/operator/ITCoherenceTests.java

@@ -4,12 +4,9 @@
 
 package oracle.kubernetes.operator;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.Map;
 import oracle.kubernetes.operator.utils.CoherenceUtils;
 import oracle.kubernetes.operator.utils.Domain;
-import oracle.kubernetes.operator.utils.ExecResult;
 import oracle.kubernetes.operator.utils.K8sTestUtils;
 import oracle.kubernetes.operator.utils.Operator;
 import oracle.kubernetes.operator.utils.TestUtils;
@@ -49,6 +46,12 @@ public static void staticPrepare() throws Exception {
     if (!QUICKTEST) {
       initialize(APP_PROPS_FILE);
 
+      // The default cmd loop sleep is too long and we could miss states like terminating. Change the
+      // sleep and iterations
+      //
+      setWaitTimePod(2);
+      setMaxIterationsPod(125);
+
       if (operator1 == null) {
         operator1 = TestUtils.createOperator(OPERATOR1_YAML);
       }
@@ -76,34 +79,30 @@ public void testRollingRestart() throws Exception {
     domain = createDomain();
     Assert.assertNotNull(domain);
 
-    //  public static void exposePod(String podName, String domainNS, String serviceName,  int port,
-    // int targetPort)  throws Exception {
+    //    String nodePortName = "coh-nodeport";
+    //    String podName = "dd";
+    //    int port = 9000;
+    //    TestUtils.exposePod(podName, domain.getDomainNS(), nodePortName, port, port);
+    //
+    utils.loadCache();
 
+    // Do the rolling restart
+    testServerPodsRestartByChangingEnvProperty();
 
-//    String nodePortName = "coh-nodeport";
-//    String podName = "dd";
-//    int port = 9000;
-//    TestUtils.exposePod(podName, domain.getDomainNS(), nodePortName, port, port);
-//
-    utils.loadCache();
     utils.validateCache();
 
     destroyDomain();
   }
 
-
   /**
    * Modify the domain scope env property on the domain resource using kubectl apply -f domain.yaml
    * Verify that all the server pods in the domain got re-started. The property tested is: env:
    * "-Dweblogic.StdoutDebugEnabled=false"--> "-Dweblogic.StdoutDebugEnabled=true"
    *
    * @throws Exception
    */
-  @Ignore
-  @Test
   public void testServerPodsRestartByChangingEnvProperty() throws Exception {
 
-
     Assume.assumeFalse(QUICKTEST);
     String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
     logTestBegin(testMethodName);
@@ -112,56 +111,34 @@ public void testServerPodsRestartByChangingEnvProperty() throws Exception {
         "About to verifyDomainServerPodRestart for Domain: "
             + domain.getDomainUid()
             + "  env property: StdoutDebugEnabled=false to StdoutDebugEnabled=true");
+
     domain.verifyDomainServerPodRestart(
         "\"-Dweblogic.StdoutDebugEnabled=false\"", "\"-Dweblogic.StdoutDebugEnabled=true\"");
 
     logger.info("SUCCESS - " + testMethodName);
   }
 
-  private static Domain createDomain() throws Exception {
-
-    Map<String, Object> domainMap = TestUtils.loadYaml(DOMAININIMAGE_WLST_YAML);
-    domainMap.put("namespace","test1");
-    domainMap.put("domainUID", "coh");
-    domain = TestUtils.createDomain(domainMap);
-    domain.verifyDomainCreated();
-    return domain;
-  }
-
   private static void destroyDomain() throws Exception {
     if (domain != null) {
       domain.destroy();
     }
   }
 
-  /**
-   * Utility method to check if a pod is in Terminating or Running status
-   *
-   * @param podName - String name of the pod to check the status for
-   * @param podStatusExpected - String the expected status of Terminating || RUnning
-   * @throws InterruptedException when thread is interrupted
-   */
-  private void verifyPodStatus(String podName, String podStatusExpected)
-      throws InterruptedException {
-    K8sTestUtils testUtil = new K8sTestUtils();
-    String domain1LabelSelector = String.format("weblogic.domainUID in (%s)", domainUid);
-    String namespace = domain.getDomainNS();
-    boolean gotExpected = false;
-    for (int i = 0; i < BaseTest.getMaxIterationsPod(); i++) {
-      if (podStatusExpected.equals("Terminating")) {
-        if (testUtil.isPodTerminating(namespace, domain1LabelSelector, podName)) {
-          gotExpected = true;
-          break;
-        }
-      } else if (podStatusExpected.equals("Running")) {
-        if (testUtil.isPodRunning(namespace, domain1LabelSelector, podName)) {
-          gotExpected = true;
-          break;
-        }
-      }
+  private Domain createDomain() throws Exception {
 
-      Thread.sleep(BaseTest.getWaitTimePod() * 1000);
-    }
-    Assert.assertTrue("Didn't get the expected pod status", gotExpected);
+    //    System.getenv().put("CUSTOM_WDT_ARCHIVE", "/Users/pmackin/archive-proxy.zip");
+
+    // create domain
+    Domain domain = null;
+    Map<String, Object> domainMap = TestUtils.loadYaml(DOMAININIMAGE_WDT_YAML);
+    domainMap.put("namespace", "test1");
+    domainMap.put("domainUID", "coh");
+    domainMap.put(
+        "customWdtTemplate",
+        BaseTest.getProjectRoot()
+            + "/integration-tests/src/test/resources/wdt/coh-wdt-config.yaml");
+    domain = TestUtils.createDomain(domainMap);
+    domain.verifyDomainCreated();
+    return domain;
   }
-}
+}

integration-tests/src/test/java/oracle/kubernetes/operator/utils/CoherenceUtils.java

@@ -32,7 +32,7 @@ public class CoherenceUtils {
   private static final String PREFIX = "Thread-";
   private static final int VALUE_SIZE = 1024;
   private static final int NUM_ITEMS =
-      200 * 1000; // Total number of items, regardless of thread count
+      100 * 1000; // Total number of items, regardless of thread count
   private static final int THREAD_COUNT = 1;
   private static final int MAX_BATCH_SIZE = 1000;
 
@@ -126,7 +126,7 @@ public void loadCache() {
     Instant finishInstant = Instant.now();
     Duration duration = Duration.between(startInstant, finishInstant);
 
-    System.out.println("Load Test elaspsed Time = " + duration.getSeconds() + " seconds");
+    System.out.println("Load Test SUCCESS - elapsed Time = " + duration.getSeconds() + " seconds");
   }
 
   public void validateCache() {
@@ -156,7 +156,7 @@ public void validateCache() {
     Instant finishInstant = Instant.now();
     Duration duration = Duration.between(startInstant, finishInstant);
 
-    System.out.println("Validate Test elaspsed Time = " + duration.getSeconds() + " seconds");
+    System.out.println("Validate Test SUCCESS - elapsed Time = " + duration.getSeconds() + " seconds");
   }
 
   /**

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -17,10 +17,13 @@
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Hashtable;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -1016,6 +1019,7 @@ public void verifyDomainServerPodRestart(String oldPropertyString, String newPro
       // verify the servers in the domain are being restarted in a sequence
       verifyAdminServerRestarted();
       verifyManagedServersRestarted();
+
       // make domain.yaml include the new changed property
       TestUtils.copyFile(
           BaseTest.getUserProjectsDir() + "/weblogic-domains/" + domainUid + "/domain_new.yaml",
@@ -1118,8 +1122,9 @@ public void verifyAdminServerRestarted() throws Exception {
     logger.info("Checking if admin pod(" + domainUid + "-" + adminServerName + ") is Terminating");
     TestUtils.checkPodTerminating(domainUid + "-" + adminServerName, domainNS);
 
-    logger.info("Checking if admin pod(" + domainUid + "-" + adminServerName + ") is Running");
+    logger.info("Checking if admin pod(" + domainUid + "-" + adminServerName + ") is Ready");
     TestUtils.checkPodCreated(domainUid + "-" + adminServerName, domainNS);
+    TestUtils.checkPodReady(domainUid + "-" + adminServerName, domainNS);
   }
 
   /**
@@ -1129,25 +1134,36 @@ public void verifyAdminServerRestarted() throws Exception {
    */
   public void verifyManagedServersRestarted() throws Exception {
     if (!serverStartPolicy.equals("ADMIN_ONLY")) {
-      // check managed server pods
+
+      // Note: Managed Servers can be stopped in any order.  Build the set of server names
+      // then loop until all servers have been recycled.  When we find one that is terminating,
+      // wait until it is running then remove it from the set.
+      Set<String> podNameSet = new HashSet<>();
       for (int i = 1; i <= initialManagedServerReplicas; i++) {
-        logger.info(
-            "Checking if managed pod("
-                + domainUid
-                + "-"
-                + managedServerNameBase
-                + i
-                + ") is Terminating");
-        TestUtils.checkPodTerminating(domainUid + "-" + managedServerNameBase + i, domainNS);
+        podNameSet.add(managedServerNameBase + i);
+      }
 
-        logger.info(
-            "Checking if managed pod("
-                + domainUid
-                + "-"
-                + managedServerNameBase
-                + i
-                + ") is Running");
-        TestUtils.checkPodCreated(domainUid + "-" + managedServerNameBase + i, domainNS);
+      // Loop until all the servers have recycled
+      //
+      while (podNameSet.size() > 0) {
+        Iterator<String> iter = podNameSet.iterator();
+        while (iter.hasNext()) {
+          String podName = iter.next();
+          if (TestUtils.checkPodTerminatingNoWait(domainUid + "-" + podName, domainNS)) {
+            // Server is terminating, wait until server running then remove it from the list
+            logger.info("Managed server pod " + podName + "  is terminating");
+            TestUtils.checkPodCreated(domainUid + "-" + adminServerName, domainNS);
+            TestUtils.checkPodReady(domainUid + "-" + podName, domainNS);
+            logger.info("Managed server pod " + podName + " has been recycled");
+            iter.remove();
+          }
+        }
+        // We iterated through all the servers and if there are some remaining then
+        // Sleep and loop through the remaining ones again.  Keep the sleep short so we don't miss the terminating status.
+        if (podNameSet.size() > 0) {
+          logger.info("Waiting for any managed server pod to be terminating...");
+          Thread.sleep(1000);
+        }
       }
     }
   }

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -82,10 +82,27 @@ public static void checkPodCreated(String podName, String domainNS) throws Excep
     StringBuffer cmd = new StringBuffer();
     cmd.append("kubectl get pod ").append(podName).append(" -n ").append(domainNS);
 
-    // check for admin pod
+    // check for pod to be running
     checkCmdInLoop(cmd.toString(), "Running", podName);
   }
 
+  /**
+   * Checks that pod is ready and running.
+   *
+   * @param podName - pod name
+   * @param domainNS - domain namespace name
+   */
+  public static void checkPodReadyAndRunning(String podName, String domainNS) throws Exception {
+
+    StringBuffer cmd = new StringBuffer();
+    cmd.append("kubectl get pod ").append(podName).append(" -n ").append(domainNS);
+
+    // check for pod to be running
+    checkCmdInLoop(cmd.toString(), "Running", podName);
+    checkCmdInLoop(cmd.toString(), "1/1", podName);
+  }
+
+
   /**
    * Checks that pod is initializing.
    *
@@ -117,6 +134,24 @@ public static void checkPodTerminating(String podName, String domainNS) throws E
     checkCmdInLoop(cmd.toString(), "Terminating", podName);
   }
 
+  /**
+   * check pod is in Terminating state without waiting
+   *
+   * @param podName - pod name
+   * @param domainNS - domain namespace name
+   *
+   * @return true if pod terminating else false
+   * @throws Exception exception
+   */
+  public static boolean checkPodTerminatingNoWait(String podName, String domainNS) throws Exception {
+
+    StringBuffer cmd = new StringBuffer();
+    cmd.append("kubectl get pod ").append(podName).append(" -n ").append(domainNS);
+
+    // check for admin pod
+    return checkPodMatch(cmd.toString(), "Terminating", podName);
+  }
+
   /**
    * Checks that service is created.
    *
@@ -1302,12 +1337,12 @@ public static void checkCmdInLoop(String cmd, String matchStr, String k8sObjName
         // check for last iteration
         if (i == (BaseTest.getMaxIterationsPod() - 1)) {
           throw new RuntimeException(
-              "FAILURE: pod " + k8sObjName + " is not running/ready, exiting!");
+              "FAILURE: Timeout - pod " + k8sObjName + " output does not contain '" + matchStr + "'");
         }
         logger.info(
             "Pod "
                 + k8sObjName
-                + " is not Running/Ready Ite ["
+                + "  output does not contain '" + matchStr + "'  Iteration ["
                 + i
                 + "/"
                 + BaseTest.getMaxIterationsPod()
@@ -1318,12 +1353,34 @@ public static void checkCmdInLoop(String cmd, String matchStr, String k8sObjName
         Thread.sleep(BaseTest.getWaitTimePod() * 1000);
         i++;
       } else {
-        logger.info("Pod " + k8sObjName + " is Running");
+        logger.info("SUCCESS: Pod " + k8sObjName + " output contains '" + matchStr + "'");
         break;
       }
     }
   }
 
+  /**
+   * Check if the pod is in a specific status.
+   *
+   * @param cmd        command to execute
+   * @param matchStr   matching string
+   * @param k8sObjName pod Name
+   *
+   * @return true for match else false
+   * @throws Exception
+   */
+  public static boolean checkPodMatch(String cmd, String matchStr, String k8sObjName)
+      throws Exception {
+      ExecResult result = ExecCommand.exec(cmd);
+      if (result.exitValue() != 0
+          || (result.exitValue() == 0 && !result.stdout().contains(matchStr))) {
+        return false;
+      } else {
+        logger.info("Pod " + k8sObjName + " match found for " + matchStr);
+        return true;
+      }
+  }
+
   private static void checkCmdInLoopForDelete(String cmd, String matchStr, String k8sObjName)
       throws Exception {
     int i = 0;

integration-tests/src/test/resources/cohdomaininimagewlst.yaml

@@ -0,0 +1,14 @@
+# Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+domainUID: domaininimagewlst
+namespace: test1
+configuredManagedServerCount: 4
+initialManagedServerReplicas: 2
+exposeAdminT3Channel: true
+exposeAdminNodePort: true
+voyagerWebPort: 30322
+domainHomeImageBuildPath: ./docker-images/OracleWebLogic/samples/12213-domain-home-in-image
+domainHomeImageBase: "container-registry.oracle.com/middleware/weblogic:12.2.1.3"
+logHomeOnPV: true
+createDomainPyScript: integration-tests/src/test/resources/domain-home-in-image/create-domain-coherence-cluster.py