Execute proxy client on the admin VM

Author: pfmackin

integration-tests/src/test/java/oracle/kubernetes/operator/ITCoherenceTests.java

@@ -5,17 +5,14 @@
 package oracle.kubernetes.operator;
 
 import java.util.Map;
-import oracle.kubernetes.operator.utils.CoherenceUtils;
 import oracle.kubernetes.operator.utils.Domain;
-import oracle.kubernetes.operator.utils.K8sTestUtils;
 import oracle.kubernetes.operator.utils.Operator;
 import oracle.kubernetes.operator.utils.TestUtils;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.FixMethodOrder;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runners.MethodSorters;
 /**
@@ -28,9 +25,14 @@ public class ITCoherenceTests extends BaseTest {
 
   private static Domain domain = null;
   private static Operator operator1;
-  private static String domainUid = "";
-  private static String restartTmpDir = "";
-  private static String originalYaml;
+
+  private static final String testAppName = "coherence-proxy-client";
+
+  private static final String PROXY_CLIENT_SCRIPT = "buildRunProxyClient.sh";
+  private static final String PROXY_CLIENT_APP_NAME = "coherence-proxy-client";
+  private static final String OP_CACHE_LOAD = "load";
+  private static final String OP_CACHE_VALIDATE = "validate";
+  private static final String PROXY_PORT = "9000";
 
   /**
    * This method gets called only once before any of the test methods are executed. It does the
@@ -46,12 +48,6 @@ public static void staticPrepare() throws Exception {
     if (!QUICKTEST) {
       initialize(APP_PROPS_FILE);
 
-      // The default cmd loop sleep is too long and we could miss states like terminating. Change the
-      // sleep and iterations
-      //
-      setWaitTimePod(2);
-      setMaxIterationsPod(125);
-
       if (operator1 == null) {
         operator1 = TestUtils.createOperator(OPERATOR1_YAML);
       }
@@ -67,26 +63,80 @@ public static void staticPrepare() throws Exception {
   public static void staticUnPrepare() throws Exception {
     if (!QUICKTEST) {
       tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
-      logger.info("SUCCESS");
     }
   }
 
   @Test
   public void testRollingRestart() throws Exception {
-
-    CoherenceUtils utils = new CoherenceUtils();
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
 
     domain = createDomain();
     Assert.assertNotNull(domain);
 
-    utils.loadCache();
+    try {
+      copyAndExecuteProxyClientInPod(OP_CACHE_LOAD);
 
-    // Do the rolling restart
-    testServerPodsRestartByChangingEnvProperty();
+      // Do the rolling restart
+      restartDomainByChangingEnvProperty();
 
-    utils.validateCache();
+      copyAndExecuteProxyClientInPod(OP_CACHE_VALIDATE);
+    } finally {
+      destroyDomain();
+    }
+    logger.info("SUCCESS - " + testMethodName);
+  }
 
-    destroyDomain();
+  /**
+   * Copy the shell script file and all App files over to the admin pod the run the script to build
+   * the proxy client and run the proxy test.
+   *
+   * @param cacheOp - cache operation
+   * @throws Exception exception
+   */
+  private static void copyAndExecuteProxyClientInPod(String cacheOp) {
+    try {
+      final String adminServerPod = domain.getDomainUid() + "-" + domain.getAdminServerName();
+
+      final String domainNS = domain.getDomainNs();
+
+      // Use the proxy running on Managed Server 1, get the internal POD IP
+      String podName = domain.getManagedSeverPodName(1);
+      String ProxyIP = TestUtils.getPodIP(domainNS, "", podName);
+
+      String cohAppLocationOnHost = BaseTest.getAppLocationOnHost() + "/" + PROXY_CLIENT_APP_NAME;
+      String cohAppLocationInPod = BaseTest.getAppLocationInPod() + "/" + PROXY_CLIENT_APP_NAME;
+      final String cohScriptPathOnHost = cohAppLocationOnHost + "/" + PROXY_CLIENT_SCRIPT;
+      final String cohScriptPathInPod = cohAppLocationInPod + "/" + PROXY_CLIENT_SCRIPT;
+      final String successMarker = "CACHE-SUCCESS";
+
+      logger.info("Copying files to admin pod for App " + PROXY_CLIENT_APP_NAME);
+
+      // Create app dir in the admin pod
+      StringBuffer mkdirCmd = new StringBuffer(" -- bash -c 'mkdir -p ");
+      mkdirCmd.append(cohAppLocationInPod).append("'");
+      TestUtils.kubectlexec(adminServerPod, domainNS, mkdirCmd.toString());
+
+      // Copy shell script to the pod
+      TestUtils.copyFileViaCat(cohScriptPathOnHost, cohScriptPathInPod, adminServerPod, domainNS);
+
+      // Copy all App files to the admin pod
+      TestUtils.copyAppFilesToPod(
+          cohAppLocationOnHost, cohAppLocationInPod, adminServerPod, domainNS);
+
+      logger.info(
+          "Executing script "
+              + PROXY_CLIENT_SCRIPT
+              + " for App "
+              + PROXY_CLIENT_APP_NAME
+              + " in the admin pod");
+
+      // Run the script to on the admin pod (note first arg is app directory is applocation in pod)
+      domain.callShellScriptInAdminPod(
+          successMarker, cohScriptPathInPod, cohAppLocationInPod, cacheOp, ProxyIP, PROXY_PORT);
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
   }
 
   /**
@@ -96,21 +146,17 @@ public void testRollingRestart() throws Exception {
    *
    * @throws Exception
    */
-  public void testServerPodsRestartByChangingEnvProperty() throws Exception {
+  private void restartDomainByChangingEnvProperty() throws Exception {
 
-    Assume.assumeFalse(QUICKTEST);
-    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
-    logTestBegin(testMethodName);
-
-    logger.info(
-        "About to verifyDomainServerPodRestart for Domain: "
-            + domain.getDomainUid()
-            + "  env property: StdoutDebugEnabled=false to StdoutDebugEnabled=true");
+    // The default cmd loop sleep is too long and we could miss states like terminating. Change
+    // the
+    // sleep and iterations
+    //
+    setWaitTimePod(2);
+    setMaxIterationsPod(125);
 
     domain.verifyDomainServerPodRestart(
         "\"-Dweblogic.StdoutDebugEnabled=false\"", "\"-Dweblogic.StdoutDebugEnabled=true\"");
-
-    logger.info("SUCCESS - " + testMethodName);
   }
 
   private static void destroyDomain() throws Exception {
@@ -136,4 +182,4 @@ private Domain createDomain() throws Exception {
     domain.verifyDomainCreated();
     return domain;
   }
-}
+}

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -709,6 +709,16 @@ public Map<String, Object> getDomainMap() {
     return domainMap;
   }
 
+  /**
+   * Get the managed server pod name for a specific index
+   *
+   * @param index  the managed server index
+   * @return the managed server pod name
+   */
+  public String getManagedSeverPodName(int index) {
+    return domainUid + "-" + managedServerNameBase + index;
+  }
+
   /**
    * delete PVC and check PV status released when weblogicDomainStorageReclaimPolicy is Recycle.
    *
@@ -1152,7 +1162,7 @@ public void verifyManagedServersRestarted() throws Exception {
           if (TestUtils.checkPodTerminatingNoWait(domainUid + "-" + podName, domainNS)) {
             // Server is terminating, wait until server running then remove it from the list
             logger.info("Managed server pod " + podName + "  is terminating");
-            TestUtils.checkPodCreated(domainUid + "-" + adminServerName, domainNS);
+            TestUtils.checkPodCreated(domainUid + "-" + podName, domainNS);
             TestUtils.checkPodReady(domainUid + "-" + podName, domainNS);
             logger.info("Managed server pod " + podName + " has been recycled");
             iter.remove();
@@ -1899,6 +1909,57 @@ public void callShellScriptToBuildDeployAppInPod(
     }
   }
 
+
+  /**
+   * Run the shell script in the admin pod.
+   *
+   * @param successMarker output string from script that indicates success
+   * @param scriptPathInPod - bash script path name in the pod
+   * @param args - optional args to add for script if needed
+   * @throws Exception exception
+   */
+  public void callShellScriptInAdminPod(
+      String successMarker,  String scriptPathInPod, String... args)
+      throws Exception {
+
+    StringBuffer cmdKubectlSh = new StringBuffer("kubectl -n ");
+    cmdKubectlSh
+        .append(domainNS)
+        .append(" exec -it ")
+        .append(domainUid)
+        .append("-")
+        .append(adminServerName)
+        .append(" -- bash -c 'chmod +x -R ")
+        .append(scriptPathInPod)
+        .append(" && sh ")
+        .append(scriptPathInPod)
+        .append(" ")
+        .append(String.join(" ", args).toString())
+        .append("'");
+
+    logger.info("Command to exec script file: " + cmdKubectlSh);
+    ExecResult result = ExecCommand.exec(cmdKubectlSh.toString());
+
+    String resultStr =
+        "Command= '"
+            + cmdKubectlSh
+            + "'"
+            + ", exitValue="
+            + result.exitValue()
+            + ", stdout='"
+            + result.stdout()
+            + "'"
+            + ", stderr='"
+            + result.stderr()
+            + "'";
+
+    // Look for positive success marker.
+    if (!resultStr.contains(successMarker)) {
+      throw new RuntimeException("FAILURE: Success marker not found after executing script" + scriptPathInPod + " in admin pod - " + resultStr);
+    }
+  }
+
+
   /**
    * create config map and label with domainUid and create secret used in custom situational
    * configuration which contains hostname, db user, db password.
@@ -2019,4 +2080,5 @@ public boolean accept(File dir, String name) {
     callShellScriptToBuildDeployAppInPod(
         appName, scriptName, username, password, infoDirName, archiveExt);
   }
+
 }

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -4,6 +4,7 @@
 
 package oracle.kubernetes.operator.utils;
 
+import io.kubernetes.client.models.V1Pod;
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
@@ -30,6 +31,7 @@
 import javax.ws.rs.core.Response;
 import oracle.kubernetes.operator.BaseTest;
 import oracle.kubernetes.operator.utils.Operator.RestCertType;
+import org.bouncycastle.i18n.MissingEntryException;
 import org.glassfish.jersey.jsonp.JsonProcessingFeature;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml;
@@ -430,6 +432,31 @@ public static String getPods(String namespace) throws Exception {
     return stdout;
   }
 
+  /**
+   * Get the POD IP.
+   *
+   * @param namespace  namespace of the POD
+   * @param labelSelectors  optional label selectors
+   * @param podName pod name
+   * @return podIP
+   */
+  public static String getPodIP(String namespace, String labelSelectors, String podName)
+      throws Exception {
+    V1Pod pod = null;
+    try {
+      labelSelectors = labelSelectors == null ? "" : labelSelectors;
+      pod = k8sTestUtils.getPod(namespace,"",podName);
+    } catch (Exception e) {
+      Exception re = new  Exception("Exception getting Pod  " + namespace  + "/" + podName);
+      re.initCause(e);
+      throw re;
+    }
+    if (pod == null) {
+      throw new Exception("Pod " + namespace  + "/" + podName  + "not found ");
+    }
+    return pod.getStatus().getPodIP();
+  }
+
   /**
    * First, kill the mgd server process in the container three times to cause the node manager to
    * mark the server 'failed not restartable'. This in turn is detected by the liveness probe, which
@@ -566,28 +593,28 @@ public static void copyAppFilesToPod(
       throws Exception {
     File appFileRoot = new File(appLocationOnHost);
     File[] appFileList = appFileRoot.listFiles();
-    String fileLocationInPod = appLocationInPod;
 
     if (appFileList == null) return;
 
     for (File file : appFileList) {
       if (file.isDirectory()) {
+
+        // Create the directory on the pod
+        String nestedDirOnPod = appLocationInPod + "/" + file.getName();
+        StringBuffer mkdirCmd = new StringBuffer(" -- bash -c 'mkdir -p ").append(nestedDirOnPod).append("'");
+        TestUtils.kubectlexec(podName, namespace, mkdirCmd.toString());
+
         // Find dir recursively
-        copyAppFilesToPod(file.getAbsolutePath(), appLocationInPod, podName, namespace);
+        copyAppFilesToPod(file.getAbsolutePath(), nestedDirOnPod, podName, namespace);
       } else {
         logger.info("Copy file: " + file.getAbsoluteFile().toString() + " to the pod: " + podName);
 
         String fileParent = file.getParentFile().getName();
         logger.fine("file Parent: " + fileParent);
 
-        if (!appLocationInPod.contains(fileParent)) {
-          // Copy files in child dir of appLocationInPod
-          fileLocationInPod = appLocationInPod + "/" + fileParent;
-        }
-
         StringBuffer copyFileCmd = new StringBuffer(" -- bash -c 'cat > ");
         copyFileCmd
-            .append(fileLocationInPod)
+            .append(appLocationInPod)
             .append("/")
             .append(file.getName())
             .append("' < ")

integration-tests/src/test/resources/apps/coherence-proxy-client/build.xml

@@ -1,5 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
+<!-- ================================================================================= -->
+<!-- Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.   -->
+<!--  Licensed under the Universal Permissive License v 1.0 as shown at                -->
+<!-- http://oss.oracle.com/licenses/upl.                                               -->
+<!-- ================================================================================= -->
+
 <!-- ====================================================================== -->
 <!-- Ant build file (http://ant.apache.org/) for Ant 1.6.2 or above.        -->
 <!-- ====================================================================== -->