Merge pull request #805 from oracle/feature/java-integration-tests

READY TO MERGE: Change java tests to use 12.2.1.3 image

Author: rjeberhard

integration-tests/README.md

@@ -166,7 +166,7 @@ Certain properties like weblogicDomainStoragePath, image, externalOperatorCert a
 
 # How does it work
 
-When the tests are run manually with mvn command on hosted Linux, WebLogic image and server jre images are pulled from a local repository wlsldi-v2.docker.oraclecorp.com. Operator image is built with the git branch from where the mvn command is executed.
+When the tests are run manually with mvn command on hosted Linux, WebLogic image store/oracle/weblogic:12.2.1.3 is pulled from docker hub or uses local image if one exists. Server jre images are pulled from a local repository wlsldi-v2.docker.oraclecorp.com. Operator image is built with the git branch from where the mvn command is executed.
 All the tests that start with IT*.java are run. The test builds the operator, runs a series of tests and archives the results into tar.gz files upon completion.
 
 Integration test classes:
@@ -194,6 +194,12 @@ Secret - to create secret
 
 Command to run the tests:
 ```
+export DOCKER_USERNAME=<docker_username>
+export DOCKER_PASSWORD=<docker_password>
+export DOCKER_EMAIL=<docker_email>
+or
+make sure the weblogic image i.e. store/oracle/weblogic:12.2.1.3 already exists locally
+
 mvn clean verify -P java-integration-tests 2>&1 | tee log.txt
 ```
 
@@ -207,7 +213,7 @@ The tests accepts optional env var overrides:
 | INGRESSPERDOMAIN  | The defult value is true. If you want to test creating LB by kubectl yaml for multiple domains, set it to false. |
 | WERCKER    | Set to true if invoking from Wercker, set to false or "" if running stand-alone or from Jenkins. Default is "". |
 | JENKINS    | Set to true if invoking from Jenkins, set to false or "" if running stand-alone or from Wercker. Default is "". |
-| NODEPORT_HOST | DNS name of a Kubernetes worker node. Default is the local host's hostname. |
+| K8S_NODEPORT_HOST | DNS name of a Kubernetes worker node. Default is the local host's hostname. |
 | BRANCH_NAME  | Git branch name.   Default is determined by calling 'git branch'. |
 | LEASE_ID   |   Set to a unique value to (A) periodically renew a lease on the k8s cluster that indicates that no other test run should attempt to use the cluster, and (B) delete this lease when the test completes. |
 

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -38,7 +38,7 @@ public class BaseTest {
   private static int waitTimePod = 5;
   private static String leaseId = "";
   private static String branchName = "";
-
+  private static String appLocationInPod = "/u01/oracle/apps";
   private static Properties appProps;
 
   public static void initialize(String appPropsFile) throws Exception {
@@ -168,12 +168,65 @@ public void testAdminT3Channel(Domain domain) throws Exception {
     Boolean exposeAdmint3Channel = (Boolean) domainMap.get("exposeAdminT3Channel");
 
     if (exposeAdmint3Channel != null && exposeAdmint3Channel.booleanValue()) {
+      ExecResult result =
+          TestUtils.kubectlexecNoCheck(
+              domain.getDomainUid() + ("-") + domainMap.get("adminServerName"),
+              "" + domainMap.get("namespace"),
+              " -- mkdir -p " + appLocationInPod);
+      if (result.exitValue() != 0) {
+        throw new RuntimeException(
+            "FAILURE: command to create directory "
+                + appLocationInPod
+                + " in the pod failed, returned "
+                + result.stderr()
+                + " "
+                + result.stdout());
+      }
+
       domain.deployWebAppViaWLST(
           TESTWEBAPP,
           getProjectRoot() + "/src/integration-tests/apps/testwebapp.war",
+          appLocationInPod,
           getUsername(),
           getPassword());
       domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+
+      /* The below check is done for domain-home-in-image domains, it needs 12.2.1.3 patched image
+       * otherwise managed servers will see unicast errors after app deployment and run as standalone servers, not in cluster.
+       * Here is the error message
+       * <Jan 18, 2019 8:54:16,214 PM GMT> <Error> <Kernel> <BEA-000802> <ExecuteRequest failed
+       * java.lang.AssertionError: LocalGroup should atleast have the local server!.
+       * java.lang.AssertionError: LocalGroup should atleast have the local server!
+       * 	at weblogic.cluster.messaging.internal.GroupImpl.send(GroupImpl.java:176)
+       * 	at weblogic.cluster.messaging.internal.server.UnicastFragmentSocket.send(UnicastFragmentSocket.java:97)
+       * 	at weblogic.cluster.FragmentSocketWrapper.send(FragmentSocketWrapper.java:84)
+       * 	at weblogic.cluster.UnicastSender.send(UnicastSender.java:53)
+       * 	at weblogic.cluster.UnicastSender.send(UnicastSender.java:21)
+       * 	Truncated. see log file for complete stacktrace
+       */
+
+      if (domainMap.containsKey("domainHomeImageBase")) {
+        if (domainMap.get("initialManagedServerReplicas") != null
+            && ((Integer) domainMap.get("initialManagedServerReplicas")).intValue() >= 1) {
+
+          result =
+              ExecCommand.exec(
+                  "kubectl logs "
+                      + domain.getDomainUid()
+                      + ("-")
+                      + domainMap.get("managedServerNameBase")
+                      + "1 -n "
+                      + domainMap.get("namespace")
+                      + " | grep BEA-000802");
+          if (result.exitValue() == 0) {
+            throw new RuntimeException(
+                "FAILURE: Managed Servers are not part of the cluster, failing with "
+                    + result.stdout()
+                    + ". \n Make sure WebLogic Server 12.2.1.3.0 with patch 29135930 applied is used.");
+          }
+        }
+      }
+
     } else {
       logger.info("exposeAdminT3Channel is false, can not test t3ChannelPort");
     }
@@ -211,7 +264,12 @@ public void testDomainLifecyle(Operator operator, Domain domain) throws Exceptio
     operator.verifyExternalRESTService();
     operator.verifyDomainExists(domain.getDomainUid());
     domain.verifyDomainCreated();
-    domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+    // if domain created with domain home in image, re-deploy the webapp and verify load balancing
+    if (domain.getDomainMap().containsKey("domainHomeImageBase")) {
+      testAdminT3Channel(domain);
+    } else {
+      domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+    }
     domain.verifyAdminServerExternalService(getUsername(), getPassword());
     logger.info("Done - testDomainLifecyle");
   }
@@ -255,7 +313,26 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
               + replicas);
     }
 
-    domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+    // make sure cluster service endpoint is updated with the scaled up servers before verifying
+    // load balancing
+    int i = 0;
+    while (i < BaseTest.getMaxIterationsPod()) {
+      int numberOfServersInEndpoint =
+          domain.getNumberOfServersInClusterServiceEndpoint((String) domainMap.get("clusterName"));
+      if (replicaCnt != numberOfServersInEndpoint) {
+        // check for last iteration
+        if (i == (BaseTest.getMaxIterationsPod() - 1)) {
+          throw new RuntimeException("FAILURE: Cluster Serice Endpoint is not updated");
+        }
+        Thread.sleep(BaseTest.getWaitTimePod() * 1000);
+        i++;
+      } else {
+        logger.info("Number of servers addresses in endpoint is same as replica count ");
+        break;
+      }
+    }
+    // commenting the load balance check, bug 29325139
+    // domain.verifyWebAppLoadBalancing(TESTWEBAPP);
 
     replicas = 2;
     podName = domainUid + "-" + managedServerNameBase + (replicas + 1);
@@ -273,8 +350,8 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
               + "/"
               + replicas);
     }
-
-    domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+    // commenting the load balance check, bug 29325139
+    // domain.verifyWebAppLoadBalancing(TESTWEBAPP);
     logger.info("Done - testClusterScaling");
   }
 
@@ -308,6 +385,7 @@ public void testWLDFScaling(Operator operator, Domain domain) throws Exception {
     domain.deployWebAppViaWLST(
         "opensessionapp",
         getProjectRoot() + "/src/integration-tests/apps/opensessionapp.war",
+        appLocationInPod,
         getUsername(),
         getPassword());
 
@@ -394,6 +472,10 @@ public static String getPassword() {
     return password;
   }
 
+  public static String getResultDir() {
+    return resultDir;
+  }
+
   public static int getMaxIterationsPod() {
     return maxIterationsPod;
   }
@@ -421,10 +503,9 @@ private void copyScalingScriptToPod(
     TestUtils.createDirUnderDomainPV(dirPathToCreate);
 
     // copy script to pod
-    TestUtils.kubectlcp(
+    TestUtils.copyFileViaCat(
         getProjectRoot() + "/src/scripts/scaling/scalingAction.sh",
         "/shared/domains/" + domainUID + "/bin/scripts/scalingAction.sh",
-        // "/shared/scalingAction.sh",
         podName,
         domainNS);
   }

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -31,6 +31,7 @@
 public class ITOperator extends BaseTest {
 
   // property file used to customize operator properties for operator inputs yaml
+
   private static String operator1File = "operator1.yaml";
   private static String operator2File = "operator2.yaml";
   private static final String operator_bcFile = "operator_bc.yaml";
@@ -42,6 +43,8 @@ public class ITOperator extends BaseTest {
   private static String domainadminonlyFile = "domainadminonly.yaml";
   private static String domainrecyclepolicyFile = "domainrecyclepolicy.yaml";
   private static String domainsampledefaultsFile = "domainsampledefaults.yaml";
+  private static String domaininimagewlstFile = "domaininimagewlst.yaml";
+  private static String domaininimagewdtFile = "domaininimagewdt.yaml";
 
   // property file used to configure constants for integration tests
   private static String appPropsFile = "OperatorIT.properties";
@@ -509,6 +512,71 @@ public void testOperatorRESTUsingCertificateChain() throws Exception {
     logger.info("SUCCESS - testOperatorRESTUsingCertificateChain");
   }
 
+  /**
+   * Create Operator and create domain using domain-in-image option. Verify the domain is started
+   * successfully and web application can be deployed and accessed.
+   *
+   * @throws Exception
+   */
+  // @Test
+  public void testDomainInImageUsingWLST() throws Exception {
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+
+    logger.info("Checking if operator1 is running, if not creating");
+    if (operator1 == null) {
+      operator1 = TestUtils.createOperator(operator1File);
+    }
+    logger.info("Creating Domain & verifing the domain creation");
+    // create domain
+    Domain domain = null;
+    boolean testCompletedSuccessfully = false;
+    try {
+      domain = TestUtils.createDomain(domaininimagewlstFile);
+      domain.verifyDomainCreated();
+
+      testBasicUseCases(domain);
+      testClusterScaling(operator1, domain);
+      testCompletedSuccessfully = true;
+    } finally {
+      if (domain != null && (JENKINS || testCompletedSuccessfully)) domain.destroy();
+    }
+    logger.info("SUCCESS - " + testMethodName);
+  }
+  /**
+   * Create Operator and create domain using domain-in-image option. Verify the domain is started
+   * successfully and web application can be deployed and accessed.
+   *
+   * @throws Exception
+   */
+  // @Test
+  public void testDomainInImageUsingWDT() throws Exception {
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+
+    logger.info("Checking if operator1 is running, if not creating");
+    if (operator1 == null) {
+      operator1 = TestUtils.createOperator(operator1File);
+    }
+    logger.info("Creating Domain & verifing the domain creation");
+    // create domain
+    Domain domain = null;
+    boolean testCompletedSuccessfully = false;
+    try {
+      domain = TestUtils.createDomain(domaininimagewdtFile);
+      domain.verifyDomainCreated();
+
+      testBasicUseCases(domain);
+      testClusterScaling(operator1, domain);
+      testCompletedSuccessfully = true;
+    } finally {
+      if (domain != null && (JENKINS || testCompletedSuccessfully)) domain.destroy();
+    }
+    logger.info("SUCCESS - " + testMethodName);
+  }
+
   private Domain testAdvancedUseCasesForADomain(Operator operator, Domain domain) throws Exception {
     if (!SMOKETEST) {
       testClusterScaling(operator, domain);