Merge pull request #519 from oracle/feature/java-integration-tests

changes in what to run in QUICKTEST and add debug for liveness probe test

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -148,6 +148,7 @@ public static void initialize(String appPropsFile) throws Exception {
    */
   public void testAdminServerExternalService(Domain domain) throws Exception {
     logger.info("Inside testAdminServerExternalService");
+    TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
     domain.verifyAdminServerExternalService(getUsername(), getPassword());
     logger.info("Done - testAdminServerExternalService");
   }
@@ -159,6 +160,7 @@ public void testAdminServerExternalService(Domain domain) throws Exception {
    */
   public void testAdminT3Channel(Domain domain) throws Exception {
     logger.info("Inside testAdminT3Channel");
+    TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
     Map<String, Object> domainMap = domain.getDomainMap();
     // check if the property is set to true
     Boolean exposeAdmint3Channel = (Boolean) domainMap.get("exposeAdminT3Channel");
@@ -203,6 +205,7 @@ public void testDomainLifecyle(Operator operator, Domain domain) throws Exceptio
    */
   public void testClusterScaling(Operator operator, Domain domain) throws Exception {
     logger.info("Inside testClusterScaling");
+    TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
     Map<String, Object> domainMap = domain.getDomainMap();
     String domainUid = domain.getDomainUid();
     String domainNS = domainMap.get("namespace").toString();
@@ -263,6 +266,7 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
    */
   public void testWLDFScaling(Operator operator, Domain domain) throws Exception {
     logger.info("Inside testWLDFScaling");
+    TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
 
     Map<String, Object> domainMap = domain.getDomainMap();
     String domainUid = domain.getDomainUid();

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -127,7 +127,6 @@ public void test3CreateDomainInTest1NS() throws Exception {
     }
     // create domain3
     Domain domain3 = testDomainCreation(domain3YamlFile);
-
     testWLDFScaling(operator1, domain3);
     logger.info("SUCCESS - test3CreateDomainInTest1NS");
   }
@@ -219,8 +218,6 @@ public void test7CreateDomainPVReclaimPolicyRecycle() throws Exception {
 
   @Test
   public void test8WlsLivenessProbe() throws Exception {
-    Assume.assumeFalse(
-        System.getenv("QUICKTEST") != null && System.getenv("QUICKTEST").equalsIgnoreCase("true"));
 
     logTestBegin("test8WlsLivenessProbe");
     logger.info("Checking if operator1 is running, if not creating");
@@ -299,9 +296,13 @@ private Domain testAllUseCasesForADomain(Operator operator, String domainYamlFil
     logger.info("Creating Domain & verifing the domain creation");
     // create domain1
     Domain domain = testDomainCreation(domainYamlFile);
-    testDomainLifecyle(operator, domain);
+    if (System.getenv("QUICKTEST") == null
+        || (System.getenv("QUICKTEST") != null
+            && !System.getenv("QUICKTEST").equalsIgnoreCase("true"))) {
+      testDomainLifecyle(operator, domain);
+      testOperatorLifecycle(operator, domain);
+    }
     testClusterScaling(operator, domain);
-    testOperatorLifecycle(operator, domain);
     return domain;
   }
 

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -230,7 +230,10 @@ public static void testWlsLivenessProbe(String domainUid, String serverName, Str
 
     // kill server process 3 times
     for (int i = 0; i < 3; i++) {
-      kubectlexecNoCheck(podName, namespace, "/shared/killserver.sh");
+      ExecResult result = kubectlexecNoCheck(podName, namespace, "/shared/killserver.sh");
+      logger.info("kill server process command exitValue " + result.exitValue());
+      logger.info(
+          "kill server process command result " + result.stdout() + " stderr " + result.stderr());
       Thread.sleep(2 * 1000);
     }
     // one more time so that liveness probe restarts
@@ -241,6 +244,7 @@ public static void testWlsLivenessProbe(String domainUid, String serverName, Str
     while (true) {
       long currentTime = System.currentTimeMillis();
       int finalRestartCnt = getPodRestartCount(podName, namespace);
+      logger.info("initialRestartCnt " + initialRestartCnt + " finalRestartCnt " + finalRestartCnt);
       if ((finalRestartCnt - initialRestartCnt) == 1) {
         logger.info("WLS liveness probe test is successful.");
         break;
@@ -301,6 +305,8 @@ public static ExecResult kubectlexecNoCheck(String podName, String namespace, St
         .append(" ")
         .append(scriptPath);
 
+    ExecResult result = ExecCommand.exec("kubectl get pods -n " + namespace);
+    logger.info("get pods before killing the server " + result.stdout() + "\n " + result.stderr());
     logger.info("Command to call kubectl sh file " + cmdKubectlSh);
     return ExecCommand.exec(cmdKubectlSh.toString());
   }
@@ -526,11 +532,11 @@ public static void renewK8sClusterLease(String projectRoot, String leaseId) thro
                 + " to try renew the lease. "
                 + "Some of the potential reasons for this failure are that another run"
                 + "may have obtained the lease, the lease may have been externally "
-                + "deleted, or the caller of run.sh may have forgotten to obtain the"
-                + "lease before calling run.sh (using 'lease.sh -o \"$LEASE_ID\"'). "
+                + "deleted, or the caller of the test may have forgotten to obtain the "
+                + "lease before calling the test (using 'lease.sh -o \"$LEASE_ID\"'). "
                 + "To force delete a lease no matter who owns the lease,"
                 + "call 'lease.sh -f' or 'kubernetes delete cm acceptance-test-lease'"
-                + "(this should only be done when sure there's no current run.sh "
+                + "(this should only be done when sure there's no current java tests "
                 + "that owns the lease).  To view the current lease holder,"
                 + "use 'lease.sh -s'.  To disable this lease check, do not set"
                 + "the LEASE_ID environment variable.");

wercker.yml

@@ -178,6 +178,11 @@ integration-test-java:
     goals: clean verify
     version: 3.5.2
     profiles: java-integration-tests
+    
+  - script:
+    name: copy pv archive 
+    code: |
+      cp /scratch/acceptance_test_pv_archive /pipeline/output/
 
   after-steps:
   - script: