Merge branch 'develop' of https://github.com/oracle/weblogic-kubernetes-operator into chshi-OWLS-67963-upd-del-sh

Author: chunshi-ora

integration-tests/README.md

@@ -268,6 +268,12 @@ mvn -Dit.test="ITOperator#test6CreateConfiguredDomainInTest2NS" -DfailIfNoTests=
 
 mvn -Dit.test="ITOperator#test6CreateConfiguredDomainInTest2NS+test7CreateDomainPVReclaimPolicyRecycle" -DfailIfNoTests=false integration-test -P java-integration-tests
 
+# How to run cleanup script
+
+cleanup script deletes the k8s artifacts, local test tmp directory, delete all helm charts and the potentially remote domain pv directories.
+cd weblogic-kubernetes-operator
+src/integration-tests/bash/cleanup.sh
+
 # Logging/Archiving
 
 Java utils logging is used, writes all the messages to console and java_test_suite.out in $RESULT_ROOT/acceptance_test_tmp directory.

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -105,6 +105,9 @@ public void test1CreateFirstOperatorAndDomain() throws Exception {
     logTestBegin("test1CreateFirstOperatorAndDomain");
     testCreateOperatorManagingDefaultAndTest1NS();
     domain1 = testAllUseCasesForADomain(operator1, domain1YamlFile);
+    domain1.testWlsLivenessProbe();
+    domain1.destroy();
+
     logger.info("SUCCESS - test1CreateFirstOperatorAndDomain");
   }
 
@@ -139,6 +142,7 @@ public void test3CreateDomainInTest1NS() throws Exception {
     // create domain3
     Domain domain3 = testDomainCreation(domain3YamlFile);
     testWLDFScaling(operator1, domain3);
+    domain3.destroy();
     logger.info("SUCCESS - test3CreateDomainInTest1NS");
   }
 
@@ -168,6 +172,8 @@ public void test5CreateConfiguredDomainInTest2NS() throws Exception {
     }
     if (domain1 == null) {
       domain1 = TestUtils.createDomain(domain1YamlFile);
+    } else {
+      domain1.create();
     }
     logger.info("Checking if operator2 is running, if not creating");
     if (operator2 == null) {
@@ -190,6 +196,8 @@ public void test5CreateConfiguredDomainInTest2NS() throws Exception {
 
     logger.info("Verify no impact on domain4");
     domain4.verifyDomainCreated();
+    domain4.destroy();
+    domain1.destroy();
     logger.info("SUCCESS - test5CreateConfiguredDomainInTest2NS");
   }
 
@@ -205,7 +213,8 @@ public void test6CreateDomainWithStartPolicyAdminOnly() throws Exception {
     }
     logger.info("Creating Domain domain5 & verifing the domain creation");
     // create domain5
-    TestUtils.createDomain(domain5YamlFile);
+    Domain domain5 = TestUtils.createDomain(domain5YamlFile);
+    domain5.destroy();
     logger.info("SUCCESS - test6CreateDomainWithStartPolicyAdminOnly");
   }
 
@@ -227,25 +236,6 @@ public void test7CreateDomainPVReclaimPolicyRecycle() throws Exception {
     logger.info("SUCCESS - test7CreateDomainPVReclaimPolicyRecycle");
   }
 
-  @Test
-  public void test8WlsLivenessProbe() throws Exception {
-
-    logTestBegin("test8WlsLivenessProbe");
-    logger.info("Checking if operator1 is running, if not creating");
-    if (operator1 == null) {
-      operator1 = TestUtils.createOperator(op1YamlFile);
-    }
-    if (domain1 == null) {
-      domain1 = TestUtils.createDomain(domain1YamlFile);
-    }
-    // test managed server1 pod auto restart
-    String domain = domain1.getDomainUid();
-    String namespace = domain1.getDomainMap().get("namespace").toString();
-    String serverName = domain1.getDomainMap().get("managedServerNameBase").toString() + "1";
-    TestUtils.testWlsLivenessProbe(domain, serverName, namespace);
-    logger.info("SUCCESS - test8WlsLivenessProbe");
-  }
-
   @Test
   public void test9CreateDomainOnExistingDir() throws Exception {
     Assume.assumeFalse(
@@ -257,10 +247,10 @@ public void test9CreateDomainOnExistingDir() throws Exception {
     }
     if (domain1 == null) {
       domain1 = TestUtils.createDomain(domain1YamlFile);
+      // create domain on existing dir
+      domain1.destroy();
     }
     logger.info("domain1 " + domain1);
-    // create domain on existing dir
-    domain1.destroy();
     domain1.createDomainOnExistingDirectory();
     logger.info("SUCCESS - test9CreateDomainOnExistingDir");
   }
@@ -278,6 +268,7 @@ public void testACreateDomainApacheLB() throws Exception {
     // create domain7
     Domain domain7 = TestUtils.createDomain(domain7YamlFile);
     domain7.verifyAdminConsoleViaLB();
+    domain7.destroy();
     logger.info("SUCCESS - testACreateDomainApacheLB");
   }
 
@@ -292,7 +283,8 @@ public void testBCreateDomainWithDefaultValuesInSampleInputs() throws Exception
     }
 
     // create domain8
-    testAllUseCasesForADomain(operator1, domain8YamlFile);
+    Domain domain8 = testAllUseCasesForADomain(operator1, domain8YamlFile);
+    domain8.destroy();
     logger.info("SUCCESS - testBCreateDomainWithDefaultValuesInSampleInputs");
   }
 
@@ -402,9 +394,9 @@ private Domain testAllUseCasesForADomain(Operator operator, String domainYamlFil
     logger.info("Creating Domain & verifing the domain creation");
     // create domain1
     Domain domain = testDomainCreation(domainYamlFile);
+    testClusterScaling(operator, domain);
     testDomainLifecyle(operator, domain);
     testOperatorLifecycle(operator, domain);
-    testClusterScaling(operator, domain);
     return domain;
   }
 

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -510,6 +510,13 @@ public String getDomainUid() {
     return domainUid;
   }
 
+  public void testWlsLivenessProbe() throws Exception {
+
+    // test managed server1 pod auto restart
+    String serverName = managedServerNameBase + "1";
+    TestUtils.testWlsLivenessProbe(domainUid, serverName, domainNS);
+  }
+
   private int getAdminSericeLBNodePort() throws Exception {
 
     String adminServerLBNodePortService = domainUid + "-apache-webtier";

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Operator.java

@@ -202,18 +202,48 @@ private void callHelmInstall() throws Exception {
     logger.info("Running " + cmd);
     ExecResult result = ExecCommand.exec(cmd.toString());
     if (result.exitValue() != 0) {
-      throw new RuntimeException(
-          "FAILURE: command "
-              + cmd
-              + " failed, returned "
-              + result.stdout()
-              + "\n"
-              + result.stderr());
+      reportHelmInstallFailure(cmd.toString(), result);
     }
     String outputStr = result.stdout().trim();
     logger.info("Command returned " + outputStr);
   }
 
+  private void reportHelmInstallFailure(String cmd, ExecResult result) throws Exception {
+    String cause = getExecFailure(cmd, result);
+    if (result.stderr().contains("Error: Job failed: BackoffLimitExceeded")) {
+      // The operator helm chart pre-install hook probably failed.
+      // This is probably because there is a problem with the values passed in,
+      // for example a listed domain namespace does not exist.
+      // Fetch the pre-install hook's results and return it too.
+      String hookResults = getOperatorHelmChartHookResults("pre-install");
+      cause = cause + "\n pre-install-hook returned\n" + hookResults;
+    }
+    throw new RuntimeException(cause);
+  }
+
+  private String getOperatorHelmChartHookResults(String hookType) throws Exception {
+    String cmd =
+        "kubectl logs job/"
+            + operatorMap.get("releaseName")
+            + "-weblogic-operator-"
+            + hookType
+            + "-hook -n kube-system";
+    ExecResult result = ExecCommand.exec(cmd);
+    if (result.exitValue() != 0) {
+      return getExecFailure(cmd, result);
+    }
+    return result.stdout();
+  }
+
+  private String getExecFailure(String cmd, ExecResult result) throws Exception {
+    return "FAILURE: command "
+        + cmd
+        + " failed, stdout:\n"
+        + result.stdout()
+        + "stderr:\n"
+        + result.stderr();
+  }
+
   private void generateInputYaml() throws Exception {
     Path parentDir =
         Files.createDirectories(Paths.get(userProjectsDir + "/weblogic-operators/" + operatorNS));

integration-tests/src/test/resources/operator1.yaml

@@ -7,4 +7,4 @@ serviceAccount: weblogic-operator
 namespace: weblogic-operator1
 domainNamespaces: [ "default", "test1" ]
 externalRestEnabled: true
-javaLoggingLevel: FINE
+javaLoggingLevel: FINE

integration-tests/src/test/resources/operator2.yaml

@@ -8,4 +8,4 @@ namespace: weblogic-operator2
 domainNamespaces: [ "test2" ]
 externalRestEnabled: true
 externalRestHttpsPort: 32001
-javaLoggingLevel: FINE
+javaLoggingLevel: FINE