Merge pull request #943 from oracle/marinatest

added usability test cases for Operator/Domain interaction

Author: rjeberhard

integration-tests/README.md

@@ -77,11 +77,13 @@ Basic Use Cases described above are verified in all the domain configurations. A
 | Two Operators using same External Https Port | create chart using same https rest port as already running first operator, verify that deployment fails with expected error |
 | Two Operators using same target domains namespace | create chart using target domains namespace as already running first operator, verify that deployment fails with expected error |
 | Operator Helm Chart using not preexisted target domains namespace | create chart using not preexisted target domains namespace as already running first operator, verify that deployment fails with expected error |
+| Operator Helm Chart add/delete target domain namespaces (domain1, domain2) | create operator helm chart managing domain1, use upgrade to add domain2. Verify that operator is able to manage added domain (domain2). Use helm upgrade to remove domain1, verify that operator not able to manage anymore the deleted one(domain1) |
+| Operator Helm Chart delete operator helm chart, leave domain running | create operator helm chart and start domain1, delete operator helm chart, verify domain1 is still functional |
 
 | Server Pods Restarted by modifying properties on the domain resource| Use Case |
 | --- | --- |
-| Server pods restarted by changing Env property | Verify admin and managed server pods being restarted by property change: "-Dweblogic.StdoutDebugEnabled=false" --> "-Dweblogic.StdoutDebugEnabled=true" |
-| Server pods restarted by changing image | Verify admin and managed server pods being restarted by property change: image: "store/oracle/weblogic:12.2.1.3" --> image: "store/oracle/weblogic:duplicate" |
+| Server pods restarted by changing Env property | Verify admin and managed server pods being restarted by property change: `-Dweblogic.StdoutDebugEnabled=false` --> `-Dweblogic.StdoutDebugEnabled=true` |
+| Server pods restarted by changing image | Verify admin and managed server pods being restarted by property change: image: `store/oracle/weblogic:12.2.1.3` --> image: `store/oracle/weblogic:duplicate` |
 | Server pods restarted by changing imagePullPolicy | Verify  admin and managed server pods being restarted by property change: imagePullPolicy: IfNotPresent --> imagePullPolicy: Never |
 | Server pods restarted by changing includeServerOutInPodLog | Verify admin and managed server pods being restarted by property change: includeServerOutInPodLog: true --> includeServerOutInPodLog: false |
 | Server pods restarted by changing logHomeEnable | Verify admin and managed server pods being restarted by property change: logHomeEnabled: true --> logHomeEnabled: false |
@@ -95,6 +97,7 @@ Configuration Overrides Usecases
 | JMS Resource Override | Override UniformDistributedTopic Delivery Failure Parameters, `redelivery-limit` and `expiration-policy`. The JMX test client verifies the serverConfig MBean tree for the expected delivery failure parameters, `redelivery-limit` and `expiration-policy`. |
 | WLDF Resource Override | Override `wldf-instrumentation-monitor` and `harvester` in a diagnostics module. The test client verifies the new instrumentation monitors/harvesters set by getting the WLDF resource from serverConfig tree with expected values.  |
 
+
 # Directory Configuration and Structure
 
 Directory structure of source code:
@@ -103,7 +106,7 @@ A new module "integration-tests" is added to the Maven project `weblogic-kuberne
 
 `weblogic-kubernetes-operator/integration-tests` - location of module pom.xml  
 `weblogic-kubernetes-operator/integration-tests/src/test/java` - integration test(JUnit) classes and utility classes  
-`weblogic-kubernetes-operator/integration-tests/src/test/resources` - properties, YAML files (see Configuration Files section) and other scripts.
+`weblogic-kubernetes-operator/integration-tests/src/test/resources` - properties, YAML files (see Configuration Files section) and other scripts
 
 Directory structure used for the test run:
 

integration-tests/src/test/java/oracle/kubernetes/operator/ITUsabilityOperatorHelmChart.java

@@ -1,11 +1,12 @@
-// Copyright 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator;
 
 import java.util.ArrayList;
 import java.util.Map;
+import oracle.kubernetes.operator.utils.Domain;
 import oracle.kubernetes.operator.utils.ExecCommand;
 import oracle.kubernetes.operator.utils.ExecResult;
 import oracle.kubernetes.operator.utils.Operator;
@@ -28,6 +29,8 @@ public class ITUsabilityOperatorHelmChart extends BaseTest {
 
   private static int number = 3;
   String oprelease = "op" + number;
+  private int waitTime = 5;
+  private int maxIterations = 60;
 
   /**
    * This method gets called only once before any of the test methods are executed. It does the
@@ -560,14 +563,6 @@ public void testCreateWithEmptyTargetDomainInstall() throws Exception {
     logger.info("SUCCESS - " + testMethodName);
   }
 
-  private void upgradeOperator(Operator operator, String upgradeSet) throws Exception {
-    Assume.assumeFalse(QUICKTEST);
-    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
-    logTestBegin(testMethodName);
-    operator.callHelmUpgrade(upgradeSet);
-    logger.info("SUCCESS - " + testMethodName);
-  }
-
   /**
    * Helm installs the operator with default target domains namespaces
    *
@@ -596,4 +591,154 @@ public void testCreateWithDefaultTargetDomainInstall() throws Exception {
     }
     logger.info("SUCCESS - " + testMethodName);
   }
+
+  /**
+   * Create operator and verify its deployed successfully. Create domain1 and verify domain is
+   * started. Call helm upgrade to add domainnew to manage, verify both domains are managed by
+   * operator Call helm upgrade to remove first domain from operator target domains, verify it can't
+   * not be managed by operator anymore
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAddRemoveDomainUpdateOperatorHC() throws Exception {
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+    logger.info("Creating Operator & waiting for the script to complete execution");
+    // create operator
+    Map<String, Object> operatorMap = TestUtils.createOperatorMap(number, true);
+    Operator operator = new Operator(operatorMap, RESTCertType.SELF_SIGNED);
+    operator.callHelmInstall();
+    Domain domain = null;
+    Domain domainnew = null;
+    boolean testCompletedSuccessfully = false;
+    try {
+      logger.info("kubectl create namespace test" + (number + 1));
+      ExecCommand.exec("kubectl create namespace test" + (number + 1));
+      domain = createVerifyDomain(number, operator);
+      ArrayList<String> targetDomainsNS =
+          (ArrayList<String>) (operator.getOperatorMap().get("domainNamespaces"));
+      targetDomainsNS.add("test" + (number + 1));
+      upgradeOperatorDomainNamespaces(operator, targetDomainsNS);
+      domainnew = createVerifyDomain(number + 1, operator);
+      logger.info("verify that old domain is managed by operator after upgrade");
+      verifyOperatorDomainManagement(operator, domain, true);
+      logger.info("Upgrade to remove first domain");
+      targetDomainsNS.remove("test" + (number));
+      upgradeOperatorDomainNamespaces(operator, targetDomainsNS);
+      logger.info("verify that old domain is not managed by operator");
+      verifyOperatorDomainManagement(operator, domain, false);
+      verifyOperatorDomainManagement(operator, domainnew, true);
+      testCompletedSuccessfully = true;
+    } finally {
+      if (domain != null) {
+        TestUtils.deleteWeblogicDomainResources(domain.getDomainUid());
+      }
+      if (domainnew != null) {
+        domainnew.destroy();
+      }
+      if (operator != null) {
+        operator.destroy();
+      }
+      number++;
+    }
+    logger.info("SUCCESS - " + testMethodName);
+  }
+
+  /**
+   * Create operator and verify its deployed successfully. Create domain1 and verify domain is
+   * started. Delete operator and make sure domain1 is still functional
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testDeleteOperatorButNotDomain() throws Exception {
+    Assume.assumeFalse(QUICKTEST);
+    String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
+    logTestBegin(testMethodName);
+    logger.info("Creating Operator & waiting for the script to complete execution");
+    // create operator
+    Operator operator = null;
+    Domain domain = null;
+    boolean testCompletedSuccessfully = false;
+    try {
+      Map<String, Object> operatorMap = TestUtils.createOperatorMap(number, true);
+      operator = new Operator(operatorMap, RESTCertType.SELF_SIGNED);
+      operator.callHelmInstall();
+      domain = createVerifyDomain(number, operator);
+      logger.info("Deleting operator to check that domain functionality is not effected");
+      operator.destroy();
+      operator = null;
+      domain.testWlsLivenessProbe();
+      testCompletedSuccessfully = true;
+    } finally {
+      if (domain != null) {
+        TestUtils.deleteWeblogicDomainResources(domain.getDomainUid());
+      }
+      if (operator != null) {
+        operator.destroy();
+      }
+      number++;
+    }
+    logger.info("SUCCESS - " + testMethodName);
+  }
+
+  private void verifyOperatorDomainManagement(
+      Operator operator, Domain domain, boolean isAccessible) throws Exception {
+    for (int i = 0; i < maxIterations; i++) {
+      try {
+        operator.verifyDomainExists(domain.getDomainUid());
+        if (!isAccessible) {
+          throw new RuntimeException("FAILURE: Operator still able to manage old namespace ");
+        } else {
+          break;
+        }
+      } catch (Exception ex) {
+        if (!isAccessible) {
+          if (!ex.getMessage()
+              .contains(
+                  "Response {\"status\":404,\"detail\":\"/operator/latest/domains/test" + number)) {
+          } else {
+            break;
+          }
+        }
+      }
+      if (i == maxIterations - 1) {
+        String errorMsg = "FAILURE: Operator can't access the domain " + domain.getDomainUid();
+        if (!isAccessible) {
+          errorMsg = "FAILURE: Operator still can access the domain " + domain.getDomainUid();
+        }
+        throw new RuntimeException(errorMsg);
+      }
+      logger.info("iteration " + i + " of " + maxIterations);
+      Thread.sleep(waitTime * 1000);
+    }
+  }
+
+  private Domain createVerifyDomain(int number, Operator operator) throws Exception {
+    logger.info("create domain with UID : test" + number);
+    Domain domain = TestUtils.createDomain(TestUtils.createDomainMap(number));
+    domain.verifyDomainCreated();
+    testAdminT3Channel(domain);
+    TestUtils.renewK8sClusterLease(getProjectRoot(), getLeaseId());
+    logger.info("verify that domain is managed by operator");
+    operator.verifyDomainExists(domain.getDomainUid());
+    return domain;
+  }
+
+  private void upgradeOperatorDomainNamespaces(
+      Operator operator, ArrayList<String> targetNamespaces) throws Exception {
+    logger.info("update operator with new target domain");
+    String upgradeSet =
+        "domainNamespaces="
+            + targetNamespaces
+                .toString()
+                .replaceAll("\\[", "{")
+                .replaceAll("\\]", "}")
+                .replaceAll(" ", "");
+    logger.info("update operator with new target domain " + upgradeSet);
+    operator.callHelmUpgrade(upgradeSet);
+    operator.getOperatorMap().replace("domainNamespaces", targetNamespaces);
+  }
 }

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Operator.java

@@ -1,4 +1,4 @@
-// Copyright 2018, Oracle Corporation and/or its affiliates.  All rights reserved.
+// Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
 // Licensed under the Universal Permissive License v 1.0 as shown at
 // http://oss.oracle.com/licenses/upl.
 
@@ -311,9 +311,9 @@ public void callHelmUpgrade(String upgradeSet) throws Exception {
         .append(" && helm upgrade ")
         .append(operatorMap.get("releaseName"))
         .append(" kubernetes/charts/weblogic-operator ")
-        .append(" --set ")
+        .append(" --set \"")
         .append(upgradeSet)
-        .append(" --reuse-values ")
+        .append("\" --reuse-values ")
         .append(" --wait --timeout 60");
     logger.info("Running " + cmd);
     ExecResult result = ExecCommand.exec(cmd.toString());
@@ -324,6 +324,22 @@ public void callHelmUpgrade(String upgradeSet) throws Exception {
     logger.info("Command returned " + outputStr);
   }
 
+  public String getHelmValues() throws Exception {
+    StringBuffer cmd = new StringBuffer("cd ");
+    cmd.append(BaseTest.getProjectRoot())
+        .append(" && helm get values ")
+        .append(operatorMap.get("releaseName"));
+
+    logger.info("Running " + cmd);
+    ExecResult result = ExecCommand.exec(cmd.toString());
+    if (result.exitValue() != 0) {
+      reportHelmFailure(cmd.toString(), result);
+    }
+    String outputStr = result.stdout().trim();
+    logger.info("Command returned " + outputStr);
+    return outputStr;
+  }
+
   private void reportHelmFailure(String cmd, ExecResult result) throws Exception {
     logger.info("reportHelmFailure " + result);
     throw new RuntimeException(getExecFailure(cmd, result));

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -517,7 +517,9 @@ public static String getLegacyAccessToken(Operator operator) throws Exception {
   }
 
   public static String getAccessToken(Operator operator) throws Exception {
-    StringBuffer secretCmd = new StringBuffer("kubectl get serviceaccount weblogic-operator ");
+    StringBuffer secretCmd =
+        new StringBuffer(
+            "kubectl get serviceaccount " + operator.getOperatorMap().get("serviceAccount"));
     secretCmd
         .append(" -n ")
         .append(operator.getOperatorNamespace())
@@ -806,6 +808,21 @@ public static Map<String, Object> createOperatorMap(int number, boolean restEnab
     return operatorMap;
   }
 
+  public static Map<String, Object> createDomainMap(int number) {
+    Map<String, Object> domainMap = new HashMap<>();
+    ArrayList<String> targetDomainsNS = new ArrayList<String>();
+    targetDomainsNS.add("test" + number);
+    domainMap.put("domainUID", "test" + number);
+    domainMap.put("namespace", "test" + number);
+    domainMap.put("configuredManagedServerCount", 4);
+    domainMap.put("initialManagedServerReplicas", 2);
+    domainMap.put("exposeAdminT3Channel", true);
+    domainMap.put("exposeAdminNodePort", true);
+    domainMap.put("adminNodePort", 30700 + number);
+    domainMap.put("t3ChannelPort", 30000 + number);
+    return domainMap;
+  }
+
   public static String callShellScriptByExecToPod(
       String scriptPath, String arguments, String podName, String namespace) throws Exception {