Merge pull request #1126 from oracle/owls75142

Add test to verify domain restart works after domain configuration change

Author: rjeberhard

integration-tests/USECASES.MD

@@ -43,7 +43,7 @@ Basic Use Cases described above are verified in all the domain configurations. A
 | Domain | Use Case |
 | --- | --- |
 | Domain on PV using WDT | WLDF scaling |
-| Domain with ADMIN_ONLY | making sure only admin server is started and managed servers are not started. Shutdown domain by deleting domain CRD. Create domain on existing PV dir, pv is already populated by a shutdown domain. |
+| Domain with ADMIN_ONLY | making sure only admin server is started and managed servers are not started. Make some changes in domain configuration by changing admin server tuning backlog configuration and restart domain. Shutdown domain by deleting domain CRD. Create domain on existing PV dir, pv is already populated by a shutdown domain. |
 | Domain with situational config | create domain with listen address not set for admin server and t3 channel/NAP and incorrect file for admin server log location. Introspector should override these with sit-config automatically. Also, with some junk value for t3 channel public address and using custom situational config override replace with valid public address using secret. Also, on Jenkins this domain uses NFS instead of HOSTPATH PV storage |	
 | Two domains managed by two operators | verify scaling and restart of one domain doesn't impact another domain. Delete domain resources using delete script from samples. |			
 | Domain with Recycle policy | create domain with pvReclaimPolicy="Recycle" and using Configured cluster. Verify that the PV is deleted once the domain and PVC are deleted |

integration-tests/src/test/java/oracle/kubernetes/operator/ItOperator.java

@@ -235,8 +235,9 @@ public void testTwoDomainsManagedByTwoOperators() throws Exception {
 
   /**
    * Create operator if its not running and create domain with serverStartPolicy="ADMIN_ONLY".
-   * Verify only admin server is created. shutdown by deleting domain CRD. Create domain on existing
-   * PV dir, pv is already populated by a shutdown domain.
+   * Verify only admin server is created. Make domain configuration change and restart the domain.
+   * shutdown by deleting domain CRD. Create domain on existing PV dir, pv is already populated by a
+   * shutdown domain.
    *
    * @throws Exception exception
    */
@@ -255,6 +256,10 @@ public void testCreateDomainWithStartPolicyAdminOnly() throws Exception {
     try {
       domain = TestUtils.createDomain(DOMAIN_ADMINONLY_YAML);
       domain.verifyDomainCreated();
+      // change domain config by modifying accept backlog on adminserver tuning
+      modifyDomainConfig(domain);
+      domain.shutdownUsingServerStartPolicy();
+      domain.restartUsingServerStartPolicy();
     } finally {
       if (domain != null) {
         // create domain on existing dir
@@ -516,4 +521,28 @@ private Domain testAdvancedUseCasesForADomain(Operator operator, Domain domain)
     }
     return domain;
   }
+
+  private void modifyDomainConfig(Domain domain) throws Exception {
+    String adminPod = domain.getDomainUid() + "-" + domain.getAdminServerName();
+    String scriptsLocInPod = "/u01/oracle";
+    TestUtils.copyFileViaCat(
+        BaseTest.getProjectRoot() + "/integration-tests/src/test/resources/modifyAcceptBacklog.py",
+        scriptsLocInPod + "/modifyAcceptBacklog.py",
+        adminPod,
+        domain.getDomainNs());
+
+    TestUtils.copyFileViaCat(
+        BaseTest.getProjectRoot() + "/integration-tests/src/test/resources/callpyscript.sh",
+        scriptsLocInPod + "/callpyscript.sh",
+        adminPod,
+        domain.getDomainNs());
+    String[] args = {
+      scriptsLocInPod + "/modifyAcceptBacklog.py",
+      BaseTest.getUsername(),
+      BaseTest.getPassword(),
+      "t3://" + adminPod + ":" + domain.getDomainMap().get("t3ChannelPort")
+    };
+    TestUtils.callShellScriptByExecToPod(
+        adminPod, domain.getDomainNs(), scriptsLocInPod, "callpyscript.sh", args);
+  }
 }

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -483,8 +483,25 @@ public void deployWebAppViaWlst(
         adminPod,
         domainNS);
 
-    callShellScriptByExecToPod(
-        username, password, webappName, appLocationInPod, useAdminPortToDeploy);
+    String t3Url = "t3://" + adminPod + ":";
+    if (useAdminPortToDeploy) {
+      t3Url = t3Url + domainMap.getOrDefault("adminPort", 7001);
+    } else {
+      t3Url = t3Url + t3ChannelPort;
+    }
+
+    String[] args = {
+      appLocationInPod + "/deploywebapp.py",
+      BaseTest.getUsername(),
+      BaseTest.getPassword(),
+      t3Url,
+      webappName,
+      appLocationInPod + "/" + webappName + ".war",
+      clusterName
+    };
+
+    TestUtils.callShellScriptByExecToPod(
+        adminPod, domainNS, appLocationInPod, "callpyscript.sh", args);
   }
 
   /**
@@ -1272,79 +1289,6 @@ protected void createLoadBalancer() throws Exception {
     new LoadBalancer(lbMap);
   }
 
-  private void callShellScriptByExecToPod(
-      String username, String password, String webappName, String appLocationInPod)
-      throws Exception {
-    callShellScriptByExecToPod(username, password, webappName, appLocationInPod, false);
-  }
-
-  private void callShellScriptByExecToPod(
-      String username,
-      String password,
-      String webappName,
-      String appLocationInPod,
-      boolean usingAdminPortToDeploy)
-      throws Exception {
-
-    StringBuffer cmdKubectlSh = new StringBuffer("kubectl -n ");
-    cmdKubectlSh
-        .append(domainNS)
-        .append(" exec -it ")
-        .append(domainUid)
-        .append("-")
-        .append(adminServerName)
-        .append(" -- bash -c 'chmod +x -R ")
-        .append(appLocationInPod)
-        .append("  && ")
-        .append(appLocationInPod)
-        .append("/callpyscript.sh ")
-        .append(appLocationInPod)
-        .append("/deploywebapp.py ")
-        .append(username)
-        .append(" ")
-        .append(password)
-        .append(" t3://")
-        // .append(TestUtils.getHostName())
-        .append(domainUid)
-        .append("-")
-        .append(adminServerName)
-        .append(":");
-
-    if (usingAdminPortToDeploy) {
-      String adminPort = (domainMap.getOrDefault("adminPort", 7001)).toString();
-      cmdKubectlSh.append(adminPort);
-    } else {
-      cmdKubectlSh.append(t3ChannelPort);
-    }
-
-    cmdKubectlSh
-        .append(" ")
-        .append(webappName)
-        .append(" ")
-        .append(appLocationInPod)
-        .append("/")
-        .append(webappName)
-        .append(".war ")
-        .append(clusterName)
-        .append("'");
-    logger.info("Command to call kubectl sh file " + cmdKubectlSh);
-    ExecResult result = ExecCommand.exec(cmdKubectlSh.toString());
-    String resultStr =
-        "Command= '"
-            + cmdKubectlSh
-            + "'"
-            + ", exitValue="
-            + result.exitValue()
-            + ", stdout='"
-            + result.stdout()
-            + "'"
-            + ", stderr='"
-            + result.stderr()
-            + "'";
-    if (result.exitValue() != 0 || !resultStr.contains("Deployment State : completed"))
-      throw new RuntimeException("FAILURE: webapp deploy failed - " + resultStr);
-  }
-
   private void callWebAppAndWaitTillReady(String curlCmd) throws Exception {
     for (int i = 0; i < maxIterations; i++) {
       ExecResult result = TestUtils.exec(curlCmd);

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -972,6 +972,38 @@ public static String callShellScriptByExecToPod(
     return result.stdout().trim();
   }
 
+  /**
+   * exec into the pod and call the shell script with given arguments.
+   *
+   * @param podName pod name
+   * @param domainNS namespace
+   * @param scriptsLocInPod script location
+   * @param shScriptName script name
+   * @param args script arguments
+   * @throws Exception exception
+   */
+  public static void callShellScriptByExecToPod(
+      String podName, String domainNS, String scriptsLocInPod, String shScriptName, String[] args)
+      throws Exception {
+    StringBuffer cmdKubectlSh = new StringBuffer("kubectl -n ");
+    cmdKubectlSh
+        .append(domainNS)
+        .append(" exec -it ")
+        .append(podName)
+        .append(" -- bash -c 'chmod +x -R ")
+        .append(scriptsLocInPod)
+        .append("  && ")
+        .append(scriptsLocInPod)
+        .append("/")
+        .append(shScriptName)
+        .append(" ")
+        .append(String.join(" ", args).toString())
+        .append("'");
+
+    logger.info("Command to call kubectl sh file " + cmdKubectlSh);
+    TestUtils.exec(cmdKubectlSh.toString());
+  }
+
   public static void createDirUnderDomainPV(String dirPath) throws Exception {
 
     String crdCmd =

integration-tests/src/test/resources/modifyAcceptBacklog.py

@@ -0,0 +1,12 @@
+# Copyright 2018, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
+
+import time as systime
+
+connect(sys.argv[1],sys.argv[2],sys.argv[3])
+edit()
+startEdit()
+cd('/Servers/admin-server')
+cmo.setAcceptBacklog(4000)
+save()
+activate()