Merge pull request #898 from oracle/voyager-java-integ-test

PR: voyager-java-integ-test

Author: rjeberhard

integration-tests/README.md

@@ -240,7 +240,8 @@ The tests accepts optional env var overrides:
 | RESULT_ROOT | The root directory to use for the tests temporary files. See "Directory Configuration and Structure" for                  defaults and a detailed description of test directories. |
 | PV_ROOT    |  The root directory on the kubernetes cluster used for persistent volumes. See "Directory Configuration and Structure" for defaults and a detailed description of test directories. |
 | QUICKTEST  | When set to "true", limits testing to a subset of the tests. |
-| INGRESSPERDOMAIN  | The defult value is true. If you want to test creating LB by kubectl yaml for multiple domains, set it to false. |
+| LB_TYPE    | The default value is "TRAEFIK". Set to "VOYAGER" if you want to use it as LB. |
+| INGRESSPERDOMAIN  | The defult value is true. If you want to test creating TRAEFIK LB by kubectl yaml for multiple domains, set it to false. |
 | WERCKER    | Set to true if invoking from Wercker, set to false or "" if running stand-alone or from Jenkins. Default is "". |
 | JENKINS    | Set to true if invoking from Jenkins, set to false or "" if running stand-alone or from Wercker. Default is "". |
 | K8S_NODEPORT_HOST | DNS name of a Kubernetes worker node. Default is the local host's hostname. |

integration-tests/src/test/java/oracle/kubernetes/operator/ITOperator.java

@@ -57,12 +57,10 @@ public class ITOperator extends BaseTest {
   private static boolean QUICKTEST;
   private static boolean SMOKETEST;
   private static boolean JENKINS;
-  private static boolean INGRESSPERDOMAIN = true;
 
   // Set QUICKTEST env var to true to run a small subset of tests.
   // Set SMOKETEST env var to true to run an even smaller subset
   // of tests, plus leave domain1 up and running when the test completes.
-  // set INGRESSPERDOMAIN to false to create LB's ingress by kubectl yaml file
   static {
     QUICKTEST =
         System.getenv("QUICKTEST") != null && System.getenv("QUICKTEST").equalsIgnoreCase("true");
@@ -72,9 +70,6 @@ public class ITOperator extends BaseTest {
     if (System.getenv("JENKINS") != null) {
       JENKINS = new Boolean(System.getenv("JENKINS")).booleanValue();
     }
-    if (System.getenv("INGRESSPERDOMAIN") != null) {
-      INGRESSPERDOMAIN = new Boolean(System.getenv("INGRESSPERDOMAIN")).booleanValue();
-    }
   }
 
   /**
@@ -239,12 +234,7 @@ public void testTwoDomainsManagedByTwoOperators() throws Exception {
       wlstDomainMap.put("domainUID", "domain1onpvwlst");
       wlstDomainMap.put("adminNodePort", new Integer("30702"));
       wlstDomainMap.put("t3ChannelPort", new Integer("30031"));
-      if (!INGRESSPERDOMAIN) {
-        wlstDomainMap.put("ingressPerDomain", new Boolean("false"));
-        logger.info(
-            "domain1onpvwlst ingressPerDomain is set to: "
-                + ((Boolean) wlstDomainMap.get("ingressPerDomain")).booleanValue());
-      }
+      wlstDomainMap.put("voyagerWebPort", new Integer("30307"));
       domain1 = TestUtils.createDomain(wlstDomainMap);
       domain1.verifyDomainCreated();
       testBasicUseCases(domain1);
@@ -260,12 +250,7 @@ public void testTwoDomainsManagedByTwoOperators() throws Exception {
       wdtDomainMap.put("adminNodePort", new Integer("30703"));
       wdtDomainMap.put("t3ChannelPort", new Integer("30041"));
       // wdtDomainMap.put("clusterType", "Configured");
-      if (!INGRESSPERDOMAIN) {
-        wdtDomainMap.put("ingressPerDomain", new Boolean("false"));
-        logger.info(
-            "domain2onpvwdt ingressPerDomain is set to: "
-                + ((Boolean) wdtDomainMap.get("ingressPerDomain")).booleanValue());
-      }
+      wdtDomainMap.put("voyagerWebPort", new Integer("30308"));
       domain2 = TestUtils.createDomain(wdtDomainMap);
       domain2.verifyDomainCreated();
       testBasicUseCases(domain2);
@@ -326,7 +311,6 @@ public void testCreateDomainWithStartPolicyAdminOnly() throws Exception {
     try {
       domain = TestUtils.createDomain(domainadminonlyFile);
       domain.verifyDomainCreated();
-
     } finally {
       if (domain != null) {
         // create domain on existing dir
@@ -439,6 +423,7 @@ public void testAutoAndCustomSitConfigOverrides() throws Exception {
       domainMap.put("domainUID", "customsitdomain");
       domainMap.put("adminNodePort", new Integer("30704"));
       domainMap.put("t3ChannelPort", new Integer("30051"));
+      domainMap.put("voyagerWebPort", new Integer("30312"));
       // use NFS for this domain on Jenkins, defaultis HOST_PATH
       if (System.getenv("JENKINS") != null && System.getenv("JENKINS").equalsIgnoreCase("true")) {
         domainMap.put("weblogicDomainStorageType", "NFS");

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -66,6 +66,12 @@ public class Domain {
   private static int maxIterations = BaseTest.getMaxIterationsPod(); // 50 * 5 = 250 seconds
   private static int waitTime = BaseTest.getWaitTimePod();
 
+  private boolean voyager;
+  // LB_TYPE is an evn var. Set to "VOYAGER" to use it as loadBalancer
+  private static String LB_TYPE;
+  // set INGRESSPERDOMAIN to false to create LB's ingress by kubectl yaml file
+  private static boolean INGRESSPERDOMAIN = true;
+
   public Domain(String inputYaml) throws Exception {
     // read input domain yaml to test
     this(TestUtils.loadYaml(inputYaml));
@@ -926,17 +932,35 @@ private void createLoadBalancer() throws Exception {
     lbMap.put("namespace", domainNS);
     lbMap.put("host", domainUid + ".org");
     lbMap.put("serviceName", domainUid + "-cluster-" + domainMap.get("clusterName"));
-    lbMap.put("loadBalancer", domainMap.getOrDefault("loadBalancer", loadBalancer));
-    lbMap.put("ingressPerDomain", domainMap.getOrDefault("ingressPerDomain", ingressPerDomain));
+    if (voyager) {
+      lbMap.put("loadBalancer", "VOYAGER");
+      lbMap.put("loadBalancerWebPort", domainMap.get("voyagerWebPort"));
+    } else {
+      lbMap.put("loadBalancer", domainMap.getOrDefault("loadBalancer", loadBalancer));
+      lbMap.put(
+          "loadBalancerWebPort",
+          domainMap.getOrDefault("loadBalancerWebPort", new Integer(loadBalancerWebPort)));
+    }
+    if (!INGRESSPERDOMAIN) {
+      lbMap.put("ingressPerDomain", new Boolean("false"));
+      logger.info("For this domain, INGRESSPERDOMAIN is set to false");
+    } else {
+      lbMap.put(
+          "ingressPerDomain",
+          domainMap.getOrDefault("ingressPerDomain", new Boolean(ingressPerDomain)));
+    }
     lbMap.put("clusterName", domainMap.get("clusterName"));
 
     loadBalancer = (String) lbMap.get("loadBalancer");
+    loadBalancerWebPort = ((Integer) lbMap.get("loadBalancerWebPort")).intValue();
     ingressPerDomain = ((Boolean) lbMap.get("ingressPerDomain")).booleanValue();
     logger.info(
         "For this domain loadBalancer is: "
             + loadBalancer
             + " ingressPerDomain is: "
-            + ingressPerDomain);
+            + ingressPerDomain
+            + " loadBalancerWebPort is: "
+            + loadBalancerWebPort);
 
     if (loadBalancer.equals("TRAEFIK") && !ingressPerDomain) {
       lbMap.put("name", "traefik-hostrouting-" + domainUid);
@@ -946,6 +970,10 @@ private void createLoadBalancer() throws Exception {
       lbMap.put("name", "traefik-ingress-" + domainUid);
     }
 
+    if (loadBalancer.equals("VOYAGER") && ingressPerDomain) {
+      lbMap.put("name", "voyager-ingress-" + domainUid);
+    }
+
     if (loadBalancer.equals("APACHE")) {
       /* lbMap.put("loadBalancerAppPrepath", "/weblogic");
       lbMap.put("loadBalancerExposeAdminPort", new Boolean(true)); */
@@ -1060,11 +1088,11 @@ private void callWebAppAndCheckForServerNameInResponse(
                 + " \n "
                 + result.stdout());
       } else {
-        logger.info("webapp invoked successfully");
+        logger.info("webapp invoked successfully for curlCmd:" + curlCmd);
       }
       if (verifyLoadBalancing) {
         String response = result.stdout().trim();
-        // logger.info("response "+ response);
+        // logger.info("response: " + response);
         for (String key : managedServers.keySet()) {
           if (response.contains(key)) {
             managedServers.put(key, new Boolean(true));
@@ -1078,6 +1106,7 @@ private void callWebAppAndCheckForServerNameInResponse(
     // error if any managedserver value is false
     if (verifyLoadBalancing) {
       for (Map.Entry<String, Boolean> entry : managedServers.entrySet()) {
+        logger.info("Load balancer will try to reach server " + entry.getKey());
         if (!entry.getValue().booleanValue()) {
           // print service and pods info for debugging
           TestUtils.describeService(domainNS, domainUid + "-cluster-" + clusterName);
@@ -1093,7 +1122,11 @@ private void initialize(Map<String, Object> inputDomainMap) throws Exception {
     domainMap = inputDomainMap;
     this.userProjectsDir = BaseTest.getUserProjectsDir();
     this.projectRoot = BaseTest.getProjectRoot();
-
+    this.voyager =
+        System.getenv("LB_TYPE") != null && System.getenv("LB_TYPE").equalsIgnoreCase("VOYAGER");
+    if (System.getenv("INGRESSPERDOMAIN") != null) {
+      INGRESSPERDOMAIN = new Boolean(System.getenv("INGRESSPERDOMAIN")).booleanValue();
+    }
     domainMap.put("domainName", domainMap.get("domainUID"));
 
     // read sample domain inputs

integration-tests/src/test/java/oracle/kubernetes/operator/utils/LoadBalancer.java

@@ -42,6 +42,22 @@ public LoadBalancer(Map lbMap) throws Exception {
         createTraefikIngressPerDomain();
       }
     }
+
+    if (lbMap.get("loadBalancer").equals("VOYAGER")) {
+      String cmdLb = "helm list voyager-operator | grep DEPLOYED";
+      logger.info("Executing cmd " + cmdLb);
+      ExecResult result = ExecCommand.exec(cmdLb);
+      if (result.exitValue() != 0) {
+        createVoyagerLoadBalancer();
+        logger.info("Sleeping for 30 seconds to ensure voyager to be ready");
+        Thread.sleep(30 * 1000);
+      }
+
+      if (((Boolean) lbMap.get("ingressPerDomain")).booleanValue()) {
+        logger.info("Is going to createVoyagerIngressPerDomain");
+        createVoyagerIngressPerDomain();
+      }
+    }
   }
 
   public void createTraefikLoadBalancer() throws Exception {
@@ -148,6 +164,90 @@ private void createTraefikIngress() throws Exception {
     logger.info("Command returned " + outputStr);
   }
 
+  public void createVoyagerLoadBalancer() throws Exception {
+
+    String cmd1 = "helm repo add appscode https://charts.appscode.com/stable/";
+    logger.info("Executing Add Appscode Chart Repository cmd " + cmd1);
+
+    executeHelmCommand(cmd1);
+
+    String cmd2 = "helm repo update";
+    logger.info("Executing Appscode Chart Repository upgrade cmd " + cmd2);
+
+    executeHelmCommand(cmd2);
+
+    String cmd3 =
+        "helm install appscode/voyager --name voyager-operator --version 7.4.0 --namespace voyage "
+            + "--set cloudProvider=baremetal --set apiserver.enableValidatingWebhook=false";
+    logger.info("Executing Install voyager operator cmd " + cmd3);
+
+    executeHelmCommand(cmd3);
+  }
+
+  private void createVoyagerIngressPerDomain() throws Exception {
+    upgradeVoyagerNamespace();
+    logger.info("Sleeping for 20 seconds after upgradeVoyagerNamespace ");
+    Thread.sleep(20 * 1000);
+    createVoyagerIngress();
+    logger.info("Sleeping for 20 seconds after createVoyagerIngress ");
+    Thread.sleep(20 * 1000);
+  }
+
+  private void upgradeVoyagerNamespace() throws Exception {
+
+    StringBuffer cmd = new StringBuffer("helm upgrade ");
+    cmd.append("--reuse-values ")
+        .append("--set ")
+        .append("\"")
+        .append("kubernetes.namespaces={voyager,")
+        .append(lbMap.get("namespace"))
+        .append("}")
+        .append("\"")
+        .append(" --version 7.4.0")
+        .append(" --set cloudProvider=baremetal")
+        .append(" --set apiserver.enableValidatingWebhook=false")
+        .append(" voyager-operator")
+        .append(" appscode/voyager");
+
+    logger.info(" upgradeVoyagerNamespace() Running " + cmd.toString());
+    executeHelmCommand(cmd.toString());
+  }
+
+  private void createVoyagerIngress() throws Exception {
+
+    String chartDir = BaseTest.getProjectRoot() + "/integration-tests/src/test/resources/charts";
+
+    StringBuffer cmd = new StringBuffer("cd ");
+    cmd.append(chartDir).append(" && helm install ingress-per-domain ");
+    cmd.append(" --name ")
+        .append(lbMap.get("name"))
+        .append(" --namespace ")
+        .append(lbMap.get("namespace"))
+        .append(" --set type=VOYAGER")
+        .append(" --set ")
+        .append("wlsDomain.domainUID=")
+        .append(lbMap.get("domainUID"))
+        .append(" --set ")
+        .append("wlsDomain.clusterName=")
+        .append(lbMap.get("clusterName"))
+        .append(" --set ")
+        .append("voyager.webPort=")
+        .append(lbMap.get("loadBalancerWebPort"));
+
+    logger.info("createVoyagerIngress() Running " + cmd.toString());
+    executeHelmCommand(cmd.toString());
+  }
+
+  private void executeHelmCommand(String cmd) throws Exception {
+    ExecResult result = ExecCommand.exec(cmd);
+    if (result.exitValue() != 0) {
+      logger.info("executeHelmCommand failed with " + cmd);
+      reportHelmInstallFailure(cmd, result);
+    }
+    String outputStr = result.stdout().trim();
+    logger.info("Command returned " + outputStr);
+  }
+
   private void reportHelmInstallFailure(String cmd, ExecResult result) throws Exception {
     throw new RuntimeException(getExecFailure(cmd, result));
   }

integration-tests/src/test/resources/charts/ingress-per-domain/values.yaml

@@ -24,4 +24,4 @@ voyager:
   # web port
   webPort: 30305
   # stats port
-  statsPort: 30315
+  #statsPort: 30315

integration-tests/src/test/resources/domainadminonly.yaml

@@ -19,3 +19,4 @@ namespace: default
 serverStartPolicy: ADMIN_ONLY
 loadBalancer: TRAEFIK
 ingressPerDomain: true
+voyagerWebPort: 30309

integration-tests/src/test/resources/domainonpvwdt.yaml

@@ -22,3 +22,4 @@ createDomainFilesDir: wdt
 #loadBalancerDashboardPort: 30321
 loadBalancer: TRAEFIK
 ingressPerDomain: true
+voyagerWebPort: 30306

integration-tests/src/test/resources/domainonpvwlst.yaml

@@ -16,4 +16,5 @@ exposeAdminNodePort: true
 namespace: default
 loadBalancer: TRAEFIK
 ingressPerDomain: true
+voyagerWebPort: 30305
 

integration-tests/src/test/resources/domainrecyclepolicy.yaml

@@ -19,3 +19,4 @@ namespace: default
 weblogicDomainStorageReclaimPolicy: Recycle
 loadBalancer: TRAEFIK
 ingressPerDomain: true
+voyagerWebPort: 30311

integration-tests/src/test/resources/domainsampledefaults.yaml

@@ -7,3 +7,4 @@ domainUID: domainsampledefaults
 configuredManagedServerCount: 4
 loadBalancer: TRAEFIK
 ingressPerDomain: true
+voyagerWebPort: 30310