Merge pull request #916 from oracle/owls-71987-debug

OWLS-71987 - Re-enable load balancing test and add logging of service and pods info if it fails

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

@@ -315,6 +315,8 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
               + replicas);
     }
 
+    domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+
     replicas = 2;
     podName = domainUid + "-" + managedServerNameBase + (replicas + 1);
     logger.info("Scale down to " + replicas + " managed servers");
@@ -331,8 +333,9 @@ public void testClusterScaling(Operator operator, Domain domain) throws Exceptio
               + "/"
               + replicas);
     }
-    // commenting the load balance check, bug 29325139
-    // domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+
+    domain.verifyWebAppLoadBalancing(TESTWEBAPP);
+
     logger.info("Done - testClusterScaling");
   }
 

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -1074,10 +1074,14 @@ private void callWebAppAndCheckForServerNameInResponse(
       }
     }
     logger.info("ManagedServers " + managedServers);
+
     // error if any managedserver value is false
     if (verifyLoadBalancing) {
       for (Map.Entry<String, Boolean> entry : managedServers.entrySet()) {
         if (!entry.getValue().booleanValue()) {
+          // print service and pods info for debugging
+          TestUtils.describeService(domainNS, domainUid + "-cluster-" + clusterName);
+          TestUtils.getPods(domainNS);
           throw new RuntimeException(
               "FAILURE: Load balancer can not reach server " + entry.getKey());
         }

integration-tests/src/test/java/oracle/kubernetes/operator/utils/TestUtils.java

@@ -253,6 +253,52 @@ public static boolean checkHasServiceChannelPort(
     return false;
   }
 
+  /**
+   * kubectl describe service serviceName -n namespace
+   *
+   * @param namespace namespace where the service is located
+   * @param serviceName name of the service to be described
+   * @return String containing output of the kubectl describe service command
+   * @throws Exception
+   */
+  public static String describeService(String namespace, String serviceName) throws Exception {
+    StringBuffer cmd = new StringBuffer("kubectl describe service ");
+    cmd.append(serviceName);
+    cmd.append(" -n ").append(namespace);
+    logger.info(
+        " Describe service "
+            + serviceName
+            + " in namespage "
+            + namespace
+            + " with command: '"
+            + cmd
+            + "'");
+
+    ExecResult result = ExecCommand.exec(cmd.toString());
+    String stdout = result.stdout();
+    logger.info(" Service " + serviceName + " found: ");
+    logger.info(stdout);
+    return stdout;
+  }
+
+  /**
+   * kubectl get pods -o wide -n namespace
+   *
+   * @param namespace namespace in which the pods are to be listed
+   * @return String containing output of the kubectl get pods command
+   * @throws Exception
+   */
+  public static String getPods(String namespace) throws Exception {
+    StringBuffer cmd = new StringBuffer("kubectl get pods -o wide ");
+    cmd.append(" -n ").append(namespace);
+    logger.info(" Get pods in namespage " + namespace + " with command: '" + cmd + "'");
+
+    ExecResult result = ExecCommand.exec(cmd.toString());
+    String stdout = result.stdout();
+    logger.info(" Pods found: ");
+    logger.info(stdout);
+    return stdout;
+  }
   /**
    * First, kill the mgd server process in the container three times to cause the node manager to
    * mark the server 'failed not restartable'. This in turn is detected by the liveness probe, which