Merge pull request #908 from oracle/int-test-instrumentation

Add more debugging output to integration tests.

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

@@ -993,14 +993,20 @@ private void callShellScriptByExecToPod(
         .append("'");
     logger.info("Command to call kubectl sh file " + cmdKubectlSh);
     ExecResult result = ExecCommand.exec(cmdKubectlSh.toString());
-    if (result.exitValue() != 0) {
-      throw new RuntimeException(
-          "FAILURE: command " + cmdKubectlSh + " failed, returned " + result.stderr());
-    }
-    String output = result.stdout().trim();
-    if (!output.contains("Deployment State : completed")) {
-      throw new RuntimeException("Failure: webapp deployment failed." + output);
-    }
+    String resultStr =
+        "Command= '"
+            + cmdKubectlSh
+            + "'"
+            + ", exitValue="
+            + result.exitValue()
+            + ", stdout='"
+            + result.stdout()
+            + "'"
+            + ", stderr='"
+            + result.stderr()
+            + "'";
+    if (result.exitValue() != 0 || !resultStr.contains("Deployment State : completed"))
+      throw new RuntimeException("FAILURE: webapp deploy failed - " + resultStr);
   }
 
   private void callWebAppAndWaitTillReady(String curlCmd) throws Exception {

integration-tests/src/test/resources/statedump.sh

@@ -39,7 +39,7 @@ function state_dump {
   kubectl get all,crd,cm,pv,pvc,ns,roles,rolebindings,clusterroles,clusterrolebindings,secrets --show-labels=true --all-namespaces=true > ${DUMP_DIR}/kgetmany.out
   kubectl get domains --show-labels=true --all-namespaces=true > ${DUMP_DIR}/kgetdomains.out
 
-  # Get all pod logs and redirect/copy to files 
+  # Get all pod logs/describes and redirect/copy to files 
 
   set +x
   local namespaces="`kubectl get namespaces | egrep -v -e "(STATUS|kube)" | awk '{ print $1 }'`"
@@ -60,6 +60,32 @@ function state_dump {
     done
   done
 
+  # Get various k8s resource describes and redirect/copy to files 
+
+  set +x
+  local ktype
+  local kobj
+  local fname
+  for namespace in $namespaces; do
+    for ktype in pod job deploy rs service pvc ingress cm secret domain; do
+      for kobj in `kubectl get $ktype -n $namespace -o=jsonpath='{range .items[*]}{" "}{.metadata.name}{end}'`; do
+        fname="${DUMP_DIR}/kubectl.describe.$ktype.$kobj.ns-$namespace"
+        echo "Generating $fname"
+        kubectl describe $ktype $kobj -n $namespace > $fname
+      done
+    done
+  done
+
+  # treat pv differently as a pv is not namespaced:
+  for ktype in pv; do
+    for kobj in `kubectl get $ktype -o=jsonpath='{range .items[*]}{" "}{.metadata.name}{end}'`; do
+      fname="${DUMP_DIR}/kubectl.describe.$ktype.$kobj"
+      echo "Generating $fname"
+      kubectl describe $ktype $kobj > $fname
+    done
+  done
+  set -x
+
   # use a job to archive PV, /scratch mounts to PV_ROOT in the K8S cluster
   echo "Archiving pv directory using a kubernetes job.  Look for it on k8s cluster in $PV_ROOT/acceptance_test_pv_archive"
   local outfile=${DUMP_DIR}/archive_pv_job.out

wercker.yml

@@ -170,6 +170,9 @@ integration-test-java:
       echo "Integration test suite running for test image: $IMAGE_NAME_OPERATOR:$IMAGE_TAG_OPERATOR"
       echo "WebLogic version is: $IMAGE_NAME_WEBLOGIC:$IMAGE_TAG_WEBLOGIC"
 
+      # Echo the image's WL version - we don't care if this fails...
+      $WERCKER_SOURCE_DIR/src/integration-tests/introspector/util_krun.sh -t 120 -i $IMAGE_NAME_WEBLOGIC:$IMAGE_TAG_WEBLOGIC -s $IMAGE_PULL_SECRET_WEBLOGIC -c 'source /u01/oracle/wlserver/server/bin/setWLSEnv.sh && java weblogic.version'
+
   - script:
     name: Install pre-reqs
     code: |