Merge pull request #575 from oracle/owls-70534-error-handling

OWLS-70534 improve create domain job error handling

Author: rjeberhard

kubernetes/samples/scripts/create-weblogic-domain/domain-home-on-pv/common/create-domain-job.sh

@@ -3,6 +3,13 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
 #
 
+function exitIfError {
+  if [ "$1" != "0" ]; then
+    echo "$2"
+    exit $1
+  fi
+}
+
 # Include common utility functions
 source ${CREATE_DOMAIN_SCRIPT_DIR}/utility.sh
 
@@ -15,9 +22,11 @@ prepareDomainHomeDir
 
 # Execute the script to create the domain
 source $script
+exitIfError $? "ERROR: $script failed."
 
 # Create the node manager
 wlst.sh -skipWLSModuleScanning ${CREATE_DOMAIN_SCRIPT_DIR}/setup-nodemanager.py
+exitIfError $? "ERROR: setup node manager failed."
 
 # DON'T REMOVE THIS
 # This script has to contain this log message. 

kubernetes/samples/scripts/create-weblogic-domain/domain-home-on-pv/create-domain.sh

@@ -469,34 +469,35 @@ function createDomainHome {
   while [ "$JOB_STATUS" != "Completed" -a $count -lt $max ] ; do
     sleep 30
     count=`expr $count + 1`
-    JOB_STATUS=`kubectl get pods --show-all -n ${namespace} | grep ${JOB_NAME} | awk ' { print $3; } '`
-    JOB_INFO=`kubectl get pods --show-all -n ${namespace} | grep ${JOB_NAME} | awk ' { print "pod", $1, "status is", $3; } '`
+    JOBS=`kubectl get pods --show-all -n ${namespace} | grep ${JOB_NAME}`
+    JOB_ERRORS=`kubectl logs jobs/$JOB_NAME -n ${namespace} | grep "ERROR:" `
+    JOB_STATUS=`echo $JOBS | awk ' { print $3; } '`
+    JOB_INFO=`echo $JOBS | awk ' { print "pod", $1, "status is", $3; } '`
     echo "status on iteration $count of $max"
     echo "$JOB_INFO"
 
     # Terminate the retry loop when a fatal error has already occurred.  Search for "ERROR:" in the job log file
     if [ "$JOB_STATUS" != "Completed" ]; then
-      JOB_ERRORS=`kubectl logs jobs/$JOB_NAME -n ${namespace} | grep "ERROR:" `
       ERR_COUNT=`echo $JOB_ERRORS | grep "ERROR:" | wc | awk ' {print $1; }'`
       if [ "$ERR_COUNT" != "0" ]; then
-        echo A failure was detected in the log file for job $JOB_NAME
-        echo $JOB_ERRORS
-        echo Check the log output for additional information
-        fail "Exiting due to failure - the job has failed"
+        echo "A failure was detected in the log file for job $JOB_NAME."
+        echo "$JOB_ERRORS"
+        echo "Check the log output for additional information."
+        fail "Exiting due to failure - the job has failed!"
       fi
     fi
   done
 
   # Confirm the job pod is status completed
-  JOB_POD=`kubectl get pods --show-all -n ${namespace} | grep ${JOB_NAME} | awk ' { print $1; } '`
   if [ "$JOB_STATUS" != "Completed" ]; then
-    echo The create domain job is not showing status completed after waiting 300 seconds
-    echo Check the log output for errors
+    echo "The create domain job is not showing status completed after waiting 300 seconds."
+    echo "Check the log output for errors."
     kubectl logs jobs/$JOB_NAME -n ${namespace}
     fail "Exiting due to failure - the job status is not Completed!"
   fi
 
   # Check for successful completion in log file
+  JOB_POD=`kubectl get pods --show-all -n ${namespace} | grep ${JOB_NAME} | awk ' { print $1; } '`
   JOB_STS=`kubectl logs $JOB_POD -n ${namespace} | grep "Successfully Completed" | awk ' { print $1; } '`
   if [ "${JOB_STS}" != "Successfully" ]; then
     echo The log file for the create domain job does not contain a successful completion status