Merge remote-tracking branch 'origin/develop' into fix-take9

Author: rjeberhard

integration-tests/src/test/java/oracle/kubernetes/operator/utils/Operator.java

@@ -202,18 +202,48 @@ private void callHelmInstall() throws Exception {
     logger.info("Running " + cmd);
     ExecResult result = ExecCommand.exec(cmd.toString());
     if (result.exitValue() != 0) {
-      throw new RuntimeException(
-          "FAILURE: command "
-              + cmd
-              + " failed, returned "
-              + result.stdout()
-              + "\n"
-              + result.stderr());
+      reportHelmInstallFailure(cmd.toString(), result);
     }
     String outputStr = result.stdout().trim();
     logger.info("Command returned " + outputStr);
   }
 
+  private void reportHelmInstallFailure(String cmd, ExecResult result) throws Exception {
+    String cause = getExecFailure(cmd, result);
+    if (result.stderr().contains("Error: Job failed: BackoffLimitExceeded")) {
+      // The operator helm chart pre-install hook probably failed.
+      // This is probably because there is a problem with the values passed in,
+      // for example a listed domain namespace does not exist.
+      // Fetch the pre-install hook's results and return it too.
+      String hookResults = getOperatorHelmChartHookResults("pre-install");
+      cause = cause + "\n pre-install-hook returned\n" + hookResults;
+    }
+    throw new RuntimeException(cause);
+  }
+
+  private String getOperatorHelmChartHookResults(String hookType) throws Exception {
+    String cmd =
+        "kubectl logs job/"
+            + operatorMap.get("releaseName")
+            + "-weblogic-operator-"
+            + hookType
+            + "-hook -n kube-system";
+    ExecResult result = ExecCommand.exec(cmd);
+    if (result.exitValue() != 0) {
+      return getExecFailure(cmd, result);
+    }
+    return result.stdout();
+  }
+
+  private String getExecFailure(String cmd, ExecResult result) throws Exception {
+    return "FAILURE: command "
+        + cmd
+        + " failed, stdout:\n"
+        + result.stdout()
+        + "stderr:\n"
+        + result.stderr();
+  }
+
   private void generateInputYaml() throws Exception {
     Path parentDir =
         Files.createDirectories(Paths.get(userProjectsDir + "/weblogic-operators/" + operatorNS));

integration-tests/src/test/resources/operator1.yaml

@@ -7,4 +7,4 @@ serviceAccount: weblogic-operator
 namespace: weblogic-operator1
 domainNamespaces: [ "default", "test1" ]
 externalRestEnabled: true
-javaLoggingLevel: FINE
+javaLoggingLevel: FINE

integration-tests/src/test/resources/operator2.yaml

@@ -8,4 +8,4 @@ namespace: weblogic-operator2
 domainNamespaces: [ "test2" ]
 externalRestEnabled: true
 externalRestHttpsPort: 32001
-javaLoggingLevel: FINE
+javaLoggingLevel: FINE

integration-tests/src/test/resources/setupenv.sh

@@ -3,98 +3,95 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
 
 function clean_jenkins {
-    echo "Cleaning."
-    /usr/local/packages/aime/ias/run_as_root "${PROJECT_ROOT}/src/integration-tests/bash/clean_docker_k8s.sh -y"
+  echo "Cleaning."
+  /usr/local/packages/aime/ias/run_as_root "${PROJECT_ROOT}/src/integration-tests/bash/clean_docker_k8s.sh -y"
 }
 
 function setup_jenkins {
-    echo "Setting up."
-    /usr/local/packages/aime/ias/run_as_root "sh ${PROJECT_ROOT}/src/integration-tests/bash/install_docker_k8s.sh -y -u wls -v ${K8S_VERSION}"
-    set +x
-    . ~/.dockerk8senv
-    set -x
-    id
-
-    docker login -u [email protected] -p $docker_pass  wlsldi-v2.docker.oraclecorp.com
-    docker images
-
-	pull_tag_images
-
-    export JAR_VERSION="`grep -m1 "<version>" pom.xml | cut -f2 -d">" | cut -f1 -d "<"`"
-    # create a docker image for the operator code being tested
-    docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}"  --build-arg VERSION=$JAR_VERSION --no-cache=true .
-	docker tag "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" wlsldi-v2.docker.oraclecorp.com/weblogic-operator:latest
-	
-    docker images
+  echo "Setting up."
+  /usr/local/packages/aime/ias/run_as_root "sh ${PROJECT_ROOT}/src/integration-tests/bash/install_docker_k8s.sh -y -u wls -v ${K8S_VERSION}"
+  set +x
+  . ~/.dockerk8senv
+  set -x
+  id
+
+  docker login -u [email protected] -p $docker_pass  wlsldi-v2.docker.oraclecorp.com
+  docker images
+
+  pull_tag_images
+
+  export JAR_VERSION="`grep -m1 "<version>" pom.xml | cut -f2 -d">" | cut -f1 -d "<"`"
+  # create a docker image for the operator code being tested
+  docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}"  --build-arg VERSION=$JAR_VERSION --no-cache=true .
+  docker tag "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}" wlsldi-v2.docker.oraclecorp.com/weblogic-operator:latest
+  
+  docker images
 
-    echo "Helm installation starts" 
-    wget -q -O  /tmp/helm-v2.8.2-linux-amd64.tar.gz https://kubernetes-helm.storage.googleapis.com/helm-v2.8.2-linux-amd64.tar.gz
-    mkdir /tmp/helm
-    tar xzf /tmp/helm-v2.8.2-linux-amd64.tar.gz -C /tmp/helm
-    chmod +x /tmp/helm/linux-amd64/helm
-    /usr/local/packages/aime/ias/run_as_root "cp /tmp/helm/linux-amd64/helm /usr/bin/"
-    rm -rf /tmp/helm
-    helm init
-    echo "Helm is configured."
+  echo "Helm installation starts" 
+  wget -q -O  /tmp/helm-v2.8.2-linux-amd64.tar.gz https://kubernetes-helm.storage.googleapis.com/helm-v2.8.2-linux-amd64.tar.gz
+  mkdir /tmp/helm
+  tar xzf /tmp/helm-v2.8.2-linux-amd64.tar.gz -C /tmp/helm
+  chmod +x /tmp/helm/linux-amd64/helm
+  /usr/local/packages/aime/ias/run_as_root "cp /tmp/helm/linux-amd64/helm /usr/bin/"
+  rm -rf /tmp/helm
+  helm init
+  echo "Helm is configured."
 }
 
 function setup_wercker {
-    echo "Perform setup for running in wercker"
-	echo "Install tiller"
-	kubectl create serviceaccount --namespace kube-system tiller
-	kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
-	
-	# Note: helm init --wait would wait until tiller is ready, and requires helm 2.8.2 or above 
-	helm init --service-account=tiller --wait
-	
-	helm version
-	
-	kubectl get po -n kube-system
-	
-	echo "Existing helm charts "
-	helm ls
-	echo "Deleting installed helm charts"
-	helm list --short | xargs -L1 helm delete --purge
-	echo "After helm delete, list of installed helm charts is: "
-	helm ls
-
-    echo "Completed setup_wercker"
+  echo "Perform setup for running in wercker"
+  echo "Install tiller"
+  kubectl create serviceaccount --namespace kube-system tiller
+  kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+  
+  # Note: helm init --wait would wait until tiller is ready, and requires helm 2.8.2 or above 
+  helm init --service-account=tiller --wait
+  
+  helm version
+  
+  kubectl get po -n kube-system
+  
+  echo "Existing helm charts "
+  helm ls
+  echo "Deleting installed helm charts"
+  helm list --short | xargs -L1 helm delete --purge
+  echo "After helm delete, list of installed helm charts is: "
+  helm ls
+
+  echo "Completed setup_wercker"
 }
 
 function pull_tag_images {
+  echo "Pull and tag the images we need"
 
-    echo "Pull and tag the images we need"
-    docker pull wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest
-    docker tag wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest store/oracle/weblogic:12.2.1.3
-
-    docker pull wlsldi-v2.docker.oraclecorp.com/weblogic:19.1.0.0
-    docker tag wlsldi-v2.docker.oraclecorp.com/weblogic:19.1.0.0 store/oracle/weblogic:19.1.0.0
+  docker pull wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest
+  docker tag wlsldi-v2.docker.oraclecorp.com/store-weblogic-12.2.1.3:latest store/oracle/weblogic:12.2.1.3
 
-    docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest
-    docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8
+  docker pull wlsldi-v2.docker.oraclecorp.com/weblogic:19.1.0.0
+  docker tag wlsldi-v2.docker.oraclecorp.com/weblogic:19.1.0.0 store/oracle/weblogic:19.1.0.0
 
-    docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest
-    docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3
+  docker pull wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest
+  docker tag wlsldi-v2.docker.oraclecorp.com/store-serverjre-8:latest store/oracle/serverjre:8
 
+  docker pull wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest
+  docker tag wlsldi-v2.docker.oraclecorp.com/weblogic-webtier-apache-12.2.1.3.0:latest store/oracle/apache:12.2.1.3
 }
 
 
 function create_image_pull_secret_jenkins {
-
-    echo "Creating Secret"
-    kubectl create secret docker-registry wlsldi-secret  \
+  echo "Creating Secret"
+  kubectl create secret docker-registry wlsldi-secret  \
     --docker-server=wlsldi-v2.docker.oraclecorp.com \
     [email protected] \
     --docker-password=$docker_pass \
     [email protected] 
 
-    echo "Checking Secret"
-    local SECRET="`kubectl get secret wlsldi-secret | grep wlsldi | wc | awk ' { print $1; }'`"
-    if [ "$SECRET" != "1" ]; then
-        echo 'secret wlsldi-secret was not created successfully'
-        exit 1
-    fi
-
+  echo "Checking Secret"
+  local SECRET="`kubectl get secret wlsldi-secret | grep wlsldi | wc | awk ' { print $1; }'`"
+  if [ "$SECRET" != "1" ]; then
+    echo 'secret wlsldi-secret was not created successfully'
+    exit 1
+  fi
 }
 
 export SCRIPTPATH="$( cd "$(dirname "$0")" > /dev/null 2>&1 ; pwd -P )"
@@ -103,21 +100,23 @@ export RESULT_ROOT=${RESULT_ROOT:-/scratch/$USER/wl_k8s_test_results}
 export PV_ROOT=${PV_ROOT:-$RESULT_ROOT}
 echo "RESULT_ROOT$RESULT_ROOT PV_ROOT$PV_ROOT"
 export BRANCH_NAME="${BRANCH_NAME:-$WERCKER_GIT_BRANCH}"
+export IMAGE_NAME_WEBLOGIC="${IMAGE_NAME_WEBLOGIC:-store/oracle/weblogic}"
+export IMAGE_TAG_WEBLOGIC="${IMAGE_TAG_WEBLOGIC:-19.1.0.0}"
 
 if [ -z "$BRANCH_NAME" ]; then
   export BRANCH_NAME="`git branch | grep \* | cut -d ' ' -f2-`"
   if [ ! "$?" = "0" ] ; then
-  	echo "Error: Could not determine branch.  Run script from within a git repo".
-  	exit 1
+    echo "Error: Could not determine branch.  Run script from within a git repo".
+    exit 1
   fi
 fi
 export IMAGE_TAG_OPERATOR=${IMAGE_TAG_OPERATOR:-`echo "test_${BRANCH_NAME}" | sed "s#/#_#g"`}
 export IMAGE_NAME_OPERATOR=${IMAGE_NAME_OPERATOR:-wlsldi-v2.docker.oraclecorp.com/weblogic-operator}
 
 cd $PROJECT_ROOT
 if [ $? -ne 0 ]; then
-    echo "Couldn't change to $PROJECT_ROOT dir"
-    exit 1
+  echo "Couldn't change to $PROJECT_ROOT dir"
+  exit 1
 fi
 
 export JAR_VERSION="`grep -m1 "<version>" pom.xml | cut -f2 -d">" | cut -f1 -d "<"`"
@@ -126,40 +125,40 @@ echo IMAGE_NAME_OPERATOR $IMAGE_NAME_OPERATOR IMAGE_TAG_OPERATOR $IMAGE_TAG_OPER
 
 if [ "$WERCKER" = "true" ]; then 
 
-	echo "Test Suite is running locally on Wercker and k8s is running on remote nodes."
+  echo "Test Suite is running locally on Wercker and k8s is running on remote nodes."
 
-	export IMAGE_PULL_SECRET_OPERATOR=$IMAGE_PULL_SECRET_OPERATOR
-	export IMAGE_PULL_SECRET_WEBLOGIC=$IMAGE_PULL_SECRET_WEBLOGIC
+  export IMAGE_PULL_SECRET_OPERATOR=$IMAGE_PULL_SECRET_OPERATOR
+  export IMAGE_PULL_SECRET_WEBLOGIC=$IMAGE_PULL_SECRET_WEBLOGIC
 
-    echo "Creating Docker Secret"
-    kubectl create secret docker-registry $IMAGE_PULL_SECRET_WEBLOGIC  \
+  echo "Creating Docker Secret"
+  kubectl create secret docker-registry $IMAGE_PULL_SECRET_WEBLOGIC  \
     --docker-server=index.docker.io/v1/ \
     --docker-username=$DOCKER_USERNAME \
     --docker-password=$DOCKER_PASSWORD \
     --docker-email=$DOCKER_EMAIL 
 
-    echo "Checking Secret"
-    SECRET="`kubectl get secret $IMAGE_PULL_SECRET_WEBLOGIC | grep $IMAGE_PULL_SECRET_WEBLOGIC | wc | awk ' { print $1; }'`"
-    if [ "$SECRET" != "1" ]; then
-        echo "secret $IMAGE_PULL_SECRET_WEBLOGIC was not created successfully"
-        exit 1
-    fi
+  echo "Checking Secret"
+  SECRET="`kubectl get secret $IMAGE_PULL_SECRET_WEBLOGIC | grep $IMAGE_PULL_SECRET_WEBLOGIC | wc | awk ' { print $1; }'`"
+  if [ "$SECRET" != "1" ]; then
+    echo "secret $IMAGE_PULL_SECRET_WEBLOGIC was not created successfully"
+    exit 1
+  fi
 
-    echo "Creating Registry Secret"
-    kubectl create secret docker-registry $IMAGE_PULL_SECRET_OPERATOR  \
+  echo "Creating Registry Secret"
+  kubectl create secret docker-registry $IMAGE_PULL_SECRET_OPERATOR  \
     --docker-server=$REPO_REGISTRY \
     --docker-username=$REPO_USERNAME \
     --docker-password=$REPO_PASSWORD \
     --docker-email=$REPO_EMAIL 
 
-    echo "Checking Secret"
-    SECRET="`kubectl get secret $IMAGE_PULL_SECRET_OPERATOR | grep $IMAGE_PULL_SECRET_OPERATOR | wc | awk ' { print $1; }'`"
-    if [ "$SECRET" != "1" ]; then
-        echo "secret $IMAGE_PULL_SECRET_OPERATOR was not created successfully"
-        exit 1
-    fi
-    
-    setup_wercker
+  echo "Checking Secret"
+  SECRET="`kubectl get secret $IMAGE_PULL_SECRET_OPERATOR | grep $IMAGE_PULL_SECRET_OPERATOR | wc | awk ' { print $1; }'`"
+  if [ "$SECRET" != "1" ]; then
+    echo "secret $IMAGE_PULL_SECRET_OPERATOR was not created successfully"
+    exit 1
+  fi
+  
+  setup_wercker
 
 elif [ "$JENKINS" = "true" ]; then
 
@@ -194,18 +193,14 @@ elif [ "$JENKINS" = "true" ]; then
   /usr/local/packages/aime/ias/run_as_root "mkdir -p $PV_ROOT/acceptance_test_pv_archive"
   /usr/local/packages/aime/ias/run_as_root "chmod 777 $PV_ROOT/acceptance_test_pv_archive"
 
-
-
 else
-	pull_tag_images
-	  
-	#docker rmi -f $(docker images -q -f dangling=true)
-	docker images --quiet --filter=dangling=true | xargs --no-run-if-empty docker rmi  -f
-	 
-	docker images	
+  pull_tag_images
+    
+  #docker rmi -f $(docker images -q -f dangling=true)
+  docker images --quiet --filter=dangling=true | xargs --no-run-if-empty docker rmi  -f
+  
+  docker images	
 
   export JAR_VERSION="`grep -m1 "<version>" pom.xml | cut -f2 -d">" | cut -f1 -d "<"`"
-	docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}"  --build-arg VERSION=$JAR_VERSION --no-cache=true .
-	
+  docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --build-arg no_proxy=$no_proxy -t "${IMAGE_NAME_OPERATOR}:${IMAGE_TAG_OPERATOR}"  --build-arg VERSION=$JAR_VERSION --no-cache=true .
 fi
-

kubernetes/charts/weblogic-operator/templates/_operator-dep.tpl

@@ -43,6 +43,8 @@ spec:
         volumeMounts:
         - name: "weblogic-operator-cm-volume"
           mountPath: "/operator/config"
+        - name: "weblogic-operator-debug-cm-volume"
+          mountPath: "/operator/debug-config"
         - name: "weblogic-operator-secrets-volume"
           mountPath: "/operator/secrets"
           readOnly: true
@@ -79,6 +81,10 @@ spec:
       - name: "weblogic-operator-cm-volume"
         configMap:
           name: "weblogic-operator-cm"
+      - name: "weblogic-operator-debug-cm-volume"
+        configMap:
+          name: "weblogic-operator-debug-cm"
+          optional: true
       - name: "weblogic-operator-secrets-volume"
         secret:
           secretName: "weblogic-operator-secrets"