|
| 1 | +#!/bin/bash |
| 2 | +# Copyright (c) 2020, Oracle Corporation and/or its affiliates. |
| 3 | +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. |
| 4 | +# |
| 5 | +# This script provisions a Kubernetes cluster using Kind (https://kind.sigs.k8s.io/) and runs the new |
| 6 | +# integration test suite against that cluster. |
| 7 | +# |
| 8 | +# To install Kind: |
| 9 | +# curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.8.0/kind-$(uname)-amd64 |
| 10 | +# chmod +x ./kind |
| 11 | +# mv ./kind /some-dir-in-your-PATH/kind |
| 12 | +# |
| 13 | +# Kind creates Kubernetes nodes on your local box by running each node as a Docker container. This |
| 14 | +# makes it very easy to setup and tear down clusters. Presently, this script creates a cluster with a |
| 15 | +# master node and a worker node. Each node will run the same Kubernetes version. Future enhancements to |
| 16 | +# this script could allow for the configuration of a different number of nodes or for nodes to run a |
| 17 | +# differing set of Kubernetes versions. |
| 18 | +# |
| 19 | +# Kind nodes do not have access to the Docker repository on the local machine. Therefore this script works |
| 20 | +# around this limitation by running a Docker registry in a Docker container and connecting the networks so |
| 21 | +# that the Kubernetes nodes have visibility to the registry. When you run tests, you will see that Docker |
| 22 | +# images are pushed to this local registry and then are pulled to the Kubernetes nodes as needed. |
| 23 | +# You can see the images on a node by running: "docker exec -it kind-worker crictl images" where kind-worker |
| 24 | +# is the name of the Docker container. |
| 25 | +# |
| 26 | +# As of May 6, 2020, the tests are clean on Kubernetes 1.16 with the following JDK workarounds: |
| 27 | +# 1. Maven must be run with OpenJDK 11.0.7, available here: https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_11.0.7_10.tar.gz |
| 28 | +# This is because of a critical bug fix. Unfortunately, the Oracle JDK 11.0.7 release was based on an earlier build and doesn't have the fix. |
| 29 | +# 2. The WebLogic Image Tool will not accept an OpenJDK JDK. Set WIT_JAVA_HOME to an Oracle JDK Java Home. |
| 30 | +# For example, "export WIT_JAVA_HOME=/usr/java/jdk-11.0.7" before running this script. |
| 31 | +# |
| 32 | +# If you want to access the cluster, use "kubectl cluster-info --context kind-kind" where the trailing "kind" is the value of the "-n" argument. |
| 33 | +set -o errexit |
| 34 | + |
| 35 | +script="${BASH_SOURCE[0]}" |
| 36 | +scriptDir="$( cd "$( dirname "${script}" )" && pwd )" |
| 37 | + |
| 38 | +function usage { |
| 39 | + echo "usage: ${script} [-v <version>] [-n <name>] [-o <directory>] [-t <tests>] [-h]" |
| 40 | + echo " -v Kubernetes version (optional) " |
| 41 | + echo " (default: 1.15.11, supported values: 1.18, 1.18.2, 1.17, 1.17.5, 1.16, 1.16.9, 1.15, 1.15.11, 1.14, 1.14.10) " |
| 42 | + echo " -n Kind cluster name (optional) " |
| 43 | + echo " (default: kind) " |
| 44 | + echo " -o Output directory (optional) " |
| 45 | + echo " (default: \${WORKSPACE}/logdir/\${BUILD_TAG}, if \${WORKSPACE} defined, else /scratch/\${USER}/kindtest) " |
| 46 | + echo " -t Test filter (optional) " |
| 47 | + echo " (default: **/It*) " |
| 48 | + echo " -h Help" |
| 49 | + exit $1 |
| 50 | +} |
| 51 | + |
| 52 | +k8s_version="1.15.11" |
| 53 | +kind_name="kind" |
| 54 | +if [[ -z "${WORKSPACE}" ]]; then |
| 55 | + outdir="/scratch/${USER}/kindtest" |
| 56 | +else |
| 57 | + outdir="${WORKSPACE}/logdir/${BUILD_TAG}" |
| 58 | +fi |
| 59 | +test_filter="**/It*" |
| 60 | + |
| 61 | +while getopts ":h:n:o:t:v:" opt; do |
| 62 | + case $opt in |
| 63 | + v) k8s_version="${OPTARG}" |
| 64 | + ;; |
| 65 | + n) kind_name="${OPTARG}" |
| 66 | + ;; |
| 67 | + o) outdir="${OPTARG}" |
| 68 | + ;; |
| 69 | + t) test_filter="${OPTARG}" |
| 70 | + ;; |
| 71 | + h) usage 0 |
| 72 | + ;; |
| 73 | + *) usage 1 |
| 74 | + ;; |
| 75 | + esac |
| 76 | +done |
| 77 | + |
| 78 | +function versionprop { |
| 79 | + grep "${1}=" "${scriptDir}/kindversions.properties"|cut -d'=' -f2 |
| 80 | +} |
| 81 | + |
| 82 | +kind_image=$(versionprop "${k8s_version}") |
| 83 | +if [ -z "${kind_image}" ]; then |
| 84 | + echo "Unsupported Kubernetes version: ${k8s_version}" |
| 85 | + exit 1 |
| 86 | +fi |
| 87 | + |
| 88 | +echo "Using Kubernetes version: ${k8s_version}" |
| 89 | + |
| 90 | +mkdir -m777 -p "${outdir}" |
| 91 | +export RESULT_ROOT="${outdir}/wl_k8s_test_results" |
| 92 | +if [ -d "${RESULT_ROOT}" ]; then |
| 93 | + rm -Rf "${RESULT_ROOT}/*" |
| 94 | +else |
| 95 | + mkdir -m777 "${RESULT_ROOT}" |
| 96 | +fi |
| 97 | + |
| 98 | +echo "Results will be in ${RESULT_ROOT}" |
| 99 | + |
| 100 | +export PV_ROOT="${outdir}/k8s-pvroot" |
| 101 | +if [ -d "${PV_ROOT}" ]; then |
| 102 | + rm -Rf "${PV_ROOT}/*" |
| 103 | +else |
| 104 | + mkdir -m777 "${PV_ROOT}" |
| 105 | +fi |
| 106 | + |
| 107 | +echo "Persistent volume files, if any, will be in ${PV_ROOT}" |
| 108 | + |
| 109 | +echo 'Remove old cluster (if any)...' |
| 110 | +kind delete cluster --name ${kind_name} --kubeconfig "${RESULT_ROOT}/kubeconfig" |
| 111 | + |
| 112 | +kind_version=$(kind version) |
| 113 | +kind_network='kind' |
| 114 | +reg_name='kind-registry' |
| 115 | +reg_port='5000' |
| 116 | +case "${kind_version}" in |
| 117 | + "kind v0.7."* | "kind v0.6."* | "kind v0.5."*) |
| 118 | + kind_network='bridge' |
| 119 | + ;; |
| 120 | +esac |
| 121 | + |
| 122 | +echo 'Create registry container unless it already exists' |
| 123 | +running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" |
| 124 | +if [ "${running}" != 'true' ]; then |
| 125 | + docker run \ |
| 126 | + -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ |
| 127 | + registry:2 |
| 128 | +fi |
| 129 | + |
| 130 | +reg_host="${reg_name}" |
| 131 | +if [ "${kind_network}" = "bridge" ]; then |
| 132 | + reg_host="$(docker inspect -f '{{.NetworkSettings.IPAddress}}' "${reg_name}")" |
| 133 | +fi |
| 134 | +echo "Registry Host: ${reg_host}" |
| 135 | + |
| 136 | +echo 'Create a cluster with the local registry enabled in containerd' |
| 137 | +cat <<EOF | kind create cluster --name "${kind_name}" --kubeconfig "${RESULT_ROOT}/kubeconfig" --config=- |
| 138 | +kind: Cluster |
| 139 | +apiVersion: kind.x-k8s.io/v1alpha4 |
| 140 | +containerdConfigPatches: |
| 141 | +- |- |
| 142 | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"] |
| 143 | + endpoint = ["http://${reg_host}:${reg_port}"] |
| 144 | +nodes: |
| 145 | + - role: control-plane |
| 146 | + image: ${kind_image} |
| 147 | + - role: worker |
| 148 | + image: ${kind_image} |
| 149 | + extraMounts: |
| 150 | + - hostPath: ${PV_ROOT} |
| 151 | + containerPath: ${PV_ROOT} |
| 152 | +EOF |
| 153 | + |
| 154 | +echo "Access your cluster in other terminals with:" |
| 155 | +echo " export KUBECONFIG=\"${RESULT_ROOT}/kubeconfig\"" |
| 156 | +echo " kubectl cluster-info --context \"kind-${kind_name}\"" |
| 157 | +export KUBECONFIG="${RESULT_ROOT}/kubeconfig" |
| 158 | +kubectl cluster-info --context "kind-${kind_name}" |
| 159 | +kubectl get node -o wide |
| 160 | + |
| 161 | +for node in $(kind get nodes --name "${kind_name}"); do |
| 162 | + kubectl annotate node "${node}" tilt.dev/registry=localhost:${reg_port}; |
| 163 | +done |
| 164 | + |
| 165 | +if [ "${kind_network}" != "bridge" ]; then |
| 166 | + containers=$(docker network inspect ${kind_network} -f "{{range .Containers}}{{.Name}} {{end}}") |
| 167 | + needs_connect="true" |
| 168 | + for c in ${containers}; do |
| 169 | + if [ "$c" = "${reg_name}" ]; then |
| 170 | + needs_connect="false" |
| 171 | + fi |
| 172 | + done |
| 173 | + if [ "${needs_connect}" = "true" ]; then |
| 174 | + docker network connect "${kind_network}" "${reg_name}" || true |
| 175 | + fi |
| 176 | +fi |
| 177 | + |
| 178 | +echo 'Set up test running ENVVARs...' |
| 179 | +export KIND_REPO="localhost:${reg_port}/" |
| 180 | +export K8S_NODEPORT_HOST=`kubectl get node kind-worker -o jsonpath='{.status.addresses[?(@.type == "InternalIP")].address}'` |
| 181 | +export JAVA_HOME="${JAVA_HOME:-`type -p java|xargs readlink -f|xargs dirname|xargs dirname`}" |
| 182 | + |
| 183 | +echo 'Clean up result root...' |
| 184 | +rm -rf "${RESULT_ROOT:?}/*" |
| 185 | + |
| 186 | +echo 'Run tests...' |
| 187 | +time mvn -Dit.test="${test_filter}" -pl new-integration-tests -P integration-tests verify 2>&1 | tee "${RESULT_ROOT}/kindtest.log" |
0 commit comments