Merge pull request #3480 from jackfrancis/cherry-pick-3456-to-release-1.8

[release-1.8] Switch k8s custom build templates to out of tree

Author: k8s-ci-robot

hack/util.sh

@@ -46,7 +46,11 @@ capz::util::should_build_ccm() {
     if [[ -n "${TEST_CCM:-}" ]]; then
         echo "true" && return
     fi
-    # If conformance is being tested with CI artifacts, CCM should be built.
+    # If testing a custom Kubernetes version, CCM should be built.
+    if [[ "$(capz::util::should_build_kubernetes)" == "true" ]]; then
+        echo "true" && return
+    fi
+    # If using Kubernetes CI artifacts, CCM should be built.
     if [[ "${E2E_ARGS:-}" == "-kubetest.use-ci-artifacts" ]]; then
         echo "true" && return
     fi

scripts/ci-build-azure-ccm.sh

@@ -46,8 +46,8 @@ setup() {
 
     # the azure-cloud-provider repo expects IMAGE_REGISTRY.
     export IMAGE_REGISTRY=${REGISTRY}
-    pushd "${AZURE_CLOUD_PROVIDER_ROOT}" && TAG=$(git rev-parse --short=7 HEAD) &&
-      IMAGE_TAG_CCM="${IMAGE_TAG_CCM:-${TAG}}" && IMAGE_TAG_CNM="${IMAGE_TAG_CNM:-${TAG}}" &&
+    pushd "${AZURE_CLOUD_PROVIDER_ROOT}" && IMAGE_TAG=$(git rev-parse --short=7 HEAD) &&
+      IMAGE_TAG_CCM="${IMAGE_TAG_CCM:-${IMAGE_TAG}}" && IMAGE_TAG_CNM="${IMAGE_TAG_CNM:-${IMAGE_TAG}}" &&
       export IMAGE_TAG_CCM && export IMAGE_TAG_CNM && popd
     echo "Image registry is ${REGISTRY}"
     echo "Image Tag CCM is ${IMAGE_TAG_CCM}"

scripts/ci-build-kubernetes.sh

@@ -71,8 +71,8 @@ setup() {
 
     # Docker tags cannot contain '+'
     # ref: https://github.com/kubernetes/kubernetes/blob/5491484aa91fd09a01a68042e7674bc24d42687a/build/lib/release.sh#L345-L346
-    export IMAGE_TAG="${KUBE_GIT_VERSION/+/_}"
-    echo "using K8s IMAGE_TAG=${IMAGE_TAG}"
+    export KUBE_IMAGE_TAG="${KUBE_GIT_VERSION/+/_}"
+    echo "using K8s KUBE_IMAGE_TAG=${KUBE_IMAGE_TAG}"
 }
 
 main() {
@@ -85,7 +85,7 @@ main() {
     if [[ "${KUBE_BUILD_CONFORMANCE:-}" =~ [yY] ]]; then
         IMAGES+=("conformance")
         # consume by the conformance test suite
-        export CONFORMANCE_IMAGE="${REGISTRY}/conformance:${IMAGE_TAG}"
+        export CONFORMANCE_IMAGE="${REGISTRY}/conformance:${KUBE_IMAGE_TAG}"
     fi
 
     if [[ "$(can_reuse_artifacts)" == "false" ]]; then
@@ -102,7 +102,7 @@ main() {
         for IMAGE_NAME in "${IMAGES[@]}"; do
             # extract docker image URL form `docker load` output
             OLD_IMAGE_URL="$(docker load --input "${KUBE_ROOT}/_output/release-images/amd64/${IMAGE_NAME}.tar" | ${GREP_BINARY} -oP '(?<=Loaded image: )[^ ]*' | head -n 1)"
-            NEW_IMAGE_URL="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"
+            NEW_IMAGE_URL="${REGISTRY}/${IMAGE_NAME}:${KUBE_IMAGE_TAG}"
             # retag and push images to ACR
             docker tag "${OLD_IMAGE_URL}" "${NEW_IMAGE_URL}" && docker push "${NEW_IMAGE_URL}"
         done
@@ -128,7 +128,7 @@ main() {
 # can_reuse_artifacts returns true if there exists Kubernetes artifacts built from a PR that we can reuse
 can_reuse_artifacts() {
     for IMAGE_NAME in "${IMAGES[@]}"; do
-        if ! docker pull "${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"; then
+        if ! docker pull "${REGISTRY}/${IMAGE_NAME}:${KUBE_IMAGE_TAG}"; then
             echo "false" && return
         fi
     done

templates/test/dev/cluster-template-custom-builds-machine-pool.yaml

@@ -1,6 +1,6 @@
 namespace: default
 resources:
-  - ../../../test/ci/prow-intree-cloud-provider-machine-pool
+  - ../../../test/ci/prow-machine-pool
 patchesStrategicMerge:
   - ../patches/control-plane-custom-builds.yaml
   - patches/custom-builds.yaml

templates/test/dev/cluster-template-custom-builds.yaml

@@ -1,6 +1,6 @@
 namespace: default
 resources:
-  - ../../../test/ci/prow-intree-cloud-provider
+  - ../../../test/ci/prow
   - ../../../addons/metrics-server/metrics-server-resource-set.yaml
 patchesStrategicMerge:
   - patches/machine-deployment-pr-version.yaml

templates/test/dev/custom-builds-machine-pool/kustomization.yaml

@@ -47,11 +47,11 @@
         rm /tmp/yq_linux_amd64.tar.gz
 
         export KUBECONFIG=/etc/kubernetes/admin.conf
-        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${IMAGE_TAG}"
+        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
         systemctl stop kubelet
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
         systemctl restart kubelet
     path: /tmp/replace-k8s-components.sh
     owner: "root:root"

templates/test/dev/custom-builds/kustomization.yaml

@@ -60,11 +60,11 @@ spec:
         rm /tmp/yq_linux_amd64.tar.gz
 
         export KUBECONFIG=/etc/kubernetes/admin.conf
-        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${IMAGE_TAG}"
+        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
         systemctl stop kubelet
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
+        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
         systemctl restart kubelet
     - path: /etc/kubernetes/azure.json
       owner: "root:root"

templates/test/dev/custom-builds/patches/kubeadm-controlplane-bootstrap.yaml

@@ -49,7 +49,7 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus
 		StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))},
 	}
 	// If testing a CI version of Kubernetes, use CCM and CNM images built from source.
-	if useCIArtifacts {
+	if useCIArtifacts || usePRArtifacts {
 		options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME")))
 		options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME")))
 		options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY")))