Convert flavor and test ci templates to external cloud-provider

Author: CecileRobertMichon

Tiltfile

@@ -365,7 +365,7 @@ def deploy_worker_templates(template, substitutions):
     else:
         calico_values = "./templates/addons/calico/values.yaml"
     flavor_cmd += "; " + helm_cmd + " repo add projectcalico https://docs.tigera.io/calico/charts; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install calico projectcalico/tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace"
-    if "external-cloud-provider" in flavor_name:
+    if "intree-cloud-provider" not in flavor_name and "ipv6" not in flavor_name:  # TODO: remove ipv6 once https://github.com/kubernetes-sigs/cloud-provider-azure/issues/3401 is fixed.
         flavor_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME}"
     local_resource(
         name = flavor_name,

docs/book/src/developers/kubernetes-developers.md

@@ -44,12 +44,11 @@ $ export CNM_IMAGE_NAME=azure-node-controller-manager
 $ export IMAGE_TAG=canary
 ```
 
-Then, use the `external-cloud-provider` flavor to create a cluster:
+Then, create a cluster:
 
 ```bash
 $ export CLUSTER_NAME=my-cluster
-$ CLUSTER_TEMPLATE=cluster-template-external-cloud-provider.yaml \
-make create-workload-cluster
+$ make create-workload-cluster
 ```
 
 Once your cluster deploys, you should receive the kubeconfig to the workload cluster. Set your `KUBECONFIG` environment variable to point to the kubeconfig file, then use the official cloud-provider-azure Helm chart to deploy the cloud-provider-azure components using your custom built images:

docs/book/src/topics/addons.md

@@ -149,9 +149,9 @@ kubectl apply -f kube-flannel.yml
 
 # External Cloud Provider
 
-To deploy a cluster using [external cloud provider](https://github.com/kubernetes-sigs/cloud-provider-azure), create a cluster configuration with the [external cloud provider template](https://raw.githubusercontent.com/kubernetes-sigs/cluster-api-provider-azure/main/templates/cluster-template-external-cloud-provider.yaml).
+The "external" or "out-of-tree" cloud provider for Azure is the recommended  cloud provider for CAPZ clusters. The "in-tree" cloud provider has been deprecated since v1.20 and only bug fixes are allowed in its Kubernetes repository directory.
 
-After the cluster has provisioned, install the `cloud-provider-azure` components using the official helm chart:
+Below are instructions to install [external cloud provider](https://github.com/kubernetes-sigs/cloud-provider-azure) components on a self-managed cluster using the official helm chart. For more information see the official [`cloud-provider-azure` helm chart documentation](https://github.com/kubernetes-sigs/cloud-provider-azure/tree/master/helm/cloud-provider-azure).
 
 Grab the CIDR ranges from your cluster by running this kubectl statement against the management cluster:
 
@@ -180,7 +180,7 @@ kube-system   cloud-node-manager-mfsdg
 kube-system   cloud-node-manager-qrz74                                            1/1     Running   0          24s
 ```
 
-For more information see the official [`cloud-provider-azure` helm chart documentation](https://github.com/kubernetes-sigs/cloud-provider-azure/tree/master/helm/cloud-provider-azure).
+To know more about configuring cloud-provider-azure, see [Configuring the Kubernetes Cloud Provider for Azure](./cloud-provider-config.md).
 
 ## Storage Drivers
 

docs/book/src/topics/machinepools.md

@@ -42,7 +42,7 @@ which provides the cloud provider-specific resource for orchestrating a group of
 
 Azure Virtual Machine Scale Sets support two orchestration modes: `Uniform` and `Flexible`. CAPZ defaults to `Uniform` mode. See [VMSS Orchestration modes in Azure](https://learn.microsoft.com/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-orchestration-modes) for more information.
 
-To use `Flexible` mode requires Kubernetes v1.26.0 or later with a workload cluster template like CAPZ's "external-cloud-provider-machinepool" flavor. Ensure that `orchestrationMode` on the `AzureMachinePool` spec is set:
+To use `Flexible` mode requires Kubernetes v1.26.0 or later. Ensure that `orchestrationMode` on the `AzureMachinePool` spec is set:
 
 ```yaml
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
@@ -190,8 +190,5 @@ spec:
     permissions: "0644"
   joinConfiguration:
     nodeRegistration:
-      kubeletExtraArgs:
-        cloud-config: /etc/kubernetes/azure.json
-        cloud-provider: azure
       name: '{{ ds.meta_data["local_hostname"] }}'
 ```

hack/util.sh

@@ -41,6 +41,16 @@ capz::util::should_build_kubernetes() {
     echo "false"
 }
 
+capz::util::should_build_ccm() {
+    if [[ -n "${TEST_CCM:-}" ]]; then
+        echo "true" && return
+    fi
+    if [[ "${E2E_ARGS:-}" == "-kubetest.use-ci-artifacts" ]]; then
+        echo "true" && return
+    fi
+    echo "false"
+}
+
 # all test regions must support AvailabilityZones
 capz::util::get_random_region() {
     local REGIONS=("canadacentral" "eastus" "eastus2" "northeurope" "uksouth" "westeurope" "westus2" "westus3")

scripts/ci-conformance.sh

@@ -55,6 +55,13 @@ else
     export KUBE_BUILD_CONFORMANCE="y"
     source "${REPO_ROOT}/scripts/ci-build-kubernetes.sh"
   fi
+
+  if [[ "$(capz::util::should_build_ccm)" == "true" ]]; then
+    # shellcheck source=scripts/ci-build-azure-ccm.sh
+    source "${REPO_ROOT}/scripts/ci-build-azure-ccm.sh"
+    echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG} cloud-controller-manager image for external cloud-provider-cluster"
+    echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG} cloud-node-manager image for external cloud-provider-azure cluster"
+  fi
 fi
 
 defaultTag=$(date -u '+%Y%m%d%H%M%S')

scripts/ci-entrypoint.sh

@@ -55,6 +55,13 @@ setup() {
         source "${REPO_ROOT}/scripts/ci-build-azure-ccm.sh"
         echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG} cloud-controller-manager image for external cloud-provider-cluster"
         echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG} cloud-node-manager image for external cloud-provider-azure cluster"
+
+        export CCM_IMG_ARGS=(--set cloudControllerManager.imageRepository="${IMAGE_REGISTRY}"
+        --set cloudNodeManager.imageRepository="${IMAGE_REGISTRY}"
+        --set cloudControllerManager.imageName="${CCM_IMAGE_NAME}"
+        --set cloudNodeManager.imageName="${CNM_IMAGE_NAME}"
+        --set-string cloudControllerManager.imageTag="${IMAGE_TAG}"
+        --set-string cloudNodeManager.imageTag="${IMAGE_TAG}")
     fi
 
     if [[ "$(capz::util::should_build_kubernetes)" == "true" ]]; then
@@ -115,11 +122,6 @@ select_cluster_template() {
         export CLUSTER_TEMPLATE="test/ci/cluster-template-prow.yaml"
     fi
 
-    if [[ -n "${TEST_CCM:-}" ]]; then
-        # replace 'prow' with 'prow-external-cloud-provider' in the template name if testing out-of-tree
-        export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-external-cloud-provider}"
-    fi
-
     if [[ "${EXP_MACHINE_POOL:-}" == "true" ]]; then
         if [[ "${CLUSTER_TEMPLATE}" =~ "prow" ]]; then
             export CLUSTER_TEMPLATE="${CLUSTER_TEMPLATE/prow/prow-machine-pool}"
@@ -148,6 +150,18 @@ get_cidrs() {
     fi
 }
 
+# get_cloud_provider determines if the Cluster is using an intree or external cloud-provider from the KubeadmConfigSpec.
+# any retryable operation in this function must return a non-zero exit code on failure so that we can
+# retry it using a `until get_cloud_provider; do sleep 5; done` pattern;
+# and any statement must be idempotent so that subsequent retry attempts can make forward progress.
+get_cloud_provider() {
+    CLOUD_PROVIDER=$("${KUBECTL}" get kubeadmcontrolplane -l cluster.x-k8s.io/cluster-name="${CLUSTER_NAME}" -o=jsonpath='{.items[0].spec.kubeadmConfigSpec.clusterConfiguration.controllerManager.extraArgs.cloud-provider}')
+    if [[ "${CLOUD_PROVIDER:-}" = "azure" ]]; then
+        IN_TREE="true"
+        export IN_TREE
+    fi
+}
+
 # install_calico installs Calico CNI componentry onto the Cluster
 # any retryable operation in this function must return a non-zero exit code on failure so that we can
 # retry it using a `until install_calico; do sleep 5; done` pattern;
@@ -205,18 +219,12 @@ install_cloud_provider_azure() {
     echo "Installing cloud-provider-azure components via helm"
     "${HELM}" upgrade cloud-provider-azure --install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure \
         --set infra.clusterName="${CLUSTER_NAME}" \
-        --set cloudControllerManager.imageRepository="${IMAGE_REGISTRY}" \
-        --set cloudNodeManager.imageRepository="${IMAGE_REGISTRY}" \
-        --set cloudControllerManager.imageName="${CCM_IMAGE_NAME}" \
-        --set cloudNodeManager.imageName="${CNM_IMAGE_NAME}" \
-        --set-string cloudControllerManager.imageTag="${IMAGE_TAG}" \
-        --set-string cloudNodeManager.imageTag="${IMAGE_TAG}" \
         --set cloudControllerManager.replicas="${CCM_COUNT}" \
         --set cloudControllerManager.enableDynamicReloading="${ENABLE_DYNAMIC_RELOADING}"  \
         --set cloudControllerManager.cloudConfig="${CLOUD_CONFIG}" \
         --set cloudControllerManager.cloudConfigSecretName="${CONFIG_SECRET_NAME}" \
         --set cloudControllerManager.logVerbosity="${CCM_LOG_VERBOSITY}" \
-        --set-string cloudControllerManager.clusterCIDR="${CCM_CLUSTER_CIDR}"
+        --set-string cloudControllerManager.clusterCIDR="${CCM_CLUSTER_CIDR}" "${CCM_IMG_ARGS[@]}"
 }
 
 # wait_for_nodes returns when all nodes in the workload cluster are Ready.
@@ -265,7 +273,10 @@ install_addons() {
         sleep 5
     done
     # install cloud-provider-azure components, if using out-of-tree
-    if [[ -n "${TEST_CCM:-}" ]]; then
+    until get_cloud_provider; do
+        sleep 5
+    done
+    if [[ -z "${IN_TREE:-}" ]]; then
         until install_cloud_provider_azure; do
             sleep 5
         done