optional flatcar tests will create a loadbalancer

- Set cloudControllerManager.caCertDir to `/usr/share/ca-certificates`
- and check if loadbalancer can spin up successfully
- add caCertDir to CCM in tiltfile
- add Note about caCertDir to CCM in azure external docs

Author: nawazkh

Tiltfile

@@ -367,6 +367,9 @@ def deploy_worker_templates(template, substitutions):
     flavor_cmd += "; " + helm_cmd + " repo add projectcalico https://docs.tigera.io/calico/charts; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install calico projectcalico/tigera-operator -f " + calico_values + " --namespace tigera-operator --create-namespace"
     if "intree-cloud-provider" not in flavor_name and "ipv6" not in flavor_name:  # TODO: remove ipv6 once https://github.com/kubernetes-sigs/cloud-provider-azure/issues/3401 is fixed.
         flavor_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME}"
+        if "flatcar" in flavor_name:  # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor
+            flavor_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates"
+
     local_resource(
         name = flavor_name,
         cmd = flavor_cmd,

docs/book/src/topics/addons.md

@@ -168,6 +168,13 @@ Then install the Helm chart on the workload cluster:
 helm install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set "cloudControllerManager.clusterCIDR=${CCM_CIDR_BLOCK}"
 ```
 
+- **Note**: 
+  When working with **Flatcar machines**, append `--set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates` to the `cloud-provider-azure` _helm_ command. The helm command to install cloud provider azure for Flatcar-flavored workload cluster will be:
+
+    ```bash
+    helm install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set "cloudControllerManager.clusterCIDR=${CCM_CIDR_BLOCK}" --set-string "cloudControllerManager.caCertDir=/usr/share/ca-certificates"
+    ```
+
 The Helm chart will pick the right version of `cloud-controller-manager` and `cloud-node-manager` to work with the version of Kubernetes your cluster is running.
 
 After running `helm install`, you should eventually see a set of pods like these in a `Running` state:

docs/book/src/topics/flatcar.md

@@ -39,3 +39,6 @@ If you would like customize your images please refer to the documentation on bui
 ## Trying it out
 
 To create a cluster using Flatcar Container Linux, use `flatcar` cluster flavor.
+
+- Note: When working with **Flatcar machines**, append `--set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates` to the `cloud-provider-azure` _helm_ command. Refer ["External Cloud Provider's Note for flatcar-flavored machine"](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/764aa1e8bd02d150dff90ff6bc7f8daa2b38810f/docs/book/src/topics/addons.md#external-cloud-provider)
+  - However, no changes are needed when using tilt to bring up flatcar-flavored workload clusters.

test/e2e/azure_test.go

@@ -297,7 +297,6 @@ var _ = Describe("Workload cluster creation", func() {
 	Context("Creating a Flatcar cluster [OPTIONAL]", func() {
 		It("With Flatcar control-plane and worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "flatcar")
-
 			clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput(
 				specName,
 				withFlavor("flatcar"),
@@ -309,7 +308,27 @@ var _ = Describe("Workload cluster creation", func() {
 				withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
 					WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
 				}),
+				withPostMachinesProvisioned(func() {
+					EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
+						return DaemonsetsSpecInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
+				}),
 			), result)
+
+			By("can create and access a load balancer", func() {
+				AzureLBSpec(ctx, func() AzureLBSpecInput {
+					return AzureLBSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+						SkipCleanup:           skipCleanup,
+					}
+				})
+			})
 		})
 	})
 

test/e2e/cloud-provider-azure.go

@@ -58,6 +58,10 @@ func InstallCalicoAndCloudProviderAzureHelmChart(ctx context.Context, input clus
 		options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM")))
 	}
 
+	if input.ConfigCluster.Flavor == "flatcar" {
+		options.StringValues = append(options.StringValues, "cloudControllerManager.caCertDir=/usr/share/ca-certificates")
+	}
+
 	clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.ConfigCluster.Namespace, input.ConfigCluster.ClusterName)
 	InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options)