add documentation on running e2e test locally

- update Makefile targets for local e2e
- Add script on peering vnets

Author: nawazkh

Makefile

@@ -186,6 +186,7 @@ E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 AZWI_SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
+MGMT_CLUSTER_TYPE ?= kind
 # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
 # to be created with extra mounts for key pairs which is not yet supported
 # by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite
@@ -317,8 +318,12 @@ install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(A
 
 .PHONY: create-management-cluster
 create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
-	# Create kind management cluster.
-	$(MAKE) kind-create
+	# Create management cluster based on type
+	@if [ "$(MGMT_CLUSTER_TYPE)" = "aks" ]; then \
+		$(MAKE) aks-create; \
+	else \
+		$(MAKE) kind-create; \
+	fi
 
 	# Install cert manager and wait for availability
 	./hack/install-cert-manager.sh
@@ -334,7 +339,9 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 	timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.2.5/addon-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done"
 
 	# Deploy CAPZ
-	$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME)
+	if [ "$(MGMT_CLUSTER_TYPE)" != "aks" ]; then \
+		$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME); \
+	fi
 	timeout --foreground 300 bash -c "until $(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true; do sleep 5; done"
 
 	# Wait for CAPI deployments
@@ -360,7 +367,10 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 	timeout --foreground 300 bash -c "until $(KUBECTL) get clusters -A; do sleep 3; done"
 	timeout --foreground 300 bash -c "until $(KUBECTL) get azureclusters -A; do sleep 3; done"
 	timeout --foreground 300 bash -c "until $(KUBECTL) get kubeadmcontrolplanes -A; do sleep 3; done"
-	@echo 'Set kubectl context to the kind management cluster by running "$(KUBECTL) config set-context kind-$(KIND_CLUSTER_NAME)"'
+
+	@if [ "$(MGMT_CLUSTER_TYPE)" != "aks" ]; then \
+		echo 'Set kubectl context to the kind management cluster by running "$(KUBECTL) config set-context kind-$(KIND_CLUSTER_NAME)"'; \
+	fi
 
 .PHONY: create-workload-cluster
 create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
@@ -726,6 +736,17 @@ test-cover: test ## Run tests with code coverage and generate reports.
 kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
 	KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh
 
+.PHONY: create-bootstrap
+create-bootstrap: $(KUBECTL) ## Create bootstrap cluster (AKS or KIND) for CAPZ testing. Default is KIND.
+	@echo "Creating bootstrap cluster with type: $(MGMT_CLUSTER_TYPE)"
+	@if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
+		MGMT_CLUSTER_NAME="$${MGMT_CLUSTER_NAME:-capz-e2e-$(shell date +%s)}" \
+		./scripts/aks-as-mgmt.sh || { echo "Failed to create AKS bootstrap cluster" >&2; exit 1; }; \
+	else \
+		KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh || { echo "Failed to create KIND bootstrap cluster" >&2; exit 1; }; \
+	fi
+	@echo "Bootstrap cluster created successfully"
+
 .PHONY: cleanup-workload-identity
 cleanup-workload-identity: ## Cleanup CI workload-identity infra
 	@if ! [ "$(AZWI_SKIP_CLEANUP)" == "true" ]; then \

docs/book/src/developers/development.md

@@ -507,6 +507,8 @@ To run E2E locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRI
 ./scripts/ci-e2e.sh
 ```
 
+Note: Users that have a restrictive environment and want to leverage API Server ILB in their flavors and to run e2e tests locally should refer to the detailed explanation in [running e2e tests locally leveraging apiserver ilb solution](./tilt-with-aks-as-mgmt-ilb.md#running-e2e-tests-locally-using-api-server-ilbs-networking-solution)
+
 You can optionally set the following variables:
 
 | Variable                   | Description                                                                                                                                                                                                                                                                                                                                                                | Default                                                                               |

docs/book/src/developers/tilt-with-aks-as-mgmt-ilb.md

@@ -124,3 +124,154 @@ Once the tilt UI is up and running
    - Flavors that leverage internal load balancer are:
      - `apiserver-ilb`
      - `windows-apiserver-ilb`
+
+### Running e2e tests locally using API Server ILB's networking solution
+
+Building upon the [challenges and solutions](#challenges-and-solutions) from above, running an e2e test locally in a restricted environment calls for some workarounds in the prow templates and the e2e test itself.
+
+1. We need to add the apiserver ILB with private endpoints and opinionated CIDRs to the workload cluster's VNet & Subnets, and pre-kubeadm commands updating the `/etc/hosts` file of the nodes of the workload cluster.
+
+2. Once the template has been modified to be run in local environment using AKS as management cluster, we need to be able to peer the vnets, create private DNS zone for the FQDN of the workload cluster and re-enable blocked NSG ports.
+
+**Note:**
+
+- The following guidance is only for testing, and is not a recommendation for any production environment.
+
+- The below steps are for self-managed templates only and do not apply to AKS workload clusters.
+
+- If you are going to run the local tests from a dev machine in Azure, you will have to use user-assigned managed identity and assign it to the management cluster. Follow the below steps before proceeding.
+  1. Create a user-assigned managed identity
+  2. Assign that managed identity a contributor role to your subscription
+  3. Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY`, and `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to the user-assigned managed identity.
+
+#### Update prow template with apiserver ILB networking solution
+
+There are three sections of a prow template that need an update.
+
+1. AzureCluster
+   - `/spec/networkSpec/apiServerLB`
+      - Add FrontendIP
+      - Add an associated private IP to be leveraged by an internal ILB
+   - `/spec/networkSpec/vnet/cidrBlocks`
+      - Add VNet CIDR
+   - `/spec/networkSpec/subnets/0/cidrBlocks`
+      - Add Subnet CIDR for the control plane
+   - `/spec/networkSpec/subnets/1/cidrBlocks`
+      - Add Subnet CIDR for the worker node
+2. `KubeadmConfigTemplate` - linux node; Identifiable by `name: .*-md-0`
+   - `/spec/template/spec/preKubeadmCommands/0`
+      - Add a prekubeadm command updating the `/etc/hosts` of worker nodes of type "linux".
+3. `KubeadmConfigTemplate` - windows node; Identifiable by `name: .*-md-win`
+   - `/spec/template/spec/preKubeadmCommands/0`
+      - Add a prekubeadm command updating the `/etc/hosts` of worker nodes of type "windows".
+
+A sample kustomize command for updating a prow template via its kustomization.yaml is pasted below.
+
+```yaml
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/apiServerLB
+      value:
+        frontendIPs:
+        - name: ${CLUSTER_NAME}-api-lb
+          publicIP:
+            dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com
+            name: ${CLUSTER_NAME}-api-lb
+        - name: ${CLUSTER_NAME}-internal-lb-private-ip
+          privateIP: ${AZURE_INTERNAL_LB_PRIVATE_IP}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/vnet/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/vnet/cidrBlocks/-
+      value: ${AZURE_VNET_CIDR}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/subnets/0/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/subnets/0/cidrBlocks/-
+      value: ${AZURE_CP_SUBNET_CIDR}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/subnets/1/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/subnets/1/cidrBlocks/-
+      value: ${AZURE_NODE_SUBNET_CIDR}
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-0
+  patch: |-
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands
+      value: []
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP}   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-win
+  patch: |-
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value:
+        powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
+```
+
+#### Peer Vnets of the management cluster and the workload cluster
+
+Peering VNets, creating a private DNS zone with the FQDN of the workload cluster, and updating NSGs of the management and workload clusters can be achieved by running `scripts/peer-vnets.sh`.
+
+This script, `scripts/peer-vnets.sh`, should be run after triggering the test run locally and in a separate terminal.
+
+#### Running the test locally
+
+We recommend running the test individually while debugging the test failure. This implies that `GINKGO_FOCUS` as unique as possible. So for instance if you want to run `periodic-cluster-api-provider-azure-e2e-main`'s "With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes" test,
+
+1. We first need to add the following environment variables to the test itself. For example:
+
+   ```go
+   Expect(os.Setenv("EXP_APISERVER_ILB", "true")).To(Succeed())
+   Expect(os.Setenv("AZURE_INTERNAL_LB_PRIVATE_IP", "10.0.0.101")).To(Succeed())
+   Expect(os.Setenv("AZURE_VNET_CIDR", "10.0.0.0/8")).To(Succeed())
+   Expect(os.Setenv("AZURE_CP_SUBNET_CIDR", "10.0.0.0/16")).To(Succeed())
+   Expect(os.Setenv("AZURE_NODE_SUBNET_CIDR", "10.1.0.0/16")).To(Succeed())
+   ```
+
+   The above lines should be added before the `clusterctl.ApplyClusterTemplateAndWait()` is invoked.
+
+
+2. Open the terminal and run the below command:
+
+   ```bash
+   GINKGO_FOCUS="With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes" USE_LOCAL_KIND_REGISTRY=false SKIP_CLEANUP="true" SKIP_LOG_COLLECTION="true" REGISTRY="<>" MGMT_CLUSTER_TYPE="aks" EXP_APISERVER_ILB=true AZURE_LOCATION="<>" ARCH="amd64" scripts/ci-e2e.sh
+   ```
+
+   **Note:**
+
+   - Set `MGMT_CLUSTER_TYPE` to `"aks"` to leverage `AKS` as the management cluster.
+   - Set `EXP_APISERVER_ILB` to `true` to enable the API Server ILB feature gate.
+   - Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY` and `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to use the user-assigned managed identity instead of the AKS-created managed identity.
+
+3. In a new terminal, wait for AzureClusters to be created by the above command. Check it using `kubectl get AzureClusters -A`. Note that this command will fail or will not output anything unless the above command, `GINKGO_FOCUS...`, has deployed the worker template and initiated workload cluster creation.
+
+   Once the worker cluster has been created, `export` the `CLUSTER_NAME` and `CLUSTER_NAMESPACE`.
+   It is recommended that `AZURE_INTERNAL_LB_PRIVATE_IP` is set an IP of `10.0.0.x`, say `10.0.0.101`, to avoid any test updates.
+
+   Then open a new terminal at the root of the cluster api provider azure repo and run the below command.
+
+   ```bash
+   AZURE_INTERNAL_LB_PRIVATE_IP="<IP of your choice>" CLUSTER_NAME="<e2e workload cluster name>" CLUSTER_NAMESPACE="<e2e cluster namespace>" ./scripts/peer-vnets.sh ./tilt-settings.yaml
+   ```
+
+You will see that the test progresses in the first terminal window that invoked `GINKGO_FOCUS=....`

e2e.mk

@@ -5,8 +5,11 @@
 
 ##@ E2E Testing:
 .PHONY: test-e2e-run
-test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
-	@$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
+test-e2e-run: generate-e2e-templates install-tools create-bootstrap ## Run e2e tests.
+	if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
+		source ./scripts/peer-vnets.sh && source_tilt_settings tilt-settings.yaml; \
+	fi; \
+	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
 	if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
 		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
 	fi; \