adding apiserver ilb soln to self-managed flavors

Author: nawazkh

Dockerfile

@@ -18,7 +18,7 @@
 ARG ARCH
 
 # Build the manager binary
-FROM golang:1.22 AS builder
+FROM golang:1.22.11 AS builder
 WORKDIR /workspace
 
 # Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy

Makefile

@@ -512,6 +512,10 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
 generate-flavors: $(KUSTOMIZE) generate-addons
 	./hack/gen-flavors.sh
 
+.PHONY: generate-flavors-internal ## Generate template flavors.
+generate-flavors: $(KUSTOMIZE) generate-addons
+	./hack/gen-flavors.sh
+
 .PHONY: generate-e2e-templates
 generate-e2e-templates: $(KUSTOMIZE) ## Generate Azure infrastructure templates for the v1beta1 CAPI test suite.
 	$(KUSTOMIZE) build $(AZURE_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(AZURE_TEMPLATES)/v1beta1/cluster-template.yaml
@@ -726,10 +730,10 @@ test-cover: test ## Run tests with code coverage and generate reports.
 .PHONY: create-bootstrap
 create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
 	if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
-		MGMT_CLUSTER_NAME=capz-e2e-4 \
-		AKS_RESOURCE_GROUP=capz-e2e-4 \
-		AKS_MGMT_VNET_NAME=capz-e2e-4-mgmt-vnet \
-		AKS_MGMT_SUBNET_NAME=capz-e2e-4-mgmt-subnet \
+		MGMT_CLUSTER_NAME=capz-e2e-6 \
+		AKS_RESOURCE_GROUP=capz-e2e-6 \
+		AKS_MGMT_VNET_NAME=capz-e2e-6-mgmt-vnet \
+		AKS_MGMT_SUBNET_NAME=capz-e2e-6-mgmt-subnet \
 		./scripts/aks-as-mgmt.sh; \
 	else \
 		KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh; \

Tiltfile

@@ -311,7 +311,14 @@ def flavors():
         os.environ.update({az_key_b64_name: base64_encode_file(default_key_path)})
         os.environ.update({az_key_name: read_file_from_path(default_key_path)})
 
-    template_list = [item for item in listdir("./templates")]
+    # Prefer templates/internal if subscription-type is msft
+    template_list = []
+    if settings.get("subscription-type", "") == "msft":
+        print("Using Azure subscription type: msft. CAPZ flavors from ./templates/internal will be used.")
+        template_list = [item for item in listdir("./templates/internal")]
+    else:
+        template_list = [item for item in listdir("./templates")]
+
     template_list = [template for template in template_list if os.path.basename(template).endswith("yaml")]
 
     for template in template_list:
@@ -389,6 +396,7 @@ def deploy_worker_templates(template, substitutions):
     flavor_name = os.path.basename(flavor)
     flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); "
 
+    # TODO: maintain a matrix for flavors and their respective internal LB IPs
     apiserver_lb_private_ip = os.getenv("AZURE_INTERNAL_LB_PRIVATE_IP", "")
     if "windows-apiserver-ilb" in flavor and apiserver_lb_private_ip == "":
         flavor_cmd += "export AZURE_INTERNAL_LB_PRIVATE_IP=\"40.0.11.100\"; "

config/capz/manager_image_patch.yaml

@@ -8,5 +8,5 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:main
+      - image: nhkregistry.azurecr.io/cluster-api-azure-controller-amd64:20250204005305
         name: manager

config/capz/manager_pull_policy.yaml

@@ -8,4 +8,4 @@ spec:
     spec:
       containers:
       - name: manager
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent

hack/gen-flavors.sh

@@ -27,6 +27,7 @@ make --directory="${REPO_ROOT}" "${KUSTOMIZE##*/}"
 flavors_dir="${REPO_ROOT}/templates/flavors/"
 ci_dir="${REPO_ROOT}/templates/test/ci/"
 dev_dir="${REPO_ROOT}/templates/test/dev/"
+internal_dir="${REPO_ROOT}/templates/internal/"
 
 for name in $(find "${flavors_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v base); do
   ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone "${flavors_dir}${name}" > "${REPO_ROOT}/templates/cluster-template-${name}.yaml"
@@ -41,3 +42,8 @@ done
 for name in $(find "${dev_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches); do
   ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone "${dev_dir}${name}" > "${dev_dir}cluster-template-${name}.yaml"
 done
+
+for name in $(find "${internal_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v base); do
+  ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone "${internal_dir}${name}" > "${REPO_ROOT}/templates/internal/cluster-template-${name}.yaml"
+done
+mv "${REPO_ROOT}/templates/internal/cluster-template-default.yaml" "${REPO_ROOT}/templates/internal/cluster-template.yaml"

templates/internal/aad/kustomization.yaml

@@ -0,0 +1,55 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+- ../../flavors/aad
+
+patches:
+- path: ../../flavors/apiserver-ilb/patches/kubeadm-config-template.yaml
+- path: ../../flavors/apiserver-ilb/patches/control-plane.yaml
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-0
+  patch: |-
+    - op: replace
+      path: /spec/template/spec/preKubeadmCommands/0
+      value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP:-50.0.0.100}   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
+  options:
+    allowResourceNotFound: true
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: replace
+      path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP
+      value: ${AZURE_INTERNAL_LB_PRIVATE_IP:-50.0.0.100}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: replace
+      path: /spec/networkSpec/vnet/cidrBlocks/0
+      value: 50.0.0.0/8
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: replace
+      path: /spec/networkSpec/subnets/0/cidrBlocks/0
+      value: 50.0.0.0/16
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: replace
+      path: /spec/networkSpec/subnets/1/cidrBlocks/0
+      value: 50.1.0.0/16
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-win
+  patch: |-
+    - op: replace
+      path: /spec/template/spec/preKubeadmCommands/0
+      value:
+        powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP:-50.0.0.100} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
+  options:
+    allowResourceNotFound: true
+
+sortOptions:
+  order: fifo

templates/internal/azure-bastion/kustomization.yaml

@@ -0,0 +1,56 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+  - ../../flavors/azure-bastion
+
+patches:
+  - path: ../../flavors/apiserver-ilb/patches/kubeadm-config-template.yaml
+  - path: ../../flavors/apiserver-ilb/patches/control-plane.yaml
+  - target:
+      kind: KubeadmConfigTemplate
+      name: .*-md-0
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/preKubeadmCommands/0
+        value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP:-60.0.0.100}   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
+    options:
+      allowResourceNotFound: true
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP
+        value: ${AZURE_INTERNAL_LB_PRIVATE_IP:-60.0.0.100}
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/vnet/cidrBlocks/0
+        value: 60.0.0.0/8
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/subnets/0/cidrBlocks/0
+        value: 60.0.0.0/16
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/subnets/1/cidrBlocks/0
+        value: 60.1.0.0/16
+  - target:
+      kind: KubeadmConfigTemplate
+      name: .*-md-win
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/preKubeadmCommands/0
+        value:
+          powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP:-60.0.0.100} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
+    options:
+      allowResourceNotFound: true
+
+
+sortOptions:
+  order: fifo

templates/internal/azure-cni-v1/kustomization.yaml

@@ -0,0 +1,56 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: default
+resources:
+- ../../flavors/azure-cni-v1
+
+patches:
+  - path: ../../flavors/apiserver-ilb/patches/kubeadm-config-template.yaml
+  - path: ../../flavors/apiserver-ilb/patches/control-plane.yaml
+  - target:
+      kind: KubeadmConfigTemplate
+      name: .*-md-0
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/preKubeadmCommands/0
+        value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP:-70.0.0.100}   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
+    options:
+      allowResourceNotFound: true
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP
+        value: ${AZURE_INTERNAL_LB_PRIVATE_IP:-70.0.0.100}
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/vnet/cidrBlocks/0
+        value: 70.0.0.0/8
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/subnets/0/cidrBlocks/0
+        value: 70.0.0.0/16
+  - target:
+      kind: AzureCluster
+    patch: |-
+      - op: replace
+        path: /spec/networkSpec/subnets/1/cidrBlocks/0
+        value: 70.1.0.0/16
+  - target:
+      kind: KubeadmConfigTemplate
+      name: .*-md-win
+    patch: |-
+      - op: replace
+        path: /spec/template/spec/preKubeadmCommands/0
+        value:
+          powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP:-70.0.0.100} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
+    options:
+      allowResourceNotFound: true
+
+
+sortOptions:
+  order: fifo