Skip to content

Commit 3d7dd30

Browse files
nawazkhnawazkh
committed
Add aks-create target to the makefile
- aks-create creates a AKS cluster that can be used as management cluster for local dev. - aks-create also updates tilt-settings.yaml with approproate Kustomize env variables - tilt-setting.yaml then gets consumed in Tiltfile. - use podidentity for aso - set right build flags when using aks cluster as management cluster - add acr-login as a target
1 parent a606de0 commit 3d7dd30

File tree

4 files changed

+246
-2
lines changed

4 files changed

+246
-2
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,4 @@ jwks.json
9494
azure_identity_id
9595
azure_wi_back_compat
9696
openid-configuration.json
97+
aks-mgmt.config

‎Makefile

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,10 @@ delete-workload-cluster: $(KUBECTL) ## Deletes the example workload Kubernetes c
388388

389389
##@ Docker:
390390

391+
.PHONY: acr-login
392+
acr-login: ## Login to Azure Container Registry.
393+
./hack/ensure-acr-login.sh
394+
391395
.PHONY: docker-pull-prerequisites
392396
docker-pull-prerequisites: ## Pull prerequisites for building controller-manager.
393397
docker pull docker/dockerfile:1.4
@@ -724,8 +728,12 @@ verify-container-images: ## Verify container images
724728
kind-create: $(KUBECTL) ## Create capz kind cluster if needed.
725729
./scripts/kind-with-registry.sh
726730

731+
.PHONY: aks-create
732+
aks-create: $(KUBECTL) ## Create aks cluster as mgmt cluster.
733+
./scripts/aks-as-mgmt.sh
734+
727735
.PHONY: tilt-up
728-
tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed.
736+
tilt-up: install-tools ## Start tilt and build kind cluster if needed.
729737
@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
730738
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
731739
fi; \
@@ -763,6 +771,7 @@ yq: $(YQ) ## Build a local copy of yq.
763771
kind: $(KIND) ## Build a local copy of kind.
764772
setup-envtest: $(SETUP_ENVTEST) ## Build a local copy of setup-envtest.
765773
codespell : $(CODESPELL) ## Build a local copy of codespell.
774+
azwi: $(AZWI) ## Build a local copy of azwi.
766775

767776
$(CONVERSION_VERIFIER): go.mod
768777
cd $(TOOLS_DIR); go build -tags=tools -o $@ sigs.k8s.io/cluster-api/hack/tools/conversion-verifier

‎Tiltfile

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ if "default_registry" in settings:
5151
default_registry(settings.get("default_registry"))
5252

5353
os_arch = str(local("go env GOARCH")).rstrip("\n")
54+
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
55+
print("Using AKS as management cluster, setting os_arch to amd64")
56+
os_arch = "amd64"
5457

5558
# deploy CAPI
5659
def deploy_capi():
@@ -242,10 +245,18 @@ def capz():
242245
if extra_args:
243246
entrypoint.extend(extra_args)
244247

248+
# use the user REGISTRY if set, otherwise use the default
249+
if settings.get("kustomize_substitutions", {}).get("REGISTRY", "") != "":
250+
registry = settings.get("kustomize_substitutions", {}).get("REGISTRY", "")
251+
print("Using REGISTRY: " + registry + " from tilt-settings.yaml")
252+
image = registry + "/cluster-api-azure-controller"
253+
else:
254+
image = "gcr.io/cluster-api-provider-azure/cluster-api-azure-controller"
255+
245256
# Set up an image build for the provider. The live update configuration syncs the output from the local_resource
246257
# build into the container.
247258
docker_build(
248-
ref = "gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller",
259+
ref = image,
249260
context = "./.tiltbuild/",
250261
dockerfile_contents = dockerfile_contents,
251262
target = "tilt",

‎scripts/aks-as-mgmt.sh

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit # exit immediately if a command exits with a non-zero status.
17+
set -o nounset # exit when script tries to use undeclared variables.
18+
set -o pipefail # make the pipeline fail if any command in it fails.
19+
20+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
21+
# shellcheck source=hack/ensure-azcli.sh
22+
source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI
23+
# shellcheck source=hack/ensure-tags.sh
24+
source "${REPO_ROOT}/hack/ensure-tags.sh" # set the right timestamp and job name
25+
26+
KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
27+
KIND="${REPO_ROOT}/hack/tools/bin/kind"
28+
AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
29+
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}" "${AZWI##*/}"
30+
31+
export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # management cluster name
32+
export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz}-${RANDOM_SUFFIX}" # resource group name
33+
export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}"
34+
export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}"
35+
export AZURE_LOCATION="${AZURE_LOCATION:-westus2}"
36+
export AKS_NODE_VM_SIZE="${AKS_NODE_VM_SIZE:-"Standard_B2s"}"
37+
export AKS_NODE_COUNT="${AKS_NODE_COUNT:-1}"
38+
export MGMT_CLUSTER_KUBECONFIG="${MGMT_CLUSTER_KUBECONFIG:-$REPO_ROOT/aks-mgmt.config}"
39+
export AZURE_IDENTITY_ID_FILEPATH="${AZURE_IDENTITY_ID_FILEPATH:-$REPO_ROOT/azure_identity_id}"
40+
export AZWI_STORAGE_ACCOUNT="capzcioidcissuer${RANDOM_SUFFIX}"
41+
export AZWI_STORAGE_CONTAINER="\$web"
42+
export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}"
43+
export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}"
44+
export REGISTRY="${REGISTRY:-}"
45+
46+
export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}"
47+
export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}"
48+
export AZURE_TENANT_ID="${AZURE_TENANT_ID:-}"
49+
50+
main() {
51+
52+
echo "--------------------------------"
53+
echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME"
54+
echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP"
55+
echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP"
56+
echo "KUBERNETES_VERSION: $KUBERNETES_VERSION"
57+
echo "AZURE_LOCATION: $AZURE_LOCATION"
58+
echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE"
59+
echo "AZURE_NODE_MACHINE_TYPE: $AZURE_NODE_MACHINE_TYPE"
60+
echo "AKS_NODE_COUNT: $AKS_NODE_COUNT"
61+
echo "MGMT_CLUSTER_KUBECONFIG: $MGMT_CLUSTER_KUBECONFIG"
62+
echo "AZURE_IDENTITY_ID_FILEPATH: $AZURE_IDENTITY_ID_FILEPATH"
63+
echo "AZWI_STORAGE_ACCOUNT: $AZWI_STORAGE_ACCOUNT"
64+
echo "AZWI_STORAGE_CONTAINER: $AZWI_STORAGE_CONTAINER"
65+
echo "SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH: $SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH"
66+
echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH"
67+
echo "REGISTRY: $REGISTRY"
68+
69+
echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID"
70+
echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID"
71+
echo "AZURE_TENANT_ID: $AZURE_TENANT_ID"
72+
echo "--------------------------------"
73+
74+
create_aks_cluster
75+
set_env_varaibles
76+
}
77+
78+
create_aks_cluster() {
79+
resource_group_exists=$(az group exists --name "${AKS_RESOURCE_GROUP}" --output tsv)
80+
if [ "${resource_group_exists}" == 'true' ]; then
81+
echo "resource group \"${AKS_RESOURCE_GROUP}\" already exists, moving on"
82+
else
83+
echo "creating resource group ${AKS_RESOURCE_GROUP}"
84+
az group create --name "${AKS_RESOURCE_GROUP}" \
85+
--location "${AZURE_LOCATION}" \
86+
--output none --only-show-errors \
87+
--tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
88+
fi
89+
90+
aks_exists=$(az aks show --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" 2>&1 || true) # true because we want to continue if the command fails
91+
if echo "$aks_exists" | grep -E -q "Resource(NotFound|GroupNotFound)"; then
92+
echo "creating aks cluster ${MGMT_CLUSTER_NAME} in the resource group ${AKS_RESOURCE_GROUP}"
93+
az aks create --name "${MGMT_CLUSTER_NAME}" \
94+
--resource-group "${AKS_RESOURCE_GROUP}" \
95+
--location "${AZURE_LOCATION}" \
96+
--kubernetes-version "${KUBERNETES_VERSION}" \
97+
--node-count "${AKS_NODE_COUNT}" \
98+
--node-vm-size "${AKS_NODE_VM_SIZE}" \
99+
--node-resource-group "${AKS_NODE_RESOURCE_GROUP}" \
100+
--vm-set-type VirtualMachineScaleSets \
101+
--generate-ssh-keys \
102+
--network-plugin azure \
103+
--tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" \
104+
--output none --only-show-errors;
105+
elif echo "$aks_exists" | grep -q "${MGMT_CLUSTER_NAME}"; then
106+
echo "cluster ${MGMT_CLUSTER_NAME} already exists in RG ${AKS_RESOURCE_GROUP}, moving on"
107+
else
108+
echo "error : ${aks_exists}"
109+
exit 1
110+
fi
111+
112+
# check and save kubeconfig
113+
echo "saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}"
114+
az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \
115+
--file "${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" --only-show-errors
116+
117+
az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \
118+
--overwrite-existing --only-show-errors
119+
120+
# echo "fetching Client ID for ${MGMT_CLUSTER_NAME}"
121+
AKS_MI_CLIENT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
122+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.clientId')
123+
export AKS_MI_CLIENT_ID
124+
echo "mgmt client identity: ${AKS_MI_CLIENT_ID}"
125+
echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
126+
127+
# echo "fetching Object ID for ${MGMT_CLUSTER_NAME}"
128+
AKS_MI_OBJECT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
129+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.objectId')
130+
export AKS_MI_OBJECT_ID
131+
echo "mgmt object identity: ${AKS_MI_OBJECT_ID}"
132+
133+
# echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}"
134+
AKS_MI_RESOURCE_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
135+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.resourceId')
136+
export AKS_MI_RESOURCE_ID
137+
echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}"
138+
139+
# save resource identity name and resource group
140+
MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" | jq -r '.name')
141+
# export MANAGED_IDENTITY_NAME
142+
echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}"
143+
USER_IDENTITY=$MANAGED_IDENTITY_NAME
144+
export USER_IDENTITY
145+
146+
MANAGED_IDENTITY_RG=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" | jq -r '.resourceGroup')
147+
export MANAGED_IDENTITY_RG
148+
echo "mgmt resource identity resource group: ${MANAGED_IDENTITY_RG}"
149+
150+
echo "assigning contributor role to the service principal"
151+
until az role assignment create --assignee-object-id "${AKS_MI_OBJECT_ID}" --role "Contributor" \
152+
--scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none \
153+
--only-show-errors; do
154+
echo "retrying to assign role to the service principal"
155+
sleep 5
156+
done
157+
158+
echo "using ASO_CREDENTIAL_SECRET_MODE as podidentity"
159+
ASO_CREDENTIAL_SECRET_MODE="podidentity"
160+
}
161+
162+
set_env_varaibles(){
163+
cat <<EOF > tilt-settings-temp.yaml
164+
kustomize_substitutions:
165+
MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}"
166+
AKS_RESOURCE_GROUP: "${AKS_RESOURCE_GROUP}"
167+
AKS_NODE_RESOURCE_GROUP: "${AKS_NODE_RESOURCE_GROUP}"
168+
MGMT_CLUSTER_KUBECONFIG: "${MGMT_CLUSTER_KUBECONFIG}"
169+
AKS_MI_CLIENT_ID: "${AKS_MI_CLIENT_ID}"
170+
AKS_MI_OBJECT_ID: "${AKS_MI_OBJECT_ID}"
171+
AKS_MI_RESOURCE_ID: "${AKS_MI_RESOURCE_ID}"
172+
MANAGED_IDENTITY_NAME: "${MANAGED_IDENTITY_NAME}"
173+
MANAGED_IDENTITY_RG: "${MANAGED_IDENTITY_RG}"
174+
AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: "${AKS_MI_CLIENT_ID}"
175+
CI_RG: "${MANAGED_IDENTITY_RG}"
176+
USER_IDENTITY: "${MANAGED_IDENTITY_NAME}"
177+
CLUSTER_IDENTITY_TYPE: "UserAssignedMSI"
178+
ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
179+
REGISTRY: "${REGISTRY}"
180+
allowed_contexts:
181+
- "$MGMT_CLUSTER_NAME"
182+
- "kind-capz"
183+
azure_location: "${AZURE_LOCATION}"
184+
EOF
185+
186+
# create tilt-settings.yaml if it does not exist
187+
if [ -f tilt-settings.yaml ]; then
188+
echo "tilt-settings.yaml exists"
189+
else
190+
echo "tilt-settings.yaml does not exist, creating one"
191+
touch tilt-settings.yaml
192+
fi
193+
194+
# copy over the existing allowed_contexts to tilt-settings.yaml if it does not exist
195+
allowed_contexts_exists=$(yq eval '.allowed_contexts' tilt-settings.yaml)
196+
if [ "$allowed_contexts_exists" == "null" ]; then
197+
yq eval '.allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
198+
fi
199+
200+
# extract allowed_contexts from tilt-settings.yaml
201+
current_contexts=$(yq eval '.allowed_contexts' tilt-settings.yaml | sort -u)
202+
203+
# extract allowed_contexts from tilt-settings-new.yaml
204+
new_contexts=$(yq eval '.allowed_contexts' tilt-settings-temp.yaml | sort -u)
205+
206+
# combine current and new contexts, keeping the union of both
207+
combined_contexts=$(echo "$current_contexts"$'\n'"$new_contexts" | sort -u)
208+
209+
# create a temporary file since env($combined_contexts) is not supported in yq
210+
echo "$combined_contexts" > combined_contexts.yaml
211+
212+
# update allowed_contexts in tilt-settings.yaml with the combined contexts
213+
yq eval --inplace ".allowed_contexts = load(\"combined_contexts.yaml\")" tilt-settings.yaml
214+
215+
# merge the updated kustomize_substitution and azure_location with the existing one in tilt-settings.yaml
216+
yq eval-all 'select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml
217+
218+
mv tilt-settings-new.yaml tilt-settings.yaml
219+
rm -r combined_contexts.yaml
220+
rm -f tilt-settings-temp.yaml
221+
}
222+
223+
main

0 commit comments

Comments
 (0)