Skip to content

Commit f4c4330

Browse files
nawazkhnawazkh
committed
update guidance on running aks-create locally when running
1 parent 702adfe commit f4c4330

File tree

2 files changed

+101
-66
lines changed

2 files changed

+101
-66
lines changed

‎docs/book/src/developers/tilt-with-aks-as-mgmt-ilb.md‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,11 @@ Building upon the [challenges and solutions](#challenges-and-solutions) from abo
139139

140140
- The below steps for self-managed templates only. Does not apply to AKS workload clusters.
141141

142+
- The below steps are for running the tests in a intuned device. If you are going to run the local tests from a dev machine in Azure, you have to first follow below steps and then proceed ahead.
143+
1. Create a managed Identity
144+
2. Assign that managed identity a contributor role to your subscription
145+
3. Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to the user assigned managed identity.
146+
142147
#### Update prow template with apiserver ILB networking solution
143148

144149
There are three sections of a prow template that needs an update.
@@ -183,7 +188,7 @@ A Sample kustomize command updating a prow template via its kustomization.yaml i
183188
path: /spec/networkSpec/vnet/cidrBlocks
184189
value: []
185190
- op: add
186-
path: /spec/networkSpec/vnet/cidrBlocks/0
191+
path: /spec/networkSpec/vnet/cidrBlocks/-
187192
value: ${AZURE_VNET_CIDR}
188193
- target:
189194
kind: AzureCluster
@@ -192,7 +197,7 @@ A Sample kustomize command updating a prow template via its kustomization.yaml i
192197
path: /spec/networkSpec/subnets/0/cidrBlocks
193198
value: []
194199
- op: add
195-
path: /spec/networkSpec/subnets/0/cidrBlocks/0
200+
path: /spec/networkSpec/subnets/0/cidrBlocks/-
196201
value: ${AZURE_CP_SUBNET_CIDR}
197202
- target:
198203
kind: AzureCluster
@@ -201,29 +206,26 @@ A Sample kustomize command updating a prow template via its kustomization.yaml i
201206
path: /spec/networkSpec/subnets/1/cidrBlocks
202207
value: []
203208
- op: add
204-
path: /spec/networkSpec/subnets/1/cidrBlocks/0
209+
path: /spec/networkSpec/subnets/1/cidrBlocks/-
205210
value: ${AZURE_NODE_SUBNET_CIDR}
206211
- target:
207212
kind: KubeadmConfigTemplate
208-
name: .*-md-win
213+
name: .*-md-0
209214
patch: |-
210215
- op: add
211216
path: /spec/template/spec/preKubeadmCommands
212217
value: []
213218
- op: add
214-
path: /spec/template/spec/preKubeadmCommands/0
215-
value:
216-
powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
219+
path: /spec/template/spec/preKubeadmCommands/-
220+
value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
217221
- target:
218222
kind: KubeadmConfigTemplate
219-
name: .*-md-0
223+
name: .*-md-win
220224
patch: |-
221225
- op: add
222-
path: /spec/template/spec/preKubeadmCommands
223-
value: []
224-
- op: add
225-
path: /spec/template/spec/preKubeadmCommands/0
226-
value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
226+
path: /spec/template/spec/preKubeadmCommands/-
227+
value:
228+
powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
227229
```
228230
229231
#### Peer Vnets of the management cluster and the workload cluster
@@ -259,6 +261,7 @@ We recommend running the test individually while debugging the test failure. Thi
259261

260262
- Set `MGMT_CLUSTER_TYPE` to `"aks"` to leverage `AKS` as the management cluster.
261263
- Set `EXP_APISERVER_ILB` to `true` to enable the API Server ILB feature gate.
264+
- Set `IS_DEV_BOX` to `"true"` to use the user assigned managed identity instead of the AKS created managed identity
262265

263266

264267

‎scripts/aks-as-mgmt.sh‎

Lines changed: 85 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -58,34 +58,38 @@ export SKIP_AKS_CREATE="${SKIP_AKS_CREATE:-false}"
5858
main() {
5959

6060
echo "--------------------------------"
61-
echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME"
62-
echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP"
63-
echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP"
64-
echo "AKS_MGMT_KUBERNETES_VERSION: $AKS_MGMT_KUBERNETES_VERSION"
65-
echo "AZURE_LOCATION: $AZURE_LOCATION"
66-
echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE"
67-
echo "AKS_NODE_COUNT: $AKS_NODE_COUNT"
68-
echo "MGMT_CLUSTER_KUBECONFIG: $MGMT_CLUSTER_KUBECONFIG"
69-
echo "AZURE_IDENTITY_ID_FILEPATH: $AZURE_IDENTITY_ID_FILEPATH"
70-
echo "REGISTRY: $REGISTRY"
71-
echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME"
72-
echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR"
73-
echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR"
74-
echo "AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP"
75-
echo "AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME"
76-
echo "AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR"
77-
78-
echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID"
79-
echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID"
80-
echo "AZURE_TENANT_ID: $AZURE_TENANT_ID"
81-
echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX"
82-
echo "AKS_MI_CLIENT_ID: $AKS_MI_CLIENT_ID"
83-
echo "AKS_MI_OBJECT_ID: $AKS_MI_OBJECT_ID"
84-
echo "AKS_MI_RESOURCE_ID: $AKS_MI_RESOURCE_ID"
85-
echo "MANAGED_IDENTITY_NAME: $MANAGED_IDENTITY_NAME"
86-
echo "MANAGED_IDENTITY_RG: $MANAGED_IDENTITY_RG"
87-
echo "ASO_CREDENTIAL_SECRET_MODE: $ASO_CREDENTIAL_SECRET_MODE"
88-
echo "SKIP_AKS_CREATE: $SKIP_AKS_CREATE"
61+
echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME"
62+
echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP"
63+
echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP"
64+
echo "AKS_MGMT_KUBERNETES_VERSION: $AKS_MGMT_KUBERNETES_VERSION"
65+
echo "AZURE_LOCATION: $AZURE_LOCATION"
66+
echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE"
67+
echo "AKS_NODE_COUNT: $AKS_NODE_COUNT"
68+
echo "MGMT_CLUSTER_KUBECONFIG: $MGMT_CLUSTER_KUBECONFIG"
69+
echo "AZURE_IDENTITY_ID_FILEPATH: $AZURE_IDENTITY_ID_FILEPATH"
70+
echo "REGISTRY: $REGISTRY"
71+
echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME"
72+
echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR"
73+
echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR"
74+
echo "AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP"
75+
echo "AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME"
76+
echo "AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR"
77+
echo
78+
echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID"
79+
echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID"
80+
echo "AZURE_TENANT_ID: $AZURE_TENANT_ID"
81+
echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX"
82+
echo "AKS_MI_CLIENT_ID: $AKS_MI_CLIENT_ID"
83+
echo "AKS_MI_OBJECT_ID: $AKS_MI_OBJECT_ID"
84+
echo "AKS_MI_RESOURCE_ID: $AKS_MI_RESOURCE_ID"
85+
echo "MANAGED_IDENTITY_NAME: $MANAGED_IDENTITY_NAME"
86+
echo "MANAGED_IDENTITY_RG: $MANAGED_IDENTITY_RG"
87+
echo "ASO_CREDENTIAL_SECRET_MODE: $ASO_CREDENTIAL_SECRET_MODE"
88+
echo "SKIP_AKS_CREATE: $SKIP_AKS_CREATE"
89+
echo "IS_DEV_BOX: $IS_DEV_BOX"
90+
echo "AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: $AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY"
91+
echo "AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY: $AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY"
92+
echo "AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID: $AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID"
8993
echo "--------------------------------"
9094

9195
# if using SKIP_AKS_CREATE=true, skip creating the AKS cluster
@@ -155,31 +159,59 @@ create_aks_cluster() {
155159
az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \
156160
--overwrite-existing --only-show-errors
157161

158-
# echo "fetching Client ID for ${MGMT_CLUSTER_NAME}"
159-
AKS_MI_CLIENT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
160-
--only-show-errors | jq -r '.identityProfile.kubeletidentity.clientId')
161-
export AKS_MI_CLIENT_ID
162-
echo "mgmt client identity: ${AKS_MI_CLIENT_ID}"
163-
echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
164-
165-
# echo "fetching Object ID for ${MGMT_CLUSTER_NAME}"
166-
AKS_MI_OBJECT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
167-
--only-show-errors | jq -r '.identityProfile.kubeletidentity.objectId')
168-
export AKS_MI_OBJECT_ID
169-
echo "mgmt object identity: ${AKS_MI_OBJECT_ID}"
170-
171-
# echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}"
172-
AKS_MI_RESOURCE_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
173-
--only-show-errors | jq -r '.identityProfile.kubeletidentity.resourceId')
174-
export AKS_MI_RESOURCE_ID
175-
echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}"
176-
177-
# save resource identity name and resource group
178-
MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.name')
179-
# export MANAGED_IDENTITY_NAME
180-
echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}"
181-
USER_IDENTITY=$MANAGED_IDENTITY_NAME
182-
export USER_IDENTITY
162+
if [[ "${IS_DEV_BOX}" == "true" ]]; then
163+
echo "using the Managed Identity created by the user instead of the one created by AKS"
164+
# echo "fetching Client ID for ${MGMT_CLUSTER_NAME}"
165+
AKS_MI_CLIENT_ID=${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}
166+
export AKS_MI_CLIENT_ID
167+
echo "mgmt client identity: ${AKS_MI_CLIENT_ID}"
168+
echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
169+
170+
# echo "fetching Object ID for ${MGMT_CLUSTER_NAME}"
171+
AKS_MI_OBJECT_ID=${AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY}
172+
export AKS_MI_OBJECT_ID
173+
echo "mgmt object identity: ${AKS_MI_OBJECT_ID}"
174+
175+
# echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}"
176+
AKS_MI_RESOURCE_ID=${AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID}
177+
export AKS_MI_RESOURCE_ID
178+
echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}"
179+
180+
# save resource identity name and resource group
181+
MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.name')
182+
# export MANAGED_IDENTITY_NAME
183+
echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}"
184+
USER_IDENTITY=$MANAGED_IDENTITY_NAME
185+
export USER_IDENTITY
186+
187+
else
188+
# echo "fetching Client ID for ${MGMT_CLUSTER_NAME}"
189+
AKS_MI_CLIENT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
190+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.clientId')
191+
export AKS_MI_CLIENT_ID
192+
echo "mgmt client identity: ${AKS_MI_CLIENT_ID}"
193+
echo "${AKS_MI_CLIENT_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
194+
195+
# echo "fetching Object ID for ${MGMT_CLUSTER_NAME}"
196+
AKS_MI_OBJECT_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
197+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.objectId')
198+
export AKS_MI_OBJECT_ID
199+
echo "mgmt object identity: ${AKS_MI_OBJECT_ID}"
200+
201+
# echo "fetching Resource ID for ${MGMT_CLUSTER_NAME}"
202+
AKS_MI_RESOURCE_ID=$(az aks show -n "${MGMT_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --output json \
203+
--only-show-errors | jq -r '.identityProfile.kubeletidentity.resourceId')
204+
export AKS_MI_RESOURCE_ID
205+
echo "mgmt resource identity: ${AKS_MI_RESOURCE_ID}"
206+
207+
# save resource identity name and resource group
208+
MANAGED_IDENTITY_NAME=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.name')
209+
# export MANAGED_IDENTITY_NAME
210+
echo "mgmt resource identity name: ${MANAGED_IDENTITY_NAME}"
211+
USER_IDENTITY=$MANAGED_IDENTITY_NAME
212+
export USER_IDENTITY
213+
214+
fi
183215

184216
MANAGED_IDENTITY_RG=$(az identity show --ids "${AKS_MI_RESOURCE_ID}" --output json | jq -r '.resourceGroup')
185217
export MANAGED_IDENTITY_RG

0 commit comments

Comments
 (0)