@@ -18,6 +18,8 @@ set -o nounset # exit when script tries to use undeclared variables.
1818set -o pipefail # make the pipeline fail if any command in it fails.
1919
2020REPO_ROOT=$( dirname " ${BASH_SOURCE[0]} " ) /..
21+ # shellcheck source=hack/common-vars.sh
22+ source " ${REPO_ROOT} /hack/common-vars.sh"
2123# shellcheck source=hack/ensure-azcli.sh
2224source " ${REPO_ROOT} /hack/ensure-azcli.sh" # install az cli and login using WI
2325# shellcheck source=hack/ensure-tags.sh
@@ -30,7 +32,7 @@ make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${AZWI##*/}"
3032export MGMT_CLUSTER_NAME=" ${MGMT_CLUSTER_NAME:- aks-mgmt-capz-${RANDOM_SUFFIX} } " # management cluster name
3133export AKS_RESOURCE_GROUP=" ${AKS_RESOURCE_GROUP:- aks-mgmt-capz-${RANDOM_SUFFIX} } " # resource group name
3234export AKS_NODE_RESOURCE_GROUP=" node-${AKS_RESOURCE_GROUP} "
33- export KUBERNETES_VERSION =" ${KUBERNETES_VERSION :- v1.30.2} "
35+ export AKS_MGMT_KUBERNETES_VERSION =" ${AKS_MGMT_KUBERNETES_VERSION :- v1.30.2} "
3436export AZURE_LOCATION=" ${AZURE_LOCATION:- westus2} "
3537export AKS_NODE_VM_SIZE=" ${AKS_NODE_VM_SIZE:- " Standard_B2s" } "
3638export AKS_NODE_COUNT=" ${AKS_NODE_COUNT:- 1} "
@@ -42,6 +44,12 @@ export AZWI_STORAGE_CONTAINER="\$web"
4244export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH=" ${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:- } "
4345export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH=" ${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:- } "
4446export REGISTRY=" ${REGISTRY:- } "
47+ export AKS_MGMT_VNET_NAME=" ${AKS_MGMT_VNET_NAME:- " aks-mgmt-vnet-${RANDOM_SUFFIX} " } "
48+ export AKS_MGMT_VNET_CIDR=" ${AKS_MGMT_VNET_CIDR:- " 20.255.0.0/16" } "
49+ export AKS_MGMT_SERVICE_CIDR=" ${AKS_MGMT_SERVICE_CIDR:- " 20.255.254.0/24" } "
50+ export AKS_MGMT_DNS_SERVICE_IP=" ${AKS_MGMT_DNS_SERVICE_IP:- " 20.255.254.100" } "
51+ export AKS_MGMT_SUBNET_NAME=" ${AKS_MGMT_SUBNET_NAME:- " aks-mgmt-subnet-${RANDOM_SUFFIX} " } "
52+ export AKS_MGMT_SUBNET_CIDR=" ${AKS_MGMT_SUBNET_CIDR:- " 20.255.0.0/24" } "
4553
4654export AZURE_SUBSCRIPTION_ID=" ${AZURE_SUBSCRIPTION_ID:- } "
4755export AZURE_CLIENT_ID=" ${AZURE_CLIENT_ID:- } "
@@ -63,7 +71,7 @@ main() {
6371 echo " MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME "
6472 echo " AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP "
6573 echo " AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP "
66- echo " KUBERNETES_VERSION : $KUBERNETES_VERSION "
74+ echo " AKS_MGMT_KUBERNETES_VERSION : $AKS_MGMT_KUBERNETES_VERSION "
6775 echo " AZURE_LOCATION: $AZURE_LOCATION "
6876 echo " AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE "
6977 echo " AZURE_NODE_MACHINE_TYPE: $AZURE_NODE_MACHINE_TYPE "
@@ -76,6 +84,12 @@ main() {
7684 echo " SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH "
7785 echo " REGISTRY: $REGISTRY "
7886 echo " APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX "
87+ echo " AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME "
88+ echo " AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR "
89+ echo " AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR "
90+ echo " AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP "
91+ echo " AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME "
92+ echo " AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR "
7993
8094 echo " AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID "
8195 echo " AZURE_CLIENT_ID: $AZURE_CLIENT_ID "
@@ -102,6 +116,17 @@ create_aks_cluster() {
102116 --location " ${AZURE_LOCATION} " \
103117 --output none --only-show-errors \
104118 --tags creationTimestamp=" ${TIMESTAMP} " jobName=" ${JOB_NAME} " buildProvenance=" ${BUILD_PROVENANCE} "
119+
120+
121+ echo " creating vnet for the resource group ${AKS_RESOURCE_GROUP} "
122+ az network vnet create \
123+ --resource-group " ${AKS_RESOURCE_GROUP} " \
124+ --name " ${AKS_MGMT_VNET_NAME} " \
125+ --address-prefix " ${AKS_MGMT_VNET_CIDR} " \
126+ --subnet-name " ${AKS_MGMT_SUBNET_NAME} " \
127+ --subnet-prefix " ${AKS_MGMT_SUBNET_CIDR} " \
128+ --output none --only-show-errors \
129+ --tags creationTimestamp=" ${TIMESTAMP} " jobName=" ${JOB_NAME} " buildProvenance=" ${BUILD_PROVENANCE} "
105130 fi
106131
107132 aks_exists=$( az aks show --name " ${MGMT_CLUSTER_NAME} " --resource-group " ${AKS_RESOURCE_GROUP} " 2>&1 || true) # true because we want to continue if the command fails
@@ -110,13 +135,16 @@ create_aks_cluster() {
110135 az aks create --name " ${MGMT_CLUSTER_NAME} " \
111136 --resource-group " ${AKS_RESOURCE_GROUP} " \
112137 --location " ${AZURE_LOCATION} " \
113- --kubernetes-version " ${KUBERNETES_VERSION } " \
138+ --kubernetes-version " ${AKS_MGMT_KUBERNETES_VERSION } " \
114139 --node-count " ${AKS_NODE_COUNT} " \
115140 --node-vm-size " ${AKS_NODE_VM_SIZE} " \
116141 --node-resource-group " ${AKS_NODE_RESOURCE_GROUP} " \
117142 --vm-set-type VirtualMachineScaleSets \
118143 --generate-ssh-keys \
119144 --network-plugin azure \
145+ --vnet-subnet-id " /subscriptions/${AZURE_SUBSCRIPTION_ID} /resourceGroups/${AKS_RESOURCE_GROUP} /providers/Microsoft.Network/virtualNetworks/${AKS_MGMT_VNET_NAME} /subnets/${AKS_MGMT_SUBNET_NAME} " \
146+ --service-cidr " ${AKS_MGMT_SERVICE_CIDR} " \
147+ --dns-service-ip " ${AKS_MGMT_DNS_SERVICE_IP} " \
120148 --tags creationTimestamp=" ${TIMESTAMP} " jobName=" ${JOB_NAME} " buildProvenance=" ${BUILD_PROVENANCE} " \
121149 --output none --only-show-errors;
122150 elif echo " $aks_exists " | grep -q " ${MGMT_CLUSTER_NAME} " ; then
@@ -127,6 +155,7 @@ create_aks_cluster() {
127155 fi
128156
129157 # check and save kubeconfig
158+ echo -e " \n"
130159 echo " saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT} /${MGMT_CLUSTER_KUBECONFIG} "
131160 az aks get-credentials --name " ${MGMT_CLUSTER_NAME} " --resource-group " ${AKS_RESOURCE_GROUP} " \
132161 --file " ${REPO_ROOT} /${MGMT_CLUSTER_KUBECONFIG} " --only-show-errors
@@ -172,11 +201,50 @@ create_aks_cluster() {
172201 sleep 5
173202 done
174203
204+ # If storage account var is set:
205+ if [ -n " ${AZURE_STORAGE_ACCOUNT} " ]; then
206+ echo " assigning storage blob data reader role to the service principal"
207+ until az role assignment create --assignee-object-id " ${AKS_MI_OBJECT_ID} " --role " Storage Blob Data Reader" \
208+ --scope " /subscriptions/${AZURE_SUBSCRIPTION_ID} /resourceGroups/${AZURE_STORAGE_ACCOUNT_RESOURCE_GROUP} /providers/Microsoft.Storage/storageAccounts/${AZURE_STORAGE_ACCOUNT} /blobServices/default/containers/${AZURE_BLOB_CONTAINER_NAME} " \
209+ --assignee-principal-type ServicePrincipal; do
210+ echo " retrying to assign storage blob data reader role to the service principal"
211+ sleep 5
212+ done
213+ fi
214+
175215 echo " using ASO_CREDENTIAL_SECRET_MODE as podidentity"
176216 ASO_CREDENTIAL_SECRET_MODE=" podidentity"
177217}
178218
179219set_env_varaibles (){
220+ rm aks-mgmt-vars.env || true
221+ cat << EOF > aks-mgmt-vars.env
222+ export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME} "
223+ export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP} "
224+ export AKS_NODE_RESOURCE_GROUP="${AKS_NODE_RESOURCE_GROUP} "
225+ export MGMT_CLUSTER_KUBECONFIG="${MGMT_CLUSTER_KUBECONFIG} "
226+ export AKS_MI_CLIENT_ID="${AKS_MI_CLIENT_ID} "
227+ export AZURE_CLIENT_ID="${AKS_MI_CLIENT_ID} "
228+ export AKS_MI_OBJECT_ID="${AKS_MI_OBJECT_ID} "
229+ export AKS_MI_RESOURCE_ID="${AKS_MI_RESOURCE_ID} "
230+ export MANAGED_IDENTITY_NAME="${MANAGED_IDENTITY_NAME} "
231+ export MANAGED_IDENTITY_RG="${MANAGED_IDENTITY_RG} "
232+ export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="${AKS_MI_CLIENT_ID} "
233+ export CI_RG="${MANAGED_IDENTITY_RG} "
234+ export USER_IDENTITY="${MANAGED_IDENTITY_NAME} "
235+ export CLUSTER_IDENTITY_TYPE="UserAssignedMSI"
236+ export ASO_CREDENTIAL_SECRET_MODE="${ASO_CREDENTIAL_SECRET_MODE} "
237+ export REGISTRY="${REGISTRY} "
238+ export APISERVER_LB_DNS_SUFFIX="${APISERVER_LB_DNS_SUFFIX} "
239+ export AZURE_LOCATION="${AZURE_LOCATION} "
240+ export AKS_MGMT_VNET_NAME="${AKS_MGMT_VNET_NAME} "
241+ export AKS_MGMT_VNET_CIDR="${AKS_MGMT_VNET_CIDR} "
242+ export AKS_MGMT_SERVICE_CIDR="${AKS_MGMT_SERVICE_CIDR} "
243+ export AKS_MGMT_DNS_SERVICE_IP="${AKS_MGMT_DNS_SERVICE_IP} "
244+ export AKS_MGMT_SUBNET_NAME="${AKS_MGMT_SUBNET_NAME} "
245+ export AKS_MGMT_SUBNET_CIDR="${AKS_MGMT_SUBNET_CIDR} "
246+ EOF
247+
180248 cat << EOF > tilt-settings-temp.yaml
181249kustomize_substitutions:
182250 MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME} "
@@ -210,28 +278,28 @@ else
210278fi
211279
212280# copy over the existing allowed_contexts to tilt-settings.yaml if it does not exist
213- allowed_contexts_exists=$( yq eval ' .allowed_contexts' tilt-settings.yaml)
281+ allowed_contexts_exists=$( ${YQ} eval ' .allowed_contexts' tilt-settings.yaml)
214282if [ " $allowed_contexts_exists " == " null" ]; then
215- yq eval ' .allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
283+ ${YQ} eval ' .allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
216284fi
217285
218286# extract allowed_contexts from tilt-settings.yaml
219- current_contexts=$( yq eval ' .allowed_contexts' tilt-settings.yaml | sort -u)
287+ current_contexts=$( ${YQ} eval ' .allowed_contexts' tilt-settings.yaml | sort -u)
220288
221289# extract allowed_contexts from tilt-settings-new.yaml
222- new_contexts=$( yq eval ' .allowed_contexts' tilt-settings-temp.yaml | sort -u)
290+ new_contexts=$( ${YQ} eval ' .allowed_contexts' tilt-settings-temp.yaml | sort -u)
223291
224292# combine current and new contexts, keeping the union of both
225293combined_contexts=$( echo " $current_contexts " $' \n ' " $new_contexts " | sort -u)
226294
227- # create a temporary file since env($combined_contexts) is not supported in yq
295+ # create a temporary file since env($combined_contexts) is not supported in ${YQ}
228296echo " $combined_contexts " > combined_contexts.yaml
229297
230298# update allowed_contexts in tilt-settings.yaml with the combined contexts
231- yq eval --inplace " .allowed_contexts = load(\" combined_contexts.yaml\" )" tilt-settings.yaml
299+ ${YQ} eval --inplace " .allowed_contexts = load(\" combined_contexts.yaml\" )" tilt-settings.yaml
232300
233301# merge the updated kustomize_substitution and azure_location with the existing one in tilt-settings.yaml
234- yq eval-all ' select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml
302+ ${YQ} eval-all ' select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml
235303
236304mv tilt-settings-new.yaml tilt-settings.yaml
237305rm -r combined_contexts.yaml
0 commit comments