Skip to content

Commit d716546

Browse files
Jont828Jont828
committed
Set up ci-entrypoint to work with AKS management cluster
1 parent d1899f8 commit d716546

File tree

3 files changed

+55
-20
lines changed

3 files changed

+55
-20
lines changed

‎Makefile‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -310,12 +310,16 @@ verify-codespell: codespell ## Verify codespell.
310310
##@ Development:
311311

312312
.PHONY: install-tools # populate hack/tools/bin
313-
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI)
313+
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI) $(YQ)
314314

315315
.PHONY: create-management-cluster
316316
create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
317-
# Create kind management cluster.
318-
$(MAKE) kind-create
317+
# Create management cluster.
318+
if [ -z "${USE_AKS_MANAGEMENT_CLUSTER}" ]; then \
319+
$(MAKE) kind-create ; \
320+
else \
321+
$(MAKE) aks-create ; \
322+
fi
319323

320324
# Install cert manager and wait for availability
321325
./hack/install-cert-manager.sh
@@ -331,7 +335,9 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
331335
timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.2.5/addon-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done"
332336

333337
# Deploy CAPZ
334-
$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME)
338+
@if [ -z "${USE_AKS_MANAGEMENT_CLUSTER}" ]; then \
339+
$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME) ; \
340+
fi
335341
timeout --foreground 300 bash -c "until $(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true; do sleep 5; done"
336342

337343
# Wait for CAPI deployments
@@ -361,16 +367,16 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
361367

362368
.PHONY: create-workload-cluster
363369
create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
364-
# Create workload Cluster.
365370
@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
366371
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
367372
fi; \
373+
# TODO: change this so it doesn't source aks-mgmt-vars.env when it is using a kind cluster
368374
if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
369-
timeout --foreground 300 bash -c "until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \
375+
timeout --foreground 300 bash -c "source aks-mgmt-vars.env && env && until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \
370376
elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \
371-
timeout --foreground 300 bash -c "until $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
377+
timeout --foreground 300 bash -c "source aks-mgmt-vars.env && env && until $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
372378
else \
373-
timeout --foreground 300 bash -c "until curl --retry "$(CURL_RETRIES)" "$(CLUSTER_TEMPLATE)" | "$(ENVSUBST)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
379+
timeout --foreground 300 bash -c "source aks-mgmt-vars.env && env && until curl --retry "$(CURL_RETRIES)" "$(CLUSTER_TEMPLATE)" | "$(ENVSUBST)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
374380
fi
375381

376382
# Wait for the kubeconfig to become available.

‎scripts/aks-as-mgmt.sh‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ set -o nounset # exit when script tries to use undeclared variables.
1818
set -o pipefail # make the pipeline fail if any command in it fails.
1919

2020
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
21+
# shellcheck source=hack/common-vars.sh
22+
source "${REPO_ROOT}/hack/common-vars.sh"
2123
# shellcheck source=hack/ensure-azcli.sh
2224
source "${REPO_ROOT}/hack/ensure-azcli.sh" # install az cli and login using WI
2325
# shellcheck source=hack/ensure-tags.sh
@@ -177,6 +179,28 @@ create_aks_cluster() {
177179
}
178180

179181
set_env_varaibles(){
182+
rm aks-mgmt-vars.env || true
183+
cat <<EOF > aks-mgmt-vars.env
184+
export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME}"
185+
export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP}"
186+
export AKS_NODE_RESOURCE_GROUP="${AKS_NODE_RESOURCE_GROUP}"
187+
export MGMT_CLUSTER_KUBECONFIG="${MGMT_CLUSTER_KUBECONFIG}"
188+
export AKS_MI_CLIENT_ID="${AKS_MI_CLIENT_ID}"
189+
export AZURE_CLIENT_ID="${AKS_MI_CLIENT_ID}"
190+
export AKS_MI_OBJECT_ID="${AKS_MI_OBJECT_ID}"
191+
export AKS_MI_RESOURCE_ID="${AKS_MI_RESOURCE_ID}"
192+
export MANAGED_IDENTITY_NAME="${MANAGED_IDENTITY_NAME}"
193+
export MANAGED_IDENTITY_RG="${MANAGED_IDENTITY_RG}"
194+
export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="${AKS_MI_CLIENT_ID}"
195+
export CI_RG="${MANAGED_IDENTITY_RG}"
196+
export USER_IDENTITY="${MANAGED_IDENTITY_NAME}"
197+
export CLUSTER_IDENTITY_TYPE="UserAssignedMSI"
198+
export ASO_CREDENTIAL_SECRET_MODE="${ASO_CREDENTIAL_SECRET_MODE}"
199+
export REGISTRY="${REGISTRY}"
200+
export APISERVER_LB_DNS_SUFFIX="${APISERVER_LB_DNS_SUFFIX}"
201+
export AZURE_LOCATION="${AZURE_LOCATION}"
202+
EOF
203+
180204
cat <<EOF > tilt-settings-temp.yaml
181205
kustomize_substitutions:
182206
MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}"
@@ -210,28 +234,28 @@ else
210234
fi
211235

212236
# copy over the existing allowed_contexts to tilt-settings.yaml if it does not exist
213-
allowed_contexts_exists=$(yq eval '.allowed_contexts' tilt-settings.yaml)
237+
allowed_contexts_exists=$(${YQ} eval '.allowed_contexts' tilt-settings.yaml)
214238
if [ "$allowed_contexts_exists" == "null" ]; then
215-
yq eval '.allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
239+
${YQ} eval '.allowed_contexts = load("tilt-settings-temp.yaml") | .allowed_contexts' tilt-settings-temp.yaml > tilt-settings.yaml
216240
fi
217241

218242
# extract allowed_contexts from tilt-settings.yaml
219-
current_contexts=$(yq eval '.allowed_contexts' tilt-settings.yaml | sort -u)
243+
current_contexts=$(${YQ} eval '.allowed_contexts' tilt-settings.yaml | sort -u)
220244

221245
# extract allowed_contexts from tilt-settings-new.yaml
222-
new_contexts=$(yq eval '.allowed_contexts' tilt-settings-temp.yaml | sort -u)
246+
new_contexts=$(${YQ} eval '.allowed_contexts' tilt-settings-temp.yaml | sort -u)
223247

224248
# combine current and new contexts, keeping the union of both
225249
combined_contexts=$(echo "$current_contexts"$'\n'"$new_contexts" | sort -u)
226250

227-
# create a temporary file since env($combined_contexts) is not supported in yq
251+
# create a temporary file since env($combined_contexts) is not supported in ${YQ}
228252
echo "$combined_contexts" > combined_contexts.yaml
229253

230254
# update allowed_contexts in tilt-settings.yaml with the combined contexts
231-
yq eval --inplace ".allowed_contexts = load(\"combined_contexts.yaml\")" tilt-settings.yaml
255+
${YQ} eval --inplace ".allowed_contexts = load(\"combined_contexts.yaml\")" tilt-settings.yaml
232256

233257
# merge the updated kustomize_substitution and azure_location with the existing one in tilt-settings.yaml
234-
yq eval-all 'select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml
258+
${YQ} eval-all 'select(fileIndex == 0) *+ {"kustomize_substitutions": select(fileIndex == 1).kustomize_substitutions, "azure_location": select(fileIndex == 1).azure_location}' tilt-settings.yaml tilt-settings-temp.yaml > tilt-settings-new.yaml
235259

236260
mv tilt-settings-new.yaml tilt-settings.yaml
237261
rm -r combined_contexts.yaml

‎scripts/ci-entrypoint.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,16 @@ select_cluster_template() {
133133

134134
create_cluster() {
135135
"${REPO_ROOT}/hack/create-dev-cluster.sh"
136-
if [ ! -f "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" ]; then
137-
echo "Unable to find kubeconfig for kind mgmt cluster ${KIND_CLUSTER_NAME}"
138-
exit 1
139-
fi
140-
"${KUBECTL}" --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" get clusters -A
136+
if [ -z "${USE_AKS_MANAGEMENT_CLUSTER}" ]; then
137+
if [ ! -f "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" ]; then
138+
echo "Unable to find kubeconfig for kind mgmt cluster ${KIND_CLUSTER_NAME}"
139+
exit 1
140+
fi
141+
"${KUBECTL}" --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" get clusters -A
142+
else
143+
"${KUBECTL}" get clusters -A
144+
fi;
145+
141146

142147
# set the SSH bastion and user that can be used to SSH into nodes
143148
KUBE_SSH_BASTION=$(${KUBECTL} get azurecluster -o json | jq '.items[0].spec.networkSpec.apiServerLB.frontendIPs[0].publicIP.dnsName' | tr -d \"):22

0 commit comments

Comments
 (0)