Merge pull request #4765 from jackfrancis/capz-e2e-workload-identity

CI: workload-identity native

Author: k8s-ci-robot

.gitignore

@@ -86,3 +86,11 @@ release-*/manifests/calico-*.yaml
 # mentioned in the capz book
 /sp.json
 /cluster.yaml
+
+# CI workload-identity
+jwks.json
+*.pub
+*.key
+azure_identity_id
+azure_wi_back_compat
+openid-configuration.json

Makefile

@@ -89,6 +89,10 @@ KUSTOMIZE_VER := v5.4.1
 KUSTOMIZE_BIN := kustomize
 KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)
 
+AZWI_VER := v1.2.2
+AZWI_BIN := azwi
+AZWI := $(TOOLS_BIN_DIR)/$(AZWI_BIN)-$(AZWI_VER)
+
 MOCKGEN_VER := v0.4.0
 MOCKGEN_BIN := mockgen
 MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN)-$(MOCKGEN_VER)
@@ -177,6 +181,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
+AZWI_SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
 # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
 # to be created with extra mounts for key pairs which is not yet supported
@@ -191,6 +196,11 @@ LDFLAGS := $(shell hack/version.sh)
 CLUSTER_TEMPLATE ?= cluster-template.yaml
 
 export KIND_CLUSTER_NAME ?= capz
+export RANDOM_SUFFIX := $(shell /bin/bash -c "echo $$RANDOM")
+export AZWI_RESOURCE_GROUP ?= capz-wi-$(RANDOM_SUFFIX)
+export CI_RG ?= $(AZWI_RESOURCE_GROUP)
+export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG))
+export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id
 
 ## --------------------------------------
 ## Binaries
@@ -287,7 +297,7 @@ verify-codespell: codespell ## Verify codespell.
 ##@ Development:
 
 .PHONY: install-tools # populate hack/tools/bin
-install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND)
+install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI)
 
 .PHONY: create-management-cluster
 create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
@@ -341,7 +351,10 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 .PHONY: create-workload-cluster
 create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
 	# Create workload Cluster.
-	@if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
+	@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
+		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
+	fi; \
+	if [ -f "$(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE)" ]; then \
 		timeout --foreground 300 bash -c "until $(ENVSUBST) < $(TEMPLATES_DIR)/$(CLUSTER_TEMPLATE) | $(KUBECTL) apply -f -; do sleep 5; done"; \
 	elif [ -f "$(CLUSTER_TEMPLATE)" ]; then \
 		timeout --foreground 300 bash -c "until $(ENVSUBST) < "$(CLUSTER_TEMPLATE)" | $(KUBECTL) apply -f -; do sleep 5; done"; \
@@ -686,7 +699,13 @@ test-cover: test ## Run tests with code coverage and generate reports.
 
 .PHONY: kind-create-bootstrap
 kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
-	export AZWI=$${AZWI:-true} KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
+	KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh
+
+.PHONY: cleanup-workload-identity
+cleanup-workload-identity: ## Cleanup CI workload-identity infra
+	@if ! [ "$(AZWI_SKIP_CLEANUP)" == "true" ]; then \
+		./scripts/cleanup-workload-identity.sh \
+	fi
 
 ## --------------------------------------
 ## Security Scanning
@@ -708,6 +727,9 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed.
 
 .PHONY: tilt-up
 tilt-up: install-tools kind-create ## Start tilt and build kind cluster if needed.
+	@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
+		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
+	fi; \
 	CLUSTER_TOPOLOGY=true EXP_CLUSTER_RESOURCE_SET=true EXP_MACHINE_POOL=true EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION=true EXP_EDGEZONE=true tilt up
 
 .PHONY: delete-cluster
@@ -792,6 +814,16 @@ $(HELM): ## Put helm into tools folder.
 	ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
 	rm -f $(TOOLS_BIN_DIR)/get_helm.sh
 
+$(AZWI): ## Put azwi into tools folder.
+	mkdir -p $(TOOLS_BIN_DIR)
+	rm -f "$(TOOLS_BIN_DIR)/$(AZWI_BIN)*"
+	curl --retry $(CURL_RETRIES) -fsSL -o $(TOOLS_BIN_DIR)/azwi.tar.gz https://github.com/Azure/azure-workload-identity/releases/download/$(AZWI_VER)/azwi-$(AZWI_VER)-$(GOOS)-$(GOARCH).tar.gz
+	tar -xf "$(TOOLS_BIN_DIR)/azwi.tar.gz" -C $(TOOLS_BIN_DIR) $(AZWI_BIN)
+	mv "$(TOOLS_BIN_DIR)/$(AZWI_BIN)" $(AZWI)
+	ln -sf $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
+	chmod +x $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
+	rm -f $(TOOLS_BIN_DIR)/azwi.tar.gz
+
 $(KIND): ## Build kind into tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER)
 

Tiltfile

@@ -35,7 +35,7 @@ settings = {
 }
 
 # Auth keys that need to be loaded from the environment
-keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_SECRET", "AZURE_CLIENT_ID"]
+keys = ["AZURE_SUBSCRIPTION_ID", "AZURE_TENANT_ID", "AZURE_CLIENT_ID"]
 
 # Get global settings from tilt-settings.yaml or tilt-settings.json
 tilt_file = "./tilt-settings.yaml" if os.path.exists("./tilt-settings.yaml") else "./tilt-settings.json"
@@ -266,13 +266,10 @@ def create_identity_secret():
 
     os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAME", "cluster-identity-secret")
     os.putenv("AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE", "default")
-    os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity")
+    os.putenv("CLUSTER_IDENTITY_NAME", "cluster-identity-ci")
     os.putenv("ASO_CREDENTIAL_SECRET_NAME", "aso-credentials")
 
-    os.putenv("AZURE_CLIENT_SECRET_B64", base64_encode(os.environ.get("AZURE_CLIENT_SECRET")))
-    local("cat templates/azure-cluster-identity/secret.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True)
     local("cat templates/flavors/aks-aso/credentials.yaml | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -", quiet = True, echo_off = True)
-    os.unsetenv("AZURE_CLIENT_SECRET_B64")
 
 def create_crs():
     # create config maps

docs/book/src/topics/workload-identity.md

@@ -169,7 +169,7 @@ to give the identity Contributor access to the Azure subscription where the work
 - At this stage, you can apply this yaml to create a workload cluster.
 
 Notes:
-- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow-workload-identity.yaml)
+- Please follow this [link](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/test/ci/cluster-template-prow.yaml)
 to see a workload cluster yaml configuration that uses workload identity.
 - Creating a workload cluster via workload identity will be
   simplified after [this](https://github.com/kubernetes-sigs/cluster-api-provider-azure/issues/3589) issue is resolved.

e2e.mk

@@ -4,15 +4,18 @@
 # long-running E2E jobs every time that file changes
 
 ##@ E2E Testing:
-
 .PHONY: test-e2e-run
 test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
-	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-    $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
-    	-e2e.artifacts-folder="$(ARTIFACTS)" \
-    	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-    	-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
-    	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
+	@$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
+	if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
+		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
+	fi; \
+	$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
+		-e2e.artifacts-folder="$(ARTIFACTS)" \
+		-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+		-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
+		-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS) \
+	$(MAKE) cleanup-workload-identity
 	$(MAKE) clean-release-git
 
 .PHONY: test-e2e

hack/create-dev-cluster.sh

@@ -38,6 +38,11 @@ export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet
 export AZURE_LOCATION="${AZURE_LOCATION:-southcentralus}"
 export AZURE_RESOURCE_GROUP=${CLUSTER_NAME}
 
+AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:=}"
+AZURE_TENANT_ID="${AZURE_TENANT_ID:=}"
+AZURE_CLIENT_ID="${AZURE_CLIENT_ID:=}"
+AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET:=}"
+
 AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')"
 AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')"
 AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')"

hack/log/redact.sh

@@ -28,10 +28,12 @@ redact_vars=(
     "${AZURE_SUBSCRIPTION_ID:-}"
     "${AZURE_TENANT_ID:-}"
     "${AZURE_JSON_B64:-}"
+    "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}"
     "$(echo -n "${AZURE_SUBSCRIPTION_ID:-}" | base64 | tr -d '\n')"
     "$(echo -n "${AZURE_TENANT_ID:-}" | base64 | tr -d '\n')"
     "$(echo -n "${AZURE_CLIENT_ID:-}" | base64 | tr -d '\n')"
     "$(echo -n "${AZURE_CLIENT_SECRET:-}" | base64 | tr -d '\n')"
+    "$(echo -n "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY:-}" | base64 | tr -d '\n')"
 )
 
 for log_file in "${log_files[@]}"; do

hack/util.sh

@@ -108,6 +108,4 @@ capz::util::generate_ssh_key() {
 capz::util::ensure_azure_envs() {
     : "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
     : "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
-    : "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
-    : "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
 }

scripts/cleanup-workload-identity.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Install kubectl and kind
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+# shellcheck source=hack/ensure-azcli.sh
+source "${REPO_ROOT}/hack/ensure-azcli.sh"
+
+AZWI_RESOURCE_GROUP="${AZWI_RESOURCE_GROUP:-}"
+
+if [[ -z "${AZWI_RESOURCE_GROUP}" ]]; then
+  echo AZWI_RESOURCE_GROUP environment variable must be set
+  exit 1
+fi
+
+echo "Cleaning up CI workload-identity infra..."
+az group delete --no-wait -y -n "${AZWI_RESOURCE_GROUP}"