CI: workload-identity native

Signed-off-by: Jack Francis <[email protected]>

Author: jackfrancis

.gitignore

@@ -86,3 +86,10 @@ release-*/manifests/calico-*.yaml
 # mentioned in the capz book
 /sp.json
 /cluster.yaml
+
+# CI workload-identity
+jwks.json
+*.pub
+*.key
+azure_identity_id
+openid-configuration.json

Makefile

@@ -89,6 +89,10 @@ KUSTOMIZE_VER := v5.4.1
 KUSTOMIZE_BIN := kustomize
 KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)
 
+AZWI_VER := v1.2.2
+AZWI_BIN := azwi
+AZWI := $(TOOLS_BIN_DIR)/$(AZWI_BIN)-$(AZWI_VER)
+
 MOCKGEN_VER := v0.4.0
 MOCKGEN_BIN := mockgen
 MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN)-$(MOCKGEN_VER)
@@ -283,7 +287,7 @@ verify-codespell: codespell ## Verify codespell.
 ##@ Development:
 
 .PHONY: install-tools # populate hack/tools/bin
-install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND)
+install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI)
 
 .PHONY: create-management-cluster
 create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
@@ -682,7 +686,11 @@ test-cover: test ## Run tests with code coverage and generate reports.
 
 .PHONY: kind-create-bootstrap
 kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
-	export AZWI=$${AZWI:-true} KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
+	KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
+
+.PHONY: cleanup-workload-identity
+cleanup-workload-identity: ## Cleanup CI workload-identity infra
+	./scripts/cleanup-workload-identity.sh
 
 ## --------------------------------------
 ## Security Scanning
@@ -788,6 +796,16 @@ $(HELM): ## Put helm into tools folder.
 	ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
 	rm -f $(TOOLS_BIN_DIR)/get_helm.sh
 
+$(AZWI): ## Put azwi into tools folder.
+	mkdir -p $(TOOLS_BIN_DIR)
+	rm -f "$(TOOLS_BIN_DIR)/$(AZWI_BIN)*"
+	curl --retry $(CURL_RETRIES) -fsSL -o $(TOOLS_BIN_DIR)/azwi.tar.gz https://github.com/Azure/azure-workload-identity/releases/download/$(AZWI_VER)/azwi-$(AZWI_VER)-$(GOOS)-$(GOARCH).tar.gz
+	tar -xf "$(TOOLS_BIN_DIR)/azwi.tar.gz" -C $(TOOLS_BIN_DIR) $(AZWI_BIN)
+	mv "$(TOOLS_BIN_DIR)/$(AZWI_BIN)" $(AZWI)
+	ln -sf $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
+	chmod +x $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
+	rm -f $(TOOLS_BIN_DIR)/azwi.tar.gz
+
 $(KIND): ## Build kind into tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER)
 

e2e.mk

@@ -4,16 +4,23 @@
 # long-running E2E jobs every time that file changes
 
 ##@ E2E Testing:
+RANDOM_SUFFIX := $(shell /bin/bash -c "echo $$RANDOM")
+export AZWI_RESOURCE_GROUP ?= capz-ci-$(RANDOM_SUFFIX)
+export CI_RG ?= $(AZWI_RESOURCE_GROUP)
+export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG))
+export AZWI_LOCATION ?= eastus
+export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id
 
 .PHONY: test-e2e-run
 test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
 	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
-    $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
+    AZURE_CLIENT_ID=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)) $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
     	-e2e.artifacts-folder="$(ARTIFACTS)" \
     	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
     	-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
     	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
 	$(MAKE) clean-release-git
+	AZWI_RESOURCE_GROUP=$(AZWI_RESOURCE_GROUP) $(MAKE) cleanup-workload-identity
 
 .PHONY: test-e2e
 test-e2e: ## Run "docker-build" and "docker-push" rules then run e2e tests.

hack/util.sh

@@ -108,6 +108,4 @@ capz::util::generate_ssh_key() {
 capz::util::ensure_azure_envs() {
     : "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
     : "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
-    : "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
-    : "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
 }

scripts/cleanup-workload-identity.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2024 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Install kubectl and kind
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+# shellcheck source=hack/ensure-azcli.sh
+source "${REPO_ROOT}/hack/ensure-azcli.sh"
+
+AZWI_RESOURCE_GROUP=${AZWI_RESOURCE_GROUP:-}
+
+if [[ -z "${AZWI_RESOURCE_GROUP}" ]]; then
+  echo AZWI_RESOURCE_GROUP environment variable must be set
+  exit 1
+fi
+
+echo "Cleaning up CI workload-identity infra..."
+az group delete --no-wait -y -n $AZWI_RESOURCE_GROUP

scripts/kind-with-registry.sh

@@ -19,9 +19,20 @@ set -o pipefail
 
 # Install kubectl and kind
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+# shellcheck source=hack/ensure-azcli.sh
+source "${REPO_ROOT}/hack/ensure-azcli.sh"
+
 KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
 KIND="${REPO_ROOT}/hack/tools/bin/kind"
-AZWI_ENABLED=${AZWI:-}
+AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
+AZWI_ENABLED=${AZWI_ENABLED:-true}
+RAND_SUFFIX=$(openssl rand -hex 4)
+AZWI_STORAGE_ACCOUNT="oidcissuer${RAND_SUFFIX}"
+AZWI_STORAGE_CONTAINER="oidc"
+SERVICE_ACCOUNT_ISSUER=${SERVICE_ACCOUNT_ISSUER:-}
+SERVICE_ACCOUNT_SIGNING_PUB=${SERVICE_ACCOUNT_SIGNING_PUB:-}
+SERVICE_ACCOUNT_SIGNING_KEY=${SERVICE_ACCOUNT_SIGNING_KEY:-}
+AZWI_JWKS_JSON_FILEPATH=${AZWI_JWKS_JSON_FILEPATH:-}
 make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}"
 
 # Export desired cluster name; default is "capz"
@@ -44,35 +55,112 @@ fi
 
 # To use workload identity, service account signing key pairs base64 encoded should be exposed via the
 # env variables. The function creates the key pair files after reading it from the env variables.
+# TODO we need to document that these env vars are a new requirement
 function checkAZWIENVPreReqsAndCreateFiles() {
-  if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then
-    echo "'SERVICE_ACCOUNT_SIGNING_PUB' is not set."
-    exit 1
+  # check if user is logged into azure cli
+  if ! az account show > /dev/null 2>&1; then
+      echo "Please login to Azure CLI using 'az login'"
+      exit 1
   fi
 
+  if [ "$(az group exists --name "${AZWI_RESOURCE_GROUP}" --output tsv)" == 'false' ]; then
+    echo "Creating resource group '${AZWI_RESOURCE_GROUP}' in '${AZWI_LOCATION}'"
+    az group create --name "${AZWI_RESOURCE_GROUP}" --location "${AZWI_LOCATION}" --output none --only-show-errors
+  fi
+  if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then
+    echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'"
+    az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --allow-blob-public-access true --output none --only-show-errors
+  fi
+  if ! az storage container show --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" > /dev/null 2>&1; then
+    echo "Creating storage container '${AZWI_STORAGE_CONTAINER}' in '${AZWI_STORAGE_ACCOUNT}'"
+    az storage container create --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" --public-access blob --output none --only-show-errors
+  fi
+  if [[ -z "${SERVICE_ACCOUNT_ISSUER}" ]]; then
+    export SERVICE_ACCOUNT_ISSUER="https://${AZWI_STORAGE_ACCOUNT}.blob.core.windows.net/${AZWI_STORAGE_CONTAINER}/"
+  fi
+  AZWI_OPENID_CONFIG_FILEPATH="${REPO_ROOT}/openid-configuration.json"
+  cat <<EOF > $AZWI_OPENID_CONFIG_FILEPATH
+{
+  "issuer": "${SERVICE_ACCOUNT_ISSUER}",
+  "jwks_uri": "${SERVICE_ACCOUNT_ISSUER}openid/v1/jwks",
+  "response_types_supported": [
+    "id_token"
+  ],
+  "subject_types_supported": [
+    "public"
+  ],
+  "id_token_signing_alg_values_supported": [
+    "RS256"
+  ]
+}
+EOF
+  if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then
+    SERVICE_ACCOUNT_SIGNING_PUB="${REPO_ROOT}/capz-ci-sa.pub"
+  fi
   if [[ -z "${SERVICE_ACCOUNT_SIGNING_KEY}" ]]; then
-    echo "'SERVICE_ACCOUNT_SIGNING_KEY' is not set."
-    exit 1
+    SERVICE_ACCOUNT_SIGNING_KEY="${REPO_ROOT}/capz-ci-sa.key"
+  fi
+  if ! [ -f $SERVICE_ACCOUNT_SIGNING_KEY ]; then
+    openssl genrsa -out $SERVICE_ACCOUNT_SIGNING_KEY 2048
+  fi
+  if ! [ -f $SERVICE_ACCOUNT_SIGNING_PUB ]; then
+    openssl rsa -in $SERVICE_ACCOUNT_SIGNING_KEY -pubout -out $SERVICE_ACCOUNT_SIGNING_PUB
+  fi
+  if [[ -z "${AZWI_JWKS_JSON_FILEPATH}" ]]; then
+    AZWI_JWKS_JSON_FILEPATH="${REPO_ROOT}/jwks.json"
   fi
-  mkdir -p "$HOME"/azwi/creds
-  echo "${SERVICE_ACCOUNT_SIGNING_PUB}" > "$HOME"/azwi/creds/sa.pub
-  echo  "${SERVICE_ACCOUNT_SIGNING_KEY}" > "$HOME"/azwi/creds/sa.key
-  SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER:-https://oidcissuercapzci.blob.core.windows.net/oidc-capzci/}"
+  if ! [ -f $AZWI_JWKS_JSON_FILEPATH ]; then
+    "${AZWI}" jwks --public-keys $SERVICE_ACCOUNT_SIGNING_PUB --output-file "${AZWI_JWKS_JSON_FILEPATH}"
+  fi
+  echo "Uploading openid-configuration document to '${AZWI_STORAGE_ACCOUNT}' storage account"
+  upload_to_blob "${AZWI_STORAGE_CONTAINER}" "${AZWI_OPENID_CONFIG_FILEPATH}" ".well-known/openid-configuration"
+  echo "Uploading jwks document to '${AZWI_STORAGE_ACCOUNT}' storage account"
+  upload_to_blob "${AZWI_STORAGE_CONTAINER}" "${AZWI_JWKS_JSON_FILEPATH}" "openid/v1/jwks"
+  az identity create -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" -l "${AZWI_LOCATION}" --output none --only-show-errors
+  AZURE_IDENTITY_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query clientId -o tsv)
+  echo $AZURE_IDENTITY_ID > $AZURE_IDENTITY_ID_FILEPATH
+  until az role assignment create --assignee $AZURE_IDENTITY_ID --role "Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --output none --only-show-errors; do
+    sleep 5
+  done
+  az identity federated-credential create -n "capz-federated-identity" \
+    --identity-name "${USER_IDENTITY}" \
+    -g "${AZWI_RESOURCE_GROUP}" \
+    --issuer "${SERVICE_ACCOUNT_ISSUER}" \
+    --subject "system:serviceaccount:capz-system:capz-manager" --output none --only-show-errors
+  az identity federated-credential create -n "aso-federated-identity" \
+    --identity-name "${USER_IDENTITY}" \
+    -g "${AZWI_RESOURCE_GROUP}" \
+    --issuer "${SERVICE_ACCOUNT_ISSUER}" \
+    --subject "system:serviceaccount:capz-system:azureserviceoperator-default" --output none --only-show-errors
+}
+
+function upload_to_blob() {
+  local container_name=$1
+  local file_path=$2
+  local blob_name=$3
+
+  echo "Uploading ${file_path} to '${AZWI_STORAGE_ACCOUNT}' storage account"
+  az storage blob upload \
+      --container-name "${container_name}" \
+      --file "${file_path}" \
+      --name "${blob_name}" \
+      --account-name "${AZWI_STORAGE_ACCOUNT}" \
+      --output none --only-show-errors
 }
 
 # This function create a kind cluster for Workload identity which requires key pairs path
 # to be mounted on the kind cluster and hence extra mount flags are required.
 function createKindForAZWI() {
-  echo "creating azwi kind"
+  echo "creating workload-identity-enabled kind configuration"
   cat <<EOF | "${KIND}" create cluster --name "${KIND_CLUSTER_NAME}" --config=-
   kind: Cluster
   apiVersion: kind.x-k8s.io/v1alpha4
   nodes:
   - role: control-plane
     extraMounts:
-      - hostPath: $HOME/azwi/creds/sa.pub
+      - hostPath: $SERVICE_ACCOUNT_SIGNING_PUB
         containerPath: /etc/kubernetes/pki/sa.pub
-      - hostPath: $HOME/azwi/creds/sa.key
+      - hostPath: $SERVICE_ACCOUNT_SIGNING_KEY
         containerPath: /etc/kubernetes/pki/sa.key
     kubeadmConfigPatches:
     - |
@@ -102,11 +190,11 @@ EOF
 # See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
 if [ "$AZWI_ENABLED" == 'true' ]
  then
-   echo "azwi is enabled..."
+   echo "workload-identity is enabled..."
    checkAZWIENVPreReqsAndCreateFiles
    createKindForAZWI
 else
-  echo "azwi is not enabled..."
+  echo "workload-identity is not enabled..."
  cat <<EOF | ${KIND} create cluster --name "${KIND_CLUSTER_NAME}" --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4

templates/azure-cluster-identity/azure-cluster-identity.yaml

@@ -6,8 +6,7 @@ metadata:
   labels:
     clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
 spec:
-  type: ServicePrincipal
+  type: WorkloadIdentity
   allowedNamespaces: {}
   tenantID: "${AZURE_TENANT_ID}"
   clientID: "${AZURE_CLIENT_ID}"
-  clientSecret: {"name":"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}","namespace":"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}"}