Skip to content

Commit 09623fe

Browse files
committed
CI: workload-identity native
Signed-off-by: Jack Francis <[email protected]>
1 parent 753edfb commit 09623fe

File tree

55 files changed

+289
-203
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+289
-203
lines changed

.gitignore

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,3 +86,10 @@ release-*/manifests/calico-*.yaml
8686
# mentioned in the capz book
8787
/sp.json
8888
/cluster.yaml
89+
90+
# CI workload-identity
91+
jwks.json
92+
*.pub
93+
*.key
94+
azure_identity_id
95+
openid-configuration.json

Makefile

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ KUSTOMIZE_VER := v5.4.1
8989
KUSTOMIZE_BIN := kustomize
9090
KUSTOMIZE := $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER)
9191

92+
AZWI_VER := v1.2.2
93+
AZWI_BIN := azwi
94+
AZWI := $(TOOLS_BIN_DIR)/$(AZWI_BIN)-$(AZWI_VER)
95+
9296
MOCKGEN_VER := v0.4.0
9397
MOCKGEN_BIN := mockgen
9498
MOCKGEN := $(TOOLS_BIN_DIR)/$(MOCKGEN_BIN)-$(MOCKGEN_VER)
@@ -283,7 +287,7 @@ verify-codespell: codespell ## Verify codespell.
283287
##@ Development:
284288

285289
.PHONY: install-tools # populate hack/tools/bin
286-
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND)
290+
install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(AZWI)
287291

288292
.PHONY: create-management-cluster
289293
create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
@@ -682,7 +686,11 @@ test-cover: test ## Run tests with code coverage and generate reports.
682686

683687
.PHONY: kind-create-bootstrap
684688
kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
685-
export AZWI=$${AZWI:-true} KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
689+
KIND_CLUSTER_NAME=capz-e2e && ./scripts/kind-with-registry.sh
690+
691+
.PHONY: cleanup-workload-identity
692+
cleanup-workload-identity: ## Cleanup CI workload-identity infra
693+
./scripts/cleanup-workload-identity.sh
686694

687695
## --------------------------------------
688696
## Security Scanning
@@ -788,6 +796,16 @@ $(HELM): ## Put helm into tools folder.
788796
ln -sf $(HELM) $(TOOLS_BIN_DIR)/$(HELM_BIN)
789797
rm -f $(TOOLS_BIN_DIR)/get_helm.sh
790798

799+
$(AZWI): ## Put azwi into tools folder.
800+
mkdir -p $(TOOLS_BIN_DIR)
801+
rm -f "$(TOOLS_BIN_DIR)/$(AZWI_BIN)*"
802+
curl --retry $(CURL_RETRIES) -fsSL -o $(TOOLS_BIN_DIR)/azwi.tar.gz https://github.com/Azure/azure-workload-identity/releases/download/$(AZWI_VER)/azwi-$(AZWI_VER)-$(GOOS)-$(GOARCH).tar.gz
803+
tar -xf "$(TOOLS_BIN_DIR)/azwi.tar.gz" -C $(TOOLS_BIN_DIR) $(AZWI_BIN)
804+
mv "$(TOOLS_BIN_DIR)/$(AZWI_BIN)" $(AZWI)
805+
ln -sf $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
806+
chmod +x $(AZWI) $(TOOLS_BIN_DIR)/$(AZWI_BIN)
807+
rm -f $(TOOLS_BIN_DIR)/azwi.tar.gz
808+
791809
$(KIND): ## Build kind into tools folder.
792810
GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) sigs.k8s.io/kind $(KIND_BIN) $(KIND_VER)
793811

e2e.mk

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,23 @@
44
# long-running E2E jobs every time that file changes
55

66
##@ E2E Testing:
7+
RANDOM_SUFFIX := $(shell /bin/bash -c "echo $$RANDOM")
8+
export AZWI_RESOURCE_GROUP ?= capz-ci-$(RANDOM_SUFFIX)
9+
export CI_RG ?= $(AZWI_RESOURCE_GROUP)
10+
export USER_IDENTITY ?= $(addsuffix $(RANDOM_SUFFIX),$(CI_RG))
11+
export AZWI_LOCATION ?= eastus
12+
export AZURE_IDENTITY_ID_FILEPATH ?= $(ROOT_DIR)/azure_identity_id
713

814
.PHONY: test-e2e-run
915
test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
1016
$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
11-
$(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
17+
AZURE_CLIENT_ID=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)) $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
1218
-e2e.artifacts-folder="$(ARTIFACTS)" \
1319
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
1420
-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
1521
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
1622
$(MAKE) clean-release-git
23+
AZWI_RESOURCE_GROUP=$(AZWI_RESOURCE_GROUP) $(MAKE) cleanup-workload-identity
1724

1825
.PHONY: test-e2e
1926
test-e2e: ## Run "docker-build" and "docker-push" rules then run e2e tests.

hack/util.sh

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,4 @@ capz::util::generate_ssh_key() {
108108
capz::util::ensure_azure_envs() {
109109
: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
110110
: "${AZURE_TENANT_ID:?Environment variable empty or not defined.}"
111-
: "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}"
112-
: "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}"
113111
}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit
17+
set -o nounset
18+
set -o pipefail
19+
20+
# Install kubectl and kind
21+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
22+
# shellcheck source=hack/ensure-azcli.sh
23+
source "${REPO_ROOT}/hack/ensure-azcli.sh"
24+
25+
AZWI_RESOURCE_GROUP=${AZWI_RESOURCE_GROUP:-}
26+
27+
if [[ -z "${AZWI_RESOURCE_GROUP}" ]]; then
28+
echo AZWI_RESOURCE_GROUP environment variable must be set
29+
exit 1
30+
fi
31+
32+
echo "Cleaning up CI workload-identity infra..."
33+
az group delete --no-wait -y -n $AZWI_RESOURCE_GROUP

scripts/kind-with-registry.sh

Lines changed: 94 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,19 @@ set -o pipefail
1919

2020
# Install kubectl and kind
2121
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
22+
# shellcheck source=hack/ensure-azcli.sh
23+
source "${REPO_ROOT}/hack/ensure-azcli.sh"
24+
2225
KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
2326
KIND="${REPO_ROOT}/hack/tools/bin/kind"
24-
AZWI_ENABLED=${AZWI:-}
27+
AZWI="${REPO_ROOT}/hack/tools/bin/azwi"
28+
AZWI_ENABLED=${AZWI_ENABLED:-true}
29+
RAND_SUFFIX=$(openssl rand -hex 4)
30+
export AZWI_STORAGE_ACCOUNT="oidcissuer${RAND_SUFFIX}"
31+
export AZWI_STORAGE_CONTAINER="oidc"
32+
export SERVICE_ACCOUNT_ISSUER=${SERVICE_ACCOUNT_ISSUER:-}
33+
export SERVICE_ACCOUNT_SIGNING_PUB=${SERVICE_ACCOUNT_SIGNING_PUB:-}
34+
export SERVICE_ACCOUNT_SIGNING_KEY=${SERVICE_ACCOUNT_SIGNING_KEY:-}
2535
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}"
2636

2737
# Export desired cluster name; default is "capz"
@@ -44,35 +54,104 @@ fi
4454

4555
# To use workload identity, service account signing key pairs base64 encoded should be exposed via the
4656
# env variables. The function creates the key pair files after reading it from the env variables.
57+
# TODO we need to document that these env vars are a new requirement
4758
function checkAZWIENVPreReqsAndCreateFiles() {
48-
if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then
49-
echo "'SERVICE_ACCOUNT_SIGNING_PUB' is not set."
50-
exit 1
59+
# check if user is logged into azure cli
60+
if ! az account show > /dev/null 2>&1; then
61+
echo "Please login to Azure CLI using 'az login'"
62+
exit 1
5163
fi
5264

65+
if [ "$(az group exists --name "${AZWI_RESOURCE_GROUP}" --output tsv)" == 'false' ]; then
66+
echo "Creating resource group '${AZWI_RESOURCE_GROUP}' in '${AZWI_LOCATION}'"
67+
az group create --name "${AZWI_RESOURCE_GROUP}" --location "${AZWI_LOCATION}" --output none --only-show-errors
68+
fi
69+
if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then
70+
echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'"
71+
az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --allow-blob-public-access true --output none --only-show-errors
72+
fi
73+
if ! az storage container show --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" > /dev/null 2>&1; then
74+
echo "Creating storage container '${AZWI_STORAGE_CONTAINER}' in '${AZWI_STORAGE_ACCOUNT}'"
75+
az storage container create --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" --public-access blob --output none --only-show-errors
76+
fi
77+
if [[ -z "${SERVICE_ACCOUNT_ISSUER}" ]]; then
78+
export SERVICE_ACCOUNT_ISSUER="https://${AZWI_STORAGE_ACCOUNT}.blob.core.windows.net/${AZWI_STORAGE_CONTAINER}/"
79+
fi
80+
AZWI_OPENID_CONFIG_FILEPATH="${REPO_ROOT}/openid-configuration.json"
81+
cat <<EOF > $AZWI_OPENID_CONFIG_FILEPATH
82+
{
83+
"issuer": "${SERVICE_ACCOUNT_ISSUER}",
84+
"jwks_uri": "${SERVICE_ACCOUNT_ISSUER}openid/v1/jwks",
85+
"response_types_supported": [
86+
"id_token"
87+
],
88+
"subject_types_supported": [
89+
"public"
90+
],
91+
"id_token_signing_alg_values_supported": [
92+
"RS256"
93+
]
94+
}
95+
EOF
96+
if [[ -z "${SERVICE_ACCOUNT_SIGNING_PUB}" ]]; then
97+
export SERVICE_ACCOUNT_SIGNING_PUB="${REPO_ROOT}/capz-ci-sa.pub"
98+
fi
5399
if [[ -z "${SERVICE_ACCOUNT_SIGNING_KEY}" ]]; then
54-
echo "'SERVICE_ACCOUNT_SIGNING_KEY' is not set."
55-
exit 1
100+
export SERVICE_ACCOUNT_SIGNING_KEY="${REPO_ROOT}/capz-ci-sa.key"
56101
fi
57-
mkdir -p "$HOME"/azwi/creds
58-
echo "${SERVICE_ACCOUNT_SIGNING_PUB}" > "$HOME"/azwi/creds/sa.pub
59-
echo "${SERVICE_ACCOUNT_SIGNING_KEY}" > "$HOME"/azwi/creds/sa.key
60-
SERVICE_ACCOUNT_ISSUER="${SERVICE_ACCOUNT_ISSUER:-https://oidcissuercapzci.blob.core.windows.net/oidc-capzci/}"
102+
openssl genrsa -out $SERVICE_ACCOUNT_SIGNING_KEY 2048
103+
openssl rsa -in $SERVICE_ACCOUNT_SIGNING_KEY -pubout -out $SERVICE_ACCOUNT_SIGNING_PUB
104+
AZWI_JWKS_JSON_FILEPATH="${REPO_ROOT}/jwks.json"
105+
"${AZWI}" jwks --public-keys $SERVICE_ACCOUNT_SIGNING_PUB --output-file "${AZWI_JWKS_JSON_FILEPATH}"
106+
echo "Uploading openid-configuration document to '${AZWI_STORAGE_ACCOUNT}' storage account"
107+
upload_to_blob "${AZWI_STORAGE_CONTAINER}" "${AZWI_OPENID_CONFIG_FILEPATH}" ".well-known/openid-configuration"
108+
echo "Uploading jwks document to '${AZWI_STORAGE_ACCOUNT}' storage account"
109+
upload_to_blob "${AZWI_STORAGE_CONTAINER}" "${AZWI_JWKS_JSON_FILEPATH}" "openid/v1/jwks"
110+
az identity create -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" -l "${AZWI_LOCATION}" --output none --only-show-errors
111+
AZURE_IDENTITY_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query clientId -o tsv)
112+
echo $AZURE_IDENTITY_ID > $AZURE_IDENTITY_ID_FILEPATH
113+
until az role assignment create --assignee $AZURE_IDENTITY_ID --role "Contributor" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --output none --only-show-errors; do
114+
sleep 5
115+
done
116+
az identity federated-credential create -n "capz-federated-identity" \
117+
--identity-name "${USER_IDENTITY}" \
118+
-g "${AZWI_RESOURCE_GROUP}" \
119+
--issuer "${SERVICE_ACCOUNT_ISSUER}" \
120+
--subject "system:serviceaccount:capz-system:capz-manager" --output none --only-show-errors
121+
az identity federated-credential create -n "aso-federated-identity" \
122+
--identity-name "${USER_IDENTITY}" \
123+
-g "${AZWI_RESOURCE_GROUP}" \
124+
--issuer "${SERVICE_ACCOUNT_ISSUER}" \
125+
--subject "system:serviceaccount:capz-system:azureserviceoperator-default" --output none --only-show-errors
126+
}
127+
128+
function upload_to_blob() {
129+
local container_name=$1
130+
local file_path=$2
131+
local blob_name=$3
132+
133+
echo "Uploading ${file_path} to '${AZWI_STORAGE_ACCOUNT}' storage account"
134+
az storage blob upload \
135+
--container-name "${container_name}" \
136+
--file "${file_path}" \
137+
--name "${blob_name}" \
138+
--account-name "${AZWI_STORAGE_ACCOUNT}" \
139+
--output none --only-show-errors
61140
}
62141

63142
# This function create a kind cluster for Workload identity which requires key pairs path
64143
# to be mounted on the kind cluster and hence extra mount flags are required.
65144
function createKindForAZWI() {
66-
echo "creating azwi kind"
145+
echo "creating workload-identity-enabled kind configuration"
67146
cat <<EOF | "${KIND}" create cluster --name "${KIND_CLUSTER_NAME}" --config=-
68147
kind: Cluster
69148
apiVersion: kind.x-k8s.io/v1alpha4
70149
nodes:
71150
- role: control-plane
72151
extraMounts:
73-
- hostPath: $HOME/azwi/creds/sa.pub
152+
- hostPath: $SERVICE_ACCOUNT_SIGNING_PUB
74153
containerPath: /etc/kubernetes/pki/sa.pub
75-
- hostPath: $HOME/azwi/creds/sa.key
154+
- hostPath: $SERVICE_ACCOUNT_SIGNING_KEY
76155
containerPath: /etc/kubernetes/pki/sa.key
77156
kubeadmConfigPatches:
78157
- |
@@ -102,11 +181,11 @@ EOF
102181
# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
103182
if [ "$AZWI_ENABLED" == 'true' ]
104183
then
105-
echo "azwi is enabled..."
184+
echo "workload-identity is enabled..."
106185
checkAZWIENVPreReqsAndCreateFiles
107186
createKindForAZWI
108187
else
109-
echo "azwi is not enabled..."
188+
echo "workload-identity is not enabled..."
110189
cat <<EOF | ${KIND} create cluster --name "${KIND_CLUSTER_NAME}" --config=-
111190
kind: Cluster
112191
apiVersion: kind.x-k8s.io/v1alpha4

templates/azure-cluster-identity/azure-cluster-identity.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ metadata:
66
labels:
77
clusterctl.cluster.x-k8s.io/move-hierarchy: "true"
88
spec:
9-
type: ServicePrincipal
9+
type: WorkloadIdentity
1010
allowedNamespaces: {}
1111
tenantID: "${AZURE_TENANT_ID}"
1212
clientID: "${AZURE_CLIENT_ID}"
13-
clientSecret: {"name":"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}","namespace":"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}"}

templates/cluster-template-aad.yaml

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

templates/cluster-template-aks-clusterclass.yaml

Lines changed: 1 addition & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

templates/cluster-template-aks.yaml

Lines changed: 1 addition & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)