Skip to content

Commit 21479a9

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4939 from nawazkh/capz_prow_2_wi
Use workload identity for azure cli when Federated token file is present
2 parents 2ed5cfc + c3f6e88 commit 21479a9

File tree

5 files changed

+47
-31
lines changed

5 files changed

+47
-31
lines changed

‎hack/ensure-acr-login.sh‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,11 @@ set +o xtrace
2222
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
2323
cd "${REPO_ROOT}" || exit 1
2424

25-
if [[ "${REGISTRY:-}" =~ capzci\.azurecr\.io ]]; then
25+
if [[ "${REGISTRY:-}" =~ \.azurecr\.io ]]; then
2626
# if we are using the prow Azure Container Registry, login.
2727
"${REPO_ROOT}/hack/ensure-azcli.sh"
2828
: "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}"
2929
az account set -s "${AZURE_SUBSCRIPTION_ID}"
30-
az acr login --name capzci
31-
# TODO(mainred): When using ACR, `az acr login` impacts the authentication of `docker buildx build --push` when the
32-
# ACR, capzci in our case, has anonymous pull enabled.
33-
# Use `docker login` as a suggested workaround and remove this target when the issue is resolved.
34-
# Issue link: https://github.com/Azure/acr/issues/582
35-
# Failed building link: https://prow.k8s.io/view/gs/kubernetes-jenkins/pr-logs/pull/kubernetes-sigs_cloud-provider-azure/974/pull-cloud-provider-azure-e2e-ccm-capz/1480459040440979456
36-
docker login -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" capzci.azurecr.io
30+
acrname="${REGISTRY%%.*}"
31+
az acr login --name "$acrname"
3732
fi

‎hack/ensure-azcli.sh‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,16 @@ if [[ -z "$(command -v az)" ]]; then
2525
AZ_REPO=$(lsb_release -cs)
2626
echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ ${AZ_REPO} main" | tee /etc/apt/sources.list.d/azure-cli.list
2727
apt-get update && apt-get install -y azure-cli
28-
az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" > /dev/null
28+
29+
if [[ -n "${AZURE_FEDERATED_TOKEN_FILE:-}" ]]; then
30+
# AZURE_CLIENT_ID has been overloaded with Azure Workload ID in the preset-azure-cred-wi.
31+
# This is done to avoid exporting Azure Workload ID as AZURE_CLIENT_ID in the test scenarios.
32+
az login --service-principal -u "${AZURE_CLIENT_ID}" -t "${AZURE_TENANT_ID}" --federated-token "$(cat "${AZURE_FEDERATED_TOKEN_FILE}")" > /dev/null
33+
34+
# Use --auth-mode "login" in az storage commands.
35+
ENABLE_AUTH_MODE_LOGIN="true"
36+
export ENABLE_AUTH_MODE_LOGIN
37+
else
38+
az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" > /dev/null
39+
fi
2940
fi

‎scripts/ci-build-azure-ccm.sh‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ source "${REPO_ROOT}/hack/ensure-go.sh"
2929
source "${REPO_ROOT}/hack/parse-prow-creds.sh"
3030

3131
: "${AZURE_STORAGE_ACCOUNT:?Environment variable empty or not defined.}"
32-
: "${AZURE_STORAGE_KEY:?Environment variable empty or not defined.}"
3332
: "${REGISTRY:?Environment variable empty or not defined.}"
3433

3534
# cloud controller manager image
@@ -69,16 +68,16 @@ main() {
6968
echo "Building and pushing Linux and Windows amd64 Azure ACR credential provider"
7069
make -C "${AZURE_CLOUD_PROVIDER_ROOT}" bin/azure-acr-credential-provider bin/azure-acr-credential-provider.exe
7170

72-
if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then
71+
if [[ "$(az storage container exists ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then
7372
echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container"
74-
az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null
75-
az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null
73+
az storage container create ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null
74+
az storage container set-permission ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null
7675
fi
7776

78-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
79-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe"
80-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/examples/out-of-tree/credential-provider-config.yaml" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
81-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/examples/out-of-tree/credential-provider-config-win.yaml" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml"
77+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider"
78+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/bin/azure-acr-credential-provider.exe" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/azure-acr-credential-provider.exe"
79+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/examples/out-of-tree/credential-provider-config.yaml" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config.yaml"
80+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${AZURE_CLOUD_PROVIDER_ROOT}/examples/out-of-tree/credential-provider-config-win.yaml" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/credential-provider-config-win.yaml"
8281
fi
8382
}
8483

@@ -102,7 +101,7 @@ can_reuse_artifacts() {
102101
fi
103102

104103
for BINARY in azure-acr-credential-provider azure-acr-credential-provider.exe credential-provider-config.yaml credential-provider-config-win.yaml; do
105-
if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then
104+
if [[ "$(az storage blob exists ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${IMAGE_TAG_ACR_CREDENTIAL_PROVIDER}/${BINARY}" --query exists --output tsv)" == "false" ]]; then
106105
echo "false" && return
107106
fi
108107
done

‎scripts/ci-build-kubernetes.sh‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ source "${REPO_ROOT}/hack/parse-prow-creds.sh"
3333
source "${REPO_ROOT}/hack/util.sh"
3434

3535
: "${AZURE_STORAGE_ACCOUNT:?Environment variable empty or not defined.}"
36-
: "${AZURE_STORAGE_KEY:?Environment variable empty or not defined.}"
3736
: "${REGISTRY:?Environment variable empty or not defined.}"
3837

3938
declare -a BINARIES=("kubeadm" "kubectl" "kubelet" "e2e.test")
@@ -80,10 +79,10 @@ setup() {
8079
}
8180

8281
main() {
83-
if [[ "$(az storage container exists --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then
82+
if [[ "$(az storage container exists ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --name "${AZURE_BLOB_CONTAINER_NAME}" --query exists --output tsv)" == "false" ]]; then
8483
echo "Creating ${AZURE_BLOB_CONTAINER_NAME} storage container"
85-
az storage container create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null
86-
az storage container set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null
84+
az storage container ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} create --name "${AZURE_BLOB_CONTAINER_NAME}" > /dev/null
85+
az storage container ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} set-permission --name "${AZURE_BLOB_CONTAINER_NAME}" --public-access container > /dev/null
8786
fi
8887

8988
if [[ "${KUBE_BUILD_CONFORMANCE:-}" =~ [yY] ]]; then
@@ -116,7 +115,7 @@ main() {
116115
for BINARY in "${BINARIES[@]}"; do
117116
BIN_PATH="${KUBE_GIT_VERSION}/bin/linux/amd64/${BINARY}"
118117
echo "uploading ${BIN_PATH}"
119-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/${BINARY}" --name "${BIN_PATH}"
118+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/linux/amd64/${BINARY}" --name "${BIN_PATH}"
120119
done
121120

122121
if [[ "${TEST_WINDOWS:-}" == "true" ]]; then
@@ -129,7 +128,7 @@ main() {
129128
for BINARY in "${WINDOWS_BINARIES[@]}"; do
130129
BIN_PATH="${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe"
131130
echo "uploading ${BIN_PATH}"
132-
az storage blob upload --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/windows/amd64/${BINARY}.exe" --name "${BIN_PATH}"
131+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --overwrite --container-name "${AZURE_BLOB_CONTAINER_NAME}" --file "${KUBE_ROOT}/_output/dockerized/bin/windows/amd64/${BINARY}.exe" --name "${BIN_PATH}"
133132
done
134133
fi
135134
fi
@@ -144,14 +143,14 @@ can_reuse_artifacts() {
144143
done
145144

146145
for BINARY in "${BINARIES[@]}"; do
147-
if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${KUBE_GIT_VERSION}/bin/linux/amd64/${BINARY}" --query exists --output tsv)" == "false" ]]; then
146+
if [[ "$(az storage blob exists ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${KUBE_GIT_VERSION}/bin/linux/amd64/${BINARY}" --query exists --output tsv)" == "false" ]]; then
148147
echo "false" && return
149148
fi
150149
done
151150

152151
if [[ "${TEST_WINDOWS:-}" == "true" ]]; then
153152
for BINARY in "${WINDOWS_BINARIES[@]}"; do
154-
if [[ "$(az storage blob exists --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe" --query exists --output tsv)" == "false" ]]; then
153+
if [[ "$(az storage blob exists ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} --container-name "${AZURE_BLOB_CONTAINER_NAME}" --name "${KUBE_GIT_VERSION}/bin/windows/amd64/${BINARY}.exe" --query exists --output tsv)" == "false" ]]; then
155154
echo "false" && return
156155
fi
157156
done

‎scripts/kind-with-registry.sh‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,22 +93,27 @@ function checkAZWIENVPreReqsAndCreateFiles() {
9393
echo "AZWI_RESOURCE_GROUP environment variable required - Azure resource group to store required Workload Identity artifacts"
9494
exit 1
9595
fi
96+
9697
if [ "$(az group exists --name "${AZWI_RESOURCE_GROUP}" --output tsv)" == 'false' ]; then
9798
echo "Creating resource group '${AZWI_RESOURCE_GROUP}' in '${AZWI_LOCATION}'"
9899
az group create --name "${AZWI_RESOURCE_GROUP}" --location "${AZWI_LOCATION}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
99100
fi
101+
100102
# Ensure that our connection to storage is inherited from the existing Azure login context
101103
unset AZURE_STORAGE_KEY
102104
unset AZURE_STORAGE_ACCOUNT
105+
103106
if ! az storage account show --name "${AZWI_STORAGE_ACCOUNT}" --resource-group "${AZWI_RESOURCE_GROUP}" > /dev/null 2>&1; then
104107
echo "Creating storage account '${AZWI_STORAGE_ACCOUNT}' in '${AZWI_RESOURCE_GROUP}'"
105108
az storage account create --resource-group "${AZWI_RESOURCE_GROUP}" --name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
106-
az storage blob service-properties update --account-name "${AZWI_STORAGE_ACCOUNT}" --static-website
109+
az storage blob service-properties ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} update --account-name "${AZWI_STORAGE_ACCOUNT}" --static-website
107110
fi
111+
108112
if ! az storage container show --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" > /dev/null 2>&1; then
109113
echo "Creating storage container '${AZWI_STORAGE_CONTAINER}' in '${AZWI_STORAGE_ACCOUNT}'"
110-
az storage container create --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors
114+
az storage container ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} create --name "${AZWI_STORAGE_CONTAINER}" --account-name "${AZWI_STORAGE_ACCOUNT}" --output none --only-show-errors
111115
fi
116+
112117
SERVICE_ACCOUNT_ISSUER=$(az storage account show --name "${AZWI_STORAGE_ACCOUNT}" -o json | jq -r .primaryEndpoints.web)
113118
export SERVICE_ACCOUNT_ISSUER
114119
AZWI_OPENID_CONFIG_FILEPATH="${REPO_ROOT}/openid-configuration.json"
@@ -131,30 +136,37 @@ EOF
131136
openssl rsa -in "${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH}" -pubout -out "${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}"
132137
AZWI_JWKS_JSON_FILEPATH="${REPO_ROOT}/jwks.json"
133138
"${AZWI}" jwks --public-keys "${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH}" --output-file "${AZWI_JWKS_JSON_FILEPATH}"
139+
134140
echo "Uploading openid-configuration document to '${AZWI_STORAGE_ACCOUNT}' storage account"
135141
upload_to_blob "${AZWI_OPENID_CONFIG_FILEPATH}" ".well-known/openid-configuration"
142+
136143
echo "Uploading jwks document to '${AZWI_STORAGE_ACCOUNT}' storage account"
137144
upload_to_blob "${AZWI_JWKS_JSON_FILEPATH}" "openid/v1/jwks"
138-
echo "Removing key access on storage account as no further data writes are required"
139-
az storage account update -n "${AZWI_STORAGE_ACCOUNT}" -g "${AZWI_RESOURCE_GROUP}" --subscription "${AZURE_SUBSCRIPTION_ID}" --allow-shared-key-access=false --output none --only-show-errors
140145
fi
146+
141147
if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then
142148
if [ -z "${USER_IDENTITY}" ]; then
143149
echo "USER_IDENTITY environment variable required if not bringing your own identity via AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY"
144150
exit 1
145151
fi
152+
146153
az identity create -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" -l "${AZWI_LOCATION}" --output none --only-show-errors --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}"
147154
AZURE_IDENTITY_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query clientId -o tsv)
148155
AZURE_IDENTITY_ID_PRINCIPAL_ID=$(az identity show -n "${USER_IDENTITY}" -g "${AZWI_RESOURCE_GROUP}" --query principalId -o tsv)
156+
149157
echo "${AZURE_IDENTITY_ID}" > "${AZURE_IDENTITY_ID_FILEPATH}"
150158
until az role assignment create --assignee-object-id "${AZURE_IDENTITY_ID_PRINCIPAL_ID}" --role "Owner" --scope "/subscriptions/${AZURE_SUBSCRIPTION_ID}" --assignee-principal-type ServicePrincipal --output none --only-show-errors; do
151159
sleep 5
152160
done
161+
162+
echo "Creating federated credentials for capz-federated-identity"
153163
az identity federated-credential create -n "capz-federated-identity" \
154164
--identity-name "${USER_IDENTITY}" \
155165
-g "${AZWI_RESOURCE_GROUP}" \
156166
--issuer "${SERVICE_ACCOUNT_ISSUER}" \
157167
--subject "system:serviceaccount:capz-system:capz-manager" --output none --only-show-errors
168+
169+
echo "Creating federated credentials for aso-federated-identity"
158170
az identity federated-credential create -n "aso-federated-identity" \
159171
--identity-name "${USER_IDENTITY}" \
160172
-g "${AZWI_RESOURCE_GROUP}" \
@@ -168,7 +180,7 @@ function upload_to_blob() {
168180
local blob_name=$2
169181

170182
echo "Uploading ${file_path} to '${AZWI_STORAGE_ACCOUNT}' storage account"
171-
az storage blob upload \
183+
az storage blob upload ${ENABLE_AUTH_MODE_LOGIN:+"--auth-mode login"} \
172184
--container-name "${AZWI_STORAGE_CONTAINER}" \
173185
--file "${file_path}" \
174186
--name "${blob_name}" \

0 commit comments

Comments
 (0)