Skip to content

Commit dca0e1a

Browse files
fxierhfxierh
authored
Add ephemeral creds jobs for Hypershift AKS (openshift#58325)
Signed-off-by: Feilian Xie <[email protected]>
1 parent 0c5addf commit dca0e1a

File tree

23 files changed

+466
-12
lines changed

23 files changed

+466
-12
lines changed

‎ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.17__amd64-nightly.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,6 +1510,16 @@ tests:
15101510
test:
15111511
- chain: openshift-e2e-test-hypershift-qe
15121512
workflow: cucushift-installer-rehearse-azure-aks-hypershift-disaster-recovery-infra-guest
1513+
- as: azure-aks-hypershift-ephemeral-creds-guest-f7
1514+
cron: 4 23 1,9,17,24 * *
1515+
steps:
1516+
cluster_profile: azure-qe
1517+
env:
1518+
TEST_FILTERS: ~ChkUpgrade&;~DisconnectedOnly&;~NonPreRelease&;~HyperShiftMGMT&;~MicroShiftOnly&;~NonHyperShiftHOST&;~Serial&;~Disruptive&
1519+
TEST_TIMEOUT: "30"
1520+
test:
1521+
- chain: openshift-e2e-test-hypershift-qe
1522+
workflow: cucushift-installer-rehearse-azure-aks-hypershift-ephemeral-creds-guest
15131523
- as: azure-aks-hypershift-etcd-disk-encryption-guest-f7
15141524
cron: 21 9 5,12,19,26 * *
15151525
steps:

‎ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.18__amd64-nightly.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1424,6 +1424,16 @@ tests:
14241424
test:
14251425
- chain: openshift-e2e-test-hypershift-qe
14261426
workflow: cucushift-installer-rehearse-azure-aks-hypershift-disaster-recovery-infra-guest
1427+
- as: azure-aks-hypershift-ephemeral-creds-guest-f7
1428+
cron: 22 15 4,12,20,27 * *
1429+
steps:
1430+
cluster_profile: azure-qe
1431+
env:
1432+
TEST_FILTERS: ~ChkUpgrade&;~DisconnectedOnly&;~NonPreRelease&;~HyperShiftMGMT&;~MicroShiftOnly&;~NonHyperShiftHOST&;~Serial&;~Disruptive&
1433+
TEST_TIMEOUT: "30"
1434+
test:
1435+
- chain: openshift-e2e-test-hypershift-qe
1436+
workflow: cucushift-installer-rehearse-azure-aks-hypershift-ephemeral-creds-guest
14271437
- as: azure-aks-hypershift-etcd-disk-encryption-guest-f7
14281438
cron: 0 15 9,16,23,30 * *
14291439
steps:

‎ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.17-periodics.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24093,6 +24093,88 @@ periodics:
2409324093
- name: result-aggregator
2409424094
secret:
2409524095
secretName: result-aggregator
24096+
- agent: kubernetes
24097+
cluster: build03
24098+
cron: 4 23 1,9,17,24 * *
24099+
decorate: true
24100+
decoration_config:
24101+
skip_cloning: true
24102+
extra_refs:
24103+
- base_ref: release-4.17
24104+
org: openshift
24105+
repo: openshift-tests-private
24106+
labels:
24107+
ci-operator.openshift.io/cloud: azure4
24108+
ci-operator.openshift.io/cloud-cluster-profile: azure-qe
24109+
ci-operator.openshift.io/variant: amd64-nightly
24110+
ci.openshift.io/generator: prowgen
24111+
job-release: "4.17"
24112+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
24113+
name: periodic-ci-openshift-openshift-tests-private-release-4.17-amd64-nightly-azure-aks-hypershift-ephemeral-creds-guest-f7
24114+
spec:
24115+
containers:
24116+
- args:
24117+
- --gcs-upload-secret=/secrets/gcs/service-account.json
24118+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
24119+
- --lease-server-credentials-file=/etc/boskos/credentials
24120+
- --oauth-token-path=/usr/local/github-credentials/oauth
24121+
- --report-credentials-file=/etc/report/credentials
24122+
- --secret-dir=/secrets/ci-pull-credentials
24123+
- --target=azure-aks-hypershift-ephemeral-creds-guest-f7
24124+
- --variant=amd64-nightly
24125+
command:
24126+
- ci-operator
24127+
image: ci-operator:latest
24128+
imagePullPolicy: Always
24129+
name: ""
24130+
resources:
24131+
requests:
24132+
cpu: 10m
24133+
volumeMounts:
24134+
- mountPath: /etc/boskos
24135+
name: boskos
24136+
readOnly: true
24137+
- mountPath: /secrets/ci-pull-credentials
24138+
name: ci-pull-credentials
24139+
readOnly: true
24140+
- mountPath: /secrets/gcs
24141+
name: gcs-credentials
24142+
readOnly: true
24143+
- mountPath: /usr/local/github-credentials
24144+
name: github-credentials-openshift-ci-robot-private-git-cloner
24145+
readOnly: true
24146+
- mountPath: /secrets/manifest-tool
24147+
name: manifest-tool-local-pusher
24148+
readOnly: true
24149+
- mountPath: /etc/pull-secret
24150+
name: pull-secret
24151+
readOnly: true
24152+
- mountPath: /etc/report
24153+
name: result-aggregator
24154+
readOnly: true
24155+
serviceAccountName: ci-operator
24156+
volumes:
24157+
- name: boskos
24158+
secret:
24159+
items:
24160+
- key: credentials
24161+
path: credentials
24162+
secretName: boskos-credentials
24163+
- name: ci-pull-credentials
24164+
secret:
24165+
secretName: ci-pull-credentials
24166+
- name: github-credentials-openshift-ci-robot-private-git-cloner
24167+
secret:
24168+
secretName: github-credentials-openshift-ci-robot-private-git-cloner
24169+
- name: manifest-tool-local-pusher
24170+
secret:
24171+
secretName: manifest-tool-local-pusher
24172+
- name: pull-secret
24173+
secret:
24174+
secretName: registry-pull-credentials
24175+
- name: result-aggregator
24176+
secret:
24177+
secretName: result-aggregator
2409624178
- agent: kubernetes
2409724179
cluster: build03
2409824180
cron: 21 9 5,12,19,26 * *

‎ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.18-periodics.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24317,6 +24317,88 @@ periodics:
2431724317
- name: result-aggregator
2431824318
secret:
2431924319
secretName: result-aggregator
24320+
- agent: kubernetes
24321+
cluster: build03
24322+
cron: 22 15 4,12,20,27 * *
24323+
decorate: true
24324+
decoration_config:
24325+
skip_cloning: true
24326+
extra_refs:
24327+
- base_ref: release-4.18
24328+
org: openshift
24329+
repo: openshift-tests-private
24330+
labels:
24331+
ci-operator.openshift.io/cloud: azure4
24332+
ci-operator.openshift.io/cloud-cluster-profile: azure-qe
24333+
ci-operator.openshift.io/variant: amd64-nightly
24334+
ci.openshift.io/generator: prowgen
24335+
job-release: "4.18"
24336+
pj-rehearse.openshift.io/can-be-rehearsed: "true"
24337+
name: periodic-ci-openshift-openshift-tests-private-release-4.18-amd64-nightly-azure-aks-hypershift-ephemeral-creds-guest-f7
24338+
spec:
24339+
containers:
24340+
- args:
24341+
- --gcs-upload-secret=/secrets/gcs/service-account.json
24342+
- --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
24343+
- --lease-server-credentials-file=/etc/boskos/credentials
24344+
- --oauth-token-path=/usr/local/github-credentials/oauth
24345+
- --report-credentials-file=/etc/report/credentials
24346+
- --secret-dir=/secrets/ci-pull-credentials
24347+
- --target=azure-aks-hypershift-ephemeral-creds-guest-f7
24348+
- --variant=amd64-nightly
24349+
command:
24350+
- ci-operator
24351+
image: ci-operator:latest
24352+
imagePullPolicy: Always
24353+
name: ""
24354+
resources:
24355+
requests:
24356+
cpu: 10m
24357+
volumeMounts:
24358+
- mountPath: /etc/boskos
24359+
name: boskos
24360+
readOnly: true
24361+
- mountPath: /secrets/ci-pull-credentials
24362+
name: ci-pull-credentials
24363+
readOnly: true
24364+
- mountPath: /secrets/gcs
24365+
name: gcs-credentials
24366+
readOnly: true
24367+
- mountPath: /usr/local/github-credentials
24368+
name: github-credentials-openshift-ci-robot-private-git-cloner
24369+
readOnly: true
24370+
- mountPath: /secrets/manifest-tool
24371+
name: manifest-tool-local-pusher
24372+
readOnly: true
24373+
- mountPath: /etc/pull-secret
24374+
name: pull-secret
24375+
readOnly: true
24376+
- mountPath: /etc/report
24377+
name: result-aggregator
24378+
readOnly: true
24379+
serviceAccountName: ci-operator
24380+
volumes:
24381+
- name: boskos
24382+
secret:
24383+
items:
24384+
- key: credentials
24385+
path: credentials
24386+
secretName: boskos-credentials
24387+
- name: ci-pull-credentials
24388+
secret:
24389+
secretName: ci-pull-credentials
24390+
- name: github-credentials-openshift-ci-robot-private-git-cloner
24391+
secret:
24392+
secretName: github-credentials-openshift-ci-robot-private-git-cloner
24393+
- name: manifest-tool-local-pusher
24394+
secret:
24395+
secretName: manifest-tool-local-pusher
24396+
- name: pull-secret
24397+
secret:
24398+
secretName: registry-pull-credentials
24399+
- name: result-aggregator
24400+
secret:
24401+
secretName: result-aggregator
2432024402
- agent: kubernetes
2432124403
cluster: build03
2432224404
cron: 0 15 9,16,23,30 * *

‎ci-operator/step-registry/azure/deprovision/sp-and-custom-role/azure-deprovision-sp-and-custom-role-commands.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ if [[ -f "${SHARED_DIR}/azure_sp_id" ]]; then
3434
echo "Deleting sp..."
3535
sp_ids=$(< "${SHARED_DIR}/azure_sp_id")
3636
for sp_id in ${sp_ids}; do
37-
cmd="az ad sp delete --id ${sp_id}"
37+
cmd="az ad app delete --id ${sp_id}"
3838
run_command "${cmd}"
3939
done
4040
fi

‎ci-operator/step-registry/azure/provision/service-principal/hypershift/OWNERS‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
approvers:
2+
- patrickdillon
3+
- yunjiang29
4+
- MayXuQQ
5+
- jianlinliu
6+
- jinyunma
7+
- fxierh

‎ci-operator/step-registry/azure/provision/service-principal/hypershift/azure-provision-service-principal-hypershift-commands.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json"
6+
AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)"
7+
AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)"
8+
AZURE_AUTH_SUBSCRIPTION_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .subscriptionId)"
9+
AZURE_AUTH_TENANT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .tenantId)"
10+
11+
az --version
12+
az cloud set --name AzureCloud
13+
az login --service-principal -u "${AZURE_AUTH_CLIENT_ID}" -p "${AZURE_AUTH_CLIENT_SECRET}" --tenant "${AZURE_AUTH_TENANT_ID}" --output none
14+
15+
set -x
16+
17+
SP_NAME_PREFIX="${NAMESPACE}-${UNIQUE_HASH}"
18+
KV_NAME=$(<"${SHARED_DIR}/azure_keyvault_name")
19+
RG_NSG=$(<"${SHARED_DIR}/resourcegroup_nsg")
20+
RG_VNET=$(<"${SHARED_DIR}/resourcegroup_vnet")
21+
RG_HC=$(<"${SHARED_DIR}/resourcegroup")
22+
COMPONENTS="azure-disk azure-file ciro cloud-provider cncc cpo ingress capz"
23+
24+
declare -A component_to_client_id
25+
declare -A component_to_cert_name
26+
27+
for component in $COMPONENTS; do
28+
name="${SP_NAME_PREFIX}-${component}"
29+
scopes="/subscriptions/$AZURE_AUTH_SUBSCRIPTION_ID/resourceGroups/$RG_HC"
30+
if [[ $component == ingress ]]; then
31+
scopes+=" /subscriptions/$AZURE_AUTH_SUBSCRIPTION_ID/resourceGroups/$RG_VNET"
32+
elif [[ $component == cloud-provider ]]; then
33+
scopes+=" /subscriptions/$AZURE_AUTH_SUBSCRIPTION_ID/resourceGroups/$RG_NSG"
34+
fi
35+
36+
client_id="$(eval "az ad sp create-for-rbac --name $name --role Contributor --scopes $scopes --create-cert --cert $name --keyvault $KV_NAME --output json --only-show-errors" | jq -r '.appId')"
37+
echo "$client_id" >> "${SHARED_DIR}/azure_sp_id"
38+
39+
component_to_client_id+=(["$component"]="$client_id")
40+
component_to_cert_name+=(["$component"]="$name")
41+
done
42+
43+
cat <<EOF >"${SHARED_DIR}"/hypershift_azure_mi_file.json
44+
{
45+
"managedIdentitiesKeyVault": {
46+
"name": "$KV_NAME",
47+
"tenantID": "$AZURE_AUTH_TENANT_ID"
48+
},
49+
"cloudProvider": {
50+
"clientID": "${component_to_client_id[cloud-provider]}",
51+
"certificateName": "${component_to_cert_name[cloud-provider]}"
52+
},
53+
"nodePoolManagement": {
54+
"clientID": "${component_to_client_id[capz]}",
55+
"certificateName": "${component_to_cert_name[capz]}"
56+
},
57+
"controlPlaneOperator": {
58+
"clientID": "${component_to_client_id[cpo]}",
59+
"certificateName": "${component_to_cert_name[cpo]}"
60+
},
61+
"imageRegistry": {
62+
"clientID": "${component_to_client_id[ciro]}",
63+
"certificateName": "${component_to_cert_name[ciro]}"
64+
},
65+
"ingress": {
66+
"clientID": "${component_to_client_id[ingress]}",
67+
"certificateName": "${component_to_cert_name[ingress]}"
68+
},
69+
"network": {
70+
"clientID": "${component_to_client_id[cncc]}",
71+
"certificateName": "${component_to_cert_name[cncc]}"
72+
},
73+
"disk": {
74+
"clientID": "${component_to_client_id[azure-disk]}",
75+
"certificateName": "${component_to_cert_name[azure-disk]}"
76+
},
77+
"file": {
78+
"clientID": "${component_to_client_id[azure-file]}",
79+
"certificateName": "${component_to_cert_name[azure-file]}"
80+
}
81+
}
82+
EOF

‎ci-operator/step-registry/azure/provision/service-principal/hypershift/azure-provision-service-principal-hypershift-ref.metadata.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"path": "azure/provision/service-principal/hypershift/azure-provision-service-principal-hypershift-ref.yaml",
3+
"owners": {
4+
"approvers": [
5+
"patrickdillon",
6+
"yunjiang29",
7+
"MayXuQQ",
8+
"jianlinliu",
9+
"jinyunma",
10+
"fxierh"
11+
]
12+
}
13+
}

‎ci-operator/step-registry/azure/provision/service-principal/hypershift/azure-provision-service-principal-hypershift-ref.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
ref:
2+
as: azure-provision-service-principal-hypershift
3+
from_image:
4+
namespace: ocp
5+
name: "4.16"
6+
tag: upi-installer
7+
timeout: 20m
8+
grace_period: 2m
9+
commands: azure-provision-service-principal-hypershift-commands.sh
10+
resources:
11+
requests:
12+
cpu: 100m
13+
memory: 100Mi
14+
documentation: |-
15+
Creates SPs required for Hypershift control plane components to authenticate to the cloud provider as MIs.

‎ci-operator/step-registry/azure/provision/vault-key/azure-provision-vault-key-commands.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ az role assignment create --assignee "$SP_ID" --scope "$KV_ID" --role "Key Vault
5656
echo "Creating Keys within the KeyVault"
5757
KEYVAULT_KEY_NAME="${KV_BASE_NAME}-key"
5858
poll "az keyvault key create --vault-name $KEYVAULT_NAME -n $KEYVAULT_KEY_NAME --protection software"
59-
KEYVAULT_KEY_URL="$(az keyvault key show --vault-name "$KEYVAULT_NAME" --name "$KEYVAULT_KEY_NAME" --query 'key.kid' -o tsv)"
59+
poll "KEYVAULT_KEY_URL=\$(az keyvault key show --vault-name \"$KEYVAULT_NAME\" --name \"$KEYVAULT_KEY_NAME\" --query 'key.kid' -o tsv)"
6060

6161
echo "Saving relevant info to \$SHARED_DIR"
6262
# Key URL format: https://<KEYVAULT_NAME>.vault.azure.net/keys/<KEYVAULT_KEY_NAME>/<KEYVAULT_KEY_VERSION>

0 commit comments

Comments
 (0)