Skip to content

Commit a30bc3c

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1578 from aramase/aramase/t/migrate_azure_jobs
ci: migrate azure jobs to eks prow cluster
2 parents 92bbbbd + 1bfd728 commit a30bc3c

14 files changed

+165
-324
lines changed

‎Makefile‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL
6161

6262
# Generate all combination of all OS, ARCH, and OSVERSIONS for iteration
6363
ALL_OS = linux windows
64-
ALL_ARCH.linux = amd64 arm64
65-
ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
66-
ALL_ARCH.windows = amd64
67-
ALL_OSVERSIONS.windows := 1809 ltsc2022
68-
ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-${osversion}-${arch}))
69-
ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})
64+
ALL_ARCH_linux ?= amd64 arm64
65+
ALL_OS_ARCH_linux = $(foreach arch, ${ALL_ARCH_linux}, linux-$(arch))
66+
ALL_ARCH_windows = amd64
67+
ALL_OSVERSIONS_windows := 1809 ltsc2022
68+
ALL_OS_ARCH_windows = $(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSIONS_windows}, windows-${osversion}-${arch}))
69+
ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH_${os}})
7070

7171
# The current context of image building
7272
# The architecture of the image
@@ -328,24 +328,24 @@ docker-buildx-builder:
328328

329329
.PHONY: container-all
330330
container-all: docker-buildx-builder
331-
for arch in $(ALL_ARCH.linux); do \
331+
for arch in $(ALL_ARCH_linux); do \
332332
ARCH=$${arch} $(MAKE) container-linux; \
333333
ARCH=$${arch} $(MAKE) crd-container-linux; \
334334
done
335-
for osversion in $(ALL_OSVERSIONS.windows); do \
335+
for osversion in $(ALL_OSVERSIONS_windows); do \
336336
OSVERSION=$${osversion} $(MAKE) container-windows; \
337337
done
338338

339339
.PHONY: push-manifest
340340
push-manifest:
341341
docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
342-
docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH.linux), $(CRD_IMAGE_TAG)-${osarch})
342+
docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH_linux), $(CRD_IMAGE_TAG)-${osarch})
343343
# add "os.version" field to windows images (based on https://github.com/kubernetes/kubernetes/blob/master/build/pause/Makefile)
344344
set -x; \
345345
registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq ".*[\/\.].*") && echo "" || echo "docker.io/"); \
346346
manifest_image_folder=`echo "$${registry_prefix}${IMAGE_TAG}" | sed "s|/|_|g" | sed "s/:/-/"`; \
347-
for arch in $(ALL_ARCH.windows); do \
348-
for osversion in $(ALL_OSVERSIONS.windows); do \
347+
for arch in $(ALL_ARCH_windows); do \
348+
for osversion in $(ALL_OSVERSIONS_windows); do \
349349
BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \
350350
full_version=`docker manifest inspect $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \
351351
sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":\"$${full_version}\"/" "${HOME}/.docker/manifests/$${manifest_image_folder}/$${manifest_image_folder}-windows-$${osversion}-$${arch}"; \
@@ -359,11 +359,12 @@ push-manifest:
359359
## --------------------------------------
360360
## E2E Testing
361361
## --------------------------------------
362+
.PHONY: e2e-install-prerequisites
363+
e2e-install-prerequisites: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ)
364+
362365
.PHONY: e2e-bootstrap
363-
e2e-bootstrap: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ) #setup all required binaries and kind cluster for testing
364-
ifndef TEST_WINDOWS
366+
e2e-bootstrap: e2e-install-prerequisites #setup all required binaries and kind cluster for testing
365367
$(MAKE) setup-kind
366-
endif
367368
docker pull $(IMAGE_TAG) || $(MAKE) e2e-container
368369

369370
.PHONY: setup-kind
@@ -378,12 +379,8 @@ setup-eks-cluster: $(HELM) $(EKSCTL) $(BATS) $(ENVSUBST) $(YQ)
378379

379380
.PHONY: e2e-container
380381
e2e-container:
381-
ifdef TEST_WINDOWS
382-
$(MAKE) container-all push-manifest
383-
else
384382
$(MAKE) container
385383
kind load docker-image --name kind $(IMAGE_TAG) $(CRD_IMAGE_TAG)
386-
endif
387384

388385
.PHONY: e2e-mock-provider-container
389386
e2e-mock-provider-container:
@@ -438,7 +435,8 @@ e2e-helm-deploy:
438435
--set rotationPollInterval=30s \
439436
--set tokenRequests[0].audience="aud1" \
440437
--set tokenRequests[1].audience="aud2" \
441-
--set tokenRequests[2].audience="conjur"
438+
--set tokenRequests[2].audience="conjur" \
439+
--set tokenRequests[3].audience="api://AzureADTokenExchange"
442440

443441
.PHONY: e2e-helm-upgrade
444442
e2e-helm-upgrade:

‎test/bats/azure.bats‎

Lines changed: 3 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,10 @@ if [ $TEST_WINDOWS ]; then
1616
NODE_SELECTOR_OS=windows
1717
fi
1818

19-
if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then
20-
export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6)
21-
fi
22-
23-
export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e}
19+
export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e}
2420
export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1}
2521
export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""}
2622
export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"}
27-
export KEY_NAME=${KEYVAULT_KEY_NAME:-key1}
28-
export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a}
29-
export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"}
3023
export LABEL_VALUE=${LABEL_VALUE:-"test"}
3124
export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
3225

@@ -35,8 +28,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
3528
export API_VERSION=$(get_secrets_store_api_version)
3629

3730
setup() {
38-
if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then
39-
echo "Error: Azure service principal is not provided" >&2
31+
if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then
32+
echo "Error: Azure managed identity id is not provided" >&2
4033
return 1
4134
fi
4235
}
@@ -55,15 +48,6 @@ setup() {
5548
kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE
5649
}
5750

58-
@test "create azure k8s secret" {
59-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET}
60-
assert_success
61-
62-
# label the node publish secret ref secret
63-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
64-
assert_success
65-
}
66-
6751
@test "deploy azure secretproviderclass crd" {
6852
envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f -
6953

@@ -92,12 +76,6 @@ setup() {
9276
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
9377
}
9478

95-
@test "CSI inline volume test with pod portability - read azure kv key from pod" {
96-
result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME)
97-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
98-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
99-
}
100-
10179
@test "CSI inline volume test with pod portability - unmount succeeds" {
10280
# On Linux a failure to unmount the tmpfs will block the pod from being
10381
# deleted.
@@ -140,10 +118,6 @@ setup() {
140118
result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias)
141119
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
142120

143-
result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME)
144-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
145-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
146-
147121
result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d)
148122
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
149123

@@ -180,13 +154,6 @@ setup() {
180154
run kubectl create ns test-ns
181155
assert_success
182156

183-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns
184-
assert_success
185-
186-
# label the node publish secret ref secret
187-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns
188-
assert_success
189-
190157
envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f -
191158

192159
kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
@@ -208,10 +175,6 @@ setup() {
208175
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias)
209176
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
210177

211-
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME)
212-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
213-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
214-
215178
result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d)
216179
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
217180

@@ -234,13 +197,6 @@ setup() {
234197
run kubectl create ns negative-test-ns
235198
assert_success
236199

237-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns
238-
assert_success
239-
240-
# label the node publish secret ref secret
241-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns
242-
assert_success
243-
244200
envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f -
245201
sleep 5
246202

@@ -280,17 +236,9 @@ setup() {
280236
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias)
281237
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
282238

283-
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME)
284-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
285-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
286-
287239
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias)
288240
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
289241

290-
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME)
291-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
292-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
293-
294242
result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d)
295243
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
296244

@@ -310,66 +258,11 @@ setup() {
310258
assert_success
311259
}
312260

313-
@test "Test auto rotation of mount contents and K8s secrets - Create deployment" {
314-
run kubectl create ns rotation
315-
assert_success
316-
317-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation
318-
assert_success
319-
320-
# label the node publish secret ref secret
321-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation
322-
assert_success
323-
324-
run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal
325-
assert_success
326-
327-
run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret
328-
assert_success
329-
330-
envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f -
331-
envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f -
332-
333-
kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation
334-
335-
run kubectl get pod/secrets-store-inline-rotation -n rotation
336-
assert_success
337-
}
338-
339-
@test "Test auto rotation of mount contents and K8s secrets" {
340-
result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
341-
[[ "${result//$'\r'}" == "secret" ]]
342-
343-
result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
344-
[[ "${result//$'\r'}" == "secret" ]]
345-
346-
run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated
347-
assert_success
348-
349-
sleep 60
350-
351-
result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
352-
[[ "${result//$'\r'}" == "rotated" ]]
353-
354-
result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
355-
[[ "${result//$'\r'}" == "rotated" ]]
356-
357-
run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME}
358-
assert_success
359-
360-
run az logout
361-
assert_success
362-
}
363-
364261
teardown_file() {
365262
archive_provider "app=csi-secrets-store-provider-azure" || true
366263
archive_info || true
367264

368265
#cleanup
369-
run kubectl delete namespace rotation
370266
run kubectl delete namespace test-ns
371-
372-
run kubectl delete secret secrets-store-creds
373-
374267
run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0
375268
}

‎test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ spec:
1313
- objectName: secretalias # name of the mounted content to sync. this could be the object name or object alias
1414
key: username
1515
parameters:
16-
usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false"
16+
clientID: "$IDENTITY_CLIENT_ID"
1717
keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
1818
objects: |
1919
array:
@@ -22,8 +22,4 @@ spec:
2222
objectType: secret # object types: secret, key or cert
2323
objectAlias: secretalias
2424
objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty
25-
- |
26-
objectName: $KEY_NAME
27-
objectType: key
28-
objectVersion: $KEY_VERSION
29-
tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
25+
tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault

‎test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ spec:
1111
- objectName: secretalias
1212
key: username
1313
parameters:
14-
usePodIdentity: "false"
14+
clientID: "$IDENTITY_CLIENT_ID"
1515
keyvaultName: "$KEYVAULT_NAME"
1616
objects: |
1717
array:
@@ -20,11 +20,7 @@ spec:
2020
objectType: secret
2121
objectVersion: $SECRET_VERSION
2222
objectAlias: secretalias
23-
- |
24-
objectName: $KEY_NAME
25-
objectType: key
26-
objectVersion: $KEY_VERSION
27-
tenantId: "$TENANT_ID"
23+
tenantId: "$AZURE_TENANT_ID"
2824
---
2925
apiVersion: $API_VERSION
3026
kind: SecretProviderClass
@@ -39,7 +35,7 @@ spec:
3935
- objectName: secretalias
4036
key: username
4137
parameters:
42-
usePodIdentity: "false"
38+
clientID: "$IDENTITY_CLIENT_ID"
4339
keyvaultName: "$KEYVAULT_NAME"
4440
objects: |
4541
array:
@@ -48,8 +44,4 @@ spec:
4844
objectType: secret
4945
objectVersion: $SECRET_VERSION
5046
objectAlias: secretalias
51-
- |
52-
objectName: $KEY_NAME
53-
objectType: key
54-
objectVersion: $KEY_VERSION
55-
tenantId: "$TENANT_ID"
47+
tenantId: "$AZURE_TENANT_ID"

‎test/bats/tests/azure/azure_v1_secretproviderclass.yaml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,12 @@ metadata:
55
spec:
66
provider: azure
77
parameters:
8-
usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false"
8+
clientID: "$IDENTITY_CLIENT_ID"
99
keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
1010
objects: |
1111
array:
1212
- |
1313
objectName: $SECRET_NAME
1414
objectType: secret # object types: secret, key or cert
1515
objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty
16-
- |
17-
objectName: $KEY_NAME
18-
objectType: key
19-
objectVersion: $KEY_VERSION
20-
tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
16+
tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault

0 commit comments

Comments
 (0)