Skip to content

Commit 280ae2b

Browse files
eyenxeyenx
committed
feat(tests): add openbao tests - highly inspired by the vault tests
Signed-off-by: Toni Tauro <[email protected]>
1 parent 792e0c7 commit 280ae2b

10 files changed

+649
-0
lines changed

‎test/bats/openbao.bats‎

Lines changed: 314 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,314 @@
1+
#!/usr/bin/env bats
2+
3+
# mostly inspired by the vault provider tests
4+
# https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/main/test/bats/vault.bats
5+
# credits to @sozercan @aramase @ritazh and the rest of the community
6+
7+
load helpers
8+
9+
BATS_TESTS_DIR=test/bats/tests/openbao
10+
WAIT_TIME=120
11+
SLEEP_TIME=1
12+
13+
export LABEL_VALUE=${LABEL_VALUE:-"test"}
14+
export ANNOTATION_VALUE=${ANNOTATION_VALUE:-"app=test"}
15+
16+
@test "install openbao provider" {
17+
# install openbao including the csi provider using helm
18+
helm repo add openbao https://openbao.github.io/openbao-helm
19+
helm repo update
20+
helm install openbao openbao/openbao -n openbao --create-namespace \
21+
--set "server.dev.enabled=true" \
22+
--set "injector.enabled=false" \
23+
--set "csi.enabled=true"
24+
25+
# wait for openbao and openbao-csi-provider pods to be running
26+
kubectl wait --for=condition=Ready --timeout=120s pods --all -n openbao
27+
}
28+
29+
@test "configure openbao" {
30+
# create the secrets pair in openbao
31+
kubectl exec openbao-0 -n openbao -- bao secrets enable -version=2 -path=secrets kv
32+
kubectl exec openbao-0 -n openbao -- bao kv put secrets/foo foo=openbao-foo
33+
kubectl exec openbao-0 -n openbao -- bao kv put secrets/bar bar=openbao-bar
34+
35+
# enable authentication
36+
kubectl exec openbao-0 -n openbao -- bao auth enable kubernetes
37+
38+
local token_reviewer_jwt="$(kubectl exec openbao-0 -n openbao -- cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
39+
local kubernetes_service_ip="$(kubectl get svc kubernetes -o go-template="{{ .spec.clusterIP }}")"
40+
# enable authentication using the kubernetes service token from openbao pod
41+
kubectl exec -i openbao-0 -n openbao -- bao write auth/kubernetes/config \
42+
issuer="https://kubernetes.default.svc.cluster.local" \
43+
token_reviewer_jwt="${token_reviewer_jwt}" \
44+
kubernetes_host="https://${kubernetes_service_ip}:443" \
45+
kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
46+
47+
# create openbao policy to allow access to created secrets
48+
kubectl exec -i openbao-0 -n openbao -- bao policy write csi - <<EOF
49+
path "secrets/data/*" {
50+
capabilities = ["read"]
51+
}
52+
EOF
53+
54+
# create authentication role
55+
kubectl exec -i openbao-0 -n openbao -- bao write auth/kubernetes/role/csi \
56+
bound_service_account_names=default \
57+
bound_service_account_namespaces=default,test-ns,negative-test-ns \
58+
policies=csi \
59+
ttl=20m
60+
}
61+
62+
@test "deploy openbao secretproviderclass crd" {
63+
kubectl apply -f $BATS_TESTS_DIR/openbao_v1_secretproviderclass.yaml
64+
kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
65+
66+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo -o yaml | grep openbao"
67+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
68+
}
69+
70+
@test "CSI inline volume test with pod portability" {
71+
kubectl apply -f $BATS_TESTS_DIR/pod-openbao-inline-volume-secretproviderclass.yaml
72+
# wait for pod to be running
73+
kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-inline
74+
75+
run kubectl get pod/secrets-store-inline
76+
assert_success
77+
}
78+
79+
@test "CSI inline volume test with pod portability - read openbao secret from pod" {
80+
result=$(kubectl exec secrets-store-inline -- cat /mnt/secrets-store/foo)
81+
[[ "$result" == "openbao-foo" ]]
82+
83+
result=$(kubectl exec secrets-store-inline -- cat /mnt/secrets-store/bar)
84+
[[ "$result" == "openbao-bar" ]]
85+
}
86+
87+
@test "CSI inline volume test with pod portability - rotation succeeds" {
88+
# seed first value
89+
kubectl exec openbao-0 -n openbao -- bao kv put secrets/rotation foo=start
90+
91+
# deploy pod
92+
kubectl apply -f $BATS_TESTS_DIR/pod-openbao-rotation.yaml
93+
kubectl wait --for=condition=Ready --timeout=60s pod/secrets-store-rotation
94+
95+
run kubectl get pod/secrets-store-rotation
96+
assert_success
97+
98+
# verify starting value
99+
result=$(kubectl exec secrets-store-rotation -- cat /mnt/secrets-store/foo)
100+
[[ "$result" == "start" ]]
101+
102+
# update the secret value
103+
kubectl exec openbao-0 -n openbao -- bao kv put secrets/rotation foo=rotated
104+
105+
sleep 60
106+
107+
# verify rotated value
108+
result=$(kubectl exec secrets-store-rotation -- cat /mnt/secrets-store/foo)
109+
[[ "$result" == "rotated" ]]
110+
}
111+
112+
@test "CSI inline volume test with pod portability - unmount succeeds" {
113+
# On Linux a failure to unmount the tmpfs will block the pod from being
114+
# deleted.
115+
run kubectl delete pod secrets-store-inline
116+
assert_success
117+
118+
run kubectl wait --for=delete --timeout=${WAIT_TIME}s pod/secrets-store-inline
119+
assert_success
120+
121+
# Sleep to allow time for logs to propagate.
122+
sleep 10
123+
124+
# save debug information to archive in case of failure
125+
archive_info
126+
127+
# On Windows, the failed unmount calls from: https://github.com/kubernetes-sigs/secrets-store-csi-driver/pull/545
128+
# do not prevent the pod from being deleted. Search through the driver logs
129+
# for the error.
130+
run bash -c "kubectl logs -l app=secrets-store-csi-driver --tail -1 -c secrets-store -n kube-system | grep '^E.*failed to clean and unmount target path.*$'"
131+
assert_failure
132+
}
133+
134+
@test "Sync with K8s secrets - create deployment" {
135+
kubectl apply -f $BATS_TESTS_DIR/openbao_synck8s_v1_secretproviderclass.yaml
136+
kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
137+
138+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo-sync -o yaml | grep openbao"
139+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
140+
141+
run kubectl apply -f $BATS_TESTS_DIR/deployment-synck8s.yaml
142+
assert_success
143+
144+
run kubectl apply -f $BATS_TESTS_DIR/deployment-two-synck8s.yaml
145+
assert_success
146+
147+
kubectl wait --for=condition=Ready --timeout=120s pod -l app=busybox
148+
}
149+
150+
@test "Sync with K8s secrets - read secret from pod, read K8s secret, read env var, check secret ownerReferences with multiple owners" {
151+
POD=$(kubectl get pod -l app=busybox -o jsonpath="{.items[0].metadata.name}")
152+
result=$(kubectl exec $POD -- cat /mnt/secrets-store/foo)
153+
[[ "$result" == "openbao-foo" ]]
154+
155+
result=$(kubectl exec $POD -- cat /mnt/secrets-store/bar)
156+
[[ "$result" == "openbao-bar" ]]
157+
158+
result=$(kubectl exec $POD -- cat /mnt/secrets-store/nested/foo)
159+
[[ "$result" == "openbao-foo" ]]
160+
161+
result=$(kubectl get secret foosecret -o jsonpath="{.data.pwd}" | base64 -d)
162+
[[ "$result" == "openbao-foo" ]]
163+
164+
result=$(kubectl get secret foosecret -o jsonpath="{.data.nested}" | base64 -d)
165+
[[ "$result" == "openbao-foo" ]]
166+
167+
result=$(kubectl exec $POD -- printenv | grep SECRET_USERNAME | awk -F"=" '{ print $2 }' | tr -d '\r\n')
168+
[[ "$result" == "openbao-bar" ]]
169+
170+
result=$(kubectl get secret foosecret -o jsonpath="{.metadata.labels.environment}")
171+
[[ "${result//$'\r'/}" == "${LABEL_VALUE}" ]]
172+
173+
result=$(kubectl get secret foosecret -o jsonpath="{.metadata.annotations.kubed\.appscode\.com\/sync}")
174+
[[ "${result//$'\r'/}" == "${ANNOTATION_VALUE}" ]]
175+
176+
result=$(kubectl get secret foosecret -o jsonpath="{.metadata.labels.secrets-store\.csi\.k8s\.io/managed}")
177+
[[ "${result//$'\r'/}" == "true" ]]
178+
179+
run wait_for_process $WAIT_TIME $SLEEP_TIME "compare_owner_count foosecret default 2"
180+
assert_success
181+
}
182+
183+
@test "Sync with K8s secrets - delete deployment, check secret is deleted" {
184+
run kubectl delete -f $BATS_TESTS_DIR/deployment-synck8s.yaml
185+
assert_success
186+
187+
run wait_for_process $WAIT_TIME $SLEEP_TIME "compare_owner_count foosecret default 1"
188+
assert_success
189+
190+
run kubectl delete -f $BATS_TESTS_DIR/deployment-two-synck8s.yaml
191+
assert_success
192+
193+
run wait_for_process $WAIT_TIME $SLEEP_TIME "check_secret_deleted foosecret default"
194+
assert_success
195+
196+
run kubectl delete -f $BATS_TESTS_DIR/openbao_synck8s_v1_secretproviderclass.yaml
197+
assert_success
198+
}
199+
200+
@test "Test Namespaced scope SecretProviderClass - create deployment" {
201+
kubectl create ns test-ns
202+
203+
kubectl apply -f $BATS_TESTS_DIR/openbao_v1_secretproviderclass_ns.yaml
204+
kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
205+
206+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo-sync -o yaml | grep openbao"
207+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
208+
209+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo-sync -n test-ns -o yaml | grep openbao"
210+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
211+
212+
kubectl apply -n test-ns -f $BATS_TESTS_DIR/deployment-synck8s.yaml
213+
214+
kubectl wait --for=condition=Ready --timeout=90s pod -l app=busybox -n test-ns
215+
}
216+
217+
@test "Test Namespaced scope SecretProviderClass - Sync with K8s secrets - read secret from pod, read K8s secret, read env var, check secret ownerReferences" {
218+
POD=$(kubectl get pod -l app=busybox -n test-ns -o jsonpath="{.items[0].metadata.name}")
219+
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/foo)
220+
[[ "$result" == "openbao-foo" ]]
221+
222+
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/bar)
223+
[[ "$result" == "openbao-bar" ]]
224+
225+
result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.pwd}" | base64 -d)
226+
[[ "$result" == "openbao-foo" ]]
227+
228+
result=$(kubectl exec -n test-ns $POD -- printenv | grep SECRET_USERNAME | awk -F"=" '{ print $2 }' | tr -d '\r\n')
229+
[[ "$result" == "openbao-bar" ]]
230+
231+
run wait_for_process $WAIT_TIME $SLEEP_TIME "compare_owner_count foosecret test-ns 1"
232+
assert_success
233+
}
234+
235+
@test "Test Namespaced scope SecretProviderClass - Sync with K8s secrets - delete deployment, check secret deleted" {
236+
run kubectl delete -f $BATS_TESTS_DIR/deployment-synck8s.yaml -n test-ns
237+
assert_success
238+
239+
run wait_for_process $WAIT_TIME $SLEEP_TIME "check_secret_deleted foosecret test-ns"
240+
assert_success
241+
}
242+
243+
@test "Test Namespaced scope SecretProviderClass - Should fail when no secret provider class in same namespace" {
244+
kubectl create ns negative-test-ns
245+
246+
kubectl apply -n negative-test-ns -f $BATS_TESTS_DIR/deployment-synck8s.yaml
247+
248+
POD=$(kubectl get pod -l app=busybox -n negative-test-ns -o jsonpath="{.items[0].metadata.name}")
249+
cmd="kubectl describe pod $POD -n negative-test-ns | grep 'FailedMount.*failed to get secretproviderclass negative-test-ns/openbao-foo-sync.*not found'"
250+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
251+
252+
run kubectl delete -f $BATS_TESTS_DIR/deployment-synck8s.yaml -n negative-test-ns
253+
assert_success
254+
255+
run kubectl delete ns negative-test-ns
256+
assert_success
257+
}
258+
259+
@test "deploy multiple openbao secretproviderclass crd" {
260+
kubectl apply -f $BATS_TESTS_DIR/openbao_v1_multiple_secretproviderclass.yaml
261+
262+
cmd="kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io"
263+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
264+
265+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo-sync-0 -o yaml | grep openbao-foo-sync-0"
266+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
267+
268+
cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/openbao-foo-sync-1 -o yaml | grep openbao-foo-sync-1"
269+
wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
270+
}
271+
272+
@test "deploy pod with multiple secret provider class" {
273+
kubectl apply -f $BATS_TESTS_DIR/pod-openbao-inline-volume-multiple-spc.yaml
274+
kubectl wait --for=condition=Ready --timeout=90s pod/secrets-store-inline-multiple-crd
275+
276+
run kubectl get pod/secrets-store-inline-multiple-crd
277+
assert_success
278+
}
279+
280+
@test "CSI inline volume test with multiple secret provider class" {
281+
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/foo)
282+
[[ "$result" == "openbao-foo" ]]
283+
284+
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/bar)
285+
[[ "$result" == "openbao-bar" ]]
286+
287+
result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.pwd}" | base64 -d)
288+
[[ "$result" == "openbao-foo" ]]
289+
290+
result=$(kubectl exec secrets-store-inline-multiple-crd -- printenv | grep SECRET_USERNAME_0 | awk -F"=" '{ print $2 }' | tr -d '\r\n')
291+
[[ "$result" == "openbao-bar" ]]
292+
293+
run wait_for_process $WAIT_TIME $SLEEP_TIME "compare_owner_count foosecret-0 default 1"
294+
assert_success
295+
296+
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/foo)
297+
[[ "$result" == "openbao-foo" ]]
298+
299+
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/bar)
300+
[[ "$result" == "openbao-bar" ]]
301+
302+
result=$(kubectl get secret foosecret-1 -o jsonpath="{.data.pwd}" | base64 -d)
303+
[[ "$result" == "openbao-foo" ]]
304+
305+
result=$(kubectl exec secrets-store-inline-multiple-crd -- printenv | grep SECRET_USERNAME_1 | awk -F"=" '{ print $2 }' | tr -d '\r\n')
306+
[[ "$result" == "openbao-bar" ]]
307+
308+
run wait_for_process $WAIT_TIME $SLEEP_TIME "compare_owner_count foosecret-1 default 1"
309+
assert_success
310+
}
311+
312+
teardown_file() {
313+
archive_info || true
314+
}

‎test/bats/tests/openbao/deployment-synck8s.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: busybox-deployment
5+
labels:
6+
app: busybox
7+
spec:
8+
replicas: 2
9+
selector:
10+
matchLabels:
11+
app: busybox
12+
template:
13+
metadata:
14+
labels:
15+
app: busybox
16+
spec:
17+
terminationGracePeriodSeconds: 0
18+
containers:
19+
- image: registry.k8s.io/e2e-test-images/busybox:1.29-4
20+
name: busybox
21+
imagePullPolicy: IfNotPresent
22+
command:
23+
- "/bin/sleep"
24+
- "10000"
25+
env:
26+
- name: SECRET_USERNAME
27+
valueFrom:
28+
secretKeyRef:
29+
name: foosecret
30+
key: username
31+
volumeMounts:
32+
- name: secrets-store-inline
33+
mountPath: "/mnt/secrets-store"
34+
readOnly: true
35+
volumes:
36+
- name: secrets-store-inline
37+
csi:
38+
driver: secrets-store.csi.k8s.io
39+
readOnly: true
40+
volumeAttributes:
41+
secretProviderClass: "openbao-foo-sync"

‎test/bats/tests/openbao/deployment-two-synck8s.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: busybox-deployment-two
5+
labels:
6+
app: busybox
7+
spec:
8+
replicas: 2
9+
selector:
10+
matchLabels:
11+
app: busybox
12+
template:
13+
metadata:
14+
labels:
15+
app: busybox
16+
spec:
17+
terminationGracePeriodSeconds: 0
18+
containers:
19+
- image: registry.k8s.io/e2e-test-images/busybox:1.29-4
20+
name: busybox
21+
imagePullPolicy: IfNotPresent
22+
command:
23+
- "/bin/sleep"
24+
- "10000"
25+
env:
26+
- name: SECRET_USERNAME
27+
valueFrom:
28+
secretKeyRef:
29+
name: foosecret
30+
key: username
31+
volumeMounts:
32+
- name: secrets-store-inline
33+
mountPath: "/mnt/secrets-store"
34+
readOnly: true
35+
volumes:
36+
- name: secrets-store-inline
37+
csi:
38+
driver: secrets-store.csi.k8s.io
39+
readOnly: true
40+
volumeAttributes:
41+
secretProviderClass: "openbao-foo-sync"

0 commit comments

Comments
 (0)