Skip to content

Commit 0f9e85c

Browse files
author
Jesus Carrillo
committed
fixed csi-sanity checks with two separate test stacks
We are testing the nodeserver separatedly of the controller the controller sanity checks exclude all the nodeserver tests. In order for the node-server sanity checks to pass we have to: 1. Add Linode token and api. 2. Exclude the controller specific tests. There's a need for the nodeserver to have linode api access because sanity checks require CreateVolume and ListVolume. By patching the node-server Daemonset and injecting the LINODE_TOKEN The Controller Code Path is able to be initialized with a proper client. This is not needed for normal nodeserver operation, however the sanity-checks expect the ability for the nodeserver to also perform controller functions. E2E tests pass without the need of LINODE_TOKEN in the nodeserver
1 parent ee860d8 commit 0f9e85c

File tree

7 files changed

+188
-54
lines changed

7 files changed

+188
-54
lines changed

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,9 @@ create-capl-cluster:
117117

118118
.PHONY: generate-csi-driver-manifests
119119
generate-csi-driver-manifests:
120-
hack/generate-yaml.sh $(IMAGE_VERSION) $(DOCKER_USER)/$(IMAGE_NAME) > csi-manifests.yaml
120+
hack/generate-yaml.sh $(IMAGE_VERSION) $(DOCKER_USER)/$(IMAGE_NAME) > hack/patch/controller/csi-manifests.yaml
121+
kustomize build hack/patch/controller/ > csi-manifests.yaml
122+
rm hack/patch/controller/csi-manifests.yaml
121123

122124
.PHONY: install-csi
123125
install-csi:
@@ -171,7 +173,8 @@ e2e-test:
171173

172174
.PHONY: csi-sanity-test
173175
csi-sanity-test:
174-
KUBECONFIG=$(KUBECONFIG) ./tests/csi-sanity/run-tests.sh
176+
KUBECONFIG=$(KUBECONFIG) ./tests/csi-sanity/run-controller-tests.sh
177+
KUBECONFIG=$(KUBECONFIG) ./tests/csi-sanity/run-nodeserver-tests.sh
175178

176179
.PHONY: upstream-e2e-tests
177180
upstream-e2e-tests:

‎hack/generate-yaml.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@ fi
1919

2020
cd $(dirname "$0")/../
2121
file=./deploy/kubernetes/overlays/release/kustomization.yaml
22-
CSI_VERSION=$TAG CSI_IMAGE_NAME=$IMAGE_NAME envsubst < "$file.template" > $file
22+
CSI_VERSION=$TAG CSI_IMAGE_NAME=$IMAGE_NAME envsubst <"$file.template" >$file
2323

2424
kustomize build "$(dirname $file)"

‎hack/patch/controller/controller-patch.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: apps/v1
2+
kind: StatefulSet
3+
metadata:
4+
name: csi-linode-controller
5+
namespace: kube-system
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: socat-csi-controller
11+
image: alpine/socat
12+
command: ["socat"]
13+
args:
14+
- "TCP-LISTEN:10001,fork,reuseaddr"
15+
- "UNIX-CONNECT:/csi/csi.sock"
16+
volumeMounts:
17+
- name: socket-dir
18+
mountPath: /csi/

‎hack/patch/controller/kustomization.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
resources:
5+
- csi-manifests.yaml
6+
7+
patches:
8+
- path: controller-patch.yaml

‎tests/csi-sanity/run-controller-tests.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
set -euf -o pipefail
3+
4+
# Define the CSI endpoint for the sanity tests
5+
CSI_ENDPOINT="dns:///127.0.0.1:10001"
6+
7+
# Define the scripts for creating and deleting directories in the pod
8+
CREATE_DIRECTORY="./tests/csi-sanity/mkdir_in_pod.sh"
9+
DELETE_DIRECTORY="./tests/csi-sanity/rmdir_in_pod.sh"
10+
11+
# Define the list of tests to skip as an array
12+
SKIP_TESTS=(
13+
# Skip all node related tests
14+
"Node.*"
15+
"VolumeStats.*"
16+
"Probe"
17+
"NodeGetVolumeStats"
18+
"WithCapacity"
19+
# Need to skip it because we do not support volume snapshots
20+
"should fail when the volume source volume is not found"
21+
# This case fails because we currently do not support read only volume creation on the linode side
22+
# but we are supporting it in the CSI driver by mounting the volume as read only
23+
"should fail when the volume is already published but is incompatible"
24+
"should create volume from an existing source volume"
25+
)
26+
27+
# Join the array into a single string with '|' as the separator
28+
SKIP_TESTS_STRING=$(
29+
IFS='|'
30+
echo "${SKIP_TESTS[*]}"
31+
)
32+
33+
# Install the latest version of csi-sanity
34+
go install github.com/kubernetes-csi/csi-test/v5/cmd/csi-sanity@latest
35+
36+
# test manifests include a socat container in the controller
37+
nohup kubectl port-forward -n kube-system statefulset/csi-linode-controller 10001:10001 >port-forward.log 2>&1 &
38+
39+
# Run the csi-sanity tests with the specified parameters
40+
csi-sanity --ginkgo.vv --ginkgo.trace --ginkgo.skip "$SKIP_TESTS_STRING" --csi.endpoint="$CSI_ENDPOINT" --csi.createstagingpathcmd="$CREATE_DIRECTORY" --csi.createmountpathcmd="$CREATE_DIRECTORY" --csi.removestagingpathcmd="$DELETE_DIRECTORY" --csi.removemountpathcmd="$DELETE_DIRECTORY"
41+
42+
# Find the process ID (PID) of the kubectl port-forward command using the specified port
43+
PID=$(lsof -t -i :10001 -sTCP:LISTEN)
44+
45+
# Check if a PID was found and kill the process if it exists
46+
if [ -z "$PID" ]; then
47+
echo "No process found on port 10001."
48+
else
49+
kill -9 "$PID"
50+
echo "Process on port 10001 with PID $PID has been killed."
51+
fi

‎tests/csi-sanity/run-nodeserver-tests.sh‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
set -euf -o pipefail
3+
4+
# Define the CSI endpoint for the sanity tests
5+
CSI_ENDPOINT="dns:///127.0.0.1:10000"
6+
7+
# Define the scripts for creating and deleting directories in the pod
8+
CREATE_DIRECTORY="./tests/csi-sanity/mkdir_in_pod.sh"
9+
DELETE_DIRECTORY="./tests/csi-sanity/rmdir_in_pod.sh"
10+
11+
# Patch the NodeServer daemonset and add the LINODE_TOKEN and LINODE_API env vars.
12+
# The csi-sanity check require CrateVolume and ListVolume calls to be working.
13+
14+
# Warning: This patch expects the csi-linode-plugin container to be the 2nd container
15+
# in the manifest
16+
17+
kubectl patch daemonset csi-linode-node \
18+
-n kube-system \
19+
--type='json' \
20+
-p='[
21+
{
22+
"op": "add",
23+
"path": "/spec/template/spec/containers/1/env/-",
24+
"value": {
25+
"name": "LINODE_URL",
26+
"value": "https://api.linode.com/v4"
27+
}
28+
},
29+
{
30+
"op": "add",
31+
"path": "/spec/template/spec/containers/1/env/-",
32+
"value": {
33+
"name": "LINODE_TOKEN",
34+
"valueFrom": {
35+
"secretKeyRef": {
36+
"name": "linode",
37+
"key": "token"
38+
}
39+
}
40+
}
41+
}
42+
]'
43+
44+
echo "Waiting for daemonset csi-linode-node to be ready"
45+
kubectl rollout status daemonset/csi-linode-node -n kube-system
46+
kubectl wait --namespace kube-system --for=condition=Ready pods --selector=app=csi-linode-node --timeout=180s
47+
48+
# Define the list of tests to skip as an array
49+
SKIP_TESTS=(
50+
"WithCapacity"
51+
# Need to skip it because we do not support volume snapshots
52+
"should fail when the volume source volume is not found"
53+
# This case fails because we currently do not support read only volume creation on the linode side
54+
# but we are supporting it in the CSI driver by mounting the volume as read only
55+
"should fail when the volume is already published but is incompatible"
56+
# Skip all the controller related tests
57+
"Controller.*"
58+
"CreateVolume.*"
59+
"DeleteVolume.*"
60+
"ControllerPublishVolume.*"
61+
"ControllerUnpublishVolume.*"
62+
"ListVolumes.*"
63+
"ControllerExpandVolume.*"
64+
"ValidateVolumeCapabilities.*"
65+
"CreateSnapshot.*"
66+
"DeleteSnapshot.*"
67+
"ListSnapshots.*"
68+
)
69+
70+
# Join the array into a single string with '|' as the separator
71+
SKIP_TESTS_STRING=$(
72+
IFS='|'
73+
echo "${SKIP_TESTS[*]}"
74+
)
75+
76+
# Install the latest version of csi-sanity
77+
go install github.com/kubernetes-csi/csi-test/v5/cmd/csi-sanity@latest
78+
79+
# Create socat statefulset
80+
kubectl apply -f tests/csi-sanity/socat.yaml
81+
82+
# Wait for pod to be ready
83+
kubectl wait --for=condition=ready --timeout=60s pods/csi-socat-0
84+
85+
# Start the port forwarding in the background and log output to a file
86+
87+
## Test the controller only
88+
nohup kubectl port-forward pods/csi-socat-0 10000:10000 >port-forward.log 2>&1 &
89+
90+
# Run the csi-sanity tests with the specified parameters
91+
csi-sanity --ginkgo.vv --ginkgo.trace --ginkgo.skip "$SKIP_TESTS_STRING" --csi.endpoint="$CSI_ENDPOINT" --csi.createstagingpathcmd="$CREATE_DIRECTORY" --csi.createmountpathcmd="$CREATE_DIRECTORY" --csi.removestagingpathcmd="$DELETE_DIRECTORY" --csi.removemountpathcmd="$DELETE_DIRECTORY"
92+
93+
# Find the process ID (PID) of the kubectl port-forward command using the specified port
94+
PID=$(lsof -t -i :10000 -sTCP:LISTEN)
95+
96+
# Check if a PID was found and kill the process if it exists
97+
if [ -z "$PID" ]; then
98+
echo "No process found on port 10000."
99+
else
100+
kill -9 "$PID"
101+
echo "Process on port 10000 with PID $PID has been killed."
102+
fi
103+
104+
# Remove the socat statefulset
105+
kubectl delete -f tests/csi-sanity/socat.yaml

‎tests/csi-sanity/run-tests.sh‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

0 commit comments

Comments
 (0)