Skip to content

Commit e293fe7

Browse files
egegunesegegunes
committed
fix pvc-resize
1 parent 7a89dcc commit e293fe7

File tree

4 files changed

+187
-43
lines changed

4 files changed

+187
-43
lines changed

‎e2e-tests/pvc-resize/conf/eks-storageclass.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: storage.k8s.io/v1
2+
kind: StorageClass
3+
metadata:
4+
name: gp2-resizable
5+
parameters:
6+
fsType: ext4
7+
type: gp2
8+
provisioner: kubernetes.io/aws-ebs
9+
reclaimPolicy: Delete
10+
volumeBindingMode: WaitForFirstConsumer
11+
allowVolumeExpansion: true

‎e2e-tests/pvc-resize/conf/some-name-eks.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
apiVersion: psmdb.percona.com/v1
2+
kind: PerconaServerMongoDB
3+
metadata:
4+
name: some-name
5+
finalizers:
6+
- delete-psmdb-pods-in-order
7+
- delete-psmdb-pvc
8+
spec:
9+
#platform: openshift
10+
image:
11+
imagePullPolicy: Always
12+
allowUnsafeConfigurations: false
13+
backup:
14+
enabled: false
15+
image: perconalab/percona-server-mongodb-operator:0.4.0-backup
16+
replsets:
17+
- name: rs0
18+
# readinessDelaySec: 40
19+
# livenessDelaySec: 120
20+
affinity:
21+
antiAffinityTopologyKey: none
22+
resources:
23+
limits:
24+
cpu: 500m
25+
memory: 0.5G
26+
requests:
27+
cpu: 100m
28+
memory: 0.1G
29+
configuration: |
30+
operationProfiling:
31+
mode: slowOp
32+
slowOpThresholdMs: 100
33+
security:
34+
enableEncryption: true
35+
redactClientLogData: false
36+
setParameter:
37+
ttlMonitorSleepSecs: 60
38+
wiredTigerConcurrentReadTransactions: 128
39+
wiredTigerConcurrentWriteTransactions: 128
40+
storage:
41+
engine: wiredTiger
42+
wiredTiger:
43+
collectionConfig:
44+
blockCompressor: snappy
45+
engineConfig:
46+
directoryForIndexes: false
47+
journalCompressor: snappy
48+
indexConfig:
49+
prefixCompression: true
50+
volumeSpec:
51+
persistentVolumeClaim:
52+
storageClassName: gp2-resizable
53+
resources:
54+
requests:
55+
storage: 1Gi
56+
size: 3
57+
secrets:
58+
users: some-users

‎e2e-tests/pvc-resize/conf/some-name.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apiVersion: psmdb.percona.com/v1
2+
kind: PerconaServerMongoDB
3+
metadata:
4+
name: some-name
5+
finalizers:
6+
- delete-psmdb-pods-in-order
7+
- delete-psmdb-pvc
8+
spec:
9+
#platform: openshift
10+
image:
11+
imagePullPolicy: Always
12+
allowUnsafeConfigurations: false
13+
backup:
14+
enabled: false
15+
image: perconalab/percona-server-mongodb-operator:0.4.0-backup
16+
replsets:
17+
- name: rs0
18+
# readinessDelaySec: 40
19+
# livenessDelaySec: 120
20+
affinity:
21+
antiAffinityTopologyKey: none
22+
resources:
23+
limits:
24+
cpu: 500m
25+
memory: 0.5G
26+
requests:
27+
cpu: 100m
28+
memory: 0.1G
29+
configuration: |
30+
operationProfiling:
31+
mode: slowOp
32+
slowOpThresholdMs: 100
33+
security:
34+
enableEncryption: true
35+
redactClientLogData: false
36+
setParameter:
37+
ttlMonitorSleepSecs: 60
38+
wiredTigerConcurrentReadTransactions: 128
39+
wiredTigerConcurrentWriteTransactions: 128
40+
storage:
41+
engine: wiredTiger
42+
wiredTiger:
43+
collectionConfig:
44+
blockCompressor: snappy
45+
engineConfig:
46+
directoryForIndexes: false
47+
journalCompressor: snappy
48+
indexConfig:
49+
prefixCompression: true
50+
volumeSpec:
51+
persistentVolumeClaim:
52+
resources:
53+
requests:
54+
storage: 1Gi
55+
size: 3
56+
secrets:
57+
users: some-users

‎e2e-tests/pvc-resize/run

Lines changed: 61 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ function get_default_storageclass() {
2121
function ensure_default_sc_allows_expansion() {
2222
local default_sc=$(get_default_storageclass)
2323

24-
echo "Checking if default storageclass ${default_sc} allows volume expansion"
24+
echo "Checking if default storageclass ${default_sc} allows volume expansion"
2525

26-
local allowVolumeExpansion=$(kubectl_bin get sc -o jsonpath='{.items[?(@.metadata.name=="'"${default_sc}"'")].allowVolumeExpansion}')
26+
local allowVolumeExpansion=$(kubectl_bin get sc -o jsonpath='{.items[?(@.metadata.name=="'"${default_sc}"'")].allowVolumeExpansion}')
2727

2828
if [[ ${allowVolumeExpansion} != "true" ]]; then
2929
echo "Default storageclass ${default_sc} does not allow volume expansion"
@@ -32,15 +32,15 @@ function ensure_default_sc_allows_expansion() {
3232
}
3333

3434
function apply_resourcequota() {
35-
local quota=$1
36-
local default_sc=$(get_default_storageclass)
35+
local quota=$1
36+
local default_sc=$(get_default_storageclass)
3737

38-
echo "Applying resourcequota for default storageclass ${default_sc} with quota ${quota}"
38+
echo "Applying resourcequota for default storageclass ${default_sc} with quota ${quota}"
3939

40-
cat ${test_dir}/conf/resourcequota.yml \
41-
| sed "s/STORAGECLASS/${default_sc}/" \
42-
| sed "s/QUOTA/${quota}/" \
43-
| kubectl_bin apply -f -
40+
cat ${test_dir}/conf/resourcequota.yml |
41+
sed "s/STORAGECLASS/${default_sc}/" |
42+
sed "s/QUOTA/${quota}/" |
43+
kubectl_bin apply -f -
4444
}
4545

4646
function wait_cluster_status() {
@@ -66,41 +66,59 @@ function wait_cluster_status() {
6666

6767
set_debug
6868

69-
ensure_default_sc_allows_expansion
69+
if [ "$EKS" == 1 ]; then
70+
echo "EKS environment detected, creating storageclass for EBS volumes"
71+
kubectl_bin apply -f ${test_dir}/conf/eks-storageclass.yml
72+
else
73+
ensure_default_sc_allows_expansion
74+
fi
7075

7176
create_infra ${namespace}
7277

7378
desc 'create secrets and psmdb client'
7479
kubectl_bin apply \
75-
-f $conf_dir/secrets.yml \
76-
-f $conf_dir/client.yml
80+
-f $conf_dir/secrets.yml \
81+
-f $conf_dir/client.yml
7782

7883
desc 'create PSMDB cluster'
7984
cluster="some-name"
80-
spinup_psmdb "${cluster}-rs0" "$conf_dir/$cluster-rs0.yml"
85+
spinup_psmdb "${cluster}-rs0" "$test_dir/conf/$cluster.yml"
8186

8287
patch_pvc_request "${cluster}" "2Gi"
8388
wait_cluster_consistency "$cluster" 3 2
8489
echo
8590

8691
for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); do
87-
retry=0
88-
echo -n "Waiting for pvc/${pvc} to be resized"
89-
until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "2Gi" ]]; do
90-
if [[ $retry -ge 60 ]]; then
91-
echo
92-
echo "pvc/${pvc} was not resized, max retries exceeded"
93-
exit 1
94-
fi
95-
echo -n "."
96-
sleep 5
97-
98-
retry=$((retry + 1))
99-
done
100-
echo
101-
echo "pvc/${pvc} was resized"
92+
retry=0
93+
echo -n "Waiting for pvc/${pvc} to be resized"
94+
until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "2Gi" ]]; do
95+
if [[ $retry -ge 60 ]]; then
96+
echo
97+
echo "pvc/${pvc} was not resized, max retries exceeded"
98+
exit 1
99+
fi
100+
echo -n "."
101+
sleep 5
102+
103+
retry=$((retry + 1))
104+
done
105+
echo
106+
echo "pvc/${pvc} was resized"
102107
done
103108

109+
if [[ "$EKS" == 1 || -n ${OPENSHIFT} ]]; then
110+
# AWS rate limits PVC expansion for the same EBS volume (1 expand operation in every 6 hours),
111+
# so we need to delete and recreate the cluster
112+
echo "Deleting and recreating PSMDB cluster ${cluster}"
113+
kubectl_bin delete psmdb ${cluster}
114+
kubectl_bin apply -f $conf_dir/secrets.yml
115+
if [ "$EKS" == 1 ]; then
116+
spinup_psmdb "${cluster}-rs0" "$test_dir/conf/$cluster-eks.yml"
117+
else
118+
spinup_psmdb "${cluster}-rs0" "$test_dir/conf/$cluster.yml"
119+
fi
120+
fi
121+
104122
desc 'create resourcequota'
105123

106124
# We're setting the quota to 7Gi, so we can only resize the first PVC to 3Gi
@@ -134,21 +152,21 @@ patch_pvc_request "${cluster}" "3Gi"
134152
wait_cluster_consistency "$cluster" 3 2
135153
echo
136154
for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); do
137-
retry=0
138-
echo -n "Waiting for pvc/${pvc} to be resized"
139-
until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "3Gi" ]]; do
140-
if [[ $retry -ge 60 ]]; then
141-
echo
142-
echo "pvc/${pvc} was not resized, max retries exceeded"
143-
exit 1
144-
fi
145-
echo -n "."
146-
sleep 5
147-
148-
retry=$((retry + 1))
149-
done
150-
echo
151-
echo "pvc/${pvc} was resized"
155+
retry=0
156+
echo -n "Waiting for pvc/${pvc} to be resized"
157+
until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "3Gi" ]]; do
158+
if [[ $retry -ge 60 ]]; then
159+
echo
160+
echo "pvc/${pvc} was not resized, max retries exceeded"
161+
exit 1
162+
fi
163+
echo -n "."
164+
sleep 5
165+
166+
retry=$((retry + 1))
167+
done
168+
echo
169+
echo "pvc/${pvc} was resized"
152170
done
153171

154172
desc "test downscale"

0 commit comments

Comments
 (0)