Skip to content

Commit 5b31035

Browse files
inelpandzicgithub-actions[bot]hors
authored
K8SPSMDB-778: Delete repset and its components properly (#1306)
* Update getting replset pods. * Logs * Improve getting stateful sets for removal. * logs * Fix getting removed sts. * fix component check * Cleanup * Fix cr.yaml * WIP: update data-sharded test * Fix removing. * Update test. * Update e2e-tests/data-sharded/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Updated data-sharded test. * Update e2e-tests/data-sharded/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/data-sharded/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/data-sharded/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * User replset label rather then extracting it from the name. * increse timeout * Refactor and fix lint error. * fix split-horizon test --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Viacheslav Sarzhan <[email protected]>
1 parent 9340bef commit 5b31035

File tree

13 files changed

+229
-196
lines changed

13 files changed

+229
-196
lines changed

‎deploy/cr.yaml

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -297,24 +297,13 @@ spec:
297297
# nodeSelector:
298298
# disktype: ssd
299299
# schedulerName: "default"
300-
resources:
301-
limits:
302-
cpu: "300m"
303-
memory: "0.5G"
304-
requests:
305-
cpu: "300m"
306-
memory: "0.5G"
307-
volumeSpec:
308-
# emptyDir: {}
309-
# hostPath:
310-
# path: /data
311-
# type: Directory
312-
persistentVolumeClaim:
313-
# storageClassName: standard
314-
# accessModes: [ "ReadWriteOnce" ]
315-
resources:
316-
requests:
317-
storage: 3Gi
300+
resources:
301+
limits:
302+
cpu: "300m"
303+
memory: "0.5G"
304+
requests:
305+
cpu: "300m"
306+
memory: "0.5G"
318307
# hostAliases:
319308
# - ip: "10.10.0.2"
320309
# hostnames:

‎e2e-tests/data-sharded/conf/some-name.yml

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ spec:
9494
requests:
9595
storage: 1Gi
9696

97-
size: 3
97+
size: 4
9898
configuration: |
9999
net:
100100
tls:
@@ -119,6 +119,9 @@ spec:
119119
journalCompressor: snappy
120120
indexConfig:
121121
prefixCompression: true
122+
arbiter:
123+
enabled: true
124+
size: 1
122125
- name: rs2
123126
affinity:
124127
antiAffinityTopologyKey: none
@@ -159,6 +162,28 @@ spec:
159162
journalCompressor: snappy
160163
indexConfig:
161164
prefixCompression: true
165+
nonvoting:
166+
enabled: true
167+
size: 3
168+
affinity:
169+
antiAffinityTopologyKey: "kubernetes.io/hostname"
170+
resources:
171+
limits:
172+
cpu: 500m
173+
memory: 1G
174+
requests:
175+
cpu: 100m
176+
memory: 0.1G
177+
volumeSpec:
178+
persistentVolumeClaim:
179+
resources:
180+
requests:
181+
storage: 2Gi
182+
volumeSpec:
183+
persistentVolumeClaim:
184+
resources:
185+
requests:
186+
storage: 1Gi
162187

163188
secrets:
164189
users: some-users

‎e2e-tests/data-sharded/run

Lines changed: 128 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -6,96 +6,138 @@ test_dir=$(realpath "$(dirname "$0")")
66
. "${test_dir}/../functions"
77
set_debug
88

9-
if [[ ${IMAGE_MONGOD} == *"percona-server-mongodb-operator"* ]]; then
10-
MONGO_VER=$(echo -n "${IMAGE_MONGOD}" | $sed -r 's/.*([0-9].[0-9])$/\1/')
11-
else
12-
MONGO_VER=$(echo -n "${IMAGE_MONGOD}" | $sed -r 's/.*:([0-9]+\.[0-9]+).*$/\1/')
13-
fi
14-
15-
deploy_cert_manager
16-
create_infra "$namespace"
17-
18-
desc 'create secrets and start client'
19-
kubectl_bin apply -f "$conf_dir/secrets.yml"
20-
kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
21-
22-
cluster="some-name"
23-
desc "create first PSMDB cluster $cluster"
24-
apply_cluster "$test_dir/conf/$cluster.yml"
25-
26-
desc 'check if all Pods started'
27-
wait_for_running $cluster-cfg 3
28-
wait_for_running $cluster-rs0 3
29-
wait_for_running $cluster-rs1 3
30-
wait_for_running $cluster-rs2 3
31-
wait_for_running $cluster-mongos 3
32-
33-
desc 'create user'
34-
run_mongos \
35-
'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' \
36-
"userAdmin:userAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
37-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
38-
sleep 2
39-
40-
desc 'set chunk size to 32 MB'
41-
run_mongos \
42-
"use config\n db.settings.save( { _id:\"chunksize\", value: 32 } )" \
43-
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
44-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
45-
sleep 2
46-
47-
desc 'write data'
48-
run_script_mongos "${test_dir}/data.js" "user:pass@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
49-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
50-
51-
desc 'shard collection'
52-
run_mongos \
53-
'sh.enableSharding("app")' \
54-
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
55-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
56-
sleep 2
57-
58-
run_mongos \
59-
'sh.shardCollection("app.city", { _id: 1 } )' \
60-
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
61-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
62-
sleep 120
63-
64-
desc 'check chunks'
65-
chunks_param1="ns"
66-
chunks_param2='"app.city"'
67-
68-
if [[ ${MONGO_VER} == "6.0" || ${MONGO_VER} == "5.0" ]]; then
69-
chunks_param1="uuid"
70-
chunks_param2=$(run_mongos \
71-
"use app\n db.getCollectionInfos({ \"name\": \"city\" })[0].info.uuid" \
72-
"user:pass@$cluster-mongos.$namespace" \
73-
'' \
74-
'' \
75-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls" \
76-
| grep "switched to db app" -A 1 | grep -v "switched to db app")
77-
fi
78-
79-
shards=0
80-
for i in "rs0" "rs1" "rs2"; do
81-
out=$(run_mongos \
82-
"use config\n db.chunks.count({\"${chunks_param1}\": ${chunks_param2}, \"shard\": \"$i\"})" \
9+
check_rs_proper_component_deletion() {
10+
local cluster="$1"
11+
local rs_name="$2"
12+
13+
rs_idx=$(kubectl_bin get psmdb ${cluster} -ojson | jq --arg RS $rs_name '.spec.replsets | map(.name == $RS) | index(true)')
14+
kubectl_bin patch psmdb ${cluster} --type=json -p="[{'op': 'remove', 'path': '/spec/replsets/$rs_idx'}]"
15+
16+
echo -n "Deleting replset $rs_name"
17+
until [[ $(kubectl_bin get sts -l app.kubernetes.io/instance=${cluster},app.kubernetes.io/replset=${rs_name} -ojson | jq '.items | length') -eq 0 ]]; do
18+
let retry+=1
19+
if [ $retry -ge 70 ]; then
20+
sts_count=$(kubectl_bin get sts -l app.kubernetes.io/instance=${cluster},app.kubernetes.io/replset=${rs_name} -ojson | jq '.items | length')
21+
echo "Replset $rs_name not properly removed, expected sts count of 0 but got $sts_count. Exiting after $retry tries..."
22+
exit 1
23+
fi
24+
echo -n .
25+
sleep 30
26+
done
27+
28+
echo "OK"
29+
}
30+
31+
main() {
32+
if [[ ${IMAGE_MONGOD} == *"percona-server-mongodb-operator"* ]]; then
33+
MONGO_VER=$(echo -n "${IMAGE_MONGOD}" | $sed -r 's/.*([0-9].[0-9])$/\1/')
34+
else
35+
MONGO_VER=$(echo -n "${IMAGE_MONGOD}" | $sed -r 's/.*:([0-9]+\.[0-9]+).*$/\1/')
36+
fi
37+
38+
deploy_cert_manager
39+
create_infra "$namespace"
40+
41+
desc 'create secrets and start client'
42+
kubectl_bin apply -f "$conf_dir/secrets.yml"
43+
kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
44+
45+
cluster="some-name"
46+
desc "create first PSMDB cluster $cluster"
47+
apply_cluster "$test_dir/conf/$cluster.yml"
48+
49+
desc 'check if all Pods started'
50+
wait_for_running $cluster-cfg 3
51+
wait_for_running $cluster-rs0 3
52+
wait_for_running $cluster-rs1 3
53+
wait_for_running $cluster-rs2 3
54+
wait_for_running $cluster-mongos 3
55+
56+
desc 'create user'
57+
run_mongos \
58+
'db.createUser({user:"user",pwd:"pass",roles:[{db:"app",role:"readWrite"}]})' \
59+
"userAdmin:userAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
60+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
61+
sleep 2
62+
63+
desc 'set chunk size to 32 MB'
64+
run_mongos \
65+
"use config\n db.settings.save( { _id:\"chunksize\", value: 32 } )" \
8366
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
84-
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls" \
85-
| grep "switched to db config" -A 1 | grep -v "switched to db config")
67+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
68+
sleep 2
8669

87-
desc "$i has $out chunks"
70+
desc 'write data'
71+
run_script_mongos "${test_dir}/data.js" "user:pass@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
72+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
73+
74+
desc 'shard collection'
75+
run_mongos \
76+
'sh.enableSharding("app")' \
77+
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
78+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
79+
sleep 2
8880

89-
if [[ $out -ne 0 ]]; then
90-
((shards = shards + 1))
81+
run_mongos \
82+
'sh.shardCollection("app.city", { _id: 1 } )' \
83+
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
84+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls"
85+
sleep 120
86+
87+
desc 'check chunks'
88+
chunks_param1="ns"
89+
chunks_param2='"app.city"'
90+
91+
if [[ ${MONGO_VER} == "6.0" || ${MONGO_VER} == "5.0" ]]; then
92+
chunks_param1="uuid"
93+
chunks_param2=$(run_mongos \
94+
"use app\n db.getCollectionInfos({ \"name\": \"city\" })[0].info.uuid" \
95+
"user:pass@$cluster-mongos.$namespace" \
96+
'' \
97+
'' \
98+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls" \
99+
| grep "switched to db app" -A 1 | grep -v "switched to db app")
91100
fi
92-
done
93101

94-
if [[ $shards -lt 3 ]]; then
95-
echo "data is only on some of the shards, maybe sharding is not working"
96-
exit 1
97-
fi
102+
shards=0
103+
for i in "rs0" "rs1" "rs2"; do
104+
out=$(run_mongos \
105+
"use config\n db.chunks.count({\"${chunks_param1}\": ${chunks_param2}, \"shard\": \"$i\"})" \
106+
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
107+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls" \
108+
| grep "switched to db config" -A 1 | grep -v "switched to db config")
109+
110+
desc "$i has $out chunks"
111+
112+
if [[ $out -ne 0 ]]; then
113+
((shards = shards + 1))
114+
fi
115+
done
116+
117+
if [[ $shards -lt 3 ]]; then
118+
echo "data is only on some of the shards, maybe sharding is not working"
119+
exit 1
120+
fi
121+
122+
# Drop non system database so we can remove shards
123+
res=$(run_mongos \
124+
"use app\n db.dropDatabase()" \
125+
"clusterAdmin:clusterAdmin123456@$cluster-mongos.$namespace" "mongodb" ".svc.cluster.local" \
126+
"--tlsCertificateKeyFile /tmp/tls.pem --tlsCAFile /etc/mongodb-ssl/ca.crt --tls")
127+
if ! echo $res | grep -q '"ok" : 1'; then
128+
echo "app database not dropped. Exiting.."
129+
exit 1
130+
fi
131+
132+
desc 'check if rs1 and all its related stateful sets are properly removed'
133+
check_rs_proper_component_deletion $cluster rs1
134+
135+
desc 'check if rs2 and all its related stateful sets are properly removed'
136+
check_rs_proper_component_deletion $cluster rs2
137+
138+
destroy "$namespace"
98139

99-
destroy "$namespace"
140+
desc 'test passed'
141+
}
100142

101-
desc 'test passed'
143+
main

‎e2e-tests/split-horizon/run

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ sleep 15
4343

4444
set -o xtrace
4545

46-
run_mongo "rs.conf().members.map(function(member) { return member.horizons })" \
46+
run_mongo "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
4747
"clusterAdmin:[email protected],some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
4848
mongodb "" "--quiet" >${tmp_dir}/horizons-3.json
4949
diff $test_dir/compare/horizons-3.json $tmp_dir/horizons-3.json
@@ -78,7 +78,7 @@ kubectl_bin patch psmdb ${cluster} \
7878
wait_for_running "${cluster}-rs0" 5
7979
wait_cluster_consistency ${cluster}
8080

81-
run_mongo "rs.conf().members.map(function(member) { return member.horizons })" \
81+
run_mongo "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
8282
"clusterAdmin:[email protected],some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
8383
mongodb "" "--quiet" >${tmp_dir}/horizons-5.json
8484
diff $test_dir/compare/horizons-5.json $tmp_dir/horizons-5.json
@@ -89,7 +89,7 @@ kubectl_bin patch psmdb ${cluster} \
8989
wait_for_running "${cluster}-rs0" 3
9090
wait_cluster_consistency ${cluster}
9191

92-
run_mongo "rs.conf().members.map(function(member) { return member.horizons })" \
92+
run_mongo "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
9393
"clusterAdmin:[email protected],some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
9494
mongodb "" "--quiet" >${tmp_dir}/horizons.json
9595
diff $test_dir/compare/horizons-3.json $tmp_dir/horizons-3.json
@@ -98,4 +98,4 @@ apply_cluster ${test_dir}/conf/${cluster}.yml
9898
wait_for_running "${cluster}-rs0" 3
9999
wait_cluster_consistency ${cluster}
100100

101-
destroy ${namespace}
101+
destroy ${namespace}

‎pkg/controller/perconaservermongodb/mgo.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCluster(ctx context.Context, cr
5050
return api.AppStateReady, nil
5151
}
5252

53-
pods, err := psmdb.GetRSPods(ctx, r.client, cr, replset.Name, false)
53+
pods, err := psmdb.GetRSPods(ctx, r.client, cr, replset.Name)
5454
if err != nil {
5555
return api.AppStateInit, errors.Wrap(err, "failed to get replset pods")
5656
}
@@ -238,7 +238,7 @@ func (r *ReconcilePerconaServerMongoDB) updateConfigMembers(ctx context.Context,
238238
// Primary with a Secondary and an Arbiter (PSA)
239239
unsafePSA := cr.Spec.UnsafeConf && rs.Arbiter.Enabled && rs.Arbiter.Size == 1 && !rs.NonVoting.Enabled && rs.Size == 2
240240

241-
pods, err := psmdb.GetRSPods(ctx, r.client, cr, rs.Name, false)
241+
pods, err := psmdb.GetRSPods(ctx, r.client, cr, rs.Name)
242242
if err != nil {
243243
return 0, errors.Wrap(err, "get rs pods")
244244
}

0 commit comments

Comments
 (0)