Skip to content

Commit 2fd8dee

Browse files
pooknullhors
pooknull
and
hors
authored
K8SPSMDB-1503: include horizon domains in certificates (#2157)
* K8SPSMDB-1503: include horizon domains in certificates https://perconadev.atlassian.net/browse/K8SPSMDB-1503 * fix `GetHorizons` method * add unit-test * add get certificate sans unit-test * prepare split-horizon e2e test * update split-horizon test * deduplicate * address copilot comments --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>
1 parent 375eb72 commit 2fd8dee

File tree

11 files changed

+441
-86
lines changed

11 files changed

+441
-86
lines changed

‎e2e-tests/split-horizon/compare/certificate_some-name-ca-cert.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
annotations: {}
5+
generation: 1
6+
labels:
7+
app.kubernetes.io/instance: some-name
8+
app.kubernetes.io/managed-by: percona-server-mongodb-operator
9+
app.kubernetes.io/name: percona-server-mongodb
10+
app.kubernetes.io/part-of: percona-server-mongodb
11+
name: some-name-ca-cert
12+
ownerReferences:
13+
- blockOwnerDeletion: true
14+
controller: true
15+
kind: PerconaServerMongoDB
16+
name: some-name
17+
spec:
18+
commonName: some-name-ca
19+
duration: 8760h0m0s
20+
isCA: true
21+
issuerRef:
22+
kind: Issuer
23+
name: some-name-psmdb-ca-issuer
24+
renewBefore: 730h0m0s
25+
secretName: some-name-ca-cert

‎e2e-tests/split-horizon/compare/certificate_some-name-ssl-internal.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
annotations: {}
5+
generation: 1
6+
labels:
7+
app.kubernetes.io/instance: some-name
8+
app.kubernetes.io/managed-by: percona-server-mongodb-operator
9+
app.kubernetes.io/name: percona-server-mongodb
10+
app.kubernetes.io/part-of: percona-server-mongodb
11+
name: some-name-ssl-internal
12+
ownerReferences:
13+
- blockOwnerDeletion: true
14+
controller: true
15+
kind: PerconaServerMongoDB
16+
name: some-name
17+
spec:
18+
commonName: some-name
19+
dnsNames:
20+
- localhost
21+
- some-name-rs0
22+
- some-name-rs0.NAME_SPACE
23+
- some-name-rs0.NAME_SPACE.svc.cluster.local
24+
- '*.some-name-rs0'
25+
- '*.some-name-rs0.NAME_SPACE'
26+
- '*.some-name-rs0.NAME_SPACE.svc.cluster.local'
27+
- some-name-rs0.NAME_SPACE.svc.clusterset.local
28+
- '*.some-name-rs0.NAME_SPACE.svc.clusterset.local'
29+
- some-name-rs0-0.clouddemo.xyz
30+
- some-name-rs0-1.clouddemo.xyz
31+
- some-name-rs0-2.clouddemo.xyz
32+
- '*.NAME_SPACE.svc.clusterset.local'
33+
- some-name-mongos
34+
- some-name-mongos.NAME_SPACE
35+
- some-name-mongos.NAME_SPACE.svc.cluster.local
36+
- '*.some-name-mongos'
37+
- '*.some-name-mongos.NAME_SPACE'
38+
- '*.some-name-mongos.NAME_SPACE.svc.cluster.local'
39+
- some-name-cfg
40+
- some-name-cfg.NAME_SPACE
41+
- some-name-cfg.NAME_SPACE.svc.cluster.local
42+
- '*.some-name-cfg'
43+
- '*.some-name-cfg.NAME_SPACE'
44+
- '*.some-name-cfg.NAME_SPACE.svc.cluster.local'
45+
- some-name-mongos.NAME_SPACE.svc.clusterset.local
46+
- '*.some-name-mongos.NAME_SPACE.svc.clusterset.local'
47+
- some-name-cfg.NAME_SPACE.svc.clusterset.local
48+
- '*.some-name-cfg.NAME_SPACE.svc.clusterset.local'
49+
duration: 2160h0m0s
50+
issuerRef:
51+
kind: Issuer
52+
name: some-name-psmdb-issuer
53+
secretName: some-name-ssl-internal
54+
subject:
55+
organizations:
56+
- PSMDB

‎e2e-tests/split-horizon/compare/certificate_some-name-ssl.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
annotations: {}
5+
generation: 1
6+
labels:
7+
app.kubernetes.io/instance: some-name
8+
app.kubernetes.io/managed-by: percona-server-mongodb-operator
9+
app.kubernetes.io/name: percona-server-mongodb
10+
app.kubernetes.io/part-of: percona-server-mongodb
11+
name: some-name-ssl
12+
ownerReferences:
13+
- blockOwnerDeletion: true
14+
controller: true
15+
kind: PerconaServerMongoDB
16+
name: some-name
17+
spec:
18+
commonName: some-name
19+
dnsNames:
20+
- localhost
21+
- some-name-rs0
22+
- some-name-rs0.NAME_SPACE
23+
- some-name-rs0.NAME_SPACE.svc.cluster.local
24+
- '*.some-name-rs0'
25+
- '*.some-name-rs0.NAME_SPACE'
26+
- '*.some-name-rs0.NAME_SPACE.svc.cluster.local'
27+
- some-name-rs0.NAME_SPACE.svc.clusterset.local
28+
- '*.some-name-rs0.NAME_SPACE.svc.clusterset.local'
29+
- some-name-rs0-0.clouddemo.xyz
30+
- some-name-rs0-1.clouddemo.xyz
31+
- some-name-rs0-2.clouddemo.xyz
32+
- '*.NAME_SPACE.svc.clusterset.local'
33+
- some-name-mongos
34+
- some-name-mongos.NAME_SPACE
35+
- some-name-mongos.NAME_SPACE.svc.cluster.local
36+
- '*.some-name-mongos'
37+
- '*.some-name-mongos.NAME_SPACE'
38+
- '*.some-name-mongos.NAME_SPACE.svc.cluster.local'
39+
- some-name-cfg
40+
- some-name-cfg.NAME_SPACE
41+
- some-name-cfg.NAME_SPACE.svc.cluster.local
42+
- '*.some-name-cfg'
43+
- '*.some-name-cfg.NAME_SPACE'
44+
- '*.some-name-cfg.NAME_SPACE.svc.cluster.local'
45+
- some-name-mongos.NAME_SPACE.svc.clusterset.local
46+
- '*.some-name-mongos.NAME_SPACE.svc.clusterset.local'
47+
- some-name-cfg.NAME_SPACE.svc.clusterset.local
48+
- '*.some-name-cfg.NAME_SPACE.svc.clusterset.local'
49+
duration: 2160h0m0s
50+
issuerRef:
51+
kind: Issuer
52+
name: some-name-psmdb-issuer
53+
secretName: some-name-ssl
54+
subject:
55+
organizations:
56+
- PSMDB

‎e2e-tests/split-horizon/compare/issuer_some-name-psmdb-ca-issuer.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
annotations: {}
5+
generation: 1
6+
labels:
7+
app.kubernetes.io/instance: some-name
8+
app.kubernetes.io/managed-by: percona-server-mongodb-operator
9+
app.kubernetes.io/name: percona-server-mongodb
10+
app.kubernetes.io/part-of: percona-server-mongodb
11+
name: some-name-psmdb-ca-issuer
12+
ownerReferences:
13+
- blockOwnerDeletion: true
14+
controller: true
15+
kind: PerconaServerMongoDB
16+
name: some-name
17+
spec:
18+
selfSigned: {}

‎e2e-tests/split-horizon/compare/issuer_some-name-psmdb-issuer.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
annotations: {}
5+
generation: 1
6+
labels:
7+
app.kubernetes.io/instance: some-name
8+
app.kubernetes.io/managed-by: percona-server-mongodb-operator
9+
app.kubernetes.io/name: percona-server-mongodb
10+
app.kubernetes.io/part-of: percona-server-mongodb
11+
name: some-name-psmdb-issuer
12+
ownerReferences:
13+
- blockOwnerDeletion: true
14+
controller: true
15+
kind: PerconaServerMongoDB
16+
name: some-name
17+
spec:
18+
ca:
19+
secretName: some-name-ca-cert

‎e2e-tests/split-horizon/run‎

Lines changed: 76 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@
33
set -o errexit
44
set -o xtrace
55

6-
test_dir=$(realpath $(dirname $0))
7-
. ${test_dir}/../functions
6+
test_dir=$(realpath "$(dirname "$0")")
7+
. "${test_dir}"/../functions
88

99
configure_client_hostAliases() {
1010
local hostAliasesJson='[]'
1111

1212
for svc in $(kubectl get svc | awk '{print $3 "|" $1}' | grep -E '^[0-9].*'); do
13-
hostname=$(echo ${svc} | awk -F '|' '{print $2}')
14-
ip=$(echo ${svc} | awk -F '|' '{print $1}')
13+
hostname=$(echo "${svc}" | awk -F '|' '{print $2}')
14+
ip=$(echo "${svc}" | awk -F '|' '{print $1}')
1515
hostAlias="{\"ip\": \"${ip}\", \"hostnames\": [\"${hostname}.clouddemo.xyz\"]}"
16-
hostAliasesJson=$(echo $hostAliasesJson | jq --argjson newAlias "$hostAlias" '. += [$newAlias]')
16+
hostAliasesJson=$(echo "$hostAliasesJson" | jq --argjson newAlias "$hostAlias" '. += [$newAlias]')
1717
done
1818

1919
kubectl_bin patch deployment psmdb-client --type='json' -p="[{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]"
@@ -22,83 +22,95 @@ configure_client_hostAliases() {
2222

2323
kubectl_bin patch deployment psmdb-client --type='json' -p="[{'op': 'replace', 'path': '/spec/template/spec/hostAliases', 'value': $hostAliasesJson}, {'op': 'replace', 'path': '/spec/replicas', 'value': 1}]"
2424

25-
wait_pod $(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
25+
wait_pod "$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')"
2626
}
2727

28-
create_infra ${namespace}
28+
main() {
29+
create_infra "${namespace}"
30+
deploy_cert_manager
2931

30-
cluster="some-name"
31-
kubectl_bin apply \
32-
-f ${conf_dir}/secrets_with_tls.yml \
33-
-f ${conf_dir}/client_with_tls.yml
32+
cluster="some-name"
33+
kubectl_bin apply \
34+
-f "${conf_dir}"/secrets.yml \
35+
-f "${conf_dir}"/client_with_tls.yml
3436

35-
apply_cluster ${test_dir}/conf/${cluster}-3horizons.yml
36-
wait_for_running "${cluster}-rs0" 3
37-
wait_cluster_consistency ${cluster}
37+
apply_cluster "${test_dir}"/conf/${cluster}-3horizons.yml
38+
wait_for_running "${cluster}-rs0" 3
39+
wait_cluster_consistency ${cluster}
3840

39-
configure_client_hostAliases
41+
desc 'compare certificates and issuers'
42+
compare_kubectl "certificate/${cluster}-ssl"
43+
compare_kubectl "certificate/${cluster}-ssl-internal"
44+
compare_kubectl "certificate/${cluster}-ca-cert"
45+
compare_kubectl "issuer/$cluster-psmdb-ca-issuer"
46+
compare_kubectl "issuer/$cluster-psmdb-issuer"
4047

41-
sleep 10 # give some time for client pod to be ready
48+
configure_client_hostAliases
4249

43-
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
44-
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
45-
mongodb "" "--quiet" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >${tmp_dir}/horizons-3.json
46-
diff $test_dir/compare/horizons-3.json $tmp_dir/horizons-3.json
50+
sleep 10 # give some time for client pod to be ready
4751

48-
isMaster=$(run_mongo_tls "db.hello().isWritablePrimary" "clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" mongodb "" "--quiet" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' | grep -v certificateNames)
49-
if [ "${isMaster}" != "true" ]; then
50-
echo "mongo client should've redirect the connection to primary"
51-
exit 1
52-
fi
52+
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
53+
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
54+
mongodb "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >"${tmp_dir}"/horizons-3.json
55+
diff "$test_dir"/compare/horizons-3.json "$tmp_dir"/horizons-3.json
5356

54-
# stepping down to ensure we haven't redirected to primary just because primary is pod-0
55-
run_mongo_tls "rs.stepDown()" \
56-
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
57-
mongodb "" "--quiet"
57+
isMaster=$(run_mongo_tls "db.hello().isWritablePrimary" "clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" mongodb "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' | grep -v certificateNames)
58+
if [ "${isMaster}" != "true" ]; then
59+
echo "mongo client should've redirect the connection to primary"
60+
exit 1
61+
fi
5862

59-
sleep 10 # give some time for re-election
63+
# stepping down to ensure we haven't redirected to primary just because primary is pod-0
64+
run_mongo_tls "rs.stepDown()" \
65+
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
66+
mongodb "" "--quiet"
6067

61-
isMaster=$(run_mongo_tls "db.hello().isWritablePrimary" "clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" mongodb "" "--quiet" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' | grep -v certificateNames)
62-
if [ "${isMaster}" != "true" ]; then
63-
echo "mongo client should've redirect the connection to primary"
64-
exit 1
65-
fi
68+
sleep 10 # give some time for re-election
6669

67-
desc "scaling up the cluster"
70+
isMaster=$(run_mongo_tls "db.hello().isWritablePrimary" "clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" mongodb "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' | grep -v certificateNames)
71+
if [ "${isMaster}" != "true" ]; then
72+
echo "mongo client should've redirect the connection to primary"
73+
exit 1
74+
fi
6875

69-
apply_cluster ${test_dir}/conf/${cluster}-5horizons.yml
70-
wait_for_running "${cluster}-rs0" 3
71-
wait_cluster_consistency ${cluster}
76+
desc "scaling up the cluster"
7277

73-
# scale up and down
74-
kubectl_bin patch psmdb ${cluster} \
75-
--type='json' \
76-
-p='[{"op": "replace", "path": "/spec/replsets/0/size", "value": 5}]'
77-
wait_for_running "${cluster}-rs0" 5
78-
wait_cluster_consistency ${cluster}
78+
apply_cluster "${test_dir}"/conf/${cluster}-5horizons.yml
79+
wait_for_running "${cluster}-rs0" 3
80+
wait_cluster_consistency ${cluster}
7981

80-
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
81-
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
82-
mongodb "" "--quiet" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >${tmp_dir}/horizons-5.json
83-
diff $test_dir/compare/horizons-5.json $tmp_dir/horizons-5.json
82+
# scale up and down
83+
kubectl_bin patch psmdb ${cluster} \
84+
--type='json' \
85+
-p='[{"op": "replace", "path": "/spec/replsets/0/size", "value": 5}]'
86+
wait_for_running "${cluster}-rs0" 5
87+
wait_cluster_consistency ${cluster}
8488

85-
desc "scaling down the cluster"
89+
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
90+
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
91+
mongodb "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >"${tmp_dir}"/horizons-5.json
92+
diff "$test_dir"/compare/horizons-5.json "$tmp_dir"/horizons-5.json
8693

87-
kubectl_bin patch psmdb ${cluster} \
88-
--type='json' \
89-
-p='[{"op": "replace", "path": "/spec/replsets/0/size", "value": 3}]'
90-
wait_for_running "${cluster}-rs0" 3
91-
wait_cluster_consistency ${cluster}
94+
desc "scaling down the cluster"
9295

93-
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
94-
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
95-
mongodb "" "--quiet" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >${tmp_dir}/horizons.json
96-
diff $test_dir/compare/horizons-3.json $tmp_dir/horizons-3.json
96+
kubectl_bin patch psmdb ${cluster} \
97+
--type='json' \
98+
-p='[{"op": "replace", "path": "/spec/replsets/0/size", "value": 3}]'
99+
wait_for_running "${cluster}-rs0" 3
100+
wait_cluster_consistency ${cluster}
97101

98-
desc "remove horizon configuration"
102+
run_mongo_tls "rs.conf().members.map(function(member) { return member.horizons }).sort((a, b) => a.external.localeCompare(b.external))" \
103+
"clusterAdmin:clusterAdmin123456@some-name-rs0-0.clouddemo.xyz,some-name-rs0-1.clouddemo.xyz,some-name-rs0-2.clouddemo.xyz" \
104+
mongodb "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|does not match the remote host name' >"${tmp_dir}"/horizons.json
105+
diff "$test_dir"/compare/horizons-3.json "$tmp_dir"/horizons-3.json
99106

100-
apply_cluster ${test_dir}/conf/${cluster}.yml
101-
wait_for_running "${cluster}-rs0" 3
102-
wait_cluster_consistency ${cluster}
107+
desc "remove horizon configuration"
103108

104-
destroy ${namespace}
109+
apply_cluster "${test_dir}"/conf/${cluster}.yml
110+
wait_for_running "${cluster}-rs0" 3
111+
wait_cluster_consistency ${cluster}
112+
113+
destroy "${namespace}"
114+
}
115+
116+
main

‎pkg/apis/psmdb/v1/psmdb_types.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -766,6 +766,35 @@ type ReplsetSpec struct {
766766
EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
767767
}
768768

769+
func (r *ReplsetSpec) GetHorizons(withPorts bool) map[string]map[string]string {
770+
horizons := make(map[string]map[string]string)
771+
for podName, m := range r.Horizons {
772+
overrides, ok := r.ReplsetOverrides[podName]
773+
hasOverrides := ok && len(overrides.Horizons) > 0
774+
775+
for h, domain := range m {
776+
if hasOverrides {
777+
if d, ok := overrides.Horizons[h]; ok {
778+
domain = d
779+
}
780+
}
781+
782+
idx := strings.IndexRune(domain, ':')
783+
if withPorts && idx == -1 {
784+
domain = fmt.Sprintf("%s:%d", domain, r.GetPort())
785+
} else if !withPorts && idx != -1 {
786+
domain = domain[:idx]
787+
}
788+
789+
if podHorizons, ok := horizons[podName]; !ok || podHorizons == nil {
790+
horizons[podName] = make(map[string]string)
791+
}
792+
horizons[podName][h] = domain
793+
}
794+
}
795+
return horizons
796+
}
797+
769798
func (r *ReplsetSpec) PodName(cr *PerconaServerMongoDB, idx int) string {
770799
return fmt.Sprintf("%s-%s-%d", cr.Name, r.Name, idx)
771800
}

0 commit comments

Comments
 (0)