Skip to content

Commit d10ea1f

Browse files
VallariAgVallariAg
authored
Merge pull request ceph#60316 from VallariAg/wip-nvmeof-teuthology-mtls-test
qa/suites/nvmeof: add mtls test
2 parents 16b44ff + fdc93ad commit d10ea1f

File tree

3 files changed

+148
-1
lines changed

3 files changed

+148
-1
lines changed

‎qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
tasks:
2+
- nvmeof:
3+
installer: host.a
4+
gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
5+
rbd:
6+
pool_name: mypool
7+
image_name_prefix: myimage
8+
gateway_config:
9+
subsystems_count: 3
10+
namespaces_count: 20
11+
cli_image: quay.io/ceph/nvmeof-cli:latest
12+
create_mtls_secrets: true
13+
14+
- cephadm.wait_for_service:
15+
service: nvmeof.mypool.mygroup0
16+
17+
- workunit:
18+
no_coverage_and_limits: true
19+
timeout: 30m
20+
clients:
21+
client.0:
22+
- nvmeof/setup_subsystem.sh
23+
- nvmeof/basic_tests.sh
24+
- nvmeof/fio_test.sh --rbd_iostat
25+
env:
26+
RBD_POOL: mypool
27+
RBD_IMAGE_PREFIX: myimage
28+
IOSTAT_INTERVAL: '10'
29+
RUNTIME: '60'
30+
31+
- workunit:
32+
no_coverage_and_limits: true
33+
timeout: 30m
34+
clients:
35+
client.0:
36+
- nvmeof/mtls_test.sh

‎qa/tasks/nvmeof.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ class Nvmeof(Task):
3232
gateway_config:
3333
namespaces_count: 10
3434
cli_version: latest
35+
create_mtls_secrets: False
3536
3637
"""
3738

@@ -69,6 +70,7 @@ def _set_defaults(self):
6970
self.serial = gateway_config.get('serial', 'SPDK00000000000001')
7071
self.port = gateway_config.get('port', '4420')
7172
self.srport = gateway_config.get('srport', '5500')
73+
self.create_mtls_secrets = gateway_config.get('create_mtls_secrets', False)
7274

7375
def deploy_nvmeof(self):
7476
"""
@@ -147,7 +149,38 @@ def deploy_nvmeof(self):
147149
started=True,
148150
)
149151
log.info("[nvmeof]: executed deploy_nvmeof successfully!")
150-
152+
153+
def write_mtls_config(self, gateway_ips):
154+
log.info("[nvmeof]: writing mtls config...")
155+
allowed_ips = ""
156+
for ip in gateway_ips:
157+
allowed_ips += ("IP:" + ip + ",")
158+
self.remote.run(
159+
args=[
160+
"sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/server.key",
161+
"-out", "/etc/ceph/server.crt", "-days", "3650", "-subj", "/CN=my.server", "-addext", f"subjectAltName={allowed_ips[:-1]}"
162+
]
163+
)
164+
self.remote.run(
165+
args=[
166+
"sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/client.key",
167+
"-out", "/etc/ceph/client.crt", "-days", "3650", "-subj", "/CN=client1"
168+
]
169+
)
170+
secrets_files = {"/etc/ceph/server.key": None,
171+
"/etc/ceph/server.crt": None,
172+
"/etc/ceph/client.key": None,
173+
"/etc/ceph/client.crt": None,
174+
}
175+
for file in secrets_files.keys():
176+
secrets_files[file] = self.remote.read_file(path=file, sudo=True)
177+
178+
for remote in self.ctx.cluster.remotes.keys():
179+
for remote_file in secrets_files.keys():
180+
data = secrets_files[remote_file]
181+
remote.sudo_write_file(path=remote_file, data=data, mode='0644')
182+
log.info("[nvmeof]: written mtls config!")
183+
151184
def set_gateway_cfg(self):
152185
log.info('[nvmeof]: running set_gateway_cfg...')
153186
ip_address = self.remote.ip_address
@@ -174,6 +207,8 @@ def set_gateway_cfg(self):
174207
data=conf_data,
175208
sudo=True
176209
)
210+
if self.create_mtls_secrets:
211+
self.write_mtls_config(gateway_ips)
177212
log.info("[nvmeof]: executed set_gateway_cfg successfully!")
178213

179214

‎qa/workunits/nvmeof/mtls_test.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
source /etc/ceph/nvmeof.env
5+
6+
# install yq
7+
wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /tmp/yq && chmod +x /tmp/yq
8+
9+
subjectAltName=$(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | sed 's/,/,IP:/g')
10+
11+
# create mtls spec files
12+
ceph orch ls nvmeof --export > /tmp/gw-conf-original.yaml
13+
sudo /tmp/yq ".spec.enable_auth=true | \
14+
.spec.root_ca_cert=\"mountcert\" | \
15+
.spec.client_cert = load_str(\"/etc/ceph/client.crt\") | \
16+
.spec.client_key = load_str(\"/etc/ceph/client.key\") | \
17+
.spec.server_cert = load_str(\"/etc/ceph/server.crt\") | \
18+
.spec.server_key = load_str(\"/etc/ceph/server.key\")" /tmp/gw-conf-original.yaml > /tmp/gw-conf-with-mtls.yaml
19+
cp /tmp/gw-conf-original.yaml /tmp/gw-conf-without-mtls.yaml
20+
sudo /tmp/yq '.spec.enable_auth=false' -i /tmp/gw-conf-without-mtls.yaml
21+
22+
wait_for_service() {
23+
MAX_RETRIES=30
24+
for ((RETRY_COUNT=1; RETRY_COUNT<=MAX_RETRIES; RETRY_COUNT++)); do
25+
26+
if ceph orch ls --refresh | grep -q "nvmeof"; then
27+
echo "Found nvmeof in the output!"
28+
break
29+
fi
30+
if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
31+
echo "Reached maximum retries ($MAX_RETRIES). Exiting."
32+
break
33+
fi
34+
sleep 5
35+
done
36+
ceph orch ps
37+
ceph orch ls --refresh
38+
}
39+
40+
# deploy mtls
41+
cat /tmp/gw-conf-with-mtls.yaml
42+
ceph orch apply -i /tmp/gw-conf-with-mtls.yaml
43+
ceph orch redeploy nvmeof.mypool.mygroup0
44+
sleep 100
45+
wait_for_service
46+
47+
48+
# test
49+
IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES"
50+
for i in "${!gateway_ips[@]}"
51+
do
52+
ip="${gateway_ips[i]}"
53+
sudo podman run -v /etc/ceph/server.crt:/server.crt:z -v /etc/ceph/client.crt:/client.crt:z \
54+
-v /etc/ceph/client.key:/client.key:z \
55+
-it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \
56+
--client-key /client.key --client-cert /client.crt --server-cert /server.crt --format json subsystem list
57+
done
58+
59+
60+
# remove mtls
61+
cat /tmp/gw-conf-without-mtls.yaml
62+
ceph orch apply -i /tmp/gw-conf-without-mtls.yaml
63+
ceph orch redeploy nvmeof.mypool.mygroup0
64+
sleep 100
65+
wait_for_service
66+
67+
68+
# test
69+
IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES"
70+
for i in "${!gateway_ips[@]}"
71+
do
72+
ip="${gateway_ips[i]}"
73+
sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \
74+
--format json subsystem list
75+
done
76+

0 commit comments

Comments
 (0)