Skip to content

Commit c8116c6

Browse files
VallariAgVallariAg
authored
Merge pull request ceph#54959 from VallariAg/wip-nvmeof-test-v2
qa: add qa/tasks/nvmeof.py
2 parents a85baa8 + 1713c48 commit c8116c6

File tree

8 files changed

+311
-121
lines changed

8 files changed

+311
-121
lines changed

‎qa/suites/rbd/nvmeof/base/install.yaml‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,4 @@ tasks:
1010
- ceph orch host ls
1111
- ceph orch device ls
1212
- ceph osd lspools
13-
# create pool
14-
- ceph osd pool create mypool
15-
- rbd pool init mypool
16-
# deploy nvmeof
17-
## Uncomment to test specific nvmeof images
18-
## - ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:latest
19-
- ceph orch apply nvmeof mypool --placement="1 $(hostname)"
20-
- ceph orch ps --refresh
2113

22-
- cephadm.wait_for_service:
23-
service: nvmeof.mypool
24-
25-
- cephadm.nvmeof_gateway_cfg:
26-
source: host.a
27-
target: client.1
28-
service: nvmeof.mypool
29-
30-
- exec:
31-
client.0:
32-
- journalctl -u $(systemctl list-units | grep nvmeof.mypool | awk '{print $1}')

‎qa/suites/rbd/nvmeof/cluster/fixed-3.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ roles:
55
- osd.0
66
- osd.1
77
- client.0
8+
- ceph.nvmeof.nvmeof.a
89
- - host.b
910
- mon.b
1011
- osd.2
1112
- osd.3
1213
- osd.4
1314
- client.1
15+
- - client.2

‎qa/suites/rbd/nvmeof/workloads/nvmeof_initiator.yaml‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,27 @@
11
tasks:
2+
- nvmeof:
3+
client: client.0
4+
version: latest # "default" uses packaged version; change to test specific nvmeof images, example "latest"
5+
rbd:
6+
pool_name: mypool
7+
image_name: myimage
8+
gateway_config:
9+
source: host.a
10+
target: client.2
11+
vars:
12+
cli_version: latest
13+
14+
- cephadm.wait_for_service:
15+
service: nvmeof.mypool
16+
217
- workunit:
318
no_coverage_and_limits: true
419
clients:
5-
client.1:
20+
client.2:
621
- rbd/nvmeof_initiator.sh
22+
- rbd/nvmeof_basic_tests.sh
23+
- rbd/nvmeof_fio_test.sh
24+
env:
25+
RBD_POOL: mypool
26+
RBD_IMAGE: myimage
27+
IOSTAT_INTERVAL: '10'

‎qa/tasks/cephadm.py‎

Lines changed: 0 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
from teuthology.orchestra import run
2222
from teuthology.orchestra.daemon import DaemonGroup
2323
from teuthology.config import config as teuth_config
24-
from teuthology.exceptions import ConfigError
2524
from textwrap import dedent
2625
from tasks.cephfs.filesystem import MDSCluster, Filesystem
2726
from tasks.util import chacra
@@ -101,43 +100,6 @@ def update_archive_setting(ctx, key, value):
101100
yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
102101

103102

104-
@contextlib.contextmanager
105-
def nvmeof_gateway_cfg(ctx, config):
106-
source_host = config.get('source')
107-
target_host = config.get('target')
108-
nvmeof_service = config.get('service')
109-
if not (source_host and target_host and nvmeof_service):
110-
raise ConfigError('nvmeof_gateway_cfg requires "source", "target", and "service"')
111-
remote = list(ctx.cluster.only(source_host).remotes.keys())[0]
112-
ip_address = remote.ip_address
113-
gateway_name = ""
114-
r = remote.run(args=[
115-
'systemctl', 'list-units',
116-
run.Raw('|'), 'grep', nvmeof_service
117-
], stdout=StringIO())
118-
output = r.stdout.getvalue()
119-
pattern_str = f"{re.escape(nvmeof_service)}(.*?)(?=\.service)"
120-
pattern = re.compile(pattern_str)
121-
match = pattern.search(output)
122-
if match:
123-
gateway_name = match.group()
124-
conf_data = dedent(f"""
125-
NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
126-
NVMEOF_GATEWAY_NAME={gateway_name}
127-
""")
128-
target_remote = list(ctx.cluster.only(target_host).remotes.keys())[0]
129-
target_remote.write_file(
130-
path='/etc/ceph/nvmeof.env',
131-
data=conf_data,
132-
sudo=True
133-
)
134-
135-
try:
136-
yield
137-
finally:
138-
pass
139-
140-
141103
@contextlib.contextmanager
142104
def normalize_hostnames(ctx):
143105
"""

‎qa/tasks/nvmeof.py‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
import logging
2+
from textwrap import dedent
3+
from teuthology.task import Task
4+
from teuthology import misc
5+
from teuthology.exceptions import ConfigError
6+
from tasks.util import get_remote_for_role
7+
from tasks.cephadm import _shell
8+
9+
log = logging.getLogger(__name__)
10+
11+
conf_file = '/etc/ceph/nvmeof.env'
12+
13+
14+
class Nvmeof(Task):
15+
"""
16+
Setup nvmeof gateway on client and then share gateway config to target host.
17+
18+
- nvmeof:
19+
client: client.0
20+
version: default
21+
rbd:
22+
pool_name: mypool
23+
image_name: myimage
24+
rbd_size: 1024
25+
gateway_config:
26+
source: host.a
27+
target: client.2
28+
vars:
29+
cli_version: latest
30+
31+
"""
32+
33+
def setup(self):
34+
super(Nvmeof, self).setup()
35+
try:
36+
self.client = self.config['client']
37+
except KeyError:
38+
raise ConfigError('nvmeof requires a client to connect with')
39+
40+
self.cluster_name, type_, self.client_id = misc.split_role(self.client)
41+
if type_ != 'client':
42+
msg = 'client role ({0}) must be a client'.format(self.client)
43+
raise ConfigError(msg)
44+
self.remote = get_remote_for_role(self.ctx, self.client)
45+
46+
def begin(self):
47+
super(Nvmeof, self).begin()
48+
self._set_defaults()
49+
self.deploy_nvmeof()
50+
self.set_gateway_cfg()
51+
52+
def _set_defaults(self):
53+
self.gateway_image = self.config.get('version', 'default')
54+
55+
rbd_config = self.config.get('rbd', {})
56+
self.poolname = rbd_config.get('pool_name', 'mypool')
57+
self.rbd_image_name = rbd_config.get('image_name', 'myimage')
58+
self.rbd_size = rbd_config.get('rbd_size', 1024*8)
59+
60+
gateway_config = self.config.get('gateway_config', {})
61+
conf_vars = gateway_config.get('vars', {})
62+
self.cli_image = conf_vars.get('cli_version', 'latest')
63+
self.bdev = conf_vars.get('bdev', 'mybdev')
64+
self.serial = conf_vars.get('serial', 'SPDK00000000000001')
65+
self.nqn = conf_vars.get('nqn', 'nqn.2016-06.io.spdk:cnode1')
66+
self.port = conf_vars.get('port', '4420')
67+
self.srport = conf_vars.get('srport', '5500')
68+
69+
def deploy_nvmeof(self):
70+
"""
71+
Deploy nvmeof gateway.
72+
"""
73+
log.info('[nvmeof]: deploying nvmeof gateway...')
74+
if not hasattr(self.ctx, 'ceph'):
75+
self.ctx.ceph = {}
76+
fsid = self.ctx.ceph[self.cluster_name].fsid
77+
78+
nodes = []
79+
daemons = {}
80+
81+
for remote, roles in self.ctx.cluster.remotes.items():
82+
for role in [r for r in roles
83+
if misc.is_type('nvmeof', self.cluster_name)(r)]:
84+
c_, _, id_ = misc.split_role(role)
85+
log.info('Adding %s on %s' % (role, remote.shortname))
86+
nodes.append(remote.shortname + '=' + id_)
87+
daemons[role] = (remote, id_)
88+
89+
if nodes:
90+
image = self.gateway_image
91+
if (image != "default"):
92+
log.info(f'[nvmeof]: ceph config set mgr mgr/cephadm/container_image_nvmeof quay.io/ceph/nvmeof:{image}')
93+
_shell(self.ctx, self.cluster_name, self.remote, [
94+
'ceph', 'config', 'set', 'mgr',
95+
'mgr/cephadm/container_image_nvmeof',
96+
f'quay.io/ceph/nvmeof:{image}'
97+
])
98+
99+
poolname = self.poolname
100+
imagename = self.rbd_image_name
101+
102+
log.info(f'[nvmeof]: ceph osd pool create {poolname}')
103+
_shell(self.ctx, self.cluster_name, self.remote, [
104+
'ceph', 'osd', 'pool', 'create', poolname
105+
])
106+
107+
log.info(f'[nvmeof]: rbd pool init {poolname}')
108+
_shell(self.ctx, self.cluster_name, self.remote, [
109+
'rbd', 'pool', 'init', poolname
110+
])
111+
112+
log.info(f'[nvmeof]: ceph orch apply nvmeof {poolname}')
113+
_shell(self.ctx, self.cluster_name, self.remote, [
114+
'ceph', 'orch', 'apply', 'nvmeof', poolname,
115+
'--placement', str(len(nodes)) + ';' + ';'.join(nodes)
116+
])
117+
118+
log.info(f'[nvmeof]: rbd create {poolname}/{imagename} --size {self.rbd_size}')
119+
_shell(self.ctx, self.cluster_name, self.remote, [
120+
'rbd', 'create', f'{poolname}/{imagename}', '--size', f'{self.rbd_size}'
121+
])
122+
123+
for role, i in daemons.items():
124+
remote, id_ = i
125+
self.ctx.daemons.register_daemon(
126+
remote, 'nvmeof', id_,
127+
cluster=self.cluster_name,
128+
fsid=fsid,
129+
logger=log.getChild(role),
130+
wait=False,
131+
started=True,
132+
)
133+
log.info("[nvmeof]: executed deploy_nvmeof successfully!")
134+
135+
def set_gateway_cfg(self):
136+
log.info('[nvmeof]: running set_gateway_cfg...')
137+
gateway_config = self.config.get('gateway_config', {})
138+
source_host = gateway_config.get('source')
139+
target_host = gateway_config.get('target')
140+
if not (source_host and target_host):
141+
raise ConfigError('gateway_config requires "source" and "target"')
142+
remote = list(self.ctx.cluster.only(source_host).remotes.keys())[0]
143+
ip_address = remote.ip_address
144+
gateway_name = ""
145+
nvmeof_daemons = self.ctx.daemons.iter_daemons_of_role('nvmeof', cluster=self.cluster_name)
146+
for daemon in nvmeof_daemons:
147+
if ip_address == daemon.remote.ip_address:
148+
gateway_name = daemon.name()
149+
conf_data = dedent(f"""
150+
NVMEOF_GATEWAY_IP_ADDRESS={ip_address}
151+
NVMEOF_GATEWAY_NAME={gateway_name}
152+
NVMEOF_CLI_IMAGE="quay.io/ceph/nvmeof-cli:{self.cli_image}"
153+
NVMEOF_BDEV={self.bdev}
154+
NVMEOF_SERIAL={self.serial}
155+
NVMEOF_NQN={self.nqn}
156+
NVMEOF_PORT={self.port}
157+
NVMEOF_SRPORT={self.srport}
158+
""")
159+
target_remote = list(self.ctx.cluster.only(target_host).remotes.keys())[0]
160+
target_remote.write_file(
161+
path=conf_file,
162+
data=conf_data,
163+
sudo=True
164+
)
165+
log.info("[nvmeof]: executed set_gateway_cfg successfully!")
166+
167+
168+
task = Nvmeof

‎qa/workunits/rbd/nvmeof_basic_tests.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/bin/bash -x
2+
3+
source /etc/ceph/nvmeof.env
4+
SPDK_CONTROLLER="SPDK bdev Controller"
5+
DISCOVERY_PORT="8009"
6+
7+
discovery() {
8+
output=$(sudo nvme discover -t tcp -a $NVMEOF_GATEWAY_IP_ADDRESS -s $DISCOVERY_PORT)
9+
expected_discovery_stdout="subtype: nvme subsystem"
10+
if ! echo "$output" | grep -q "$expected_discovery_stdout"; then
11+
return 1
12+
fi
13+
}
14+
15+
connect() {
16+
sudo nvme connect -t tcp --traddr $NVMEOF_GATEWAY_IP_ADDRESS -s $NVMEOF_PORT -n $NVMEOF_NQN
17+
output=$(sudo nvme list)
18+
if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
19+
return 1
20+
fi
21+
}
22+
23+
disconnect_all() {
24+
sudo nvme disconnect-all
25+
output=$(sudo nvme list)
26+
if echo "$output" | grep -q "$SPDK_CONTROLLER"; then
27+
return 1
28+
fi
29+
}
30+
31+
connect_all() {
32+
sudo nvme connect-all --traddr=$NVMEOF_GATEWAY_IP_ADDRESS --transport=tcp
33+
output=$(sudo nvme list)
34+
if ! echo "$output" | grep -q "$SPDK_CONTROLLER"; then
35+
return 1
36+
fi
37+
}
38+
39+
list_subsys() {
40+
expected_count=$1
41+
output=$(sudo nvme list-subsys --output-format=json)
42+
multipath=$(echo $output | grep -c '"tcp"')
43+
if [ "$multipath" -ne "$expected_count" ]; then
44+
return 1
45+
fi
46+
}
47+
48+
49+
test_run() {
50+
echo "[nvmeof] Running test: $1"
51+
$1 "${@:2}" # execute func
52+
if [ $? -eq 0 ]; then
53+
echo "[nvmeof] $1 test passed!"
54+
else
55+
echo "[nvmeof] $1 test failed!"
56+
exit 1
57+
fi
58+
}
59+
60+
61+
test_run disconnect_all
62+
test_run discovery
63+
test_run connect
64+
test_run list_subsys 1
65+
test_run disconnect_all
66+
test_run list_subsys 0
67+
test_run connect_all
68+
test_run list_subsys 1
69+
70+
71+
echo "-------------Test Summary-------------"
72+
echo "[nvmeof] All nvmeof basic tests passed!"

‎qa/workunits/rbd/nvmeof_fio_test.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash -ex
2+
3+
sudo yum -y install fio
4+
sudo yum -y install sysstat
5+
6+
fio_file=$(mktemp -t nvmeof-fio-XXXX)
7+
drives_list=$(sudo nvme list --output-format=json | jq -r '.Devices | .[] | select(.ModelNumber == "SPDK bdev Controller") | .DevicePath')
8+
9+
RUNTIME=${RUNTIME:-600}
10+
# IOSTAT_INTERVAL=10
11+
12+
13+
cat >> $fio_file <<EOF
14+
[nvmeof-fio-test]
15+
ioengine=${IO_ENGINE:-sync}
16+
bsrange=${BS_RANGE:-4k-64k}
17+
numjobs=${NUM_OF_JOBS:-1}
18+
size=${SIZE:-1G}
19+
time_based=1
20+
runtime=$RUNTIME
21+
rw=${RW:-randrw}
22+
filename=$(echo "$drives_list" | tr '\n' ':' | sed 's/:$//')
23+
verify=md5
24+
verify_fatal=1
25+
EOF
26+
27+
fio --showcmd $fio_file
28+
sudo fio $fio_file &
29+
30+
if [ -n "$IOSTAT_INTERVAL" ]; then
31+
iostat_count=$(( RUNTIME / IOSTAT_INTERVAL ))
32+
iostat -d $IOSTAT_INTERVAL $iostat_count -h
33+
fi
34+
wait
35+
36+
echo "[nvmeof] fio test successful!"

0 commit comments

Comments
 (0)