Skip to content

Commit 6f35d34

Browse files
authored
Merge pull request ceph#56369 from adk3798/nvmeof-new-fields
mgr/cephadm: add some NVMEof gateway fields to be configurable by cephadm Reviewed-by: Gil Bregman <[email protected]> Reviewed-by: John Mulligan <[email protected]>
2 parents 0d302a8 + a16ac81 commit 6f35d34

File tree

4 files changed

+150
-26
lines changed

4 files changed

+150
-26
lines changed

src/pybind/mgr/cephadm/services/nvmeof.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,9 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
4343
'name': name,
4444
'addr': host_ip,
4545
'port': spec.port,
46-
'log_level': 'WARN',
47-
'rpc_socket': '/var/tmp/spdk.sock',
46+
'spdk_log_level': 'WARNING',
47+
'rpc_socket_dir': '/var/tmp/',
48+
'rpc_socket_name': 'spdk.sock',
4849
'transport_tcp_options': transport_tcp_options,
4950
'rados_id': rados_id
5051
}

src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,33 @@
33
name = {{ name }}
44
group = {{ spec.group if spec.group is not none else '' }}
55
addr = {{ addr }}
6-
port = {{ port }}
6+
port = {{ spec.port }}
77
enable_auth = {{ spec.enable_auth }}
8-
state_update_notify = True
9-
state_update_interval_sec = 5
10-
min_controller_id = {{ spec.min_controller_id }}
11-
max_controller_id = {{ spec.max_controller_id }}
8+
state_update_notify = {{ spec.state_update_notify }}
9+
state_update_interval_sec = {{ spec.state_update_interval_sec }}
1210
enable_spdk_discovery_controller = {{ spec.enable_spdk_discovery_controller }}
13-
enable_prometheus_exporter = True
11+
enable_prometheus_exporter = {{ spec.enable_prometheus_exporter }}
1412
prometheus_exporter_ssl = False
1513
prometheus_port = 10008
14+
verify_nqns = {{ spec.verify_nqns }}
15+
omap_file_lock_duration = {{ spec.omap_file_lock_duration }}
16+
omap_file_lock_retries = {{ spec.omap_file_lock_retries }}
17+
omap_file_lock_retry_sleep_interval = {{ spec.omap_file_lock_retry_sleep_interval }}
18+
omap_file_update_reloads = {{ spec.omap_file_update_reloads }}
19+
20+
[gateway-logs]
21+
log_level = {{ spec.log_level }}
22+
log_files_enabled = {{ spec.log_files_enabled }}
23+
log_files_rotation_enabled = {{ spec.log_files_rotation_enabled }}
24+
verbose_log_messages = {{ spec.verbose_log_messages }}
25+
max_log_file_size_in_mb = {{ spec.max_log_file_size_in_mb }}
26+
max_log_files_count = {{ spec.max_log_files_count }}
27+
max_log_directory_backups = {{ spec.max_log_directory_backups }}
28+
log_directory = {{ spec.log_directory }}
29+
30+
[discovery]
31+
addr = {{ addr }}
32+
port = {{ spec.discovery_port }}
1633

1734
[ceph]
1835
pool = {{ spec.pool }}
@@ -27,9 +44,11 @@ client_cert = {{ spec.client_cert }}
2744

2845
[spdk]
2946
tgt_path = {{ spec.tgt_path }}
30-
rpc_socket = {{ rpc_socket }}
31-
timeout = {{ spec.timeout }}
32-
log_level = {{ log_level }}
47+
rpc_socket_dir = {{ spec.rpc_socket_dir }}
48+
rpc_socket_name = {{ spec.rpc_socket_name }}
49+
timeout = {{ spec.spdk_timeout }}
50+
bdevs_per_cluster = {{ spec.bdevs_per_cluster }}
51+
log_level = {{ spec.spdk_log_level }}
3352
conn_retries = {{ spec.conn_retries }}
3453
transports = {{ spec.transports }}
3554
{% if transport_tcp_options %}
@@ -38,3 +57,7 @@ transport_tcp_options = {{ transport_tcp_options }}
3857
{% if spec.tgt_cmd_extra_args %}
3958
tgt_cmd_extra_args = {{ spec.tgt_cmd_extra_args }}
4059
{% endif %}
60+
61+
[monitor]
62+
timeout = {{ spec.monitor_timeout }}
63+

src/pybind/mgr/cephadm/tests/test_services.py

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -390,12 +390,29 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
390390
enable_auth = False
391391
state_update_notify = True
392392
state_update_interval_sec = 5
393-
min_controller_id = 1
394-
max_controller_id = 65519
395393
enable_spdk_discovery_controller = False
396394
enable_prometheus_exporter = True
397395
prometheus_exporter_ssl = False
398396
prometheus_port = 10008
397+
verify_nqns = True
398+
omap_file_lock_duration = 60
399+
omap_file_lock_retries = 15
400+
omap_file_lock_retry_sleep_interval = 5
401+
omap_file_update_reloads = 10
402+
403+
[gateway-logs]
404+
log_level = INFO
405+
log_files_enabled = True
406+
log_files_rotation_enabled = True
407+
verbose_log_messages = True
408+
max_log_file_size_in_mb = 10
409+
max_log_files_count = 20
410+
max_log_directory_backups = 10
411+
log_directory = /var/log/ceph/
412+
413+
[discovery]
414+
addr = 192.168.100.100
415+
port = 8009
399416
400417
[ceph]
401418
pool = {pool}
@@ -410,13 +427,18 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
410427
411428
[spdk]
412429
tgt_path = /usr/local/bin/nvmf_tgt
413-
rpc_socket = /var/tmp/spdk.sock
414-
timeout = 60
415-
log_level = WARN
430+
rpc_socket_dir = /var/tmp/
431+
rpc_socket_name = spdk.sock
432+
timeout = 60.0
433+
bdevs_per_cluster = 32
434+
log_level = WARNING
416435
conn_retries = 10
417436
transports = tcp
418437
transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}}
419-
tgt_cmd_extra_args = {tgt_cmd_extra_args}\n"""
438+
tgt_cmd_extra_args = {tgt_cmd_extra_args}
439+
440+
[monitor]
441+
timeout = 1.0\n"""
420442

421443
with with_host(cephadm_module, 'test'):
422444
with with_service(cephadm_module, NvmeofServiceSpec(service_id=pool,

src/python-common/ceph/deployment/service_spec.py

Lines changed: 87 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1267,21 +1267,41 @@ def __init__(self,
12671267
port: Optional[int] = None,
12681268
pool: Optional[str] = None,
12691269
enable_auth: bool = False,
1270-
min_controller_id: Optional[str] = '1',
1271-
max_controller_id: Optional[str] = '65519',
1270+
state_update_notify: Optional[bool] = True,
1271+
state_update_interval_sec: Optional[int] = 5,
12721272
enable_spdk_discovery_controller: Optional[bool] = False,
1273+
omap_file_lock_duration: Optional[int] = 60,
1274+
omap_file_lock_retries: Optional[int] = 15,
1275+
omap_file_lock_retry_sleep_interval: Optional[int] = 5,
1276+
omap_file_update_reloads: Optional[int] = 10,
1277+
enable_prometheus_exporter: Optional[bool] = True,
1278+
bdevs_per_cluster: Optional[int] = 32,
1279+
verify_nqns: Optional[bool] = True,
12731280
server_key: Optional[str] = None,
12741281
server_cert: Optional[str] = None,
12751282
client_key: Optional[str] = None,
12761283
client_cert: Optional[str] = None,
12771284
spdk_path: Optional[str] = None,
12781285
tgt_path: Optional[str] = None,
1279-
timeout: Optional[int] = 60,
1286+
spdk_timeout: Optional[float] = 60.0,
1287+
spdk_log_level: Optional[str] = 'WARNING',
1288+
rpc_socket_dir: Optional[str] = '/var/tmp/',
1289+
rpc_socket_name: Optional[str] = 'spdk.sock',
12801290
conn_retries: Optional[int] = 10,
12811291
transports: Optional[str] = 'tcp',
12821292
transport_tcp_options: Optional[Dict[str, int]] =
12831293
{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
12841294
tgt_cmd_extra_args: Optional[str] = None,
1295+
discovery_port: Optional[int] = None,
1296+
log_level: Optional[str] = 'INFO',
1297+
log_files_enabled: Optional[bool] = True,
1298+
log_files_rotation_enabled: Optional[bool] = True,
1299+
verbose_log_messages: Optional[bool] = True,
1300+
max_log_file_size_in_mb: Optional[int] = 10,
1301+
max_log_files_count: Optional[int] = 20,
1302+
max_log_directory_backups: Optional[int] = 10,
1303+
log_directory: Optional[str] = '/var/log/ceph/',
1304+
monitor_timeout: Optional[float] = 1.0,
12851305
placement: Optional[PlacementSpec] = None,
12861306
unmanaged: bool = False,
12871307
preview_only: bool = False,
@@ -1310,12 +1330,26 @@ def __init__(self,
13101330
self.group = group
13111331
#: ``enable_auth`` enables user authentication on nvmeof gateway
13121332
self.enable_auth = enable_auth
1313-
#: ``min_controller_id`` minimum controller id used by SPDK, essential for multipath
1314-
self.min_controller_id = min_controller_id
1315-
#: ``max_controller_id`` maximum controller id used by SPDK, essential for multipath
1316-
self.max_controller_id = max_controller_id
1333+
#: ``state_update_notify`` enables automatic update from OMAP in nvmeof gateway
1334+
self.state_update_notify = state_update_notify
1335+
#: ``state_update_interval_sec`` number of seconds to check for updates in OMAP
1336+
self.state_update_interval_sec = state_update_interval_sec
13171337
#: ``enable_spdk_discovery_controller`` SPDK or ceph-nvmeof discovery service
13181338
self.enable_spdk_discovery_controller = enable_spdk_discovery_controller
1339+
#: ``enable_prometheus_exporter`` enables Prometheus exporter
1340+
self.enable_prometheus_exporter = enable_prometheus_exporter
1341+
#: ``verify_nqns`` enables verification of subsystem and host NQNs for validity
1342+
self.verify_nqns = verify_nqns
1343+
#: ``omap_file_lock_duration`` number of seconds before automatically unlock OMAP file lock
1344+
self.omap_file_lock_duration = omap_file_lock_duration
1345+
#: ``omap_file_lock_retries`` number of retries to lock OMAP file before giving up
1346+
self.omap_file_lock_retries = omap_file_lock_retries
1347+
#: ``omap_file_lock_retry_sleep_interval`` seconds to wait before retrying to lock OMAP
1348+
self.omap_file_lock_retry_sleep_interval = omap_file_lock_retry_sleep_interval
1349+
#: ``omap_file_update_reloads`` number of attempt to reload OMAP when it differs from local
1350+
self.omap_file_update_reloads = omap_file_update_reloads
1351+
#: ``bdevs_per_cluster`` number of bdevs per cluster
1352+
self.bdevs_per_cluster = bdevs_per_cluster
13191353
#: ``server_key`` gateway server key
13201354
self.server_key = server_key or './server.key'
13211355
#: ``server_cert`` gateway server certificate
@@ -1328,8 +1362,14 @@ def __init__(self,
13281362
self.spdk_path = spdk_path or '/usr/local/bin/nvmf_tgt'
13291363
#: ``tgt_path`` nvmeof target path
13301364
self.tgt_path = tgt_path or '/usr/local/bin/nvmf_tgt'
1331-
#: ``timeout`` ceph connectivity timeout
1332-
self.timeout = timeout
1365+
#: ``spdk_timeout`` SPDK connectivity timeout
1366+
self.spdk_timeout = spdk_timeout
1367+
#: ``spdk_log_level`` the SPDK log level
1368+
self.spdk_log_level = spdk_log_level or 'WARNING'
1369+
#: ``rpc_socket_dir`` the SPDK socket file directory
1370+
self.rpc_socket_dir = rpc_socket_dir or '/var/tmp/'
1371+
#: ``rpc_socket_name`` the SPDK socket file name
1372+
self.rpc_socket_name = rpc_socket_name or 'spdk.sock'
13331373
#: ``conn_retries`` ceph connection retries number
13341374
self.conn_retries = conn_retries
13351375
#: ``transports`` tcp
@@ -1338,6 +1378,26 @@ def __init__(self,
13381378
self.transport_tcp_options: Optional[Dict[str, int]] = transport_tcp_options
13391379
#: ``tgt_cmd_extra_args`` extra arguments for the nvmf_tgt process
13401380
self.tgt_cmd_extra_args = tgt_cmd_extra_args
1381+
#: ``discovery_port`` port of the discovery service
1382+
self.discovery_port = discovery_port or 8009
1383+
#: ``log_level`` the nvmeof gateway log level
1384+
self.log_level = log_level or 'INFO'
1385+
#: ``log_files_enabled`` enables the usage of files to keep the nameof gateway log
1386+
self.log_files_enabled = log_files_enabled
1387+
#: ``log_files_rotation_enabled`` enables rotation of log files when pass the size limit
1388+
self.log_files_rotation_enabled = log_files_rotation_enabled
1389+
#: ``verbose_log_messages`` add more details to the nvmeof gateway log message
1390+
self.verbose_log_messages = verbose_log_messages
1391+
#: ``max_log_file_size_in_mb`` max size in MB before starting a new log file
1392+
self.max_log_file_size_in_mb = max_log_file_size_in_mb
1393+
#: ``max_log_files_count`` max log files to keep before overriding them
1394+
self.max_log_files_count = max_log_files_count
1395+
#: ``max_log_directory_backups`` max directories for old gateways with same name to keep
1396+
self.max_log_directory_backups = max_log_directory_backups
1397+
#: ``log_directory`` directory for keeping nameof gateway log files
1398+
self.log_directory = log_directory or '/var/log/ceph/'
1399+
#: ``monitor_timeout`` monitor connectivity timeout
1400+
self.monitor_timeout = monitor_timeout
13411401

13421402
def get_port_start(self) -> List[int]:
13431403
return [5500, 4420, 8009]
@@ -1357,6 +1417,24 @@ def validate(self) -> None:
13571417
if self.transports not in ['tcp']:
13581418
raise SpecValidationError('Invalid transport. Valid values are tcp')
13591419

1420+
if self.log_level:
1421+
if self.log_level not in ['debug', 'DEBUG',
1422+
'info', 'INFO',
1423+
'warning', 'WARNING',
1424+
'error', 'ERROR',
1425+
'critical', 'CRITICAL']:
1426+
raise SpecValidationError(
1427+
'Invalid log level. Valid values are: debug, info, warning, error, critial')
1428+
1429+
if self.spdk_log_level:
1430+
if self.spdk_log_level not in ['debug', 'DEBUG',
1431+
'info', 'INFO',
1432+
'warning', 'WARNING',
1433+
'error', 'ERROR',
1434+
'notice', 'NOTICE']:
1435+
raise SpecValidationError(
1436+
'Invalid SPDK log level. Valid values are: DEBUG, INFO, WARNING, ERROR, NOTICE')
1437+
13601438

13611439
yaml.add_representer(NvmeofServiceSpec, ServiceSpec.yaml_representer)
13621440

0 commit comments

Comments
 (0)