Adding support for extra args by using the field tgt_cmd_extra_args

Signed-off-by: Redouane Kachach <[email protected]>

Author: rkachach

src/cephadm/cephadm.py

@@ -1024,8 +1024,8 @@ def get_container_mounts(data_dir: str) -> Dict[str, str]:
         mounts['/etc/ceph/ceph.client.admin.keyring'] = '/etc/ceph/keyring:z'  # TODO: FIXME
         mounts[os.path.join(data_dir, 'ceph-nvmeof.conf')] = '/src/ceph-nvmeof.conf:z'
         mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
-        # mounts[log_dir] = '/var/log:z' # TODO: would we need a logdir?
-        mounts['/dev'] = '/dev'
+        mounts['/dev/hugepages'] = '/dev/hugepages'
+        mounts['/dev/vfio/vfio'] = '/dev/vfio/vfio'
         return mounts
 
     @staticmethod
@@ -1104,7 +1104,7 @@ def configfs_mount_umount(data_dir, mount=True):
     @staticmethod
     def get_sysctl_settings() -> List[str]:
         return [
-            'vm.nr_hugepages = 2048',
+            'vm.nr_hugepages = 4096',
         ]
 
 
@@ -6552,7 +6552,7 @@ def _dispatch_deploy(
                       ports=daemon_ports)
     elif daemon_type == CephNvmeof.daemon_type:
         config, keyring = get_config_and_keyring(ctx)
-        uid, gid = 65534, 65534  # TODO: check this
+        uid, gid = 167, 167  # TODO: need to get properly the uid/gid
         c = get_deployment_container(ctx, ctx.fsid, daemon_type, daemon_id)
         deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
                       config=config, keyring=keyring,

src/pybind/mgr/cephadm/services/nvmeof.py

@@ -1,5 +1,6 @@
 import errno
 import logging
+import json
 from typing import List, cast, Optional
 
 from mgr_module import HandleCommandResult
@@ -32,14 +33,17 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD
                                               'mds', 'allow *',
                                               'mgr', 'allow *',
                                               'osd', 'allow *'])
+
+        # TODO: check if we can force jinja2 to generate dicts with double quotes instead of using json.dumps
+        transport_tcp_options = json.dumps(spec.transport_tcp_options) if spec.transport_tcp_options else None
         context = {
             'spec': spec,
             'name': '{}.{}'.format(utils.name_to_config_section('nvmeof'), igw_id),
             'addr': self.mgr.get_mgr_ip(),
             'port': spec.port,
-            'tgt_cmd_extra_args': None,
             'log_level': 'WARN',
             'rpc_socket': '/var/tmp/spdk.sock',
+            'transport_tcp_options': transport_tcp_options
         }
         gw_conf = self.mgr.template.render('services/nvmeof/ceph-nvmeof.conf.j2', context)
 

src/pybind/mgr/cephadm/templates/services/nvmeof/ceph-nvmeof.conf.j2

@@ -25,6 +25,9 @@ timeout = {{ spec.timeout }}
 log_level = {{ log_level }}
 conn_retries = {{ spec.conn_retries }}
 transports = {{ spec.transports }}
-{% if tgt_cmd_extra_args %}
-tgt_cmd_extra_args = {{ tgt_cmd_extra_args }}
+{% if transport_tcp_options %}
+transport_tcp_options = {{ transport_tcp_options }}
+{% endif %}
+{% if spec.tgt_cmd_extra_args %}
+tgt_cmd_extra_args = {{ spec.tgt_cmd_extra_args | tojson }}
 {% endif %}

src/pybind/mgr/cephadm/tests/test_services.py

@@ -392,7 +392,8 @@ def test_nvmeof_config(self, _get_name, _run_cephadm, cephadm_module: CephadmOrc
 timeout = 60
 log_level = WARN
 conn_retries = 10
-transports = tcp\n"""
+transports = tcp
+transport_tcp_options = {{"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7}}\n"""
 
         with with_host(cephadm_module, 'test'):
             with with_service(cephadm_module, NvmeofServiceSpec(service_id=pool,

src/python-common/ceph/deployment/service_spec.py

@@ -1126,14 +1126,17 @@ def __init__(self,
                  tgt_path: Optional[str] = None,
                  timeout: Optional[int] = 60,
                  conn_retries: Optional[int] = 10,
-                 transports: Optional[str] = "tcp",
+                 transports: Optional[str] = 'tcp',
+                 transport_tcp_options: Optional[Dict[str, int]] =
+                 {"in_capsule_data_size": 8192, "max_io_qpairs_per_ctrlr": 7},
+                 tgt_cmd_extra_args: Optional[str] = None,
                  placement: Optional[PlacementSpec] = None,
                  unmanaged: bool = False,
                  preview_only: bool = False,
                  config: Optional[Dict[str, str]] = None,
                  networks: Optional[List[str]] = None,
-                 extra_container_args: Optional[List[str]] = None,
-                 extra_entrypoint_args: Optional[List[str]] = None,
+                 extra_container_args: Optional[GeneralArgList] = None,
+                 extra_entrypoint_args: Optional[GeneralArgList] = None,
                  custom_configs: Optional[List[CustomConfig]] = None,
                  ):
         assert service_type == 'nvmeof'
@@ -1171,8 +1174,12 @@ def __init__(self,
         self.timeout = timeout
         #: ``conn_retries`` ceph connection retries number
         self.conn_retries = conn_retries
-        #: ``transports`` TODO
+        #: ``transports`` tcp
         self.transports = transports
+        #: List of extra arguments for transports in the form opt=value
+        self.transport_tcp_options: Optional[Dict[str, int]] = transport_tcp_options
+        #: ``tgt_cmd_extra_args`` extra arguments for the nvmf_tgt process
+        self.tgt_cmd_extra_args = tgt_cmd_extra_args
 
     def get_port_start(self) -> List[int]:
         return [5500, 4420, 8009]