Merge pull request ceph#59178 from VallariAg/wip-nvmeof-teuthology-v6

VallariAg · web-flow · commit 53cc78b22241 · 2024-09-03T11:29:01.000+05:30
qa: add namespace and scale testing for nvmeof teuthology suite
diff --git a/qa/suites/nvmeof/basic/base/install.yaml b/qa/suites/nvmeof/basic/base/install.yaml
@@ -3,8 +3,7 @@ tasks:
 - install:
     extra_packages:
         - nvme-cli
-- cephadm: 
-    watchdog_setup:
+- cephadm:
 - cephadm.shell:
     host.a:
     # get state before nvmeof deployment
diff --git a/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml b/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml
@@ -1,21 +1,26 @@
 roles:
+- - client.0
+- - client.1
 - - host.a
   - mon.a
   - mgr.x
   - osd.0
-  - osd.1
-  - client.0
+  - client.2
   - ceph.nvmeof.nvmeof.a
 - - host.b
   - mon.b
+  - osd.1
+  - client.3
+  - ceph.nvmeof.nvmeof.b
+- - host.c
   - mon.c
   - osd.2
+  - client.4
+  - ceph.nvmeof.nvmeof.c
+- - host.d
   - osd.3
-  - osd.4
-  - client.1
-  - ceph.nvmeof.nvmeof.b
-- - client.2
-- - client.3
+  - client.5
+  - ceph.nvmeof.nvmeof.d
 
 overrides:
   ceph:
diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_initiator.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_initiator.yaml
@@ -1,6 +1,6 @@
 tasks:
 - nvmeof:
-    client: client.0
+    installer: host.a
     gw_image: quay.io/ceph/nvmeof:1.2 # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
     rbd:
       pool_name: mypool
@@ -16,8 +16,8 @@ tasks:
 - workunit:
     no_coverage_and_limits: true
     clients:
-      client.2:
-        - rbd/nvmeof_setup_subsystem.sh
+      client.0:
+        - nvmeof/setup_subsystem.sh
     env:
       RBD_POOL: mypool
       RBD_IMAGE_PREFIX: myimage
@@ -26,12 +26,12 @@ tasks:
     no_coverage_and_limits: true
     timeout: 30m
     clients:
-      client.2:
-        - rbd/nvmeof_basic_tests.sh
-        - rbd/nvmeof_fio_test.sh --start_ns 1 --end_ns 30 --rbd_iostat
-      client.3:
-        - rbd/nvmeof_basic_tests.sh
-        - rbd/nvmeof_fio_test.sh --start_ns 31 --end_ns 60
+      client.0:
+        - nvmeof/basic_tests.sh
+        - nvmeof/fio_test.sh --start_ns 1 --end_ns 30 --rbd_iostat
+      client.1:
+        - nvmeof/basic_tests.sh
+        - nvmeof/fio_test.sh --start_ns 31 --end_ns 60
     env:
       RBD_POOL: mypool
       IOSTAT_INTERVAL: '10'
diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_namespaces.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_namespaces.yaml
@@ -0,0 +1,39 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:1.2 # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 3
+      namespaces_count: 20
+      cli_image: quay.io/ceph/nvmeof-cli:1.2
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/basic_tests.sh
+        - nvmeof/fio_test.sh --rbd_iostat
+      client.1:
+        - nvmeof/basic_tests.sh
+        - nvmeof/namespace_test.sh
+    env:
+      RBD_POOL: mypool
+      IOSTAT_INTERVAL: '10'
+      RUNTIME: '120'
+      NEW_NAMESPACES_COUNT: '5'
diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml
@@ -0,0 +1,38 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:1.2 # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 3
+      namespaces_count: 20
+      cli_image: quay.io/ceph/nvmeof-cli:1.2
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+        - nvmeof/fio_test.sh --rbd_iostat
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+      IOSTAT_INTERVAL: '10'
+      RUNTIME: '60'
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/scalability_test.sh nvmeof.a,nvmeof.b
+        - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c,nvmeof.d
+    env:
+      SCALING_DELAYS: '50'
diff --git a/qa/suites/nvmeof/thrash/clusters/4-gateways-1-initiator.yaml b/qa/suites/nvmeof/thrash/clusters/4-gateways-1-initiator.yaml
@@ -1,26 +1,30 @@
 roles:
+- - client.0 # initiator
 - - host.a
   - mon.a
   - mgr.x
   - osd.0
   - osd.1
-  - client.0
+  - client.1
   - ceph.nvmeof.nvmeof.a
 - - host.b
   - mon.b
   - osd.2
   - osd.3
   - osd.4
-  - client.1
+  - client.2
   - ceph.nvmeof.nvmeof.b
 - - host.c
   - mon.c
   - osd.5
   - osd.6
   - osd.7
-  - client.2
+  - client.3
   - ceph.nvmeof.nvmeof.c
-- - client.3 # initiator
+- - host.d
+  - client.4
+  - ceph.nvmeof.nvmeof.d
+
 
 overrides:
   ceph:
diff --git a/qa/suites/nvmeof/thrash/gateway-initiator-setup/3-subsys-60-namespace.yaml b/qa/suites/nvmeof/thrash/gateway-initiator-setup/3-subsys-60-namespace.yaml
@@ -1,6 +1,6 @@
 tasks:
 - nvmeof:
-    client: client.0
+    installer: host.a
     gw_image: quay.io/ceph/nvmeof:1.2 # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
     rbd:
       pool_name: mypool
@@ -16,9 +16,9 @@ tasks:
 - workunit:
     no_coverage_and_limits: true
     clients:
-      client.3:
-        - rbd/nvmeof_setup_subsystem.sh
-        - rbd/nvmeof_basic_tests.sh
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
     env:
       RBD_POOL: mypool
       RBD_IMAGE_PREFIX: myimage
diff --git a/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml b/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml
@@ -13,7 +13,7 @@ overrides:
 
 tasks:
 - nvmeof.thrash:
-    checker_host: 'client.3'
+    checker_host: 'client.0'
     switch_thrashers: True
 
 - mon_thrash:
diff --git a/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml b/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml
@@ -8,4 +8,4 @@ overrides:
 
 tasks:
 - nvmeof.thrash:
-    checker_host: 'client.3'
+    checker_host: 'client.0'
diff --git a/qa/suites/nvmeof/thrash/workloads/fio.yaml b/qa/suites/nvmeof/thrash/workloads/fio.yaml
@@ -3,8 +3,8 @@ tasks:
     no_coverage_and_limits: true
     timeout: 30m
     clients:
-      client.3:
-        - rbd/nvmeof_fio_test.sh --rbd_iostat
+      client.0:
+        - nvmeof/fio_test.sh --rbd_iostat
     env:
       RBD_POOL: mypool
       IOSTAT_INTERVAL: '10'
diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py
@@ -17,14 +17,14 @@
 log = logging.getLogger(__name__)
 
 conf_file = '/etc/ceph/nvmeof.env'
-
+gw_yaml_file = '/etc/ceph/nvmeof-gw.yaml'
 
 class Nvmeof(Task):
     """
     Setup nvmeof gateway on client and then share gateway config to target host.
 
         - nvmeof:
-            client: client.0
+            installer: host.a     // or 'nvmeof.nvmeof.a' 
             version: default
             rbd:
                 pool_name: mypool
@@ -38,15 +38,11 @@ class Nvmeof(Task):
     def setup(self):
         super(Nvmeof, self).setup()
         try:
-            self.client = self.config['client']
+            host = self.config['installer']
         except KeyError:
-            raise ConfigError('nvmeof requires a client to connect with')
-
-        self.cluster_name, type_, self.client_id = misc.split_role(self.client)
-        if type_ != 'client':
-            msg = 'client role ({0}) must be a client'.format(self.client)
-            raise ConfigError(msg)
-        self.remote = get_remote_for_role(self.ctx, self.client)
+            raise ConfigError('nvmeof requires a installer host to deploy service') 
+        self.cluster_name, _, _ = misc.split_role(host)
+        self.remote = get_remote_for_role(self.ctx, host)  
 
     def begin(self):
         super(Nvmeof, self).begin()
@@ -139,6 +135,9 @@ def deploy_nvmeof(self):
 
         for role, i in daemons.items():
             remote, id_ = i
+            _shell(self.ctx, self.cluster_name, remote, [
+                'ceph', 'orch', 'ls', 'nvmeof', '--export', run.Raw('>'), gw_yaml_file
+            ])
             self.ctx.daemons.register_daemon(
                 remote, 'nvmeof', id_,
                 cluster=self.cluster_name,
@@ -343,6 +342,37 @@ def switch_task(self):
                 self.log('switch_task: done waiting for the other thrasher')
                 other_thrasher.switch_thrasher.clear()
 
+    def kill_daemon(self, daemon):
+        kill_methods = [
+            "ceph_daemon_stop", "systemctl_stop",
+            "daemon_remove",
+        ]
+        chosen_method = self.rng.choice(kill_methods)
+        d_name = '%s.%s' % (daemon.type_, daemon.id_)
+        if chosen_method == "ceph_daemon_stop": 
+            daemon.remote.run(args=[
+                "ceph", "orch", "daemon", "stop",
+                d_name
+            ], check_status=False)
+        elif chosen_method == "systemctl_stop":
+            daemon.stop()
+        elif chosen_method == "daemon_remove":
+            daemon.remote.run(args=[
+                "ceph", "orch", "daemon", "rm",
+                d_name
+            ], check_status=False)
+        return chosen_method
+
+    def revive_daemon(self, daemon, killed_method):
+        if killed_method == "ceph_daemon_stop":
+            name = '%s.%s' % (daemon.type_, daemon.id_)
+            daemon.remote.run(args=[
+                "ceph", "orch", "daemon", "restart",
+                name
+            ])
+        elif killed_method == "systemctl_stop":
+            daemon.restart() 
+
     def do_thrash(self):
         self.log('start thrashing')
         self.log(f'seed: {self.random_seed}, , '\
@@ -354,7 +384,7 @@ def do_thrash(self):
         summary = []
 
         while not self.stopping.is_set():
-            killed_daemons = []
+            killed_daemons = defaultdict(list)
 
             weight = 1.0 / len(self.daemons)
             count = 0
@@ -380,9 +410,10 @@ def do_thrash(self):
                         continue
 
                 self.log('kill {label}'.format(label=daemon.id_))
-                daemon.stop()
+                # daemon.stop()
+                kill_method = self.kill_daemon(daemon)
 
-                killed_daemons.append(daemon)
+                killed_daemons[kill_method].append(daemon)
                 daemons_thrash_history[daemon.id_] += [datetime.now()]
 
                 # only thrash max_thrash_daemons amount of daemons
@@ -391,7 +422,10 @@ def do_thrash(self):
                     break
 
             if killed_daemons:
-                summary += ["killed: " + ", ".join([d.id_ for d in killed_daemons])]
+                iteration_summary = "thrashed- "
+                for kill_method in killed_daemons:
+                    iteration_summary += (", ".join([d.id_ for d in killed_daemons[kill_method]]) + f" (by {kill_method}); ") 
+                summary += [iteration_summary]
                 # delay before reviving
                 revive_delay = self.min_revive_delay
                 if self.randomize:
@@ -405,9 +439,11 @@ def do_thrash(self):
                 self.switch_task()
 
                 # revive after thrashing
-                for daemon in killed_daemons:
-                    self.log('reviving {label}'.format(label=daemon.id_))
-                    daemon.restart()
+                for kill_method in killed_daemons:
+                    for daemon in killed_daemons[kill_method]:
+                        self.log('reviving {label}'.format(label=daemon.id_))
+                        # daemon.restart()
+                        self.revive_daemon(daemon, kill_method)
                 
                 # delay before thrashing
                 thrash_delay = self.min_thrash_delay
diff --git a/qa/workunits/nvmeof/basic_tests.sh b/qa/workunits/nvmeof/basic_tests.sh
diff --git a/qa/workunits/nvmeof/fio_test.sh b/qa/workunits/nvmeof/fio_test.sh
diff --git a/qa/workunits/nvmeof/namespace_test.sh b/qa/workunits/nvmeof/namespace_test.sh
diff --git a/qa/workunits/nvmeof/scalability_test.sh b/qa/workunits/nvmeof/scalability_test.sh
diff --git a/qa/workunits/nvmeof/setup_subsystem.sh b/qa/workunits/nvmeof/setup_subsystem.sh
