Merge pull request ceph#61357 from VallariAg/wip-nvmeof-teuthology-test-fix-ha

qa: fix nvmeof teuthology thrasher fix

Author: VallariAg

qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml

@@ -36,6 +36,6 @@ tasks:
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c,nvmeof.d
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c
     env:
-      SCALING_DELAYS: '50'
+      SCALING_DELAYS: '120'
       RBD_POOL: mypool
       NVMEOF_GROUP: mygroup0

qa/suites/nvmeof/thrash/base/install.yaml

@@ -4,7 +4,6 @@ tasks:
     extra_packages:
         - nvme-cli
 - cephadm:
-    watchdog_setup:
 - cephadm.shell:
     host.a:
     # get state before nvmeof deployment

qa/suites/nvmeof/thrash/gateway-initiator-setup/2-subsys-8-namespace.yaml

@@ -0,0 +1,24 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 2
+      namespaces_count: 8 # each subsystem
+      cli_image: quay.io/ceph/nvmeof-cli:latest
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage

qa/tasks/nvmeof.py

@@ -209,6 +209,22 @@ def set_gateway_cfg(self):
         if self.create_mtls_secrets: 
             self.write_mtls_config(gateway_ips)
         log.info("[nvmeof]: executed set_gateway_cfg successfully!")
+    
+    def teardown(self):
+        log.info("[nvmeof] Removing nvmeof service")
+        _shell(self.ctx, self.cluster_name, self.remote, [
+            'ceph', 'orch', 'host', 'ls'
+        ])
+        for i in range(self.groups_count):
+            group_name = self.groups_prefix + str(i)
+            service_name = f"nvmeof.{self.poolname}.{group_name}"
+            _shell(self.ctx, self.cluster_name, self.remote, [
+                'ceph', 'orch', 'rm', service_name
+            ])
+        _shell(self.ctx, self.cluster_name, self.remote, [
+            'ceph', 'orch', 'host', 'ls'
+        ])
+        log.info("[nvmeof] Nvmeof teardown completed!")
 
 
 class NvmeofThrasher(Thrasher, Greenlet):
@@ -334,26 +350,41 @@ def _run(self): # overriding
     def stop(self):
         self.stopping.set()
 
+    def stop_and_join(self):
+        """
+        Stop the thrashing process and join the thread.
+        """
+        self.stop()
+        return self.join()
+
     def do_checks(self):
         """
         Run some checks to see if everything is running well during thrashing.
         """
         self.log('display and verify stats:')
-        for d in self.daemons:
-            d.remote.sh(d.status_cmd, check_status=False)
-        check_cmd = [
-            'ceph', 'orch', 'ls',
-            run.Raw('&&'), 'ceph', 'orch', 'ps', '--daemon-type', 'nvmeof',
-            run.Raw('&&'), 'ceph', 'health', 'detail',
-            run.Raw('&&'), 'ceph', '-s',
-            run.Raw('&&'), 'sudo', 'nvme', 'list',
-        ]
-        for dev in self.devices:
-            check_cmd += [
-                run.Raw('&&'), 'sudo', 'nvme', 'list-subsys', dev,
-                run.Raw('|'), 'grep', 'live optimized'
-            ] 
-        self.checker_host.run(args=check_cmd).wait()        
+        for retry in range(5):
+            try: 
+                random_gateway_host = None
+                initiator_host = self.checker_host 
+                for d in self.daemons:
+                    random_gateway_host = d.remote
+                    d.remote.sh(d.status_cmd, check_status=False)
+                random_gateway_host.run(args=['ceph', 'orch', 'ls', '--refresh'])
+                random_gateway_host.run(args=['ceph', 'orch', 'ps', '--daemon-type', 'nvmeof', '--refresh'])
+                random_gateway_host.run(args=['ceph', 'health', 'detail'])
+                random_gateway_host.run(args=['ceph', '-s'])
+                random_gateway_host.run(args=['ceph', 'nvme-gw', 'show', 'mypool', 'mygroup0'])
+
+                initiator_host.run(args=['sudo', 'nvme', 'list'])
+                for dev in self.devices:
+                    device_check_cmd = [
+                        'sudo', 'nvme', 'list-subsys', dev,
+                        run.Raw('|'), 'grep', 'live optimized'
+                    ]
+                    initiator_host.run(args=device_check_cmd)
+                break
+            except run.CommandFailedError:
+                self.log(f"retry do_checks() for {retry} time")
 
     def switch_task(self):
         """
@@ -373,13 +404,14 @@ def switch_task(self):
             ):
                 other_thrasher = t
                 self.log('switch_task: waiting for other thrasher')
-                other_thrasher.switch_thrasher.wait(300)
+                other_thrasher.switch_thrasher.wait(600)
                 self.log('switch_task: done waiting for the other thrasher')
                 other_thrasher.switch_thrasher.clear()
 
     def kill_daemon(self, daemon):
         kill_methods = [
-            "ceph_daemon_stop", "systemctl_stop",
+            "ceph_daemon_stop", 
+            # "systemctl_stop",
             "daemon_remove",
         ]
         chosen_method = self.rng.choice(kill_methods)
@@ -390,7 +422,8 @@ def kill_daemon(self, daemon):
                 d_name
             ], check_status=False)
         elif chosen_method == "systemctl_stop":
-            daemon.stop()
+            # To bypass is_started logic of CephadmUnit
+            daemon.remote.sh(daemon.stop_cmd, check_status=False)
         elif chosen_method == "daemon_remove":
             daemon.remote.run(args=[
                 "ceph", "orch", "daemon", "rm",
@@ -399,14 +432,24 @@ def kill_daemon(self, daemon):
         return chosen_method
 
     def revive_daemon(self, daemon, killed_method):
+        name = '%s.%s' % (daemon.type_, daemon.id_)
         if killed_method == "ceph_daemon_stop":
-            name = '%s.%s' % (daemon.type_, daemon.id_)
             daemon.remote.run(args=[
                 "ceph", "orch", "daemon", "restart",
                 name
             ])
+        # note: temporarily use 'daemon start' to restart
+        # daemons instead of 'systemctl start'
         elif killed_method == "systemctl_stop":
-            daemon.restart() 
+            daemon.remote.run(args=[
+                "ceph", "orch", "daemon", "start",
+                name
+            ])
+        else:
+            daemon.remote.run(args=[
+                "ceph", "orch", "daemon", "start",
+                name
+            ])
 
     def do_thrash(self):
         self.log('start thrashing')
@@ -515,6 +558,9 @@ def end(self):
         self.thrasher.join()
         log.info('done joining')
 
+    def teardown(self):
+        log.info('tearing down nvmeof thrasher...')
+
 
 task = Nvmeof
 thrash = ThrashTest

qa/workunits/nvmeof/fio_test.sh

@@ -50,7 +50,8 @@ fi
 
 
 RUNTIME=${RUNTIME:-600}
-
+filename=$(echo "$selected_drives" | sed -z 's/\n/:\/dev\//g' | sed 's/:\/dev\/$//')
+filename="/dev/$filename"
 
 cat >> $fio_file <<EOF
 [nvmeof-fio-test]
@@ -61,7 +62,7 @@ size=${SIZE:-1G}
 time_based=1
 runtime=$RUNTIME
 rw=${RW:-randrw}
-filename=$(echo "$selected_drives" | tr '\n' ':' | sed 's/:$//')
+filename=${filename}
 verify=md5
 verify_fatal=1
 direct=1
@@ -79,6 +80,5 @@ if [ "$rbd_iostat" = true  ]; then
 fi
 fio --showcmd $fio_file
 sudo fio $fio_file 
-wait
 
 echo "[nvmeof.fio] fio test successful!"

qa/workunits/nvmeof/scalability_test.sh

@@ -32,10 +32,7 @@ status_checks() {
         return 1
     fi
 
-    ceph_status=$(ceph -s)
-    if ! echo "$ceph_status" | grep -q "HEALTH_OK"; then
-        return 1
-    fi
+    ceph -s
 }
 
 total_gateways_count=$(( $(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | tr -cd ',' | wc -c) + 1 ))