Merge pull request ceph#62725 from VallariAg/nvmeof-teuthology-fio

qa/suites/nvmeof: Fix thrasher and fio script

Author: VallariAg

qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml

@@ -29,4 +29,8 @@ overrides:
         # cephadm can take up to 5 minutes to bring up remaining mons
         mon down mkfs grace: 300
     log-ignorelist:
+      - OSD_DOWN
+      - OSD_HOST_DOWN
+      - OSD_ROOT_DOWN
+      - PG_DEGRADED
       - NVMEOF_SINGLE_GATEWAY

qa/suites/nvmeof/thrash/gateway-initiator-setup/16-subsys-4-namespace.yaml

@@ -0,0 +1,24 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 16
+      namespaces_count: 4 # each subsystem
+      cli_image: quay.io/ceph/nvmeof-cli:latest
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage

qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml

@@ -1,6 +1,11 @@
 overrides:
   ceph:
-    log-ignorelist: 
+    log-ignorelist:
+      # general ignorelist
+      - OSD_DOWN
+      - OSD_HOST_DOWN
+      - OSD_ROOT_DOWN
+      - PG_DEGRADED
       # mon thrashing
       - MON_DOWN
       - mons down

qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml

@@ -1,6 +1,11 @@
 overrides:
   ceph:
     log-ignorelist:  
+      # general ignorelist
+      - OSD_DOWN
+      - OSD_HOST_DOWN
+      - OSD_ROOT_DOWN
+      - PG_DEGRADED
       # nvmeof daemon thrashing
       - CEPHADM_FAILED_DAEMON
       - NVMEOF_SINGLE_GATEWAY

qa/suites/nvmeof/thrash/workloads/fio.yaml

@@ -9,4 +9,4 @@ tasks:
       RBD_POOL: mypool
       NVMEOF_GROUP: mygroup0
       IOSTAT_INTERVAL: '10'
-      RUNTIME: '1800'
+      RUNTIME: '1200'

qa/tasks/nvmeof.py

@@ -321,7 +321,7 @@ def __init__(self, ctx, config, daemons) -> None:
         self.max_thrash_daemons = int(self.config.get('max_thrash', len(self.daemons) - 1))
 
         # Limits on thrashing each daemon
-        self.daemon_max_thrash_times = int(self.config.get('daemon_max_thrash_times', 5))
+        self.daemon_max_thrash_times = int(self.config.get('daemon_max_thrash_times', 4))
         self.daemon_max_thrash_period = int(self.config.get('daemon_max_thrash_period', 30 * 60)) # seconds
 
         self.min_thrash_delay = int(self.config.get('min_thrash_delay', 60))
@@ -436,7 +436,7 @@ def revive_daemon(self, daemon, killed_method):
         if killed_method == "ceph_daemon_stop":
             daemon.remote.run(args=[
                 "ceph", "orch", "daemon", "restart",
-                name
+                name, "--force"
             ])
         # note: temporarily use 'daemon start' to restart
         # daemons instead of 'systemctl start'

qa/workunits/nvmeof/fio_test.sh

@@ -54,20 +54,31 @@ filename=$(echo "$selected_drives" | sed -z 's/\n/:\/dev\//g' | sed 's/:\/dev\/$
 filename="/dev/$filename"
 
 cat >> $fio_file <<EOF
-[nvmeof-fio-test]
+[global]
 ioengine=${IO_ENGINE:-sync}
 bsrange=${BS_RANGE:-4k-64k}
 numjobs=${NUM_OF_JOBS:-1}
 size=${SIZE:-1G}
 time_based=1
 runtime=$RUNTIME
 rw=${RW:-randrw}
-filename=${filename}
 verify=md5
 verify_fatal=1
+do_verify=1
+serialize_overlap=1
+group_reporting
 direct=1
+
 EOF
 
+for i in $selected_drives; do
+  echo "[job-$i]" >> "$fio_file"
+  echo "filename=/dev/$i" >> "$fio_file"
+  echo "" >> "$fio_file"  # Adds a blank line
+done
+
+cat $fio_file
+
 status_log() {
     POOL="${RBD_POOL:-mypool}"
     GROUP="${NVMEOF_GROUP:-mygroup0}"
@@ -79,6 +90,7 @@ status_log() {
     ceph nvme-gw show $POOL $GROUP
     sudo nvme list
     sudo nvme list | wc -l
+    sudo nvme list-subsys
     for device in $selected_drives; do
         echo "Processing device: $device"
         sudo nvme list-subsys /dev/$device
@@ -103,6 +115,7 @@ fio --showcmd $fio_file
 set +e 
 sudo fio $fio_file
 if [ $? -ne 0 ]; then
+    echo "[nvmeof.fio]: fio failed!" 
     status_log
     exit 1
 fi