Merge pull request ceph#61788 from VallariAg/wip-nvmeof-teuthology-improve

qa/suite/nvmeof: Add asserts to scalability_test and extra logs in fio_test

Author: VallariAg

qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml

@@ -31,11 +31,12 @@ tasks:
     no_coverage_and_limits: true
     timeout: 30m
     clients:
+      client.0:
+        - nvmeof/fio_test.sh
       client.3:
         - nvmeof/scalability_test.sh nvmeof.a,nvmeof.b
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c,nvmeof.d
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c
     env:
-      SCALING_DELAYS: '120'
-      RBD_POOL: mypool
-      NVMEOF_GROUP: mygroup0
+      SCALING_DELAYS: '300'
+      RUNTIME: '1200'

qa/suites/nvmeof/thrash/workloads/fio.yaml

@@ -7,5 +7,6 @@ tasks:
         - nvmeof/fio_test.sh --random_devices 200
     env:
       RBD_POOL: mypool
+      NVMEOF_GROUP: mygroup0
       IOSTAT_INTERVAL: '10'
       RUNTIME: '1800'

qa/workunits/nvmeof/fio_test.sh

@@ -68,6 +68,26 @@ verify_fatal=1
 direct=1
 EOF
 
+status_log() {
+    POOL="${RBD_POOL:-mypool}"
+    GROUP="${NVMEOF_GROUP:-mygroup0}"
+    ceph -s
+    ceph orch host ls
+    ceph orch ls 
+    ceph orch ps
+    ceph health detail
+    ceph nvme-gw show $POOL $GROUP
+    sudo nvme list
+    sudo nvme list | wc -l
+    for device in $selected_drives; do
+        echo "Processing device: $device"
+        sudo nvme list-subsys /dev/$device
+        sudo nvme id-ns /dev/$device
+    done
+    
+}
+
+
 echo "[nvmeof.fio] starting fio test..."
 
 if [ -n "$IOSTAT_INTERVAL" ]; then
@@ -79,6 +99,13 @@ if [ "$rbd_iostat" = true  ]; then
     timeout 20 rbd perf image iostat $RBD_POOL --iterations $iterations &
 fi
 fio --showcmd $fio_file
-sudo fio $fio_file 
+
+set +e 
+sudo fio $fio_file
+if [ $? -ne 0 ]; then
+    status_log
+    exit 1
+fi
+
 
 echo "[nvmeof.fio] fio test successful!"

qa/workunits/nvmeof/scalability_test.sh

@@ -16,10 +16,40 @@ status_checks() {
     expected_count=$1
 
     output=$(ceph nvme-gw show $POOL $GROUP) 
-    nvme_show=$(echo $output | grep -o '"AVAILABLE"' | wc -l)
-    if [ "$nvme_show" -ne "$expected_count" ]; then
-        return 1
+    # nvme_show=$(echo $output | grep -o '"AVAILABLE"' | wc -l)
+    # if [ "$nvme_show" -ne "$expected_count" ]; then
+    #    return 1
+    # fi
+    total_ns=$(echo $output | jq '.["num-namespaces"]')
+    total_gws=$(echo $output | jq '.["num gws"]')
+    expected_avg_ns=$((total_ns / total_gws))
+
+    if [ "$total_gws" -ne "$expected_count" ]; then
+       return 1
     fi
+    expected_ns_count=$(( $NVMEOF_NAMESPACES_COUNT * $NVMEOF_SUBSYSTEMS_COUNT )) 
+    if [ "$total_ns" -ne "$expected_ns_count" ]; then
+       return 1
+    fi
+
+    gateways=$(echo "$output" | jq -c '.["Created Gateways:"][]')
+
+    echo "$gateways" | while read -r gw; do
+        gw_id=$(echo "$gw" | jq -r '.["gw-id"]')
+        availability=$(echo "$gw" | jq -r '.["Availability"]')
+        num_namespaces=$(echo "$gw" | jq '.["num-namespaces"]')
+
+        if [[ "$availability" != "AVAILABLE" ]]; then
+            echo "Gateway $gw_id is not AVAILABLE."
+            exit 1
+        fi
+
+        diff=$((num_namespaces - expected_avg_ns))
+        if [[ $diff -lt -1 || $diff -gt 1 ]]; then
+            echo "Gateway $gw_id has num-namespaces ($num_namespaces), expected around $expected_ns. Indicates a problem in ns load-balancing."
+            exit 1
+        fi
+    done
 
     orch_ls=$(ceph orch ls)
     if ! echo "$orch_ls" | grep -q "$expected_count/$expected_count"; then