Merge pull request ceph#58781 from idryomov/wip-67097

qa/suites/krbd: stress test for recovering from watch errors for -o exclusive

Reviewed-by: Ramana Raja <[email protected]>

Author: idryomov

qa/suites/krbd/mirror/tasks/compare-mirror-image-alternate-primary.yaml

@@ -1,8 +1,7 @@
 overrides:
   install:
-    ceph:
-      extra_system_packages:
-        - pv
+    extra_system_packages:
+      - pv
 tasks:
 - workunit:
     clients:

qa/suites/krbd/mirror/tasks/compare-mirror-images.yaml

@@ -1,8 +1,7 @@
 overrides:
   install:
-    ceph:
-      extra_system_packages:
-        - pv
+    extra_system_packages:
+      - pv
 tasks:
 - workunit:
     clients:

qa/suites/krbd/singleton/tasks/krbd_watch_errors_exclusive.yaml

@@ -0,0 +1,19 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd pool default size: 1
+      osd:
+        osd shutdown pgref assert: true
+roles:
+- [mon.a, mgr.x, osd.0, client.0]
+
+tasks:
+- install:
+    extra_system_packages:
+      - fio
+- ceph:
+- workunit:
+    clients:
+      all:
+        - rbd/krbd_watch_errors_exclusive.sh

qa/suites/krbd/thrash/workloads/krbd_diff_continuous.yaml

@@ -1,8 +1,7 @@
 overrides:
   install:
-    ceph:
-      extra_system_packages:
-        - pv
+    extra_system_packages:
+      - pv
 tasks:
 - workunit:
     clients:

qa/workunits/rbd/krbd_watch_errors_exclusive.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ex
+set -o pipefail
+
+readonly IMAGE_NAME="watch-errors-exclusive-test"
+
+rbd create -s 1G --image-feature exclusive-lock,object-map "${IMAGE_NAME}"
+
+# induce a watch error every 30 seconds
+dev="$(sudo rbd device map -o exclusive,osdkeepalive=60 "${IMAGE_NAME}")"
+dev_id="${dev#/dev/rbd}"
+
+sudo dmesg -C
+
+# test that a workload doesn't encounter EIO errors
+fio --name test --filename="${dev}" --ioengine=libaio --direct=1 \
+    --rw=randwrite --norandommap --randrepeat=0 --bs=512 --iodepth=128 \
+    --time_based --runtime=1h --eta=never
+
+num_errors="$(dmesg | grep -c "rbd${dev_id}: encountered watch error")"
+echo "Recorded ${num_errors} watch errors"
+
+sudo rbd device unmap "${dev}"
+
+if ((num_errors < 60)); then
+    echo "Too few watch errors"
+    exit 1
+fi
+
+echo OK