qa: add "failover / failback loop" test for rbd-mirror

For snapshot-based mirroring, check that demote (or other mirror
snapshots) don't pile up.  Nothing in particular to assert on for
journal-based mirroring but the test is still useful.

Signed-off-by: Ilya Dryomov <[email protected]>

Author: idryomov

qa/workunits/rbd/rbd_mirror_helpers.sh

@@ -1169,6 +1169,16 @@ wait_for_snap_removed_from_trash()
     return 1
 }
 
+count_mirror_snaps()
+{
+    local cluster=$1
+    local pool=$2
+    local image=$3
+
+    rbd --cluster ${cluster} snap ls ${pool}/${image} --all |
+        grep -c -F " mirror ("
+}
+
 write_image()
 {
     local cluster=$1

qa/workunits/rbd/rbd_mirror_journal.sh

@@ -214,7 +214,29 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
-# force promote
+testlog "TEST: failover / failback loop"
+for i in `seq 1 20`; do
+  demote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
+  demote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+done
+
+testlog "TEST: force promote"
 force_promote_image=test_force_promote
 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100

qa/workunits/rbd/rbd_mirror_snapshot.sh

@@ -220,7 +220,32 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 compare_images ${POOL} ${image}
 
-# force promote
+testlog "TEST: failover / failback loop"
+for i in `seq 1 20`; do
+  demote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying'
+  demote_image ${CLUSTER1} ${POOL} ${image}
+  wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
+  promote_image ${CLUSTER2} ${POOL} ${image}
+  wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+  wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+  wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
+  wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
+done
+# check that demote (or other mirror snapshots) don't pile up
+test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3
+test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3
+
+testlog "TEST: force promote"
 force_promote_image=test_force_promote
 create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image}
 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100