Merge #151573

151573: interactive_tests: wait for liveness upreplication r=rafiss a=pav-kv

This commit fixes `test_demo_node_cmds` so that it waits for liveness range to have 5 voters, before shutting down and decommissioning a node. This avoids loss of availability and test stalls in cases when the liveness range has < 3 voters, and the killed node is one of them.

Fixes #147867
Fixes #149151 (via backport)

Co-authored-by: Pavel Kalinnikov <[email protected]>

Author: craig[bot]

pkg/cli/interactive_tests/test_demo_node_cmds.tcl

@@ -65,7 +65,7 @@ send "\\demo restart 3\r"
 eexpect "node 3 has been restarted"
 eexpect "defaultdb>"
 
-set timeout 1
+set timeout 2
 set stmt "select node_id, draining, membership from crdb_internal.kv_node_liveness ORDER BY node_id;\r"
 send $stmt
 expect {
@@ -90,6 +90,26 @@ eexpect "4 |    f     | active"
 eexpect "5 |    f     | active"
 eexpect "defaultdb>"
 
+# Wait for the liveness range to have the default 5 voters. If its replication
+# factor is too low, shutting down the node below can cause it to lose quorum
+# and stall the decommissioning command (example: #147867).
+set timeout 2
+set stmt "select range_id, array_length(voting_replicas,1) from crdb_internal.ranges where range_id=2;\r"
+send $stmt
+expect {
+    "2 |            5" {
+        puts "\rliveness range has 5 voters"
+    }
+    timeout {
+        puts "\rliveness range does not yet have 5 voters"
+        sleep 2
+        send $stmt
+        exp_continue
+    }
+}
+# Reset timeout back to 45 to match common.tcl.
+set timeout 45
+
 # Try decommissioning commands
 send "\\demo decommission 4\r"
 eexpect "node is draining"