acceptance: deflake test_demo_node_cmds

Previously the acceptance test `test_demo_node_cmds`, which attempts to
shutdown and decommission some nodes using the `cockroach demo` CLI,
would sometimes be flaky due to delays in propagating information via
gossip. This change fixes these flakes by utilizing the virtual table
`crdb_internal.kv_node_liveness` rather than the gossip-based
`gossip_liveness` virtual table.

Fixes: cockroachdb#76391

Release note: None

Author: AlexTalks

pkg/cli/interactive_tests/test_demo_node_cmds.tcl

@@ -67,31 +67,26 @@ send "\\demo restart 3\r"
 eexpect "node 3 has been restarted"
 eexpect "defaultdb>"
 
-# NB: this is flaky, sometimes n3 is still marked as draining due to
-# gossip propagation delays. See:
-# https://github.com/cockroachdb/cockroach/issues/76391
-# send "select node_id, draining, decommissioning, membership from crdb_internal.gossip_liveness ORDER BY node_id;\r"
-# eexpect "1 |  false   |      false      | active"
-# eexpect "2 |  false   |      false      | active"
-# eexpect "3 |  false   |      false      | active"
-# eexpect "4 |  false   |      false      | active"
-# eexpect "5 |  false   |      false      | active"
-# eexpect "defaultdb>"
-
-# Try commissioning commands
+send "select node_id, draining, decommissioning, membership from crdb_internal.gossip_liveness ORDER BY node_id;\r"
+eexpect "1 |    f     |        f        | active"
+eexpect "2 |    f     |        f        | active"
+eexpect "3 |    f     |        f        | active"
+eexpect "4 |    f     |        f        | active"
+eexpect "5 |    f     |        f        | active"
+eexpect "defaultdb>"
+
+# Try decommissioning commands
 send "\\demo decommission 4\r"
 eexpect "node 4 has been decommissioned"
 eexpect "defaultdb>"
 
-# NB: skipping this out of an abundance of caution, see:
-# https://github.com/cockroachdb/cockroach/issues/76391
-# send "select node_id, draining, decommissioning, membership from crdb_internal.gossip_liveness ORDER BY node_id;\r"
-# eexpect "1 |  false   |      false      | active"
-# eexpect "2 |  false   |      false      | active"
-# eexpect "3 |  false   |      false      | active"
-# eexpect "4 |  false   |      true       | decommissioned"
-# eexpect "5 |  false   |      false      | active"
-# eexpect "defaultdb>"
+send "select node_id, draining, membership from crdb_internal.kv_node_liveness ORDER BY node_id;\r"
+eexpect "1 |    f     | active"
+eexpect "2 |    f     | active"
+eexpect "3 |    f     | active"
+eexpect "4 |    f     | decommissioned"
+eexpect "5 |    f     | active"
+eexpect "defaultdb>"
 
 send "\\demo recommission 4\r"
 eexpect "can only recommission a decommissioning node"
@@ -128,17 +123,16 @@ eexpect "node 6 has been shutdown"
 set timeout 30
 eexpect "defaultdb>"
 
-# By now the node should have stabilized in gossip which allows us to query the more detailed information there.
-# NB: skip this to avoid flakes, see:
-# https://github.com/cockroachdb/cockroach/issues/76391
-# send "select node_id, draining, decommissioning, membership from crdb_internal.gossip_liveness ORDER BY node_id;\r"
-# eexpect "1 |  false   |      false      | active"
-# eexpect "2 |  false   |      false      | active"
-# eexpect "3 |  false   |      false      | active"
-# eexpect "4 |  false   |      true       | decommissioned"
-# eexpect "5 |  false   |      false      | active"
-# eexpect "6 |   true   |      false      | active"
-# eexpect "defaultdb>"
+# NB: use kv_node_liveness to avoid flakes due to gossip delays.
+# See https://github.com/cockroachdb/cockroach/issues/76391
+send "select node_id, draining, membership from crdb_internal.kv_node_liveness ORDER BY node_id;\r"
+eexpect "1 |    f     | active"
+eexpect "2 |    f     | active"
+eexpect "3 |    f     | active"
+eexpect "4 |    f     | decommissioned"
+eexpect "5 |    f     | active"
+eexpect "6 |    t     | active"
+eexpect "defaultdb>"
 
 send "\\q\r"
 eexpect eof