Stop using DEBUG LOADAOF on replica in ASM tests (#2719)

DEBUG LOADAOF sometimes works but it results in -LOADING responses to
the primary so there are lots of race conditions. It isn't something we
should be doing anyways. To test, I just disconnect the replica before
loading the AOF, then reconnect it.

Fixes #2712

Signed-off-by: Jacob Murphy <[email protected]>
(cherry picked from commit dbcf022)
Signed-off-by: cherukum-amazon <[email protected]>

Author: murphyjacob4

tests/unit/cluster/cluster-migrateslots.tcl

@@ -2183,11 +2183,21 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster aofrw} overrides {
         # Make sure the replicas have it
         wait_for_countkeysinslot 3 16383 500
 
-        # First, reload the AOF, without restart. It should still keep the
-        # migration state.
+        # Disconnect the replica and load the AOF, it should still be in there
+        # but should fail and cleanup on load.
+        assert_match "OK" [R 3 CLUSTER REPLICATE NO ONE]
         assert_match "OK" [R 3 DEBUG LOADAOF]
-        wait_for_migration_field 3 $jobname state occurring-on-primary
+        wait_for_migration_field 3 $jobname state failed
         assert_match "0" [R 3 DBSIZE]
+        assert_match "0" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
+
+        # Reconnect the replica and wait for AOF rewrite
+        assert_match "OK" [R 3 CLUSTER REPLICATE $node0_id]
+        wait_for_sync [srv -3 client] 50 1000
+        assert_match "occurring-on-primary" [dict get [get_migration_by_name 3 $jobname] state]
+        assert_match "0" [R 3 DBSIZE]
+        assert_match "500" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
+        waitForBgrewriteaof [srv -3 client]
 
         # Restart the replica and wait for resync
         do_node_restart 3
@@ -2197,22 +2207,30 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster aofrw} overrides {
         # persist the replication ID, this is because of a full resync.
         wait_for_migration_field 3 $jobname state occurring-on-primary
         assert_match "0" [R 3 DBSIZE]
+        assert_match "500" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
 
-        # Fail the migration and reload the AOF. We should have no migrations on
-        # the replica
+        # Fail the migration and wait for AOF rewrite
         assert_match "OK" [R 2 CLUSTER CANCELSLOTMIGRATIONS]
         wait_for_migration_field 3 $jobname state failed
         assert_match "0" [R 3 DBSIZE]
         assert_match "0" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
-
-        # Load the AOF, it should still be in there. First ensure the AOF is
-        # done being rewritten with the latest full sync.
         waitForBgrewriteaof [srv -3 client]
+
+        # Disconnect the replica and load the AOF, it should still be in there
+        # but in failed state.
+        assert_match "OK" [R 3 CLUSTER REPLICATE NO ONE]
         assert_match "OK" [R 3 DEBUG LOADAOF]
         assert_match "failed" [dict get [get_migration_by_name 3 $jobname] state]
         assert_match "0" [R 3 DBSIZE]
         assert_match "0" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
 
+        # Reconnect the replica and wait for AOF rewrite
+        assert_match "OK" [R 3 CLUSTER REPLICATE $node0_id]
+        wait_for_sync [srv -3 client] 50 1000
+        assert_match "0" [R 3 DBSIZE]
+        assert_match "0" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
+        waitForBgrewriteaof [srv -3 client]
+
         # Cleanup
         set_debug_prevent_pause 0
         assert_match "OK" [R 2 FLUSHALL SYNC]
@@ -2288,9 +2306,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster aofrw} overrides {
         assert_match "0" [R 0 CLUSTER COUNTKEYSINSLOT 16383]
         R 2 DEBUG LOADAOF
         assert_match "500" [R 2 CLUSTER COUNTKEYSINSLOT 16383]
-        R 3 DEBUG LOADAOF
         assert_match "0" [R 3 CLUSTER COUNTKEYSINSLOT 16383]
-        R 5 DEBUG LOADAOF
         assert_match "500" [R 5 CLUSTER COUNTKEYSINSLOT 16383]
         assert_equal $slots_start [R 0 CLUSTER SLOTS]
         assert_match "OK" [R 2 FLUSHDB SYNC]