Merge pull request #1247 from basho/at-cluster-check-on-repl-fullsync

Check stats on all nodes for REPL fullsync success

Author: Andy Till

tests/repl_util.erl

@@ -235,7 +235,9 @@ start_and_wait_until_fullsync_complete(Node, Cluster) ->
 start_and_wait_until_fullsync_complete(Node, Cluster, NotifyPid) ->
     start_and_wait_until_fullsync_complete(Node, Cluster, NotifyPid, 20).
 
-start_and_wait_until_fullsync_complete(Node, Cluster, NotifyPid, Retries) ->
+start_and_wait_until_fullsync_complete(Node, Cluster, NotifyPid, Retries) when is_atom(Node) ->
+    start_and_wait_until_fullsync_complete([Node], Cluster, NotifyPid, Retries);
+start_and_wait_until_fullsync_complete([Node|_] = Nodes, Cluster, NotifyPid, Retries) ->
     Status0 = rpc:call(Node, riak_repl_console, status, [quiet]),
     Count0 = proplists:get_value(server_fullsyncs, Status0),
     Count = fullsync_count(Count0, Status0, Cluster),
@@ -252,7 +254,7 @@ start_and_wait_until_fullsync_complete(Node, Cluster, NotifyPid, Retries) ->
     %% Send message to process and notify fullsync has began.
     fullsync_notify(NotifyPid),
 
-    case rt:wait_until(make_fullsync_wait_fun(Node, Count), 100, 1000) of
+    case rt:wait_until(make_fullsync_wait_fun(Nodes, Count), 100, 1000) of
         ok ->
             ok;
         _  when Retries > 0 ->
@@ -281,19 +283,24 @@ fullsync_notify(NotifyPid) when is_pid(NotifyPid) ->
 fullsync_notify(_) ->
     ok.
 
-make_fullsync_wait_fun(Node, Count) ->
+make_fullsync_wait_fun(Cluster, Count) when is_list(Cluster) ->
     fun() ->
-            Status = rpc:call(Node, riak_repl_console, status, [quiet]),
-            case Status of
-                {badrpc, _} ->
-                    false;
+        make_fullsync_wait_fun2(Cluster, Count)
+    end.
+
+make_fullsync_wait_fun2([], _) ->
+    false;
+make_fullsync_wait_fun2([Node|Tail], Count) when is_atom(Node) ->
+    Status = rpc:call(Node, riak_repl_console, status, [quiet]),
+    case Status of
+        {badrpc, _} ->
+            false;
+        _ ->
+            case proplists:get_value(server_fullsyncs, Status) of
+                C when C >= Count ->
+                    true;
                 _ ->
-                    case proplists:get_value(server_fullsyncs, Status) of
-                        C when C >= Count ->
-                            true;
-                        _ ->
-                            false
-                    end
+                    make_fullsync_wait_fun2(Tail, Count)
             end
     end.
 

tests/ts_cluster_replication.erl

@@ -207,7 +207,7 @@ full_sync_replication_test([AFirst|_]=ANodes, [BFirst|_]=BNodes, LeaderA, PortB,
     lager:info("Starting and waiting for fullsync"),
     connect_clusters(ANodes, BNodes, LeaderA, PortB),
     start_mdc(ANodes, LeaderA, "B", true),
-    repl_util:start_and_wait_until_fullsync_complete(hd(ANodes), "B"),
+    repl_util:start_and_wait_until_fullsync_complete(ANodes, "B"),
 
     lager:info("Verifying first 100 keys present on 2nd cluster (non-w1c)"),
     ?assertEqual(100, kv_num_objects_present(BNode, 1, 100, KVBucket)),
@@ -221,7 +221,7 @@ full_sync_replication_test([AFirst|_]=ANodes, [BFirst|_]=BNodes, LeaderA, PortB,
     timer:sleep(500),
     lager:info("Verifying record is no longer on Cluster A"),
     ?assertEqual(0, ts_num_records_present(AFirst, 23, 23, Table)),
-    repl_util:start_and_wait_until_fullsync_complete(hd(ANodes), "B"),
+    repl_util:start_and_wait_until_fullsync_complete(ANodes, "B"),
     lager:info("Verifying record is no longer on Cluster B"),
     ?assertEqual(0, ts_num_records_present(BFirst, 23, 23, Table)),