Make KV rebalance details available in stage_info output.

Helps with,
MB-30803: Collect rebalance related stats
CBSE-1097: Visualize bucket movements during rebalance

Part of EPIC,
MB-30894: Rebalance visibility and reporting

Change-Id: Ic2709fdb9196fc6d4bc6b70e592b40da1e3132f7
Reviewed-on: http://review.couchbase.org/101822
Reviewed-by: Aliaksey Artamonau <[email protected]>
Well-Formed: Build Bot <[email protected]>
Tested-by: Aliaksey Artamonau <[email protected]>

Author: anuthan

src/ns_rebalance_observer.erl

@@ -84,7 +84,7 @@ get_aggregated_progress(Timeout) ->
     generic_get_call(get_aggregated_progress, Timeout).
 
 get_stage_info() ->
-    generic_get_call(get_stage_info).
+    generic_get_call({get_stage_info, []}).
 
 update_progress(Stage, StageProgress) ->
     gen_server:cast(?SERVER, {update_progress, Stage, StageProgress}).
@@ -167,9 +167,11 @@ handle_call(get_detailed_progress, _From, State) ->
 handle_call(get_aggregated_progress, _From,
             #state{stage_info = StageInfo} = State) ->
     {reply, dict:to_list(rebalance_stage_info:get_progress(StageInfo)), State};
-handle_call(get_stage_info, _From,
+handle_call({get_stage_info, Options}, _From,
             #state{stage_info = StageInfo} = State) ->
-    {reply, rebalance_stage_info:get_stage_info(StageInfo), State};
+    StageDetails = get_all_stage_rebalance_details(State, Options),
+    {reply, rebalance_stage_info:get_stage_info(StageInfo, StageDetails),
+     State};
 handle_call(Req, From, State) ->
     ?log_error("Got unknown request: ~p from ~p", [Req, From]),
     {reply, unknown_request, State}.
@@ -660,3 +662,78 @@ update_vbucket_level_info_inner(
             BucketLevelInfo#bucket_level_info{
               vbucket_level_info = NewVBLevelInfo}
     end.
+
+construct_bucket_level_info_json(
+  #bucket_level_info{bucket_name = BucketName,
+                     vbucket_level_info = VBLevelInfo}, Options) ->
+    case dict:is_empty(VBLevelInfo#vbucket_level_info.vbucket_info) of
+        true ->
+            undefined;
+        false ->
+            {BucketName, {[{vbucketLevelInfo,
+                            construct_vbucket_level_info_json(VBLevelInfo,
+                                                              Options)}]}}
+    end.
+
+construct_stat_info_json(#stat_info{start_time = false}) ->
+    {[{startTime, rebalance_stage_info:binarify_timestamp(false)}]};
+construct_stat_info_json(#stat_info{start_time = ST,
+                                    end_time = ET}) ->
+    {[{startTime, rebalance_stage_info:binarify_timestamp(ST)},
+      {endTime, rebalance_stage_info:binarify_timestamp(ET)},
+      {timeTaken, rebalance_stage_info:diff_timestamp(ET, ST)}]}.
+
+average(_, 0) ->
+    0;
+average(Total, Count) ->
+    Total/Count.
+
+construct_total_stat_info_json(#total_stat_info{total_time = TT,
+                                                completed_count = CC}) ->
+    {[{averageTime, average(TT, CC)}]}.
+
+construct_replica_building_stats_json(#replica_building_stats{node = Node,
+                                                              docs_total = DT,
+                                                              docs_left = DL}) ->
+    {Node, {[{node, Node},
+             {docsTotal, DT},
+             {docsLeft, DL}]}}.
+
+construct_vbucket_info_json(Id, #vbucket_info{before_chain = BC,
+                                              after_chain = AC,
+                                              stats = RBS,
+                                              move = Move}) ->
+    {[{id, Id},
+      {beforeChain, BC},
+      {afterChain, AC},
+      {stats, {[construct_replica_building_stats_json(X) || X <- RBS]}},
+      {move, construct_stat_info_json(Move)}]}.
+
+construct_vbucket_level_info_json(
+  #vbucket_level_info{move = Move,
+                      vbucket_info = AllVBInfo}, Options) ->
+    VBI = case proplists:get_bool(add_vbucket_info, Options) of
+              true ->
+                  [{vbucketInfo,
+                    dict:fold(fun (VB, Info, Acc) ->
+                                      [construct_vbucket_info_json(VB, Info) | Acc]
+                              end, [], AllVBInfo)}];
+              _ ->
+                  []
+          end,
+    {[{move, construct_total_stat_info_json(Move)}]
+     ++ VBI}.
+
+get_all_stage_rebalance_details(#state{bucket_info = BucketLevelInfo},
+                                Options) ->
+    RV = dict:fold(
+           fun (_Key, BLI, Acc) ->
+                   case construct_bucket_level_info_json(BLI, Options) of
+                       undefined -> Acc;
+                       Val -> [Val | Acc]
+                   end
+           end, [], BucketLevelInfo),
+    case RV of
+        [] -> [];
+        _ -> [{kv, {RV}}]
+    end.

src/rebalance_stage_info.erl

@@ -16,7 +16,7 @@
 -module(rebalance_stage_info).
 
 -export([init/1,
-         get_stage_info/1,
+         get_stage_info/2,
          get_progress/1,
          update_progress/3,
          update_stage_info/3,
@@ -104,8 +104,8 @@ aggregate(PerStage) ->
                      Sum / Count
              end, TmpAggr).
 
-get_stage_info(StageInfo) ->
-    {get_per_stage_info(StageInfo)}.
+get_stage_info(StageInfo, AllStageDetails) ->
+    {get_per_stage_info(StageInfo, AllStageDetails)}.
 
 diff_timestamp(false, false) ->
     false;
@@ -121,17 +121,19 @@ binarify_timestamp(Time) ->
 
 get_per_stage_info(#stage_info{
                       per_stage_progress = PerStageProgress,
-                      per_stage_info = PerStageInfo}) ->
+                      per_stage_info = PerStageInfo}, AllStageDetails) ->
     AllStageProgress = get_per_stage_progress(PerStageProgress),
     lists:map(
       fun ({Stage, StageInfo}) ->
               construct_per_stage_json(AllStageProgress,
+                                       AllStageDetails,
                                        Stage,
                                        StageInfo)
       end, PerStageInfo).
 
-construct_per_stage_json(AllStageProgress, Stage, StageInfo) ->
+construct_per_stage_json(AllStageProgress, AllStageDetails, Stage, StageInfo) ->
     {ok, PerNodeProgress} = dict:find(Stage, AllStageProgress),
+    StageDetails = construct_per_stage_details_json(Stage, AllStageDetails),
     TotalStageProgress = case lists:foldl(fun ({_, P}, {Total, Count}) ->
                                                   {Total + P, Count + 1}
                                           end, {0, 0}, PerNodeProgress) of
@@ -143,8 +145,10 @@ construct_per_stage_json(AllStageProgress, Stage, StageInfo) ->
     ProgressInfoJson = construct_per_stage_progress_json(TotalStageProgress,
                                                          PerNodeProgress,
                                                          StageInfo),
-    StageInfoJson = construct_per_stage_info_json(StageInfo, AllStageProgress),
-    {get_stage_name(Stage), {ProgressInfoJson ++ StageInfoJson}}.
+    StageInfoJson = construct_per_stage_info_json(StageInfo, AllStageProgress,
+                                                  AllStageDetails),
+    {get_stage_name(Stage), {ProgressInfoJson ++ StageInfoJson ++
+                             StageDetails}}.
 
 get_stage_name(Stage) ->
     Stages = [{kv, data},
@@ -159,6 +163,14 @@ get_stage_name(Stage) ->
         false -> Stage
     end.
 
+construct_per_stage_details_json(Stage, AllStageDetails) ->
+    case lists:keyfind(Stage, 1, AllStageDetails) of
+        {Stage, StageDetails} ->
+            [{details, StageDetails}];
+        false ->
+            []
+    end.
+
 construct_per_stage_progress_json(
   TSP, _, #stage_details{complete_time = false}) when TSP == 0 ->
     [];
@@ -176,11 +188,12 @@ construct_per_stage_info_json(#stage_details{
                                  complete_time = EndTime,
                                  sub_stages = SubStages,
                                  notable_events = NotableEvents},
-                              AllStageProgress) ->
+                              AllStageProgress, AllStageDetails) ->
     SubStagesInfo = case SubStages of
                         [] -> [];
                         _ -> [{subStages,
                                {[construct_per_stage_json(AllStageProgress,
+                                                          AllStageDetails,
                                                           SubStage,
                                                           SubStageInfo) ||
                                  {SubStage, SubStageInfo} <- SubStages]}}]