MB-50712: Correct max possible replica count for bucket

If we are rack aware, i.e. have more than one rack configured, the max
replicas possible are the minimum number of nodes each node in a
server-group can replicate to.

A node 'n1' in a server-group 'sg1' can replicate to (NumTotalNodes -
NumServerGroupNodes) Nodes. Where NumServerGroupNodes are the number of nodes
in server-group sg1.

Change-Id: Ic6aeee71a69bd3ed4105e6424d21e287d06baae5
Reviewed-on: https://review.couchbase.org/c/ns_server/+/175371
Well-Formed: Restriction Checker
Well-Formed: Build Bot <[email protected]>
Tested-by: Hareen Kancharla <[email protected]>
Reviewed-by: Steve Watanabe <[email protected]>

Author: hareen-kancharla

src/menelaus_web_buckets.erl

@@ -476,7 +476,8 @@ init_bucket_validation_context(IsNew, BucketName, AllBuckets,
     MaxReplicas =
         case ns_cluster_membership:rack_aware(KvServerGroups) of
             true ->
-                min(length(KvServerGroups), NumKvNodes) - 1;
+                ns_cluster_membership:get_max_replicas(NumKvNodes,
+                                                       KvServerGroups);
             false ->
                 NumKvNodes - 1
         end,

src/ns_cluster_membership.erl

@@ -80,7 +80,8 @@
          should_run_service/2,
          should_run_service/3,
          user_friendly_service_name/1,
-         json_service_name/1]).
+         json_service_name/1,
+         get_max_replicas/2]).
 
 fetch_snapshot(Txn) ->
     Snapshot =
@@ -198,6 +199,12 @@ get_nodes_server_groups(Nodes, Groups) ->
               end
       end, Groups).
 
+get_max_replicas(NumKvNodes, KvServerGroups) ->
+    lists:foldl(fun (Group, MaxReplicas) ->
+                    NumGroupNodes = length(proplists:get_value(nodes, Group)),
+                    min(MaxReplicas, NumKvNodes - NumGroupNodes)
+                end, NumKvNodes - 1, KvServerGroups).
+
 system_joinable() ->
     nodes_wanted() =:= [node()].