Skip to content

Commit a6fa7c1

Browse files
anuthananuthan
committed
MB-46881: REST API for get/set scope limits
All API's for get/modifying collection manifest support setting limits. Change-Id: I1bb3575b4afb3bc377323ddc399c744dbeccb332 Reviewed-on: http://review.couchbase.org/c/ns_server/+/159832 Tested-by: Abhijeeth Nuthan <[email protected]> Well-Formed: Build Bot <[email protected]> Reviewed-by: Artem Stemkovski <[email protected]>
1 parent 9b7bd7d commit a6fa7c1

File tree

2 files changed

+146
-41
lines changed

2 files changed

+146
-41
lines changed

‎src/collections.erl‎

Lines changed: 102 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@
2929
uid/2,
3030
manifest_json/2,
3131
manifest_json/3,
32-
create_scope/2,
32+
create_scope/3,
33+
update_limits/3,
3334
create_collection/4,
3435
drop_scope/2,
3536
drop_collection/3,
@@ -235,11 +236,19 @@ jsonify_manifest(Manifest, WithDefaults) ->
235236
ScopesJson =
236237
lists:map(
237238
fun ({ScopeName, Scope}) ->
239+
LimitsJson = case get_limits(Scope) of
240+
[] ->
241+
[];
242+
Limits ->
243+
[{limits,
244+
{[{S, {L}} || {S, L} <- Limits]}}]
245+
end,
238246
{[{name, list_to_binary(ScopeName)},
239247
{uid, uid(Scope)},
240248
{collections,
241249
[collection_to_memcached(CollName, Coll, WithDefaults) ||
242-
{CollName, Coll} <- get_collections(Scope)]}]}
250+
{CollName, Coll} <- get_collections(Scope)]}] ++
251+
LimitsJson}
243252
end, get_scopes(Manifest)),
244253
{[{uid, uid(Manifest)}, {scopes, ScopesJson}]}.
245254

@@ -248,8 +257,11 @@ get_max_supported(num_scopes) ->
248257
get_max_supported(num_collections) ->
249258
ns_config:read_key_fast(max_collections_count, ?MAX_COLLECTIONS_SUPPORTED).
250259

251-
create_scope(Bucket, Name) ->
252-
update(Bucket, {create_scope, Name}).
260+
update_limits(Bucket, Name, Limits) ->
261+
update(Bucket, {update_limits, Name, Limits}).
262+
263+
create_scope(Bucket, Name, Limits) ->
264+
update(Bucket, {create_scope, Name, Limits}).
253265

254266
create_collection(Bucket, Scope, Name, Props) ->
255267
update(Bucket, {create_collection, Scope, Name,
@@ -444,30 +456,33 @@ get_operations(CurrentScopes, RequiredScopes) ->
444456
fun ({delete, ScopeName, _}) ->
445457
{drop_scope, ScopeName};
446458
({add, ScopeName, ScopeProps}) ->
447-
[{create_scope, ScopeName} |
459+
Limits = get_limits(ScopeProps),
460+
[{create_scope, ScopeName, Limits} |
448461
[{create_collection, ScopeName, CollectionName,
449462
remove_defaults(CollectionProps)} ||
450463
{CollectionName, CollectionProps}
451464
<- get_collections(ScopeProps)]];
452465
({modify, ScopeName, ScopeProps, CurrentScopeProps}) ->
453-
get_operations(
454-
fun ({delete, CollectionName, _}) ->
455-
{drop_collection, ScopeName, CollectionName};
456-
({add, CollectionName, CollectionProps}) ->
457-
{create_collection, ScopeName, CollectionName,
458-
remove_defaults(CollectionProps)};
459-
({modify, CollectionName, CollectionProps,
460-
CurrentCollectionProps}) ->
461-
case lists:sort(remove_defaults(CollectionProps)) =:=
462-
lists:sort(lists:keydelete(
463-
uid, 1, CurrentCollectionProps)) of
464-
false ->
465-
{modify_collection, ScopeName, CollectionName};
466-
true ->
467-
[]
468-
end
469-
end, get_collections(CurrentScopeProps),
470-
get_collections(ScopeProps))
466+
Limits = [{update_limits, ScopeName, get_limits(ScopeProps)}],
467+
(Limits ++
468+
get_operations(
469+
fun ({delete, CollectionName, _}) ->
470+
{drop_collection, ScopeName, CollectionName};
471+
({add, CollectionName, CollectionProps}) ->
472+
{create_collection, ScopeName, CollectionName,
473+
remove_defaults(CollectionProps)};
474+
({modify, CollectionName, CollectionProps,
475+
CurrentCollectionProps}) ->
476+
case lists:sort(remove_defaults(CollectionProps)) =:=
477+
lists:sort(lists:keydelete(
478+
uid, 1, CurrentCollectionProps)) of
479+
false ->
480+
{modify_collection, ScopeName, CollectionName};
481+
true ->
482+
[]
483+
end
484+
end, get_collections(CurrentScopeProps),
485+
get_collections(ScopeProps)))
471486
end, CurrentScopes, RequiredScopes).
472487

473488
diff_scopes(CurrentScopes, RequiredScopes) ->
@@ -528,6 +543,14 @@ compile_operation({set_manifest, Roles, RequiredScopes, CheckUid},
528543
compile_operation(Oper, _Bucket, _Manifest) ->
529544
[Oper].
530545

546+
verify_oper({update_limits, Name, _Limits}, Manifest) ->
547+
Scopes = get_scopes(Manifest),
548+
case find_scope(Name, Scopes) of
549+
undefined ->
550+
{scope_not_found, Name};
551+
_ ->
552+
ok
553+
end;
531554
verify_oper({check_uid, CheckUid}, Manifest) ->
532555
ManifestUid = proplists:get_value(uid, Manifest),
533556
case CheckUid =:= ManifestUid orelse CheckUid =:= undefined of
@@ -536,7 +559,7 @@ verify_oper({check_uid, CheckUid}, Manifest) ->
536559
false ->
537560
uid_mismatch
538561
end;
539-
verify_oper({create_scope, Name}, Manifest) ->
562+
verify_oper({create_scope, Name, _Limits}, Manifest) ->
540563
Scopes = get_scopes(Manifest),
541564
case find_scope(Name, Scopes) of
542565
undefined ->
@@ -568,12 +591,14 @@ verify_oper({modify_collection, ScopeName, Name}, _Manifest) ->
568591
verify_oper(bump_epoch, _Manifest) ->
569592
ok.
570593

594+
handle_oper({update_limits, Name, Limits}, Manifest) ->
595+
do_update_limits(Manifest, Name, Limits);
571596
handle_oper({check_uid, _CheckUid}, Manifest) ->
572597
Manifest;
573-
handle_oper({create_scope, Name}, Manifest) ->
598+
handle_oper({create_scope, Name, Limits}, Manifest) ->
574599
functools:chain(
575600
Manifest,
576-
[add_scope(_, Name),
601+
[add_scope(_, Name, Limits),
577602
bump_id(_, next_scope_uid),
578603
update_counter(_, num_scopes, 1)]);
579604
handle_oper({drop_scope, Name}, Manifest) ->
@@ -639,9 +664,41 @@ get_scopes(Manifest) ->
639664
find_scope(Name, Scopes) ->
640665
proplists:get_value(Name, Scopes).
641666

642-
add_scope(Manifest, Name) ->
667+
sort_limits(Limits) ->
668+
lists:usort([{S, lists:usort(L)} || {S, L} <- Limits]).
669+
670+
do_update_limits(Manifest, ScopeName, Limits) ->
671+
SortedLimits = sort_limits(Limits),
672+
on_scopes(
673+
fun (Scopes) ->
674+
Scope = find_scope(ScopeName, Scopes),
675+
CurLimits = get_limits(Scope),
676+
case CurLimits =:= SortedLimits of
677+
true ->
678+
Scopes;
679+
false ->
680+
NScope = case SortedLimits of
681+
[] ->
682+
lists:keydelete(limits, 1, Scope);
683+
_ ->
684+
lists:keystore(limits, 1, Scope,
685+
{limits, SortedLimits})
686+
end,
687+
lists:keystore(ScopeName, 1, Scopes,
688+
{ScopeName, NScope})
689+
end
690+
end, Manifest).
691+
692+
add_scope(Manifest, Name, Limits) ->
643693
Uid = proplists:get_value(next_scope_uid, Manifest),
644-
on_scopes([{Name, [{uid, Uid}, {collections, []}]} | _], Manifest).
694+
Extra = case Limits of
695+
no_limits ->
696+
[];
697+
_ ->
698+
[{limits, sort_limits(Limits)}]
699+
end,
700+
on_scopes([{Name, [{uid, Uid}, {collections, []}] ++ Extra} | _],
701+
Manifest).
645702

646703
delete_scope(Manifest, Name) ->
647704
on_scopes(lists:keydelete(Name, 1, _), Manifest).
@@ -654,6 +711,9 @@ on_scopes(Fun, Manifest) ->
654711
NewScopes = Fun(Scopes),
655712
update_scopes(NewScopes, Manifest).
656713

714+
get_limits(Scope) ->
715+
proplists:get_value(limits, Scope, []).
716+
657717
get_collections(Scope) ->
658718
proplists:get_value(collections, Scope, []).
659719

@@ -777,7 +837,8 @@ set_manifest(Bucket, Identity, RequiredScopes, RequestedUid) ->
777837
Scopes =
778838
[{proplists:get_value(name, Scope),
779839
[{collections, [extract_name(Props) ||
780-
{Props} <- get_collections(Scope)]}]} ||
840+
{Props} <- get_collections(Scope)]},
841+
{limits, get_limits(Scope)}]} ||
781842
{Scope} <- RequiredScopes],
782843
update(Bucket, {set_manifest, Roles, Scopes, Uid})
783844
end.
@@ -870,7 +931,7 @@ get_operations_test_() ->
870931
[{"Create scopes and collections commands in the correct order",
871932
fun () ->
872933
?assertEqual(
873-
[{create_scope, "s1"},
934+
[{create_scope, "s1", []},
874935
{create_collection, "s1", "c1", []},
875936
{create_collection, "s1", "c2", [{maxTTL, 8}]}],
876937
get_operations(
@@ -881,7 +942,10 @@ get_operations_test_() ->
881942
{"Drop/create collections",
882943
fun () ->
883944
?assertListsEqual(
884-
[{create_collection, "s2", "c3", []},
945+
[{update_limits, "s1", []},
946+
{update_limits, "s2", []},
947+
{update_limits, "_default", []},
948+
{create_collection, "s2", "c3", []},
885949
{create_collection, "s1", "c2", []},
886950
{drop_collection, "s1", "c1"},
887951
{drop_collection, "_default", "_default"}],
@@ -897,7 +961,8 @@ get_operations_test_() ->
897961
{"Drop scope with collection present.",
898962
fun () ->
899963
?assertListsEqual(
900-
[{create_collection, "s1", "c2", []},
964+
[{update_limits, "s1", []},
965+
{create_collection, "s1", "c2", []},
901966
{drop_scope, "s2"}],
902967
get_operations(
903968
[{"s1", [{collections, [{"c1", []}]}]},
@@ -909,15 +974,18 @@ get_operations_test_() ->
909974
{"Modify collection.",
910975
fun () ->
911976
?assertListsEqual(
912-
[{modify_collection, "s3", "ic2"},
977+
[{update_limits, "s3", []},
978+
{update_limits, "s1", [{"l1", 1}, {"l2", 2}]},
979+
{modify_collection, "s3", "ic2"},
913980
{create_collection, "s1", "c2", []},
914981
{drop_scope, "s2"}],
915982
get_operations(
916983
[{"s1", [{collections, [{"c1", []}]}]},
917984
{"s2", [{collections, [{"c1", []}, {"c2", []}]}]},
918985
{"s3", [{collections, [{"ic1", []},
919986
{"ic2", [{maxTTL, 10}]}]}]}],
920-
[{"s1", [{collections, [{"c1", []}, {"c2", []}]}]},
987+
[{"s1", [{limits, [{"l1", 1}, {"l2", 2}]},
988+
{collections, [{"c1", []}, {"c2", []}]}]},
921989
{"s3", [{collections, [{"ic1", [{maxTTL, 0}]},
922990
{"ic2", [{maxTTL, 0}]}]}]}]))
923991
end}]}.

‎src/menelaus_web_collections.erl‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,19 @@ handle_post_scope(Bucket, Req) ->
3737
validator:handle(
3838
fun (Values) ->
3939
Name = proplists:get_value(name, Values),
40-
RV = collections:create_scope(Bucket, Name),
41-
maybe_audit(RV, Req, ns_audit:create_scope(_, Bucket, Name, _)),
42-
handle_rv(RV, Req)
43-
end, Req, form, scope_validators(default_not_allowed)).
40+
Limits = proplists:get_value(limits, Values, no_limits),
41+
RV = collections:create_scope(Bucket, Name, Limits),
42+
case {RV, Limits} of
43+
{{scope_already_exists, _}, L} when L =/= no_limits ->
44+
RV1 = collections:update_limits(Bucket, Name, Limits),
45+
handle_rv(RV1, Req);
46+
_ ->
47+
maybe_audit(RV, Req,
48+
ns_audit:create_scope(_, Bucket, Name, _)),
49+
handle_rv(RV, Req)
50+
end
51+
end, Req, form,
52+
scope_limit_validators(raw) ++ scope_validators(default_not_allowed)).
4453

4554
maybe_audit({ok, Uid}, Req, SuccessfulAuditFun) ->
4655
SuccessfulAuditFun(Req, Uid);
@@ -49,6 +58,31 @@ maybe_audit(forbidden, Req, _SuccessfulAuditFun) ->
4958
maybe_audit(_, _Req, _SuccessfulAuditFun) ->
5059
ok.
5160

61+
service_scope_limit_validators(clusterManager) ->
62+
[validator:integer(num_collections, 1, infinity, _),
63+
validator:unsupported(_)];
64+
service_scope_limit_validators(fts) ->
65+
[validator:integer(num_fts_indexes, 1, infinity, _),
66+
validator:unsupported(_)];
67+
service_scope_limit_validators(index) ->
68+
[validator:integer(num_indexes, 1, infinity, _),
69+
validator:unsupported(_)];
70+
service_scope_limit_validators(kv) ->
71+
[validator:integer(data_size, 1, infinity, _),
72+
validator:unsupported(_)].
73+
74+
scope_limit_validators(Type) ->
75+
Validators = [validator:decoded_json(
76+
Service, service_scope_limit_validators(Service), _) ||
77+
Service <- [clusterManager, fts, index, kv]] ++
78+
[validator:unsupported(_)],
79+
[case Type of
80+
decoded ->
81+
validator:decoded_json(limits, Validators, _);
82+
raw ->
83+
validator:json(limits, Validators, _)
84+
end].
85+
5286
scope_validators(default_not_allowed) ->
5387
scope_validators([]);
5488
scope_validators(default_allowed) ->
@@ -129,9 +163,12 @@ check_duplicates(Name, State) ->
129163

130164
validate_scopes(Name, State) ->
131165
validator:json_array(
132-
Name, [validate_collections(collections, _),
133-
check_duplicates(collections, _) |
134-
scope_validators(default_allowed)], State).
166+
Name,
167+
scope_limit_validators(decoded) ++
168+
[validate_collections(collections, _),
169+
check_duplicates(collections, _) |
170+
scope_validators(default_allowed)],
171+
State).
135172

136173
validate_collections(Name, State) ->
137174
validator:json_array(Name, collection_validators(default_allowed), State).

0 commit comments

Comments
 (0)