MB-62509: Add query completed-stream-size ...

... settings to the query_settings_manager. Also refactors common
upgrade code to reduce duplication.

Change-Id: Iafc796c0c434537e219646e3d038311ee5be3dc9
Reviewed-on: https://review.couchbase.org/c/ns_server/+/221041
Tested-by: Bryan McCoid <[email protected]>
Well-Formed: Build Bot <[email protected]>
Reviewed-by: Steve Watanabe <[email protected]>

Author: bryandmc

apps/ns_server/src/analytics_settings_manager.erl

@@ -30,7 +30,8 @@
          config_upgrade_to_76/1]).
 
 -import(json_settings_manager,
-        [id_lens/1]).
+        [id_lens/1,
+         config_upgrade_settings/5]).
 
 -define(ANALYTICS_CONFIG_KEY, {metakv, <<"/analytics/settings/config">>}).
 
@@ -121,15 +122,11 @@ general_settings_lens_props(ClusterVersion) ->
                 []
         end.
 
-config_upgrade_settings(Config, OldVersion, NewVersion) ->
-    NewSettings = general_settings_defaults(NewVersion) --
-        general_settings_defaults(OldVersion),
-    json_settings_manager:upgrade_existing_key(
-      ?MODULE, Config, [{generalSettings, NewSettings}],
-      known_settings(NewVersion)).
-
 config_upgrade_to_76(Config) ->
-    config_upgrade_settings(Config, ?VERSION_72, ?VERSION_76).
+    config_upgrade_settings(?MODULE, Config,
+                            general_settings_defaults(?VERSION_72),
+                            general_settings_defaults(?VERSION_76),
+                            known_settings(?VERSION_76)).
 
 -ifdef(TEST).
 defaults_test() ->

apps/ns_server/src/json_settings_manager.erl

@@ -18,6 +18,7 @@
          fetch_settings_json/2,
          decode_settings_json/1,
          id_lens/1,
+         config_upgrade_settings/5,
          allow_missing_lens/1,
          props_lens/1,
          update/2,
@@ -213,3 +214,7 @@ props_lens(Props) ->
                   lens_set_many(Props, Values, Map)
           end,
     {Get, Set}.
+
+config_upgrade_settings(M, Config, DefaultsOld, DefaultsNew, Known) ->
+    NewSettings = DefaultsNew -- DefaultsOld,
+    upgrade_existing_key(M, Config, [{generalSettings, NewSettings}], Known).

apps/ns_server/src/menelaus_web_queries.erl

@@ -67,6 +67,14 @@ settings_post_validators() ->
             false ->
                 []
         end ++
+        case cluster_compat_mode:is_cluster_morpheus() of
+            true ->
+                [validator:integer(queryCompletedStreamSize, _),
+                 validator:range(queryCompletedStreamSize, 0,
+                                 ?MAX_64BIT_UNSIGNED_INT, _)];
+            false ->
+                []
+        end ++
         [validator:unsupported(_)].
 
 validate_tmp_space_size(Name, State) ->

apps/ns_server/src/ns_online_config_upgrader.erl

@@ -94,4 +94,5 @@ upgrade(?VERSION_76, Config) ->
     {?VERSION_MORPHEUS,
      menelaus_web_alerts_srv:config_upgrade_to_morpheus(Config) ++
          menelaus_web_auto_failover:config_upgrade_to_morpheus(Config) ++
-         menelaus_web_guardrails:config_upgrade_to_morpheus(Config)}.
+         menelaus_web_guardrails:config_upgrade_to_morpheus(Config) ++
+         query_settings_manager:config_upgrade_to_morpheus(Config)}.

apps/ns_server/src/query_settings_manager.erl

@@ -24,15 +24,17 @@
          update/2,
          update_txn/1,
          config_default/0,
-         config_upgrade_to_76/1]).
+         config_upgrade_to_76/1,
+         config_upgrade_to_morpheus/1]).
 
 -export([cfg_key/0,
          is_enabled/0,
          known_settings/0,
          on_update/2]).
 
 -import(json_settings_manager,
-        [id_lens/1]).
+        [id_lens/1,
+         config_upgrade_settings/5]).
 
 -define(QUERY_CONFIG_KEY, {metakv, <<"/query/settings/config">>}).
 
@@ -77,12 +79,18 @@ config_default() ->
                           maps:new(),
                           known_settings(?MIN_SUPPORTED_VERSION))}.
 
+
+config_upgrade_to_morpheus(Config) ->
+    config_upgrade_settings(?MODULE, Config,
+                            general_settings_defaults(?VERSION_76),
+                            general_settings_defaults(?VERSION_MORPHEUS),
+                            known_settings(?VERSION_MORPHEUS)).
+
 config_upgrade_to_76(Config) ->
-    NewSettings = general_settings_defaults(?VERSION_76) --
-        general_settings_defaults(?MIN_SUPPORTED_VERSION),
-    json_settings_manager:upgrade_existing_key(
-      ?MODULE, Config, [{generalSettings, NewSettings}],
-      known_settings(?VERSION_76), fun functools:id/1).
+    config_upgrade_settings(?MODULE, Config,
+                            general_settings_defaults(?VERSION_72),
+                            general_settings_defaults(?VERSION_76),
+                            known_settings(?VERSION_76)).
 
 known_settings() ->
     known_settings(cluster_compat_mode:get_ns_config_compat_version()).
@@ -147,19 +155,27 @@ general_settings(Ver) ->
      {queryCleanupClientAttempts, "cleanupclientattempts", true},
      {queryCleanupLostAttempts, "cleanuplostattempts", true},
      {queryCleanupWindow,      "cleanupwindow",       <<"60s">>},
-     {queryNumAtrs,            "numatrs",             1024}] ++
-    case cluster_compat_mode:is_version_76(Ver) of
-        true ->
-            [{queryNodeQuota, "node-quota", ?QUERY_NODE_QUOTA_DEFAULT},
-             {queryUseReplica, "use-replica", <<"unset">>},
-             {queryNodeQuotaValPercent,
-              "node-quota-val-percent", 67},
-             {queryNumCpus, "num-cpus", 0},
-             {queryCompletedMaxPlanSize,
-              "completed-max-plan-size", 262144}];
-        false ->
-            []
-    end ++ n1ql_feature_ctrl_setting(Ver).
+     {queryNumAtrs,            "numatrs",             1024}]
+        ++ case cluster_compat_mode:is_version_76(Ver) of
+               true ->
+                   [{queryNodeQuota, "node-quota", ?QUERY_NODE_QUOTA_DEFAULT},
+                    {queryUseReplica, "use-replica", <<"unset">>},
+                    {queryNodeQuotaValPercent,
+                     "node-quota-val-percent", 67},
+                    {queryNumCpus, "num-cpus", 0},
+                    {queryCompletedMaxPlanSize,
+                     "completed-max-plan-size", 262144}];
+               false ->
+                   []
+           end
+        ++ case cluster_compat_mode:is_version_morpheus(Ver) of
+               true ->
+                   [{queryCompletedStreamSize,
+                     "completed-stream-size", 0}];
+               false ->
+                   []
+           end
+        ++ n1ql_feature_ctrl_setting(Ver).
 
 curl_whitelist_settings_len_props() ->
     [{queryCurlWhitelist, id_lens(<<"query.settings.curl_whitelist">>)}].
@@ -211,7 +227,13 @@ config_upgrade_test() ->
                    "\"use-replica\":\"unset\"}">>,
                  Data1),
 
-    meck:unload(config_profile).
+    meck:unload(config_profile),
+
+    CmdList2 = config_upgrade_to_morpheus([]),
+    [{set, {metakv, Meta2}, Data2}] = CmdList2,
+    ?assertEqual(<<"/query/settings/config">>, Meta2),
+    ?assertEqual(<<"{\"completed-stream-size\":0}">>,
+                 Data2).
 
 create_test_config_n1ql_quotas(NodeQuotaValue) when is_number(NodeQuotaValue) ->
     WOutNodeQuota = proplists:delete(queryNodeQuota,

cluster_tests/testsets/settings_managers_tests.py

@@ -86,6 +86,7 @@ def shard_affinty_blob(true_false):
     "queryNodeQuotaValPercent": create_inner(67),
     "queryNumCpus": create_inner(0),
     "queryCompletedMaxPlanSize": create_inner(262144),
+    "queryCompletedStreamSize": create_inner(0),
     "queryCurlWhitelist": create_inner({"all_access": False, "allowed_urls": [],
                                         "disallowed_urls": []})
 }