Merge remote-tracking branch 'couchbase/trinity' into phoenix

* MB-67082: Add online eviction policy test cases
* MB-67082: Add eviction_policy tests for storage migration
* MB-67758: Support UI pop ups for auto-failover alerts
* MB-67082: Add support for online eviction_policy changes
* MB-67082: Add --no-restart option for eviction policy changes

Change-Id: Ic4374cc178ee8f6afa0e52825c75ce82ff573b00

Author: neelima32

apps/ns_server/src/auto_failover.erl

@@ -212,9 +212,10 @@ cast(Call) ->
     misc:wait_for_global_name(?MODULE),
     gen_server:cast(?SERVER, Call).
 
--define(log_info_and_email(Alert, Fmt, Args),
-        ale:info(?USER_LOGGER, Fmt, Args),
-        ns_email_alert:alert(Alert, Fmt, Args)).
+-define(log_and_alert(Alert, Fmt, Args),
+        Message = list_to_binary(lists:flatten(io_lib:format(Fmt, Args))),
+        %% The alert server sends emails, raises pop ups, etc as needed.
+        menelaus_web_alerts_srv:global_alert(Alert, Message)).
 
 %% @doc Returns a list of all alerts that might send out an email notification.
 -spec alert_keys() -> [atom()].
@@ -550,7 +551,7 @@ trim_nodes_for_failover(Nodes, S) ->
 
 process_action({mail_too_small, Service, SvcNodes, {Node, _UUID}},
                S, _, _, _) ->
-    ?log_info_and_email(
+    ?log_and_alert(
        auto_failover_cluster_too_small,
        "Could not auto-failover node (~p). "
        "Number of remaining nodes that are running ~s service is ~p. "
@@ -561,28 +562,28 @@ process_action({mail_too_small, Service, SvcNodes, {Node, _UUID}},
         auto_failover_logic:service_failover_min_node_count()]),
     S;
 process_action({mail_down_warning, {Node, _UUID}}, S, _, _, _) ->
-    ?log_info_and_email(
+    ?log_and_alert(
        auto_failover_other_nodes_down,
        "Could not auto-failover node (~p). "
        "There was at least another node down.",
        [Node]),
     S;
 process_action({mail_down_warning_multi_node, {Node, _UUID}}, S, _, _, _) ->
-    ?log_info_and_email(
+    ?log_and_alert(
        auto_failover_other_nodes_down,
        "Could not auto-failover node (~p). "
        "The list of nodes being down has changed.",
        [Node]),
     S;
 process_action({mail_auto_failover_disabled, Service, {Node, _}}, S, _, _, _) ->
-    ?log_info_and_email(
+    ?log_and_alert(
        auto_failover_disabled,
        "Could not auto-failover node (~p). "
        "Auto-failover for ~s service is disabled.",
        [Node, ns_cluster_membership:user_friendly_service_name(Service)]),
     S;
 process_action({mail_kv_not_fully_failed_over, {Node, _}}, S, _, _, _) ->
-    ?log_info_and_email(
+    ?log_and_alert(
        auto_failover_other_nodes_down,
        "Could not auto-failover service node (~p). "
        "One of the data service nodes cannot be automatically failed over.",
@@ -644,12 +645,12 @@ maybe_report_max_node_reached(AllNodes, NotFailedOver, ErrMsg, S) ->
         true ->
             case AllNodes -- NotFailedOver of
                 [] ->
-                    ?log_info_and_email(
+                    ?log_and_alert(
                        auto_failover_maximum_reached,
                        "Could not auto-failover more nodes (~p). ~s",
                        [NotFailedOver, ErrMsg]);
                 RemainingNodes ->
-                    ?log_info_and_email(
+                    ?log_and_alert(
                        auto_failover_maximum_reached,
                        "Could not auto-failover nodes (~p). ~s Continuing to "
                        "auto-failover nodes ~p",
@@ -685,12 +686,12 @@ log_failover_success(Node, DownNodes, NodeStatuses) ->
     case failover_reason(Node, DownNodes, NodeStatuses) of
         {Reason, MA} ->
             master_activity_events:note_autofailover_done(Node, MA),
-            ?log_info_and_email(
+            ?log_and_alert(
                auto_failover_node,
                "Node (~p) was automatically failed over. Reason: ~s",
                [Node, Reason]);
         Reason ->
-            ?log_info_and_email(
+            ?log_and_alert(
                auto_failover_node,
                "Node (~p) was automatically failed over.~n~p", [Node, Reason])
 
@@ -717,7 +718,7 @@ log_unsafe_node({Node, {Service, Error}}, State) ->
     Flag = {Node, Service, Error},
     case should_report(Flag, State) of
         true ->
-            ?log_info_and_email(
+            ?log_and_alert(
                auto_failover_node,
                "Could not automatically fail over node (~p) due to operation "
                "being unsafe for service ~p. ~s",
@@ -777,7 +778,7 @@ process_failover_error(last_node, Nodes, S) ->
 report_failover_error(Flag, ErrMsg, Nodes, State) ->
     case should_report(Flag, State) of
         true ->
-            ?log_info_and_email(
+            ?log_and_alert(
                auto_failover_node,
                "Could not automatically fail over nodes (~p). ~s",
                [Nodes, ErrMsg]),

apps/ns_server/src/memcached_bucket_config.erl

@@ -136,7 +136,7 @@ get_continuous_backup_interval(BucketConfig) ->
 get_eviction_policy(Persistent, BucketConfig) ->
     case ns_bucket:is_persistent(BucketConfig) of
         Persistent ->
-            case ns_bucket:eviction_policy(BucketConfig) of
+            case ns_bucket:node_eviction_policy(BucketConfig) of
                 nru_eviction ->
                     auto_delete;
                 no_eviction ->

apps/ns_server/src/menelaus_web_buckets.erl

@@ -803,7 +803,13 @@ handle_bucket_update_inner(BucketId, Req, Params, Limit) ->
         {false, _, {ok, ParsedProps, _}} ->
             BucketType = proplists:get_value(bucketType, ParsedProps),
             UpdatedProps = ns_bucket:extract_bucket_props(ParsedProps),
-            case update_bucket(Ctx, BucketId, BucketType, UpdatedProps, Req) of
+            Options =
+                case proplists:get_value(no_restart, ParsedProps, false) of
+                    true -> [no_restart];
+                    false -> []
+                end,
+            case update_bucket(Ctx, BucketId, BucketType, UpdatedProps,
+                               Options, Req) of
                 retry ->
                     handle_bucket_update_inner(BucketId, Req, Params,
                                                Limit - 1);
@@ -849,6 +855,10 @@ storage_mode_migration_error(history_retention_enabled_on_bucket) ->
     {"Cannot migrate storage mode. history_retention enabled on bucket.", 400};
 storage_mode_migration_error(history_retention_enabled_on_collections) ->
     {"Cannot migrate storage mode. history_retention enabled on collections.",
+     400};
+storage_mode_migration_error(eviction_policy_no_restart_required) ->
+    {"Eviction policy changes during storage mode migration require "
+     "--no-restart option.",
      400}.
 
 reply_storage_mode_migration_error(Req, Error) ->
@@ -875,14 +885,10 @@ maybe_update_cas_props(BucketId, BucketConfig, UpdatedProps, true = _CcvEn) ->
 maybe_update_cas_props(_, _, UpdatedProps, false = _CcEn) ->
     {ok, UpdatedProps}.
 
-update_via_orchestrator(Req, BucketId, StorageMode, BucketType, UpdatedProps) ->
-    update_via_orchestrator(Req, BucketId, StorageMode, BucketType,
-                            UpdatedProps, true).
-
 update_via_orchestrator(Req, BucketId, StorageMode, BucketType, UpdatedProps,
-                        CanRetry) ->
+                        CanRetry, Options) ->
     case ns_orchestrator:update_bucket(BucketType, StorageMode,
-                                       BucketId, UpdatedProps) of
+                                       BucketId, UpdatedProps, Options) of
         ok ->
             ns_audit:modify_bucket(Req, BucketId, BucketType, UpdatedProps),
             DisplayBucketType = ns_bucket:display_type(BucketType,
@@ -911,7 +917,11 @@ update_via_orchestrator(Req, BucketId, StorageMode, BucketType, UpdatedProps,
         {error, {storage_mode_migration, janitor_not_run}} when CanRetry ->
             ns_orchestrator:ensure_janitor_run({bucket, BucketId}, 5000),
             update_via_orchestrator(Req, BucketId, StorageMode, BucketType,
-                                    UpdatedProps, false);
+                                    UpdatedProps, false, Options);
+        {error, {eviction_policy_change, janitor_not_run}} when CanRetry ->
+            ns_orchestrator:ensure_janitor_run({bucket, BucketId}, 5000),
+            update_via_orchestrator(Req, BucketId, StorageMode, BucketType,
+                                    UpdatedProps, false, Options);
         {error, {storage_mode_migration, Error}} ->
             reply_storage_mode_migration_error(Req, Error);
         {error, secret_not_found} ->
@@ -930,7 +940,7 @@ update_via_orchestrator(Req, BucketId, StorageMode, BucketType, UpdatedProps,
             end
     end.
 
-update_bucket(Ctx, BucketId, BucketType, UpdatedProps, Req) ->
+update_bucket(Ctx, BucketId, BucketType, UpdatedProps, Options, Req) ->
     #bv_ctx{bucket_config = BucketConfig} = Ctx,
     StorageMode = ns_bucket:storage_mode(BucketConfig),
     CcvEn = proplists:get_value(cross_cluster_versioning_enabled,
@@ -939,7 +949,7 @@ update_bucket(Ctx, BucketId, BucketType, UpdatedProps, Req) ->
     case maybe_update_cas_props(BucketId, BucketConfig, UpdatedProps, CcvEn) of
         {ok, UpdateProps1} ->
             update_via_orchestrator(Req, BucketId, StorageMode, BucketType,
-                                    UpdateProps1);
+                                    UpdateProps1, true, Options);
         {error, max_cas_vbucket_retrieval_no_map} ->
             reply_text(Req, "Unable to retrieve max_cas due to no vBucket map",
                             503);
@@ -1625,7 +1635,8 @@ basic_bucket_params_screening(Ctx, Params) ->
     CommonParams = validate_common_params(Ctx, Params),
     TypeSpecificParams =
         validate_bucket_type_specific_params(CommonParams, Params, Ctx),
-    Candidates = CommonParams ++ TypeSpecificParams,
+    NoRestartParam = parse_validate_no_restart(Params),
+    Candidates = CommonParams ++ TypeSpecificParams ++ [NoRestartParam],
     assert_candidates(Candidates),
     %% Basic parameter checking has been done. Take the non-error key/values
     %% and do additional checking (e.g. relationships between different
@@ -3730,6 +3741,38 @@ parse_validate_workload_pattern_default(Params) ->
                "'mixed'">>}
     end.
 
+parse_validate_no_restart(Params) ->
+    case proplists:get_value("noRestart", Params) of
+        undefined ->
+            ignore;
+        Value ->
+            case ns_config:read_key_fast(allow_online_eviction_policy_change,
+                                         false) of
+                true ->
+                    case menelaus_util:parse_validate_boolean(Value) of
+                        {ok, BoolValue} ->
+                            case proplists:get_value("evictionPolicy",
+                                                     Params) of
+                                undefined ->
+                                    {error, no_restart,
+                                     <<"noRestart option has no effect unless "
+                                       "evictionPolicy is being changed">>};
+                                _ ->
+                                    {ok, no_restart, BoolValue}
+                            end;
+                        _Error ->
+                            {error, no_restart,
+                             <<"noRestart must be a boolean (true/false)">>}
+                    end;
+                false ->
+                    {error, no_restart,
+                     <<"noRestart option is not supported. Enable "
+                       "allow_online_eviction_policy_change to use this "
+                       "feature">>}
+            end
+    end.
+
+
 handle_compact_bucket(_PoolId, Bucket, Req) ->
     ok = compaction_api:force_compact_bucket(Bucket),
     reply(Req, 200).

apps/ns_server/src/menelaus_web_node.erl

@@ -424,6 +424,20 @@ do_build_nodes_info_fun(#ctx{ns_config = Config,
                 fun (_, _) -> [] end
         end,
 
+    PerNodeEvictionPolicyBuilder =
+        case WithBucket of
+            true ->
+                fun (_Node, undefined) ->
+                        [];
+                    (Node, BucketName) ->
+                        {ok, BucketConfig} =
+                            ns_bucket:get_bucket(BucketName, Snapshot),
+                        build_eviction_policy(Node, BucketConfig)
+                end;
+            false ->
+                fun (_, _) -> [] end
+        end,
+
     fun(WantENode, Bucket) ->
             InfoNode = ns_doctor:get_node(WantENode, NodeStatuses),
             StableInfo =
@@ -448,7 +462,8 @@ do_build_nodes_info_fun(#ctx{ns_config = Config,
                  build_failover_status(Snapshot, WantENode),
                  LimitsAndBucketPlacerInfoBuilder(WantENode)] ++
                  build_encryption_at_rest_info(Bucket, Snapshot, InfoNode) ++
-                   PerNodeStorageBackendBuilder(WantENode, Bucket),
+                   PerNodeStorageBackendBuilder(WantENode, Bucket) ++
+                   PerNodeEvictionPolicyBuilder(WantENode, Bucket),
 
             NodeHash = erlang:phash2(StableInfo),
 
@@ -505,6 +520,28 @@ build_storage_backend(Node, BucketConfig) ->
                 [{storageBackend, NodeStorageBackend}]
         end.
 
+build_eviction_policy(Node, BucketConfig) ->
+    case ns_config:read_key_fast(allow_online_eviction_policy_change, false) of
+        true ->
+            NodeEvictionPolicy = ns_bucket:node_eviction_policy_override(
+                                   Node, BucketConfig),
+            case NodeEvictionPolicy of
+                undefined ->
+                    [];
+                _ ->
+                    EvictionPolicyBinary =
+                        case NodeEvictionPolicy of
+                            value_only -> <<"valueOnly">>;
+                            full_eviction -> <<"fullEviction">>;
+                            no_eviction -> <<"noEviction">>;
+                            nru_eviction -> <<"nruEviction">>
+                        end,
+                    [{evictionPolicy, EvictionPolicyBinary}]
+            end;
+        false ->
+            []
+    end.
+
 build_failover_status(Snapshot, Node) ->
     PrevFailoverNodes = chronicle_master:get_prev_failover_nodes(Snapshot),
     case lists:member(Node, PrevFailoverNodes) of