Merge remote-tracking branch 'couchbase/cheshire-cat'

* MB-47702: Optionally register node with epmd.
* MB-47702: Conditionally kill epmd on settings change and restart

Change-Id: I3df2ba6a72835fe9951c082899cf5e79561bc823

Author: anuthan

couchbase-server.sh.in

@@ -79,6 +79,9 @@ export ERL_CRASH_DUMP
 ERL_FULLSWEEP_AFTER=512
 export ERL_FULLSWEEP_AFTER
 
+ERL_EPMD_RELAXED_COMMAND_CHECK=1
+export ERL_EPMD_RELAXED_COMMAND_CHECK
+
 # For some obscure reason erl requires HOME environment variable to be set.
 if [ -z "$HOME" ]
 then
@@ -143,11 +146,14 @@ _prepare_datadir () {
 }
 
 _maybe_start_epmd () {
-    # Initialize distributed erlang on the system (i.e. epmd)
-    erl -noshell -setcookie nocookie -sname init -run init stop 2>&1 > /dev/null
-    if [ $? -ne 0 ]
+    if [ ! -f $CB_DATA_DIR/no_epmd ]
     then
-        exit 1
+        # Initialize distributed erlang on the system (i.e. epmd)
+        erl -noshell -setcookie nocookie -sname init -run init stop 2>&1 > /dev/null
+        if [ $? -ne 0 ]
+        then
+            exit 1
+        fi
     fi
 }
 
@@ -211,6 +217,7 @@ _start() {
         -name '[email protected]' \
         -proto_dist cb \
         -epmd_module cb_epmd \
+        -start_epmd false \
         -ssl_dist_optfile $SSL_DIST_OPTFILE \
         -setcookie nocookie \
         $* \
@@ -236,6 +243,7 @@ _stop() {
                 inetrc "\"$HOSTS_CFG_FILE\"" \
         -proto_dist cb \
         -epmd_module cb_epmd \
+        -start_epmd false \
         -ssl_dist_optfile $SSL_DIST_OPTFILE \
         -noshell \
         -hidden \

pylib/cluster_run_lib.py

@@ -263,6 +263,7 @@ def erlang_args_for_node(i, ebin_path, extra_args, args_prefix, root_dir):
         "-proto_dist", "cb",
         "-ssl_dist_optfile", ssloptfile,
         "-epmd_module", "cb_epmd",
+        "-start_epmd", "false",
         "-hidden",
         "-kernel", "dist_config_file", quote_string_for_erl(cb_dist_config),
         "-kernel", "inetrc", f"\"{hosts_file}\"",
@@ -454,6 +455,7 @@ def start_node(node_num):
         params['env']['ERL_CRASH_DUMP'] = crash_dump_base + '.babysitter'
 
         params['env']['COUCHBASE_SMALLER_PKEYS'] = '1'
+        params['env']['ERL_EPMD_RELAXED_COMMAND_CHECK'] = '1'
 
         params['close_fds'] = True
         if platform.system() == "Windows":
@@ -477,6 +479,11 @@ def start_node(node_num):
             params['stdout'] = subprocess.DEVNULL
             params['stderr'] = subprocess.DEVNULL
 
+        if not os.path.isfile(abs_path_join(root_dir, 'data', f'n_{node_num}',
+                                            'no_epmd')):
+            subprocess.run(["erl", "-noshell", "-setcookie", "nocookie",
+                            "-sname", "init", "-run", "init", "stop"],
+                           env=params['env'])
         pr = subprocess.Popen(args, **params)
         if w is not None:
             os.close(r)

src/cb_dist.erl

@@ -597,10 +597,8 @@ listen_proto({AddrType, Module}, NodeName) ->
         fun () ->
                 case Module:listen(NodeName) of
                     {ok, _} = Res ->
-                        case maybe_register_on_epmd(Module, NodeName, Port) of
-                            ok -> Res;
-                            {error, _} = Error -> Error
-                        end;
+                        maybe_register_on_epmd(Module, NodeName, Port),
+                        Res;
                     Error -> Error
                 end
         end,
@@ -636,9 +634,14 @@ maybe_register_on_epmd(Module, NodeName, PortNo)
         ns_server ->
             Family = proto_to_family(Module),
             case erl_epmd:register_node(NameStr, PortNo, Family) of
-                {ok, _} -> ok;
-                {error, already_registered} -> ok;
-                Error -> Error
+                {ok, _} ->
+                    ok;
+                {error, already_registered} ->
+                    ok;
+                Error ->
+                    info_msg("Failed to register ~p with epmd. Reason ~p",
+                             [{NodeName, PortNo, Family}, Error]),
+                    Error
             end;
         _ ->
             ok

src/menelaus_event.erl

@@ -203,6 +203,7 @@ maybe_restart(#state{webconfig = WebConfigOld,
          AFROld =:= AFRNew of
         true -> State;
         false -> {ok, _} = menelaus_web_sup:restart_web_servers(),
+                 netconfig_updater:maybe_kill_epmd(),
                  State#state{webconfig = WebConfigNew,
                              disable_non_ssl_ports = DisableNew,
                              afamily_requirement = AFRNew}

src/netconfig_updater.erl

@@ -12,6 +12,7 @@
 
 %% API
 -export([start_link/0,
+         maybe_kill_epmd/0,
          apply_config/1,
          change_external_listeners/2,
          ensure_tls_dist_started/1]).
@@ -25,6 +26,8 @@
 -include_lib("kernel/include/net_address.hrl").
 -include("ns_common.hrl").
 
+-define(CAN_KILL_EPMD, ?get_param(can_kill_epmd, true)).
+
 %%%===================================================================
 %%% API
 %%%===================================================================
@@ -49,6 +52,10 @@ init([]) ->
     ServerName = ?MODULE,
     register(ServerName, self()),
     ensure_ns_config_settings_in_order(),
+    %% We choose to kill epmd at startup if required. This is mainly required
+    %% for windows as for unix systems epmd will not be started because of
+    %% no_epmd file.
+    misc:is_windows() andalso maybe_kill_epmd(),
     proc_lib:init_ack({ok, self()}),
     case misc:consult_marker(update_marker_path()) of
         {ok, [Cmd]} ->
@@ -371,6 +378,59 @@ check_nodename_resolvable(Node, AFamily) ->
             {error, iolist_to_binary(M)}
     end.
 
+epmd_executable() ->
+    case misc:is_windows() of
+        true ->
+            %% Epmd doesn't exist in the bin path for windows so we pass the
+            %% erts_bin_path env to point us to it.
+            {ok, ERTSPath} = application:get_env(ns_server,
+                                                 erts_bin_path),
+            filename:join(ERTSPath, "epmd.exe");
+        false ->
+            path_config:component_path(bin, "epmd")
+    end.
+
+kill_epmd() ->
+    Path = epmd_executable(),
+    Port = erlang:open_port({spawn_executable, Path},
+                            [stderr_to_stdout, binary,
+                             stream, exit_status, hide,
+                             {args, ["-kill"]}]),
+    {ExitStatus, Output} = wait_for_exit(Port, []),
+    case ExitStatus of
+        0 ->
+            ok;
+        _ ->
+            ?log_error("Failed to kill epmd: ~p", [{ExitStatus, Output}]),
+            error
+    end.
+
+wait_for_exit(Port, Output) ->
+    receive
+        {Port, {data, Data}} ->
+            wait_for_exit(Port, Output ++ binary_to_list(Data));
+        {Port, {exit_status, Status}} ->
+            {Status, Output}
+    end.
+
+maybe_kill_epmd() ->
+    NoEpmdFile = path_config:component_path(data, "no_epmd"),
+    case ?CAN_KILL_EPMD andalso cluster_compat_mode:is_cluster_70() andalso
+         (misc:get_afamily_only() orelse misc:disable_non_ssl_ports()) of
+        true ->
+            try
+                misc:create_marker(NoEpmdFile),
+                ?log_info("Killing epmd ..."),
+                kill_epmd()
+            catch
+                T:E:S ->
+                    ?log_error("Exception while killing epmd ~p", [{T, E, S}])
+            end;
+        false ->
+            file:delete(NoEpmdFile),
+            ok
+    end.
+
 %% This function is needed in two cases:
 %%  - migration for address family settings to 6.5
 %%  - allow manual changes in dist_cfg file