Add description of backup server usage

Author: marcosbento

docs/client_api/build.py

@@ -147,6 +147,7 @@ def render_index_rst():
     :maxdepth: 1
     
     desc/cli_scripting_in_batch
+    desc/using_backup_servers
 """
 
     txt += """

docs/client_api/desc/using_backup_servers.rst

@@ -0,0 +1,36 @@
+.. _using_backup_servers:
+
+Using backup servers
+////////////////////
+
+
+The :code:`ecflow_client`, when running :term:`Task commands <child command>`, can be configured to contact
+alternate backup servers in case the primary server is not available -- this typically most useful in Operational environments.
+
+.. important::
+
+    The use of backup servers is only supported for :term:`Task commands <child command>`, and not for User commands such as :code:`ecflow_client --ping` or :code:`ecflow_client --alter`.
+
+A list of backup servers can be defined in a file, by convention located at :code:`$HOME/.ecf_hostfile`, with the following format:
+
+.. code-block:: shell
+
+   # This is a comment
+   host1             # port 3141 is used by default, when not specified
+   host2:port2
+   host3:port3
+
+
+To enable the :code:`ecflow_client` to read this file, and use the servers listed in it as backup servers, the following environment variable must be set before running the :code:`ecflow_client` command:
+
+.. code-block:: shell
+
+  export ECF_HOSTFILE=$HOME/.ecf_hostfile
+
+.. warning::
+
+    When performing a Task command, the :code:`ecflow_client` it will always first try to connect to the primary server, as defined by command line options or :code:`ECF_HOST`::code:`ECF_PORT`.
+    If the first attemp to contact the primary server fails, the client will automatically retry contacting the primary server after waiting for a retry period of 10 seconds.
+    Only after this second attempt has failed, will the :code:`ecflow_client` then immediatelly try to connect to the backup servers listed in the :code:`ECF_HOSTFILE`.
+
+    This implies that the :code:`ecflow_client` will not try to connect to the backup servers immediately, and thus contacting the backup server incurs in a minimum 10 seconds delay.

docs/client_api/index.rst

@@ -33,6 +33,7 @@ The list of commands, amongst other details, can be displayed by using the optio
     :maxdepth: 1
 
     desc/cli_scripting_in_batch
+    desc/using_backup_servers
 
 
 .. rubric:: Reference Commands

libs/client/src/ecflow/client/ClientInvoker.cpp

@@ -594,25 +594,34 @@ int ClientInvoker::do_invoke_cmd(Cmd_ptr cts_cmd) const {
 
                 // Wait a bit before trying to connect again, but only if no_of_tries > 0
                 no_of_tries--;
-                if (no_of_tries > 0)
+                if (no_of_tries > 0) {
                     sleep(retry_connection_period);
+                }
             }
 
-            // Don't bother with other hosts when:
+            //
+            // The ecFlow client may try to connect to multiple hosts, as specified in the ECF_HOSTFILE
+            //
+            // Attempting other hosts only happens for Task commands.
+            //
+            // The multiple hosts connection is ***NOT*** attempted when:
             //  1/ Testing
-            //  2/ ping-ing
-            //  3/ ECF_DENIED has been set
-            //  4/ Dealing with non tasks based request
+            //  2/ Executing Ping command
+            //  3/ Environment variable ECF_DENIED is set
+            //  4/ Executing any User command
+            //
             if (!cts_cmd->connect_to_different_servers() || test_ || cts_cmd->ping_cmd() || clientEnv_.denied()) {
                 std::stringstream ss;
                 ss << TimeStamp::now() << "Request( " << cts_cmd->print_short() << " )";
-                if (clientEnv_.denied())
+                if (clientEnv_.denied()) {
                     ss << " ECF_DENIED ";
+                }
                 ss << ", Failed to connect to " << client_env_host_port() << ". After " << connection_attempts_
                    << " attempts. Is the server running ?\n";
                 // Only print client environment if not pinging
-                if (!cts_cmd->ping_cmd())
+                if (!cts_cmd->ping_cmd()) {
                     ss << clientEnv_.toString() << endl;
+                }
                 server_reply_.set_error_msg(ss.str());
                 return 1;
             }