Tree: fix error handling when gateway channel is closing

Fix PropagationChannel.ev_close() where gateway channel termination is
handled.
If we get an actual rc > 0, that comes from the gateway command itself
and that means the gateway is defective/misconfigured, in that case, we
mark it as unreachable at the Task level.
In addition, in that case, if we have not launched the remote commands
yet, they are redistributed to other available gateways.

rc=None is now handled as a normal termination of the propagation
channel and the corresponding gateway is not marked as unreachable
anymore.

Fixes #566.

Author: thiell

lib/ClusterShell/Propagation.py

@@ -394,6 +394,7 @@ def recv_ctl(self, msg):
 
     def ev_hup(self, worker, node, rc):
         """Channel command is closing"""
+        self.logger.debug("ev_hup gateway=%s %s", str(worker.nodes), self)
         self._rc = rc
 
     def ev_close(self, worker, timedout):
@@ -402,16 +403,24 @@ def ev_close(self, worker, timedout):
         # common stream names
         gateway = str(worker.nodes)
         self.logger.debug("ev_close gateway=%s %s", gateway, self)
+
+        # NOTE: self._rc is set None when we abort the channel
         self.logger.debug("ev_close rc=%s", self._rc) # may be None
 
-        # NOTE: self._rc may be None if the communication channel has aborted
-        if self._rc != 0:
-            self.logger.debug("error on gateway %s (setup=%s)", gateway,
-                              self.setup)
+        if self._rc is None and not self.setup:
+            # aborting before the channel is setup would be weird...
+            self.logger.warning("ev_close rc=%s and self.setup=%s", self._rc,
+                                self.setup)
+
+        if self._rc is not None and self._rc != 0:
+            # handle gateway channel error
+            self.logger.debug("error on gateway %s (rc=%s, setup=%s)", gateway,
+                              self._rc, self.setup)
             self.task.router.mark_unreachable(gateway)
             self.logger.debug("gateway %s now set as unreachable", gateway)
 
             if not self.setup:
-                # channel was not set up: we can safely repropagate commands
+                # channel was not set up: we can safely redistribute commands
+                self.logger.debug("channel was not set up: redistributing...")
                 for mw in set(self.task.gateways[gateway][1]):
                     mw._relaunch(gateway)

lib/ClusterShell/Worker/Tree.py

@@ -430,8 +430,8 @@ def _on_remote_node_msgline(self, node, msg, sname, gateway):
     def _on_remote_node_close(self, node, rc, gateway):
         """remote node closing with return code"""
         DistantWorker._on_node_close(self, node, rc)
-        self.logger.debug("_on_remote_node_close %s %s via gw %s", node,
-                          self._close_count, gateway)
+        self.logger.debug("_on_remote_node_close %s %s via gw %s rc=%s", node,
+                          self._close_count, gateway, rc)
 
         # finalize rcopy: extract tar data
         if self.source and self.reverse: