Merge pull request #68277 from bluesliverx/3007.x

Fix 66568: Prevent unclosed TCP channels and avoid additional errors

Author: dwoz

changelog/66568.fixed.md

@@ -0,0 +1 @@
+Fix closing of TCP transport channels and avoid additional errors

salt/channel/client.py

@@ -556,7 +556,10 @@ def connect_callback(self, result):
             # may have been restarted
             yield self.send_id(self.token, self._reconnected)
             self.connected = True
-            self.event.fire_event({"master": self.opts["master"]}, "__master_connected")
+            if self.event:
+                self.event.fire_event(
+                    {"master": self.opts["master"]}, "__master_connected"
+                )
             if self._reconnected:
                 # On reconnects, fire a master event to notify that the minion is
                 # available.

salt/transport/tcp.py

@@ -235,7 +235,6 @@ def __init__(self, opts, io_loop, **kwargs):  # pylint: disable=W0231
         self.connected = False
         self._closing = False
         self._stream = None
-        self._closing = False
         self._closed = False
         self.backoff = opts.get("tcp_reconnect_backoff", 1)
         self.resolver = kwargs.get("resolver")
@@ -1769,6 +1768,7 @@ def __init__(self, opts, io_loop, **kwargs):  # pylint: disable=W0231
         self._closed = False
         self._stream_return_running = False
         self._stream = None
+        self.task = None
         self.disconnect_callback = _null_callback
         self.connect_callback = _null_callback
         self.backoff = opts.get("tcp_reconnect_backoff", 1)
@@ -1837,7 +1837,7 @@ async def _stream_return(self):
                                 message_id,
                             )
             except tornado.iostream.StreamClosedError as e:
-                log.error(
+                log.debug(
                     "tcp stream to %s:%s closed, unable to recv",
                     self.host,
                     self.port,
@@ -1879,6 +1879,8 @@ async def _stream_return(self):
                     stream.close()
                 unpacker = salt.utils.msgpack.Unpacker()
                 await self.connect()
+            except asyncio.CancelledError:
+                log.debug("Stream return cancelled")
         self._stream_return_running = False
 
     def _message_id(self):
@@ -1927,9 +1929,20 @@ async def _do_send():
     def close(self):
         if self._closing:
             return
+        self._closing = True
         if self._stream is not None:
             self._stream.close()
             self._stream = None
+        if self.task is not None:
+            self.task.cancel()
+            # Wait for the task to finish via asyncio
+            group = asyncio.gather(self.task)
+            try:
+                self.task.get_loop().run_until_complete(group)
+            except RuntimeError:
+                # Ignore event loop was already running message
+                pass
+            self.task = None
 
     def __enter__(self):
         return self

tests/pytests/functional/transport/server/test_request_server.py

@@ -44,10 +44,16 @@ async def handler(message):
     try:
 
         ret = await req_client.send({"req": "test"})
+        if transport == "tcp":
+            assert req_client.task is not None
         assert [reqmsg] == requests
         assert repmsg == ret
     finally:
         req_client.close()
+        if transport == "tcp":
+            # Ensure that issue 68277 is fixed
+            assert req_client.task is None or req_client.task.done()
+            assert req_client._closing
         req_server.close()
 
     # Yield to loop in order to allow background close methods to finish.