feat: add netstat connection listing and refactor tools

- Added `get_netstat_conn` to fetch netstat connections
- Refactored `get_netstat_out` to `get_netstat_listen`
- Updated `kill_old_cluster` to use new netstat functions

Author: mkoura

cardano_node_tests/cluster_management/cluster_getter.py

@@ -208,7 +208,7 @@ def _netstat_log_func(msg: str) -> None:
             if _cluster_started:
                 break
 
-            netstat_out = netstat_tools.get_netstat_out()
+            netstat_out = netstat_tools.get_netstat_conn()
             self.log(
                 f"c{self.cluster_instance_num}: failed to start cluster:\n{excp}"
                 f"\nnetstat:\n{netstat_out}"

cardano_node_tests/cluster_management/netstat_tools.py

@@ -12,8 +12,8 @@
 LOGGER = logging.getLogger(__name__)
 
 
-def get_netstat_out() -> str:
-    """Get output of the `netstat` command."""
+def get_netstat_listen() -> str:
+    """Get listing of listening services from the `netstat` command."""
     try:
         return helpers.run_command(
             "netstat -pant | grep -E 'LISTEN|TIME_WAIT'", ignore_fail=True, shell=True
@@ -23,11 +23,23 @@ def get_netstat_out() -> str:
         return ""
 
 
+def get_netstat_conn() -> str:
+    """Get listing of connections from the `netstat` command."""
+    try:
+        return helpers.run_command("netstat -pant", ignore_fail=True).decode()
+    except Exception as excp:
+        LOGGER.error(f"Failed to fetch netstat output: {excp}")  # noqa: TRY400
+        return ""
+
+
 def kill_old_cluster(instance_num: int, log_func: tp.Callable[[str], None]) -> None:  # noqa: C901
     """Attempt to kill all processes left over from a previous cluster instance."""
 
-    def _get_netstat_split() -> list[str]:
-        return get_netstat_out().replace("\t", "    ").splitlines()
+    def _get_listen_split() -> list[str]:
+        return get_netstat_listen().replace("\t", "    ").splitlines()
+
+    def _get_conn_split() -> list[str]:
+        return get_netstat_conn().replace("\t", "    ").splitlines()
 
     def _get_pid(line: str) -> int | None:
         try:
@@ -64,11 +76,12 @@ def _get_proc_cmdline(pid: int) -> str:
             *port_nums.node_ports,
         )
     ]
+    ports_re = re.compile(r"|".join(re.escape(p) for p in port_strs))
 
     # Attempt to kill the `supervisord` process first. If successful, this will also kill all the
     # processes started by supervisor.
     port_supervisor_str = port_strs[0]
-    for line in _get_netstat_split():
+    for line in _get_listen_split():
         if port_supervisor_str not in line:
             continue
         pid = _get_pid(line)
@@ -79,10 +92,9 @@ def _get_proc_cmdline(pid: int) -> str:
         break
 
     # Kill all the leftover processes, if possible, and wait for them to finish
-    ports_re = re.compile(r"|".join(re.escape(p) for p in port_strs))
     for _ in range(5):
         found = False
-        for line in _get_netstat_split():
+        for line in _get_listen_split():
             if not ports_re.search(line):
                 continue
             found = True
@@ -95,3 +107,15 @@ def _get_proc_cmdline(pid: int) -> str:
             break
         if not found:
             break
+
+    # Wait until all connections are closed
+    for _ in range(3):
+        found = False
+        for line in _get_conn_split():
+            if not ports_re.search(line):
+                continue
+            found = True
+            time.sleep(5)
+            break
+        if not found:
+            break