test: dump debug information whenever ssh.check_output fails

Print microvm logs and stack backtraces whenever running a command via
SSH inside the guest fails (instead of only doing this if
`wait_for_ssh_up` fails).

To this by adding an `on_error` hook that gets called from
`SSHConnection.check_output` if running the SSH command fails.
Potentially, this could later be extended to automatically snapshot the
microvm inside the hook, but for now, let's only print some data.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/framework/microvm.py

@@ -983,6 +983,7 @@ def ssh_iface(self, iface_idx=0):
             ssh_key=self.ssh_key,
             user="root",
             host=guest_ip,
+            on_error=self._dump_debug_information,
         )
 
     @property
@@ -1002,19 +1003,23 @@ def thread_backtraces(self):
                 )
         return "\n".join(backtraces)
 
+    def _dump_debug_information(self, exc: Exception):
+        """
+        Dumps debug information about this microvm
+
+        Used for example when running a command inside the guest via `SSHConnection.check_output` fails.
+        """
+        print(
+            f"Failure executing command via SSH in microVM: {exc}\n\n"
+            f"Firecracker logs:\n{self.log_data}\n"
+            f"Thread backtraces:\n{self.thread_backtraces}"
+        )
+
     def wait_for_ssh_up(self):
         """Wait for guest running inside the microVM to come up and respond."""
-        try:
-            # Ensure that we have an initialized SSH connection to the guest that can
-            # run commands. The actual connection retry loop happens in SSHConnection._init_connection
-            self.ssh_iface(0)
-        except Exception as exc:
-            print(
-                f"Failed to establish SSH connection to guest: {exc}\n\n"
-                f"Firecracker logs:\n{self.log_data}\n"
-                f"Thread backtraces:\n{self.thread_backtraces}"
-            )
-            raise
+        # Ensure that we have an initialized SSH connection to the guest that can
+        # run commands. The actual connection retry loop happens in SSHConnection._init_connection
+        _ = self.ssh_iface(0)
 
 
 class MicroVMFactory:

tests/host_tools/network.py

@@ -25,7 +25,7 @@ class SSHConnection:
     ssh -i ssh_key_path username@hostname
     """
 
-    def __init__(self, netns, ssh_key: Path, host, user):
+    def __init__(self, netns, ssh_key: Path, host, user, *, on_error=None):
         """Instantiate a SSH client and connect to a microVM."""
         self.netns = netns
         self.ssh_key = ssh_key
@@ -37,6 +37,8 @@ def __init__(self, netns, ssh_key: Path, host, user):
         self.host = host
         self.user = user
 
+        self._on_error = None
+
         self.options = [
             "-o",
             "LogLevel=ERROR",
@@ -52,7 +54,18 @@ def __init__(self, netns, ssh_key: Path, host, user):
             str(self.ssh_key),
         ]
 
-        self._init_connection()
+        # _init_connection loops until it can connect to the guest
+        # dumping debug state on every iteration is not useful or wanted, so
+        # only dump it once if _all_ iterations fail.
+        try:
+            self._init_connection()
+        except Exception as exc:
+            if on_error:
+                on_error(exc)
+
+            raise
+
+        self._on_error = on_error
 
     def remote_path(self, path):
         """Convert a path to remote"""
@@ -123,7 +136,13 @@ def _exec(self, cmd, timeout=None, check=False):
         if self.netns is not None:
             cmd = ["ip", "netns", "exec", self.netns] + cmd
 
-        return utils.run_cmd(cmd, check=check, timeout=timeout)
+        try:
+            return utils.run_cmd(cmd, check=check, timeout=timeout)
+        except Exception as exc:
+            if self._on_error:
+                self._on_error(exc)
+
+            raise
 
 
 def mac_from_ip(ip_address):