Stabilize SUT reboot during tests execution

Sometimes SUT class might have communication problems at the very first
start after restarting it. With this patch we retry 10 times before
considering SUT communication broken.

Signed-off-by: Andrea Cervesato <[email protected]>
Reviewed-by: Cyril Hrubis <[email protected]>
Tested-by: Cyril Hrubis <[email protected]>

Author: acerv

ltp/dispatcher.py

@@ -317,9 +317,10 @@ def _reboot_sut(self, force: bool = False) -> None:
         else:
             self._sut.stop(timeout=360)
 
-        self._sut.communicate(
+        self._sut.ensure_communicate(
             timeout=3600,
-            iobuffer=RedirectStdout(self._sut))
+            iobuffer=RedirectStdout(self._sut),
+            force=force)
 
         self._logger.info("SUT rebooted")
 

ltp/qemu.py

@@ -495,7 +495,10 @@ def communicate(
                     self._wait_for("Password:", 5, iobuffer)
                     self._write_stdin(f"{self._password}\n")
 
+                time.sleep(0.2)
+
                 self._wait_for("#", 5, iobuffer)
+                time.sleep(0.2)
 
                 self._write_stdin("stty -echo; stty cols 1024\n")
                 self._wait_for("#", 5, None)

ltp/session.py

@@ -185,7 +185,7 @@ def _start_sut(
 
         ltp.events.fire("sut_start", sut.name)
 
-        sut.communicate(
+        sut.ensure_communicate(
             timeout=3600,
             iobuffer=Printer(sut, False))
 

ltp/sut.py

@@ -186,6 +186,44 @@ def fetch_file(
         """
         raise NotImplementedError()
 
+    def ensure_communicate(
+            self,
+            timeout: float = 3600,
+            iobuffer: IOBuffer = None,
+            retries: int = 10,
+            force: bool = False) -> None:
+        """
+        Ensure that `communicate` is completed, retrying as many times we
+        want in case of `LTPException` error. After each `communicate` error
+        the SUT is stopped and a new communication is tried.
+        :param timeout: timeout to complete communication in seconds
+        :type timeout: float
+        :param iobuffer: buffer used to write SUT stdout
+        :type iobuffer: IOBuffer
+        :param retries: number of times we retry communicating with SUT
+        :type retries: int
+        :param force: use `force_stop` instead of `stop` before communicating
+            again with the SUT
+        :type force: bool
+        """
+        retries = max(retries, 1)
+
+        for retry in range(retries):
+            try:
+                self.communicate(
+                    timeout=timeout,
+                    iobuffer=iobuffer)
+
+                break
+            except LTPException as err:
+                if retry >= retries - 1:
+                    raise err
+
+                if force:
+                    self.force_stop(timeout=timeout, iobuffer=iobuffer)
+                else:
+                    self.stop(timeout=timeout, iobuffer=iobuffer)
+
     def get_info(self) -> dict:
         """
         Return SUT information.

ltp/tests/sut.py

@@ -90,6 +90,17 @@ def test_communicate(self, sut):
             sut.communicate(iobuffer=Printer())
         sut.stop()
 
+    def test_ensure_communicate(self, sut):
+        """
+        Test ensure_communicate method.
+        """
+        sut.ensure_communicate(iobuffer=Printer())
+        with pytest.raises(SUTError):
+            sut.ensure_communicate(iobuffer=Printer(), retries=1)
+
+        sut.ensure_communicate(iobuffer=Printer(), retries=10)
+        sut.stop()
+
     @pytest.fixture
     def sut_stop_sleep(self, request):
         """