Keep VM alive if a particular error string is encountered (#159)

Apply suggestions from code review

Co-authored-by: Daniil Anfimov <64651799+anfimovdm@users.noreply.github.com>

Author: Kwaizer

alts/shared/constants.py

@@ -63,6 +63,10 @@
     'publickey',
 ]
 
+ERROR_STRINGS = (
+    'Error: Failed to wait virtual machine to be in RUNNING state',
+)
+
 
 class TapStatusEnum(IntEnum):
     FAILED = 0

alts/shared/utils/log_utils.py

@@ -2,6 +2,7 @@
 from tempfile import NamedTemporaryFile
 from typing import IO, Tuple
 
+from alts.shared.constants import ERROR_STRINGS
 
 def get_temp_log_files(prefix: str) -> Tuple[IO, IO]:
     temp_file_kwargs = {
@@ -27,3 +28,22 @@ def read_and_cleanup_temp_log_files(
         file.close()
         os.unlink(file.name)
     return out, err
+
+
+def check_for_error_string(stage_data: dict) -> bool:
+    """
+    Checks if we encounter errors during testing that worth keepin VM alive.
+
+    Parameters
+    ----------
+    stage_data: dict
+
+    Returns
+    -------
+    bool
+        True if we encounter any error from the list. False otherwise
+    """
+    err = stage_data.get('stderr', '')
+    if any(error_str in err for error_str in ERROR_STRINGS):
+        return True
+    return False

alts/worker/runners/base.py

@@ -57,6 +57,7 @@
 from alts.shared.utils.log_utils import (
     get_temp_log_files,
     read_and_cleanup_temp_log_files,
+    check_for_error_string,
 )
 from alts.shared.utils.plumbum_utils import wait_bg_process
 from alts.worker import CONFIG, RESOURCES_DIR
@@ -1554,6 +1555,11 @@ def setup(self, skip_provision: bool = False):
 
     def teardown(self, publish_artifacts: bool = True):
         try:
+            if not self.vm_alive:
+                for _, stage_data in self.artifacts.items():
+                    self.vm_alive = check_for_error_string(stage_data)
+                    for _, inner_data in stage_data.items():
+                        self.vm_alive = check_for_error_string(inner_data)
             self.stop_env()
         except Exception as e:
             self._logger.exception('Error while stop environment: %s', e)