Merge pull request #109 from AlmaLinux/fix-stop-env

Fix stopping environments in Opennebula

Author: Korulag

alts/worker/runners/base.py

@@ -1479,6 +1479,25 @@ def write_to_file(file_base_name: str, artifacts_section: dict):
         except UploadError as e:
             raise PublishArtifactsError from e
 
+    def _stop_env(self):
+        if not os.path.exists(self._work_dir):
+            return 0, '', f'Working directory {self._work_dir} does not exist'
+        self._logger.info(
+            'Destroying the environment %s...',
+            self.env_name,
+        )
+        self._logger.debug(
+            'Running "terraform destroy --auto-approve" command'
+        )
+        cmd_args = ['destroy', '--auto-approve', '-no-color']
+        if self.TF_VARIABLES_FILE:
+            cmd_args.extend(['--var-file', self.TF_VARIABLES_FILE])
+        return local['terraform'].with_cwd(self._work_dir).run(
+            args=cmd_args,
+            retcode=None,
+            timeout=CONFIG.provision_timeout,
+        )
+
     # After: install_package and run_tests
     @command_decorator(
         'stop_environment',
@@ -1487,35 +1506,23 @@ def write_to_file(file_base_name: str, artifacts_section: dict):
         is_abortable=False,
     )
     def stop_env(self):
-        if os.path.exists(self._work_dir):
-            self._logger.info(
-                'Destroying the environment %s...',
-                self.env_name,
-            )
-            self._logger.debug(
-                'Running "terraform destroy --auto-approve" command'
-            )
-            cmd_args = ['destroy', '--auto-approve', '-no-color']
-            if self.TF_VARIABLES_FILE:
-                cmd_args.extend(['--var-file', self.TF_VARIABLES_FILE])
-            return local['terraform'].with_cwd(self._work_dir).run(
-                args=cmd_args,
-                retcode=None,
-                timeout=CONFIG.provision_timeout,
-            )
+        return self._stop_env()
 
     def erase_work_dir(self):
-        if self._work_dir and os.path.exists(self._work_dir):
-            self._logger.info('Erasing working directory...')
-            try:
-                shutil.rmtree(self._work_dir)
-            except Exception as e:
-                self._logger.error(
-                    'Error while erasing working directory: %s',
-                    e,
-                )
-            else:
-                self._logger.info('Working directory was successfully removed')
+        if not self._work_dir:
+            return
+        if self._work_dir and not os.path.exists(self._work_dir):
+            return
+        self._logger.info('Erasing working directory...')
+        try:
+            shutil.rmtree(self._work_dir)
+        except Exception as e:
+            self._logger.error(
+                'Error while erasing working directory: %s',
+                e,
+            )
+        else:
+            self._logger.info('Working directory was successfully removed')
 
     def setup(self, skip_provision: bool = False):
         self._stats['started_at'] = datetime.datetime.utcnow().isoformat()

alts/worker/runners/docker.py

@@ -20,7 +20,6 @@
 from alts.shared.exceptions import (
     PackageIntegrityTestsError,
     ProvisionError,
-    StopEnvironmentError,
     ThirdPartyTestError,
 )
 from alts.shared.uploaders.base import BaseLogsUploader
@@ -351,20 +350,14 @@ def clone_third_party_repo(
         ])
         return test_repo_path
 
-    @command_decorator(
-        'stop_environment',
-        'Cannot destroy environment',
-        exception_class=StopEnvironmentError,
-    )
-    def stop_env(self):
+    def _stop_env(self):
         _, container_id, _ = local['terraform'].with_cwd(
             self._work_dir).run(
             args=('output', '-raw', '-no-color', 'container_id'),
             retcode=None,
             timeout=CONFIG.provision_timeout,
         )
-        try:
-            return super().stop_env()
-        except StopEnvironmentError:
-            # Attempt to delete environment via plain docker command
+        exit_code, out, err = super()._stop_env()
+        if exit_code != 0:
             return self.exec_command('rm', '-f', container_id)
+        return exit_code, out, err

alts/worker/runners/opennebula.py

@@ -21,11 +21,10 @@
 from alts.shared.exceptions import (
     OpennebulaVMStopError,
     VMImageNotFound,
-    StopEnvironmentError,
 )
 from alts.shared.uploaders.base import BaseLogsUploader
 from alts.worker import CONFIG
-from alts.worker.runners.base import GenericVMRunner, command_decorator
+from alts.worker.runners.base import GenericVMRunner
 
 __all__ = ['OpennebulaRunner']
 
@@ -242,24 +241,24 @@ def recover_delete():
             )
             recover_delete()
 
-    @command_decorator(
-        'stop_environment',
-        'Cannot destroy environment',
-        exception_class=StopEnvironmentError,
-        is_abortable=False,
-    )
-    def stop_env(self):
+    def _stop_env(self):
+        stop_exit_code, stop_out, stop_err = super()._stop_env()
+        if stop_exit_code == 0:
+            return stop_exit_code, stop_out, stop_err
+
+        self._logger.warning(
+            'Cannot stop VM conventionally. Output:\n%s\nStderr:\n%s',
+            stop_out, stop_err
+        )
         id_exit_code, vm_id, id_stderr = local['terraform'].with_cwd(
             self._work_dir).run(
             args=('output', '-raw', '-no-color', 'vm_id'),
             retcode=None,
             timeout=CONFIG.provision_timeout,
         )
-        if id_exit_code != 0:
+        self._logger.debug('VM ID: %s', vm_id)
+        if id_exit_code != 0 or not vm_id:
             self._logger.warning('Cannot get VM ID: %s', id_stderr)
-        try:
-            return super().stop_env()
-        except StopEnvironmentError:
-            if vm_id:
-                self.destroy_vm_via_api(int(vm_id.strip()))
-                return 0, f'{vm_id} is destroyed via API', ''
+            return id_exit_code, 'Cannot get VM ID', id_stderr
+        self.destroy_vm_via_api(int(vm_id.strip()))
+        return 0, f'{vm_id} is destroyed via API', ''