Merge pull request #3534 from vkarak/bugfix/handle-sighup

[bugfix] Properly treat forceful termination due to `SIGTERM` and `SIGHUP`

Author: vkarak

reframe/frontend/executors/__init__.py

@@ -591,8 +591,13 @@ def on_task_success(self, task):
         '''Called when a regression test has succeeded.'''
 
 
-def _handle_sigterm(signum, frame):
-    raise ForceExitError('received TERM signal')
+def _force_exit(signum, frame):
+    # ReFrame will exit, ignore all other signals that cause a graceful
+    # termination
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGHUP, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    raise ForceExitError(f'received signal {signum}')
 
 
 class Runner:
@@ -620,7 +625,8 @@ def __init__(self, policy, printer=None, max_retries=0,
         self._policy.printer = self._printer
         self._policy.max_failures = max_failures
 
-        signal.signal(signal.SIGTERM, _handle_sigterm)
+        signal.signal(signal.SIGHUP, _force_exit)
+        signal.signal(signal.SIGTERM, _force_exit)
 
     @property
     def max_failures(self):

unittests/conftest.py

@@ -10,6 +10,7 @@
 import contextlib
 import copy
 import pytest
+import signal
 import tempfile
 
 import reframe.core.settings as settings
@@ -105,9 +106,17 @@ def _make_loader(check_search_path, *args, **kwargs):
     return _make_loader
 
 
+@pytest.fixture
+def restore_signals():
+    yield
+    signal.signal(signal.SIGTERM, signal.SIG_DFL)
+    signal.signal(signal.SIGINT, signal.SIG_DFL)
+    signal.signal(signal.SIGHUP, signal.SIG_DFL)
+
+
 @pytest.fixture(params=[policies.SerialExecutionPolicy,
                         policies.AsynchronousExecutionPolicy])
-def make_runner(request):
+def make_runner(request, restore_signals):
     '''Test runner with all the execution policies'''
 
     def _make_runner(*args, **kwargs):
@@ -120,7 +129,7 @@ def _make_runner(*args, **kwargs):
 
 
 @pytest.fixture
-def make_async_runner():
+def make_async_runner(restore_signals):
     def _make_runner(*args, **kwargs):
         policy = policies.AsynchronousExecutionPolicy()
         policy._pollctl.SLEEP_MIN = 0.001

unittests/resources/checks/frontend_checks.py

@@ -8,7 +8,6 @@
 #
 
 import os
-import signal
 import sys
 import time
 
@@ -190,10 +189,13 @@ class SelfKillCheck(rfm.RunOnlyRegressionTest, special=True):
     executable = 'echo'
     sanity_patterns = sn.assert_true(1)
 
+    def __init__(self, signum):
+        self.signum = signum
+
     def run(self):
         super().run()
         time.sleep(0.5)
-        os.kill(os.getpid(), signal.SIGTERM)
+        os.kill(os.getpid(), self.signum)
 
 
 class CompileFailureCheck(rfm.RegressionTest):

unittests/test_policies.py

@@ -6,6 +6,7 @@
 import contextlib
 import os
 import pytest
+import signal
 
 import reframe as rfm
 import reframe.core.runtime as rt
@@ -310,11 +311,16 @@ def test_pass_in_retries(make_runner, make_cases, tmp_path, common_exec_ctx):
     assert 0 == len(runner.stats.failed())
 
 
-def test_sigterm_handling(make_runner, make_cases, common_exec_ctx):
+@pytest.fixture(params=[signal.SIGTERM, signal.SIGHUP])
+def signum(request):
+    return request.param
+
+
+def test_signal_handling(signum, make_runner, make_cases, common_exec_ctx):
     runner = make_runner()
     with pytest.raises(ForceExitError,
-                       match='received TERM signal'):
-        runner.runall(make_cases([SelfKillCheck()]))
+                       match=f'received signal {signum}'):
+        runner.runall(make_cases([SelfKillCheck(signum)]))
 
     assert_all_dead(runner)
     assert runner.stats.num_cases() == 1
@@ -442,7 +448,7 @@ def max_jobs_opts(n):
 
 
 @pytest.fixture
-def make_async_runner():
+def make_async_runner(restore_signals):
     # We need to have control in the unit tests where the policy is created,
     # because in some cases we need it to be initialized after the execution
     # context. For this reason, we use a constructor fixture here.