Provide better visibility on jobs timing out

Job timeouts are handled by setting an alarm to fire after the
desired timeout, however there's currently nothing to explicitly
log the failure.

Introduce a handler which logs these instances but otherwise does
not change the behaviour.

Author: PeterJCLaw

django_lightweight_queue/worker.py

@@ -159,13 +159,25 @@ def configure_cancellation(self, timeout: Optional[int], sigkill_on_stop: bool)
             signal.signal(signal.SIGUSR2, self._handle_sigusr2)
 
         if timeout is not None:
+            signal.signal(signal.SIGALRM, self._handle_alarm)
             # alarm(3) takes whole seconds
             alarm_duration = int(math.ceil(timeout))
             signal.alarm(alarm_duration)
         else:
             # Cancel any scheduled alarms
             signal.alarm(0)
 
+    def _handle_alarm(self, signal_number: int, frame: object) -> None:
+        # Log for observability
+        self.log(logging.ERROR, "Alarm received: job has timed out")
+
+        # Disconnect ourselves then re-signal so that Python does what it
+        # normally would. We could raise an exception here, however raising
+        # exceptions from signal handlers is generally discouraged.
+        signal.signal(signal.SIGALRM, signal.SIG_DFL)
+        # TODO(python-upgrade): use signal.raise_signal on Python 3.8+
+        os.kill(os.getpid(), signal.SIGALRM)
+
     def set_process_title(self, *titles: str) -> None:
         set_process_title(self.name, *titles)