Fix test-run crash when default server is crashed

Before this change, test-run crashed if the default tarantool server
had crashed with a segmentation fault, and as a result, this caused an
unhandled error with the broken tarantool console handshake. So such a
situation wasn't properly processed by test-run and it failed with the
following traceback:

    Traceback (most recent call last):
      File "/tarantool/test-run/lib/worker.py", line 283, in __init__
        self.inspector = suite.start_server(self.server)
      File "/tarantool/test-run/lib/test_suite.py", line 244, in start_server
        server.deploy(silent=False)
      File "/tarantool/test-run/lib/tarantool_server.py", line 784, in deploy
        self.start(silent=silent, **kwargs)
      File "/tarantool/test-run/lib/tarantool_server.py", line 911, in start
        self.wait_until_started(wait_load, deadline)
      File "/tarantool/test-run/lib/tarantool_server.py", line 1157, in wait_until_started
        ans = yaml.safe_load(temp.execute('box.info.status'))[0]
      File "/tarantool/test-run/lib/tarantool_connection.py", line 161, in execute
        self.opt_reconnect()
      File "/tarantool/test-run/lib/tarantool_connection.py", line 150, in opt_reconnect
        self.reconnect()
      File "/tarantool/test-run/lib/tarantool_connection.py", line 138, in reconnect
        self.connect()
      File "/tarantool/test-run/lib/admin_connection.py", line 96, in connect
        raise RuntimeError('Broken tarantool console handshake')
    RuntimeError: Broken tarantool console handshake

    During handling of the above exception, another exception occurred:

    Traceback (most recent call last):
      File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
        self.run()
      File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
        self._target(*self._args, **self._kwargs)
      File "/tarantool/test-run/dispatcher.py", line 386, in _run_worker
        worker = self.gen_worker(worker_id)
      File "/tarantool/test-run/lib/worker.py", line 297, in __init__
        self.stop_server(cleanup=False)
      File "/tarantool/test-run/lib/worker.py", line 301, in stop_server
        self.suite.stop_server(self.server, self.inspector, silent=silent,
      File "/tarantool/test-run/lib/test_suite.py", line 249, in stop_server
        server.stop(silent=silent)
      File "/tarantool/test-run/lib/tarantool_server.py", line 1092, in stop
        save_join(self.crash_detector)
      File "/tarantool/test-run/lib/tarantool_server.py", line 57, in save_join
        green_obj.get(timeout=timeout)
      File "src/gevent/greenlet.py", line 829, in gevent._gevent_cgreenlet.Greenlet.get
      File "src/gevent/greenlet.py", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception
      File "/home/tarantool/.local/lib/python3.10/site-packages/gevent/_compat.py", line 65, in reraise
        raise value.with_traceback(tb)
      File "src/gevent/greenlet.py", line 906, in gevent._gevent_cgreenlet.Greenlet.run
      File "/tarantool/test-run/lib/test.py", line 35, in _run
        self.callable(*self.callable_args, **self.callable_kwargs)
      File "/tarantool/test-run/lib/tarantool_server.py", line 967, in crash_detect
        self.kill_current_test()
      File "/tarantool/test-run/lib/tarantool_server.py", line 1032, in kill_current_test
        if self.current_test.current_test_greenlet:
    AttributeError: 'NoneType' object has no attribute 'current_test_greenlet'

Now test-run handles the segmentation fault of the default server and
the error with the broken tarantool console handshake properly.

Fixes tarantool#398

Author: ylobankov

lib/admin_connection.py

@@ -80,6 +80,10 @@ def cmd(self, socket, cmd, silent):
         return res
 
 
+class BrokenConsoleHandshake(RuntimeError):
+    pass
+
+
 class AdminConnection(TarantoolConnection, ExecMixIn):
     def execute_no_reconnect(self, command, silent):
         if not command:
@@ -93,7 +97,7 @@ def connect(self):
         super(AdminConnection, self).connect()
         handshake = get_handshake(self.socket)
         if not re.search(r'^Tarantool.*console.*', str(handshake)):
-            raise RuntimeError('Broken tarantool console handshake')
+            raise BrokenConsoleHandshake('Broken tarantool console handshake')
 
 
 class AdminAsyncConnection(TarantoolAsyncConnection, ExecMixIn):

lib/tarantool_server.py

@@ -26,7 +26,7 @@
     # Python 3
     from io import StringIO
 
-from lib.admin_connection import AdminConnection, AdminAsyncConnection
+from lib.admin_connection import AdminConnection, AdminAsyncConnection, BrokenConsoleHandshake
 from lib.box_connection import BoxConnection
 from lib.colorer import color_stdout
 from lib.colorer import color_log
@@ -421,14 +421,17 @@ def execute(self, server):
 
 
 class TarantoolStartError(OSError):
-    def __init__(self, name=None, timeout=None):
+    def __init__(self, name=None, timeout=None, reason=None):
         self.name = name
         self.timeout = timeout
+        self.reason = reason
 
     def __str__(self):
         message = '[Instance "{}"] Failed to start'.format(self.name)
         if self.timeout:
-            return "\n{} within {} seconds\n".format(message, self.timeout)
+            message = "{} within {} seconds".format(message, self.timeout)
+        if self.reason:
+            message = "{}: {}".format(message, self.reason)
         return "\n{}\n".format(message)
 
 
@@ -957,7 +960,7 @@ def crash_detect(self):
             if self.process.returncode is None:
                 gevent.sleep(0.1)
 
-        if self.process.returncode in [0, -signal.SIGKILL, -signal.SIGTERM]:
+        if self.process.returncode in [0, -signal.SIGABRT, -signal.SIGKILL, -signal.SIGTERM]:
             return
 
         self.kill_current_test()
@@ -1168,6 +1171,8 @@ def wait_until_started(self, wait_load=True, deadline=None):
                     gevent.sleep(0.1)
                     continue
                 raise
+            except BrokenConsoleHandshake as e:
+                raise TarantoolStartError(self.name, reason=e)
         else:
             raise TarantoolStartError(
                 self.name, Options().args.server_start_timeout)

lib/worker.py

@@ -13,6 +13,7 @@
 from lib.colorer import color_log
 from lib.colorer import color_stdout
 from lib.tarantool_server import TarantoolServer
+from lib.tarantool_server import TarantoolStartError
 from lib.test import get_result
 from lib.test_suite import TestSuite
 from lib.utils import safe_makedirs
@@ -285,6 +286,9 @@ def __init__(self, suite, _id):
         except KeyboardInterrupt:
             self.report_keyboard_interrupt()
             self.stop_server(cleanup=False)
+        except TarantoolStartError as e:
+            color_stdout(e, schema='error')
+            self.stop_server(cleanup=False)
         except Exception as e:
             color_stdout('Worker "%s" cannot start tarantool server; '
                          'the tasks will be ignored...\n' % self.name,