refactor autobahn test runners

  * client & server:  keep running on exception during case run
  * client: display case ID's

Author: belm0

autobahn/client.py

@@ -3,7 +3,9 @@
 test_client.py in wsproto.
 '''
 import argparse
+import json
 import logging
+import sys
 
 import trio
 from trio_websocket import open_websocket_url, ConnectionClosed
@@ -23,6 +25,12 @@ async def get_case_count(url):
     return int(case_count)
 
 
+async def get_case_info(url, case):
+    url = f'{url}/getCaseInfo?case={case}'
+    async with open_websocket_url(url) as conn:
+        return json.loads(await conn.get_message())
+
+
 async def run_case(url, case):
     url = f'{url}/runCase?case={case}&agent={AGENT}'
     try:
@@ -52,13 +60,23 @@ async def run_tests(args):
         logging.getLogger('trio-websocket').setLevel(logging.DEBUG)
     else:
         case_count = await get_case_count(args.url)
-        test_cases = range(1, case_count + 1)
+        test_cases = list(range(1, case_count + 1))
+    exception_cases = []
     for case in test_cases:
+        case_id = (await get_case_info(args.url, case))['id']
         if case_count:
-            logger.info("Running test case %d of %d", case, case_count)
+            logger.info("Running test case %s (%d of %d)", case_id, case, case_count)
         else:
-            logger.info("Debugging test case %d", case)
-        await run_case(args.url, case)
+            logger.info("Debugging test case %s (%d)", case_id, case)
+        try:
+            await run_case(args.url, case)
+        except Exception:  # pylint: disable=broad-exception-caught
+            logger.exception('  runtime exception during test case %s (%d)', case_id, case)
+            exception_cases.append(case_id)
+    if exception_cases:
+        logger.error('Runtime exception in %d of %d test cases: %s',
+                     len(exception_cases), len(test_cases), exception_cases)
+        sys.exit(1)
     if not args.debug_cases:
         # Don't update report when debugging a single test case. It adds noise
         # to the debug logging.
@@ -71,6 +89,7 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Autobahn client for'
         ' trio-websocket')
     parser.add_argument('url', help='WebSocket URL for server')
+    # TODO: accept case ID's rather than indices
     parser.add_argument('debug_cases', type=int, nargs='*', help='Run'
         ' individual test cases with debug logging (optional)')
     return parser.parse_args()

autobahn/server.py

@@ -12,8 +12,7 @@
 import logging
 
 import trio
-from trio_websocket import serve_websocket, ConnectionClosed
-
+from trio_websocket import serve_websocket, ConnectionClosed, WebSocketRequest
 
 BIND_IP = '0.0.0.0'
 BIND_PORT = 9000
@@ -31,7 +30,7 @@ async def main():
                           max_message_size=MAX_MESSAGE_SIZE)
 
 
-async def handler(request):
+async def handler(request: WebSocketRequest):
     ''' Reverse incoming websocket messages and send them back. '''
     global connection_count  # pylint: disable=global-statement
     connection_count += 1
@@ -43,6 +42,8 @@ async def handler(request):
             await ws.send_message(message)
         except ConnectionClosed:
             break
+        except Exception:  # pylint: disable=broad-exception-caught
+            logger.exception('  runtime exception handling connection #%d', connection_count)
 
 
 def parse_args():

docs/contributing.rst

@@ -151,7 +151,7 @@ In the second terminal, you will run the Docker image::
           -p 9000:9000 \
           --name autobahn \
           crossbario/autobahn-testsuite \
-          /usr/local/bin/wstest --mode fuzzingclient --spec /config/fuzzingclient.json
+          wstest --mode fuzzingclient --spec /config/fuzzingclient.json
 
 If a test fails, ``server.py`` does not support the same ``debug_cases``
 argument as ``client.py``, but you can modify `fuzzingclient.json` to specify a