Various fixes for the compliance test suite.

Author: aaugustin

compliance/README.rst

@@ -38,7 +38,7 @@ To test the servers:
         crossbario/autobahn-testsuite \
         wstest --mode fuzzingclient --spec /config/fuzzingclient.json
 
-    $ open reports/servers/index.html
+    $ open compliance/reports/servers/index.html
 
 To test the clients:
 
@@ -54,7 +54,7 @@ To test the clients:
     $ PYTHONPATH=src python compliance/asyncio/client.py
     $ PYTHONPATH=src python compliance/sync/client.py
 
-    $ open reports/clients/index.html
+    $ open compliance/reports/clients/index.html
 
 Conformance notes
 -----------------
@@ -67,6 +67,13 @@ In 3.2, 3.3, 4.1.3, 4.1.4, 4.2.3, 4.2.4, and 5.15 websockets notices the
 protocol error and closes the connection at the library level before the
 application gets a chance to echo the previous frame.
 
-In 6.4.3 and 6.4.4, even though it uses an incremental decoder, websockets
-doesn't notice the invalid utf-8 fast enough to get a "Strict" pass. These tests
-are more strict than the RFC.
+In 6.4.1, 6.4.2, 6.4.3, and 6.4.4, even though it uses an incremental decoder,
+websockets doesn't notice the invalid utf-8 fast enough to get a "Strict" pass.
+These tests are more strict than the RFC.
+
+Test case 7.1.5 fails because websockets treats closing the connection in the
+middle of a fragmented message as a protocol error. As a consequence, it sends
+a close frame with code 1002. The test suite expects a close frame with code
+1000, echoing the close code that it sent. This isn't required. RFC 6455 states
+that "the endpoint typically echos the status code it received", which leaves
+the possibility to send a close frame with a different status code.

compliance/asyncio/client.py

@@ -8,7 +8,9 @@
 
 logging.basicConfig(level=logging.WARNING)
 
-SERVER = "ws://127.0.0.1:9001"
+SERVER = "ws://localhost:9001"
+
+AGENT = "websockets.asyncio"
 
 
 async def get_case_count():
@@ -18,16 +20,21 @@ async def get_case_count():
 
 async def run_case(case):
     async with connect(
-        f"{SERVER}/runCase?case={case}",
-        user_agent_header="websockets.asyncio",
+        f"{SERVER}/runCase?case={case}&agent={AGENT}",
         max_size=2**25,
     ) as ws:
-        async for msg in ws:
-            await ws.send(msg)
+        try:
+            async for msg in ws:
+                await ws.send(msg)
+        except WebSocketException:
+            pass
 
 
 async def update_reports():
-    async with connect(f"{SERVER}/updateReports", open_timeout=60):
+    async with connect(
+        f"{SERVER}/updateReports?agent={AGENT}",
+        open_timeout=60,
+    ):
         pass
 
 
@@ -43,7 +50,7 @@ async def main():
             print(f"FAIL: {type(exc).__name__}: {exc}")
         else:
             print("OK")
-    print("Ran {cases} test cases")
+    print(f"Ran {cases} test cases")
     await update_reports()
     print("Updated reports")
 

compliance/asyncio/server.py

@@ -2,6 +2,7 @@
 import logging
 
 from websockets.asyncio.server import serve
+from websockets.exceptions import WebSocketException
 
 
 logging.basicConfig(level=logging.WARNING)
@@ -10,8 +11,11 @@
 
 
 async def echo(ws):
-    async for msg in ws:
-        await ws.send(msg)
+    try:
+        async for msg in ws:
+            await ws.send(msg)
+    except WebSocketException:
+        pass
 
 
 async def main():

compliance/config/fuzzingclient.json

@@ -5,7 +5,7 @@
    }, {
       "url": "ws://host.docker.internal:9003"
    }],
-   "outdir": "./reports/servers",
+   "outdir": "/reports/servers",
    "cases": ["*"],
    "exclude-cases": []
 }

compliance/config/fuzzingserver.json

@@ -1,7 +1,7 @@
 
 {
    "url": "ws://localhost:9001",
-   "outdir": "./reports/clients",
+   "outdir": "/reports/clients",
    "cases": ["*"],
    "exclude-cases": []
 }

compliance/sync/client.py

@@ -7,7 +7,9 @@
 
 logging.basicConfig(level=logging.WARNING)
 
-SERVER = "ws://127.0.0.1:9001"
+SERVER = "ws://localhost:9001"
+
+AGENT = "websockets.sync"
 
 
 def get_case_count():
@@ -17,16 +19,21 @@ def get_case_count():
 
 def run_case(case):
     with connect(
-        f"{SERVER}/runCase?case={case}",
-        user_agent_header="websockets.sync",
+        f"{SERVER}/runCase?case={case}&agent={AGENT}",
         max_size=2**25,
     ) as ws:
-        for msg in ws:
-            ws.send(msg)
+        try:
+            for msg in ws:
+                ws.send(msg)
+        except WebSocketException:
+            pass
 
 
 def update_reports():
-    with connect(f"{SERVER}/updateReports", open_timeout=60):
+    with connect(
+        f"{SERVER}/updateReports?agent={AGENT}",
+        open_timeout=60,
+    ):
         pass
 
 
@@ -42,7 +49,7 @@ def main():
             print(f"FAIL: {type(exc).__name__}: {exc}")
         else:
             print("OK")
-    print("Ran {cases} test cases")
+    print(f"Ran {cases} test cases")
     update_reports()
     print("Updated reports")
 

compliance/sync/server.py

@@ -1,5 +1,6 @@
 import logging
 
+from websockets.exceptions import WebSocketException
 from websockets.sync.server import serve
 
 
@@ -9,8 +10,11 @@
 
 
 def echo(ws):
-    for msg in ws:
-        ws.send(msg)
+    try:
+        for msg in ws:
+            ws.send(msg)
+    except WebSocketException:
+        pass
 
 
 def main():