add WebTransport handshake, stream and datagram tests

Author: marten-seemann

.gitignore

@@ -2,7 +2,7 @@ certs/
 logs/
 logs_*/
 *.json
-!implementations_quic.json
+!implementations_*.json
 web/latest
 
 *.egg-info/

docker-compose.yml

@@ -42,6 +42,8 @@ services:
       - SSLKEYLOGFILE=/logs/keys.log
       - QLOGDIR=/logs/qlog/
       - TESTCASE=$TESTCASE_SERVER
+      - REQUESTS=$REQUESTS_SERVER
+      - PROTOCOLS=$PROTOCOLS_SERVER
     depends_on:
       - sim
     cap_add:
@@ -74,7 +76,8 @@ services:
       - SSLKEYLOGFILE=/logs/keys.log
       - QLOGDIR=/logs/qlog/
       - TESTCASE=$TESTCASE_CLIENT
-      - REQUESTS=$REQUESTS
+      - REQUESTS=$REQUESTS_CLIENT
+      - PROTOCOLS=$PROTOCOLS_CLIENT
     depends_on:
       - sim
     cap_add:

empty.env

@@ -12,3 +12,5 @@ TESTCASE_CLIENT=""
 TESTCASE_SERVER=""
 WWW=""
 WAITFORSERVER=""
+PROTOCOLS_CLIENT=""
+PROTOCOLS_SERVER=""

implementations.py

@@ -10,8 +10,16 @@ class Role(Enum):
 
 
 def get_quic_implementations() -> Dict[str, Dict[str, str | Role]]:
+    return get_implementations("implementations_quic.json")
+
+
+def get_webtransport_implementations() -> Dict[str, Dict[str, str | Role]]:
+    return get_implementations("implementations_webtransport.json")
+
+
+def get_implementations(filename: str) -> Dict[str, Dict[str, str | Role]]:
     implementations: Dict[str, Dict[str, str | Role]] = {}
-    with open("implementations_quic.json", "r") as f:
+    with open(filename, "r") as f:
         data = json.load(f)
         for name, val in data.items():
             implementations[name] = {"image": val["image"], "url": val["url"]}

implementations_webtransport.json

@@ -0,0 +1,12 @@
+{
+  "webtransport-go": {
+    "image": "martenseemann/webtransport-go-interop:latest",
+    "url": "https://github.com/quic-go/webtransport-go",
+    "role": "both"
+  },
+  "chrome": {
+    "image": "martenseemann/chrome-webtransport-interop-runner",
+    "url": "https://github.com/quic-interop/chrome-webtransport-interop-runner",
+    "role": "client"
+  }
+}

interop.py

@@ -413,7 +413,7 @@ def _run_test(
             + str(test)
         )
 
-        reqs = " ".join([test.urlprefix() + p for p in test.get_paths()])
+        reqs = test.get_paths()
         logging.debug("Requests: %s", reqs)
         params = (
             "WAITFORSERVER=server:443 "
@@ -429,7 +429,8 @@ def _run_test(
             'SCENARIO="{}" '
             "CLIENT=" + self._implementations[client]["image"] + " "
             "SERVER=" + self._implementations[server]["image"] + " "
-            'REQUESTS="' + reqs + '" '
+            'REQUESTS_CLIENT="' + " ".join(reqs) + '" '
+            'REQUESTS_SERVER="' + " ".join(test.get_paths_server()) + '" '
         ).format(test.scenario())
         params += " ".join(test.additional_envs())
         containers = "sim client server " + " ".join(test.additional_containers())
@@ -476,9 +477,20 @@ def _run_test(
 
         if not expired:
             lines = output.splitlines()
+            exit_lines = [
+                str(line) for line in lines if "exited with code" in str(line)
+            ]
+            if exit_lines:
+                logging.debug("Container exits: %s", " | ".join(exit_lines))
+            client_exited_ok = any(
+                "client exited with code 0" in str(line) for line in lines
+            )
+            server_exited_ok = any(
+                "server exited with code 0" in str(line) for line in lines
+            )
             if self._is_unsupported(lines):
                 status = TestResult.UNSUPPORTED
-            elif any("client exited with code 0" in str(line) for line in lines):
+            elif client_exited_ok or server_exited_ok:
                 try:
                     status = test.check()
                 except FileNotFoundError as e:

pull.py

@@ -2,31 +2,44 @@
 import os
 import sys
 
-from implementations import get_quic_implementations
-
-print("Pulling the simulator...")
-os.system("docker pull martenseemann/quic-network-simulator")
-
-print("\nPulling the iperf endpoint...")
-os.system("docker pull martenseemann/quic-interop-iperf-endpoint")
+from implementations import get_quic_implementations, get_webtransport_implementations
 
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-p",
+        "--protocol",
+        default="quic",
+        help="quic / webtransport",
+    )
     parser.add_argument("-i", "--implementations", help="implementations to pull")
     return parser.parse_args()
 
 
-impls = get_quic_implementations()
+args = get_args()
+if args.protocol == "quic":
+    impls = get_quic_implementations()
+elif args.protocol == "webtransport":
+    impls = get_webtransport_implementations()
+else:
+    sys.exit("Unknown protocol: " + args.protocol)
 implementations = {}
-if get_args().implementations:
-    for s in get_args().implementations.split(","):
+if args.implementations:
+    for s in args.implementations.split(","):
         if s not in [n for n, _ in impls.items()]:
             sys.exit("implementation " + s + " not found.")
         implementations[s] = impls[s]
 else:
     implementations = impls
 
+print("Pulling the simulator...")
+os.system("docker pull martenseemann/quic-network-simulator")
+
+if args.protocol == "quic":
+    print("\nPulling the iperf endpoint...")
+    os.system("docker pull martenseemann/quic-interop-iperf-endpoint")
+
 for name, value in implementations.items():
     print("\nPulling " + name + "...")
     os.system("docker pull " + value["image"])

run.py

@@ -5,30 +5,25 @@
 from typing import List, Tuple
 
 import testcase
-from implementations import Role, get_quic_implementations
+from implementations import (
+    Role,
+    get_quic_implementations,
+    get_webtransport_implementations,
+)
 from interop import InteropRunner
 from testcases_quic import MEASUREMENTS, TESTCASES_QUIC
+from testcases_webtransport import TESTCASES_WEBTRANSPORT
 
 
 def main():
-    impls = get_quic_implementations()
-    implementations_all = {
-        name: {"image": value["image"], "url": value["url"]}
-        for name, value in impls.items()
-    }
-    client_implementations = [
-        name
-        for name, value in impls.items()
-        if value["role"] == Role.BOTH or value["role"] == Role.CLIENT
-    ]
-    server_implementations = [
-        name
-        for name, value in impls.items()
-        if value["role"] == Role.BOTH or value["role"] == Role.SERVER
-    ]
-
     def get_args():
         parser = argparse.ArgumentParser()
+        parser.add_argument(
+            "-p",
+            "--protocol",
+            default="quic",
+            help="quic / webtransport",
+        )
         parser.add_argument(
             "-d",
             "--debug",
@@ -86,6 +81,29 @@ def get_args():
         )
         return parser.parse_args()
 
+    protocol = get_args().protocol
+    if protocol == "quic":
+        impls = get_quic_implementations()
+    elif protocol == "webtransport":
+        impls = get_webtransport_implementations()
+    else:
+        sys.exit("Unknown protocol: " + protocol)
+
+    implementations_all = {
+        name: {"image": value["image"], "url": value["url"]}
+        for name, value in impls.items()
+    }
+    client_implementations = [
+        name
+        for name, value in impls.items()
+        if value["role"] == Role.BOTH or value["role"] == Role.CLIENT
+    ]
+    server_implementations = [
+        name
+        for name, value in impls.items()
+        if value["role"] == Role.BOTH or value["role"] == Role.SERVER
+    ]
+
     replace_arg = get_args().replace
     if replace_arg:
         for s in replace_arg.split(","):
@@ -122,19 +140,23 @@ def get_impl_pairs(clients, servers, must_include) -> List[Tuple[str, str]]:
     def get_tests_and_measurements(
         arg,
     ) -> Tuple[List[testcase.TestCase], List[testcase.TestCase]]:
+        if protocol == "quic":
+            testcases = TESTCASES_QUIC
+        elif protocol == "webtransport":
+            testcases = TESTCASES_WEBTRANSPORT
         if arg is None:
-            return TESTCASES_QUIC, MEASUREMENTS
+            return testcases, MEASUREMENTS
         elif arg == "onlyTests":
-            return TESTCASES_QUIC, []
+            return testcases, []
         elif arg == "onlyMeasurements":
             return [], MEASUREMENTS
         elif not arg:
             return []
         tests = []
         measurements = []
         for t in arg.split(","):
-            if t in [tc.name() for tc in TESTCASES_QUIC]:
-                tests += [tc for tc in TESTCASES_QUIC if tc.name() == t]
+            if t in [tc.name() for tc in testcases]:
+                tests += [tc for tc in testcases if tc.name() == t]
             elif t in [tc.name() for tc in MEASUREMENTS]:
                 measurements += [tc for tc in MEASUREMENTS if tc.name() == t]
             else:
@@ -145,7 +167,7 @@ def get_tests_and_measurements(
                         "Available measurements: {}"
                     ).format(
                         t,
-                        ", ".join([t.name() for t in TESTCASES_QUIC]),
+                        ", ".join([t.name() for t in testcases]),
                         ", ".join([t.name() for t in MEASUREMENTS]),
                     )
                 )

testcase.py

@@ -189,12 +189,18 @@ def _server_trace(self):
             self._cached_server_trace = TraceAnalyzer(trace, self._keylog_file())
         return self._cached_server_trace
 
-    def _generate_random_file(self, size: int, filename: str = None) -> str:
+    def _generate_random_file(
+        self, size: int, filename: str = None, directory: str = None
+    ) -> str:
         if filename is None:
             filename = generate_slug()
         # see https://www.stefanocappellini.it/generate-pseudorandom-bytes-with-python/ for benchmarks
         enc = AES.new(os.urandom(32), AES.MODE_OFB, b"a" * 16)
-        f = open(self.server_www_dir() + filename, "wb")
+        if directory is not None:
+            path = os.path.join(directory, filename)
+        else:
+            path = os.path.join(self.server_www_dir(), filename)
+        f = open(path, "wb")
         f.write(enc.encrypt(b" " * size))
         f.close()
         logging.debug("Generated random file: %s of size: %d", filename, size)
@@ -213,29 +219,40 @@ def _check_version_and_files(self) -> bool:
             return False
         return self._check_files()
 
-    def _check_files(self) -> bool:
-        if len(self._files) == 0:
+    def _check_files(
+        self, source_dir: str = None, download_dir: str = None, files: List[str] = None
+    ) -> bool:
+        if source_dir is None:
+            source_dir = self.server_www_dir()
+        if download_dir is None:
+            download_dir = self.client_download_dir()
+        if files is None:
+            files = self._files
+
+        if len(files) == 0:
             raise Exception("No test files generated.")
-        files = [
+
+        downloaded_files = [
             n
-            for n in os.listdir(self.client_download_dir())
-            if os.path.isfile(os.path.join(self.client_download_dir(), n))
+            for n in os.listdir(download_dir)
+            if os.path.isfile(os.path.join(download_dir, n))
         ]
-        too_many = [f for f in files if f not in self._files]
+        too_many = [f for f in downloaded_files if f not in files]
         if len(too_many) != 0:
             logging.info("Found unexpected downloaded files: %s", too_many)
-        too_few = [f for f in self._files if f not in files]
+        too_few = [f for f in files if f not in downloaded_files]
         if len(too_few) != 0:
             logging.info("Missing files: %s", too_few)
         if len(too_many) != 0 or len(too_few) != 0:
             return False
-        for f in self._files:
-            fp = self.client_download_dir() + f
+
+        for f in files:
+            fp = os.path.join(download_dir, f)
             if not os.path.isfile(fp):
                 logging.info("File %s does not exist.", fp)
                 return False
             try:
-                size = os.path.getsize(self.server_www_dir() + f)
+                size = os.path.getsize(os.path.join(source_dir, f))
                 downloaded_size = os.path.getsize(fp)
                 if size != downloaded_size:
                     logging.info(
@@ -245,13 +262,13 @@ def _check_files(self) -> bool:
                         downloaded_size,
                     )
                     return False
-                if not filecmp.cmp(self.server_www_dir() + f, fp, shallow=False):
+                if not filecmp.cmp(os.path.join(source_dir, f), fp, shallow=False):
                     logging.info("File contents of %s do not match.", fp)
                     return False
             except Exception as exception:
                 logging.info(
                     "Could not compare files %s and %s: %s",
-                    self.server_www_dir() + f,
+                    os.path.join(source_dir, f),
                     fp,
                     exception,
                 )
@@ -299,9 +316,14 @@ def cleanup(self):
             self._server_download_dir.cleanup()
             self._server_download_dir = None
 
-    @abc.abstractmethod
     def get_paths(self):
-        pass
+        return []
+
+    def get_paths_raw(self):
+        return []
+
+    def get_paths_server(self):
+        return []
 
     @abc.abstractmethod
     def check(self) -> TestResult: