generalize setup to allow for testing of other protocols

Author: marten-seemann

.gitignore

@@ -2,7 +2,7 @@ certs/
 logs/
 logs_*/
 *.json
-!implementations.json
+!implementations_quic.json
 web/latest
 
 *.egg-info/

implementations.py

@@ -1,7 +1,6 @@
 import json
 from enum import Enum
-
-IMPLEMENTATIONS = {}
+from typing import Dict
 
 
 class Role(Enum):
@@ -10,16 +9,19 @@ class Role(Enum):
     CLIENT = "client"
 
 
-with open("implementations.json", "r") as f:
-    data = json.load(f)
-    for name, val in data.items():
-        IMPLEMENTATIONS[name] = {"image": val["image"], "url": val["url"]}
-        role = val["role"]
-        if role == "server":
-            IMPLEMENTATIONS[name]["role"] = Role.SERVER
-        elif role == "client":
-            IMPLEMENTATIONS[name]["role"] = Role.CLIENT
-        elif role == "both":
-            IMPLEMENTATIONS[name]["role"] = Role.BOTH
-        else:
-            raise Exception("unknown role: " + role)
+def get_quic_implementations() -> Dict[str, Dict[str, str | Role]]:
+    implementations: Dict[str, Dict[str, str | Role]] = {}
+    with open("implementations_quic.json", "r") as f:
+        data = json.load(f)
+        for name, val in data.items():
+            implementations[name] = {"image": val["image"], "url": val["url"]}
+            role = val["role"]
+            if role == "server":
+                implementations[name]["role"] = Role.SERVER
+            elif role == "client":
+                implementations[name]["role"] = Role.CLIENT
+            elif role == "both":
+                implementations[name]["role"] = Role.BOTH
+            else:
+                raise Exception("unknown role: " + role)
+        return implementations

implementations.json implementations_quic.jsonimplementations.json renamed to implementations_quic.json

@@ -10,13 +10,12 @@
 import tempfile
 from datetime import datetime
 from typing import Callable, List, Tuple
-
 import prettytable
 from termcolor import colored
 
-import testcases
+import testcases_quic
 from result import TestResult
-from testcases import Perspective
+from testcase import Perspective
 
 
 class MeasurementResult:
@@ -50,8 +49,8 @@ def __init__(
         self,
         implementations: dict,
         client_server_pairs: List[Tuple[str, str]],
-        tests: List[testcases.TestCase],
-        measurements: List[testcases.Measurement],
+        tests: List[testcases_quic.TestCase],
+        measurements: List[testcases_quic.Measurement],
         output: str,
         markdown: bool,
         debug: bool,
@@ -117,7 +116,7 @@ def _check_impl_is_compliant(self, name: str, role: Perspective) -> bool:
             dir="/tmp", prefix="compliance_downloads_"
         )
 
-        testcases.generate_cert_chain(certs_dir.name)
+        testcases_quic.generate_cert_chain(certs_dir.name)
 
         if role == Perspective.CLIENT:
             # check that the client is capable of returning UNSUPPORTED
@@ -305,8 +304,8 @@ def _export_results(self):
                 }
                 for x in self._tests + self._measurements
             },
-            "quic_draft": testcases.QUIC_DRAFT,
-            "quic_version": testcases.QUIC_VERSION,
+            "quic_draft": testcases_quic.QUIC_DRAFT,
+            "quic_version": testcases_quic.QUIC_VERSION,
             "results": [],
             "measurements": [],
         }
@@ -367,7 +366,7 @@ def _copy_logs(self, container: str, dir: tempfile.TemporaryDirectory):
             )
 
     def _run_testcase(
-        self, server: str, client: str, test: Callable[[], testcases.TestCase]
+        self, server: str, client: str, test: Callable[[], testcases_quic.TestCase]
     ) -> TestResult:
         return self._run_test(server, client, None, test)[0]
 
@@ -376,7 +375,7 @@ def _run_test(
         server: str,
         client: str,
         log_dir_prefix: None,
-        test: Callable[[], testcases.TestCase],
+        fn: Callable[[], testcases_quic.TestCase],
     ) -> Tuple[TestResult, float]:
         start_time = datetime.now()
         sim_log_dir = tempfile.TemporaryDirectory(dir="/tmp", prefix="logs_sim_")
@@ -390,7 +389,7 @@ def _run_test(
         log_handler.setFormatter(formatter)
         logging.getLogger().addHandler(log_handler)
 
-        testcase = test(
+        test = fn(
             sim_log_dir=sim_log_dir,
             client_keylog_file=client_log_dir.name + "/keys.log",
             server_keylog_file=server_log_dir.name + "/keys.log",
@@ -401,27 +400,27 @@ def _run_test(
             + ". Client: "
             + client
             + ". Running test case: "
-            + str(testcase)
+            + str(test)
         )
 
-        reqs = " ".join([testcase.urlprefix() + p for p in testcase.get_paths()])
+        reqs = " ".join([test.urlprefix() + p for p in test.get_paths()])
         logging.debug("Requests: %s", reqs)
         params = (
             "WAITFORSERVER=server:443 "
-            "CERTS=" + testcase.certs_dir() + " "
-            "TESTCASE_SERVER=" + testcase.testname(Perspective.SERVER) + " "
-            "TESTCASE_CLIENT=" + testcase.testname(Perspective.CLIENT) + " "
-            "WWW=" + testcase.www_dir() + " "
-            "DOWNLOADS=" + testcase.download_dir() + " "
+            "CERTS=" + test.certs_dir() + " "
+            "TESTCASE_SERVER=" + test.testname(Perspective.SERVER) + " "
+            "TESTCASE_CLIENT=" + test.testname(Perspective.CLIENT) + " "
+            "WWW=" + test.www_dir() + " "
+            "DOWNLOADS=" + test.download_dir() + " "
             "SERVER_LOGS=" + server_log_dir.name + " "
             "CLIENT_LOGS=" + client_log_dir.name + " "
             'SCENARIO="{}" '
             "CLIENT=" + self._implementations[client]["image"] + " "
             "SERVER=" + self._implementations[server]["image"] + " "
             'REQUESTS="' + reqs + '" '
-        ).format(testcase.scenario())
-        params += " ".join(testcase.additional_envs())
-        containers = "sim client server " + " ".join(testcase.additional_containers())
+        ).format(test.scenario())
+        params += " ".join(test.additional_envs())
+        containers = "sim client server " + " ".join(test.additional_containers())
         cmd = (
             params
             + " docker compose --env-file empty.env up --abort-on-container-exit --timeout 1 "
@@ -438,7 +437,7 @@ def _run_test(
                 shell=True,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
-                timeout=testcase.timeout(),
+                timeout=test.timeout(),
             )
             output = r.stdout
         except subprocess.TimeoutExpired as ex:
@@ -448,7 +447,7 @@ def _run_test(
         logging.debug("%s", output.decode("utf-8", errors="replace"))
 
         if expired:
-            logging.debug("Test failed: took longer than %ds.", testcase.timeout())
+            logging.debug("Test failed: took longer than %ds.", test.timeout())
             r = subprocess.run(
                 "docker compose --env-file empty.env stop " + containers,
                 shell=True,
@@ -469,7 +468,7 @@ def _run_test(
                 status = TestResult.UNSUPPORTED
             elif any("client exited with code 0" in str(line) for line in lines):
                 try:
-                    status = testcase.check()
+                    status = test.check()
                 except FileNotFoundError as e:
                     logging.error(f"testcase.check() threw FileNotFoundError: {e}")
                     status = TestResult.FAILED
@@ -478,41 +477,41 @@ def _run_test(
         logging.getLogger().removeHandler(log_handler)
         log_handler.close()
         if status == TestResult.FAILED or status == TestResult.SUCCEEDED:
-            log_dir = self._log_dir + "/" + server + "_" + client + "/" + str(testcase)
+            log_dir = self._log_dir + "/" + server + "_" + client + "/" + str(test)
             if log_dir_prefix:
                 log_dir += "/" + log_dir_prefix
             shutil.copytree(server_log_dir.name, log_dir + "/server")
             shutil.copytree(client_log_dir.name, log_dir + "/client")
             shutil.copytree(sim_log_dir.name, log_dir + "/sim")
             shutil.copyfile(log_file.name, log_dir + "/output.txt")
             if self._save_files and status == TestResult.FAILED:
-                shutil.copytree(testcase.www_dir(), log_dir + "/www")
+                shutil.copytree(test.www_dir(), log_dir + "/www")
                 try:
-                    shutil.copytree(testcase.download_dir(), log_dir + "/downloads")
+                    shutil.copytree(test.download_dir(), log_dir + "/downloads")
                 except Exception as exception:
                     logging.info("Could not copy downloaded files: %s", exception)
 
-        testcase.cleanup()
+        test.cleanup()
         server_log_dir.cleanup()
         client_log_dir.cleanup()
         sim_log_dir.cleanup()
         logging.debug(
             "Test: %s took %ss, status: %s",
-            str(testcase),
+            str(test),
             (datetime.now() - start_time).total_seconds(),
             str(status),
         )
 
         # measurements also have a value
-        if hasattr(testcase, "result"):
-            value = testcase.result()
+        if hasattr(test, "result"):
+            value = test.result()
         else:
             value = None
 
         return status, value
 
     def _run_measurement(
-        self, server: str, client: str, test: Callable[[], testcases.Measurement]
+        self, server: str, client: str, test: Callable[[], testcases_quic.Measurement]
     ) -> MeasurementResult:
         values = []
         for i in range(0, test.repetitions()):

interop.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from implementations import IMPLEMENTATIONS
+from implementations import get_quic_implementations
 
 print("Pulling the simulator...")
 os.system("docker pull martenseemann/quic-network-simulator")
@@ -17,14 +17,15 @@ def get_args():
     return parser.parse_args()
 
 
+impls = get_quic_implementations()
 implementations = {}
 if get_args().implementations:
     for s in get_args().implementations.split(","):
-        if s not in [n for n, _ in IMPLEMENTATIONS.items()]:
+        if s not in [n for n, _ in impls.items()]:
             sys.exit("implementation " + s + " not found.")
-        implementations[s] = IMPLEMENTATIONS[s]
+        implementations[s] = impls[s]
 else:
-    implementations = IMPLEMENTATIONS
+    implementations = impls
 
 for name, value in implementations.items():
     print("\nPulling " + name + "...")

pull.py

@@ -4,28 +4,29 @@
 import sys
 from typing import List, Tuple
 
-import testcases
-from implementations import IMPLEMENTATIONS, Role
+import testcase
+from implementations import Role, get_quic_implementations
 from interop import InteropRunner
-from testcases import MEASUREMENTS, TESTCASES
-
-implementations = {
-    name: {"image": value["image"], "url": value["url"]}
-    for name, value in IMPLEMENTATIONS.items()
-}
-client_implementations = [
-    name
-    for name, value in IMPLEMENTATIONS.items()
-    if value["role"] == Role.BOTH or value["role"] == Role.CLIENT
-]
-server_implementations = [
-    name
-    for name, value in IMPLEMENTATIONS.items()
-    if value["role"] == Role.BOTH or value["role"] == Role.SERVER
-]
+from testcases_quic import MEASUREMENTS, TESTCASES_QUIC
 
 
 def main():
+    impls = get_quic_implementations()
+    implementations_all = {
+        name: {"image": value["image"], "url": value["url"]}
+        for name, value in impls.items()
+    }
+    client_implementations = [
+        name
+        for name, value in impls.items()
+        if value["role"] == Role.BOTH or value["role"] == Role.CLIENT
+    ]
+    server_implementations = [
+        name
+        for name, value in impls.items()
+        if value["role"] == Role.BOTH or value["role"] == Role.SERVER
+    ]
+
     def get_args():
         parser = argparse.ArgumentParser()
         parser.add_argument(
@@ -46,7 +47,7 @@ def get_args():
             "-t",
             "--test",
             help="test cases (comma-separatated). Valid test cases are: "
-            + ", ".join([x.name() for x in TESTCASES + MEASUREMENTS]),
+            + ", ".join([x.name() for x in TESTCASES_QUIC + MEASUREMENTS]),
         )
         parser.add_argument(
             "-r",
@@ -92,9 +93,9 @@ def get_args():
             if len(pair) != 2:
                 sys.exit("Invalid format for replace")
             name, image = pair[0], pair[1]
-            if name not in IMPLEMENTATIONS:
+            if name not in impls:
                 sys.exit("Implementation " + name + " not found.")
-            implementations[name]["image"] = image
+            implementations_all[name]["image"] = image
 
     def get_impls(arg, availableImpls, role) -> List[str]:
         if not arg:
@@ -120,20 +121,20 @@ def get_impl_pairs(clients, servers, must_include) -> List[Tuple[str, str]]:
 
     def get_tests_and_measurements(
         arg,
-    ) -> Tuple[List[testcases.TestCase], List[testcases.TestCase]]:
+    ) -> Tuple[List[testcase.TestCase], List[testcase.TestCase]]:
         if arg is None:
-            return TESTCASES, MEASUREMENTS
+            return TESTCASES_QUIC, MEASUREMENTS
         elif arg == "onlyTests":
-            return TESTCASES, []
+            return TESTCASES_QUIC, []
         elif arg == "onlyMeasurements":
             return [], MEASUREMENTS
         elif not arg:
             return []
         tests = []
         measurements = []
         for t in arg.split(","):
-            if t in [tc.name() for tc in TESTCASES]:
-                tests += [tc for tc in TESTCASES if tc.name() == t]
+            if t in [tc.name() for tc in TESTCASES_QUIC]:
+                tests += [tc for tc in TESTCASES_QUIC if tc.name() == t]
             elif t in [tc.name() for tc in MEASUREMENTS]:
                 measurements += [tc for tc in MEASUREMENTS if tc.name() == t]
             else:
@@ -144,7 +145,7 @@ def get_tests_and_measurements(
                         "Available measurements: {}"
                     ).format(
                         t,
-                        ", ".join([t.name() for t in TESTCASES]),
+                        ", ".join([t.name() for t in TESTCASES_QUIC]),
                         ", ".join([t.name() for t in MEASUREMENTS]),
                     )
                 )
@@ -160,7 +161,7 @@ def get_tests_and_measurements(
         if len(kind) == 1:
             no_auto_unsupported.add(kind[0])
     return InteropRunner(
-        implementations=implementations,
+        implementations=implementations_all,
         client_server_pairs=get_impl_pairs(clients, servers, get_args().must_include),
         tests=t[0],
         measurements=t[1],