Extend tests summary table (#626)

Author: MakisH

tools/tests/build_docker_images.py

@@ -74,7 +74,7 @@ def main():
         t = time.perf_counter()
         result = systemtest.run_only_build(run_directory)
         elapsed_time = time.perf_counter() - t
-        logging.info(f"Building image for {systemtest} took {elapsed_time} seconds")
+        logging.info(f"Building image for {systemtest} took {elapsed_time:^.1f} seconds")
         results.append(result)
 
     build_docker_success = True

tools/tests/generate_reference_results.py

@@ -123,7 +123,7 @@ def main():
         t = time.perf_counter()
         result = systemtest.run_for_reference_results(run_directory)
         elapsed_time = time.perf_counter() - t
-        logging.info(f"Running {systemtest} took {elapsed_time} seconds")
+        logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds")
         if not result.success:
             raise RuntimeError(f"Failed to execute {systemtest}")
         reference_result_per_tutorial[systemtest.tutorial] = []

tools/tests/systemtests.py

@@ -74,7 +74,7 @@ def main():
         t = time.perf_counter()
         result = systemtest.run(run_directory)
         elapsed_time = time.perf_counter() - t
-        logging.info(f"Running {systemtest} took {elapsed_time} seconds")
+        logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds")
         results.append(result)
 
     system_test_success = True

tools/tests/systemtests/Systemtest.py

@@ -84,27 +84,36 @@ def _get_length_of_name(results: List[SystemtestResult]) -> int:
     max_name_length = _get_length_of_name(results)
 
     header = f"| {'systemtest':<{max_name_length + 2}} | {'success':^7} | {'building time [s]':^17} | {'solver time [s]':^15} | {'fieldcompare time [s]':^21} |"
-    separator = "+-" + "-" * (max_name_length + 2) + \
+    separator_plaintext = "+-" + "-" * (max_name_length + 2) + \
         "-+---------+-------------------+-----------------+-----------------------+"
+    separator_markdown = "| --- | --- | --- | --- | --- |"
 
-    print(separator)
+    print(separator_plaintext)
     print(header)
-    print(separator)
+    print(separator_plaintext)
 
     if "GITHUB_STEP_SUMMARY" in os.environ:
         with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
-            print(separator, file=f)
             print(header, file=f)
-            print(separator, file=f)
+            print(separator_markdown, file=f)
 
     for result in results:
-        row = f"| {str(result.systemtest):<{max_name_length + 2}} | {result.success:^7} | {result.build_time:^17.2f} | {result.solver_time:^15.2f} | {result.fieldcompare_time:^21.2f} |"
+        row = f"| {str(result.systemtest):<{max_name_length + 2}} | {result.success:^7} | {result.build_time:^17.1f} | {result.solver_time:^15.1f} | {result.fieldcompare_time:^21.1f} |"
         print(row)
-        print(separator)
+        print(separator_plaintext)
         if "GITHUB_STEP_SUMMARY" in os.environ:
             with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
                 print(row, file=f)
-                print(separator, file=f)
+
+    if "GITHUB_STEP_SUMMARY" in os.environ:
+        with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
+            print("\n\n", file=f)
+            print(
+                "In case a test fails, download the archive from the bottom of this page and look into each `stdout.log` and `stderr.log`. The time spent in each step might already give useful hints.",
+                file=f)
+            print(
+                "See the [documentation](https://precice.org/dev-docs-system-tests.html#understanding-what-went-wrong).",
+                file=f)
 
 
 @dataclass