🐛 🧪 Fix e2e client tests (#73)

* avoid import of private method

* fix postprocessing and generation of html table

* minor change

Author: bisgaard-itis

clients/python/test/e2e/ci/check_for_failures.py

@@ -23,7 +23,9 @@ def main():
         raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
     for pth in result_jsons:
         df = pd.read_json(pth)
-        df = df == pytest.ExitCode.TESTS_FAILED
+        df = (df != pytest.ExitCode.OK) & (
+            df != E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER
+        )
         if df.to_numpy().flatten().any():
             raise typer.Exit(code=pytest.ExitCode.TESTS_FAILED)
 

clients/python/test/e2e/ci/generate_html_table.py

@@ -4,30 +4,29 @@
 import pytest
 import typer
 from _utils import E2eExitCodes
+from postprocess_e2e import exit_code_valid
 
 
 def exitcode_to_text(exitcode: int) -> str:
     """Turn exitcodes to string"""
-    if exitcode == E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER:
-        return "incompatible"
-    elif exitcode == pytest.ExitCode.OK:
-        return "pass"
-    elif exitcode == pytest.ExitCode.TESTS_FAILED:
-        return "fail"
+    if exitcode in set(E2eExitCodes):
+        return E2eExitCodes(exitcode).name
+    elif exitcode in set(pytest.ExitCode):
+        return pytest.ExitCode(exitcode).name
     else:
         raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
 
 
 def make_pretty(entry: str):
     color: str
-    if entry == "incompatible":
+    if entry == E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER.name:
         color = "#999999"
-    elif entry == "pass":
+    elif entry == pytest.ExitCode.OK.name:
         color = "#99FF99"
-    elif entry == "fail":
+    elif entry == pytest.ExitCode.TESTS_FAILED.name:
         color = "#FF9999"
     else:
-        raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
+        color = "#FF00FF"
     return "background-color: %s" % color
 
 
@@ -40,9 +39,10 @@ def main(e2e_artifacts_dir: str) -> None:
     df: pd.DataFrame = pd.DataFrame()
     for file in artifacts.glob("*.json"):
         df = pd.concat([df, pd.read_json(file)], axis=1)
-    any_failure: bool = bool(
-        (df == pytest.ExitCode.TESTS_FAILED).to_numpy().flatten().any()
-    )
+
+    for exit_code in df.to_numpy().flatten():
+        if not exit_code_valid(exit_code):
+            raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
 
     style = [
         {
@@ -62,9 +62,6 @@ def main(e2e_artifacts_dir: str) -> None:
     s.set_table_styles(style)
     s.set_caption("OSPARC e2e python client vs server tests")
     s.to_html(artifacts / "test_results.html")
-    raise typer.Exit(
-        code=pytest.ExitCode.TESTS_FAILED if any_failure else pytest.ExitCode.OK
-    )
 
 
 if __name__ == "__main__":

clients/python/test/e2e/ci/postprocess_e2e.py

@@ -1,6 +1,5 @@
 import warnings
 from pathlib import Path
-from typing import Set
 
 import pandas as pd
 import pytest
@@ -23,6 +22,16 @@ def log(exit_code: int):
     print_line()
 
 
+def exit_code_valid(exit_code: int) -> bool:
+    if exit_code not in set(pytest.ExitCode).union(E2eExitCodes):
+        warnings.warn(
+            f"Received unexpected exitcode {exit_code}. See https://docs.pytest.org/en/7.1.x/reference/exit-codes.html",
+            E2eScriptFailure,
+        )
+        return False
+    return True
+
+
 def main(exit_code: int) -> None:
     """
     Postprocess results from e2e pytests
@@ -41,17 +50,8 @@ def main(exit_code: int) -> None:
         None
     """
     log(exit_code)
-    expected_exitcodes: Set = {
-        E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER,
-        pytest.ExitCode.OK,
-        pytest.ExitCode.TESTS_FAILED,
-    }
-    if exit_code not in expected_exitcodes:
-        warnings.warn(
-            f"Received unexpected exitcode {exit_code}. See https://docs.pytest.org/en/7.1.x/reference/exit-codes.html",
-            E2eScriptFailure,
-        )
-        typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
+    if not exit_code_valid(exit_code):
+        raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
 
     # get config
     try:

clients/python/test/e2e/test_files_api.py

@@ -4,7 +4,6 @@
 import osparc
 import pytest
 from conftest import _KB
-from osparc._utils import compute_sha256
 from packaging.version import Version
 
 
@@ -51,8 +50,7 @@ def test_upload_file(tmp_file: Path, cfg: osparc.Configuration) -> None:
 def test_search_files(
     tmp_file: Path, cfg: osparc.Configuration, use_checksum: bool, use_id: bool
 ) -> None:
-    checksum: str = compute_sha256(tmp_file)
-    assert checksum == _hash_file(tmp_file), "Could not compute correct checksum"
+    checksum: str = _hash_file(tmp_file)
     results: osparc.PaginationGenerator
     with osparc.ApiClient(configuration=cfg) as api_client:
         files_api: osparc.FilesApi = osparc.FilesApi(api_client=api_client)