152 improve client e2e tests (#153)

* update workflow before publishing python package

* fix dependency issue and bump version

* point to website in project description

* fix broken dependency

* improve doc

* add github token to download artifacts

* ensure only read-access @wvangeit

* yet another attempt at downloading artifacts

* make sure to use repo that ran the trigger wf

* another attempt at fixing

* change owner

* allow publishing to testpypi also when pr

* minor change

* revert minor (but breaking) change

* minor fix

* add debug messages

* another debug message

* hopefully the final version

* final fix

* minor fix

* move master and tag to individual jobs

* add debug messages

* dev->post

* add python script for determining semantic version

* minor changes

* minor changes

* improve error handling and add version file to artifacts

* check if release

* minor fix

* ensure to enter venv

* also when tagging

* source venv in publishin workflow

* ensure only master

* add script for testing 'pure' semver

* adapt workflows to new python script

* minor change

* attempt to evaluate expressions correctly

* several fixes to fix tests

* ensure repo is checked out in publish workflow

* several small fixes

* cleanup

* debug

* minor cleanup

* mionr changes

* add debug message

* minor change

* minor change

* yet another try

* minor change

* minor change

* minor change

* mionr change

* minor changes

* correct workflow run id

* cosmetic change

* avoid using gh

* change to a single job for publishing

* minor cleanup

* swap loops in clean up jobs

* correction

* update server compatibility to new url

* minor change to trigger ci

* improve name of skipping tests

* improve naming

* print html table

* pretty print test result table

* improve printing of html table

* remove redudnant file

* cosmetic change

* improve table

* @pcrespov dont cast generator to list

Author: bisgaard-itis

clients/python/test/e2e/ci/e2e/e2e/_utils.py

@@ -35,7 +35,7 @@ class E2eExitCodes(IntEnum):
     """
 
     CI_SCRIPT_FAILURE = 100
-    INCOMPATIBLE_CLIENT_SERVER = 101
+    SKIPPING_TESTS = 101
     INVALID_JSON_DATA = 102
 
 

clients/python/test/e2e/ci/e2e/e2e/postprocess.py

@@ -111,22 +111,20 @@ def single_testrun(exit_code: int) -> None:
 
 @cli.command()
 @handle_validation_error
-def check_for_failure():
+def check_for_failure(e2e_artifacts_dir: str):
     """Loop through all json artifacts and fail in case of testfailure"""
-    pytest_ini: PytestIniFile = PytestIniFile.create_from_file()
-    artifacts: Artifacts = pytest_ini.artifacts
-    if not artifacts.log_dir.is_dir():
-        raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
-    result_jsons = list(Path(artifacts.artifact_dir).glob("*.json"))
-    if not len(result_jsons):
+    artifacts: Path = Path(e2e_artifacts_dir)
+    if not artifacts.is_dir():
         raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
-    for pth in result_jsons:
+    count = 0
+    for pth in artifacts.glob("*.json"):
+        count += 1
         df = pd.read_json(pth)
-        df = (df != pytest.ExitCode.OK) & (
-            df != E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER
-        )
+        df = (df != pytest.ExitCode.OK) & (df != E2eExitCodes.SKIPPING_TESTS)
         if df.to_numpy().flatten().any():
             raise typer.Exit(code=pytest.ExitCode.TESTS_FAILED)
+    if count == 0:
+        raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
 
 
 def _exitcode_to_text(exitcode: int) -> str:
@@ -141,7 +139,7 @@ def _exitcode_to_text(exitcode: int) -> str:
 
 def _make_pretty(entry: str):
     color: str
-    if entry == E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER.name:
+    if entry == E2eExitCodes.SKIPPING_TESTS.name:
         color = "#999999"
     elif entry == pytest.ExitCode.OK.name:
         color = "#99FF99"
@@ -185,6 +183,8 @@ def generate_html_table(e2e_artifacts_dir: str) -> None:
     s.set_table_styles(style)
     s.set_caption("OSPARC e2e python client vs server tests")
     s.to_html(artifacts / "test_results.html")
+    df.rename_axis("OSPARC e2e python client vs server test")
+    typer.echo(df.to_markdown(tablefmt="double_grid"))
 
 
 @cli.command()

clients/python/test/e2e/ci/e2e/e2e/preprocess.py

@@ -116,6 +116,6 @@ def check_compatibility() -> None:
         raise typer.Exit(code=E2eExitCodes.CI_SCRIPT_FAILURE)
 
     if not is_compatible:
-        raise typer.Exit(code=E2eExitCodes.INCOMPATIBLE_CLIENT_SERVER)
+        raise typer.Exit(code=E2eExitCodes.SKIPPING_TESTS)
     else:
         raise typer.Exit(code=pytest.ExitCode.OK)