Skip to content

Commit f674691

Browse files
bisgaard-itisbisgaard-itis
authored
🔨 Clean up e2e testing framework (#65)
* begin improving logging and refactoring of tests * minor change * several more changes * enter test folder directly * cosmetic change * make sure there is a make recipe for running tests * make postprocess job fail in case of failure * small correction * change name of generated html * change to new compatibility.json * modify compatibility json and script * fix compatibility script * improve readability * adapt proprocessing bash script * minor fix * minor change * correct indexing * fix line issue * test tb flag * reset tb flag * make sure pytest.ini is really in the right locatoin * explicitely add config file * minor change to generated html * minor change * minor change in options * reset options * add another flag * reset flag * try short tb instead of no tb * make sure no traceback is shown * small change * make sure to enter e2e dir already in gitlab * remove not needed var * correct folder * throw error if result jsons are not found * quiet pytest * do logging from python * improve logging yet another time * disable pytest logging from bash * cosmetic * rename enum value * small fix * further minor changes * generate junit.xml file * further changes according to PR feedback
1 parent 37a765b commit f674691

16 files changed

+457
-310
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,4 +78,4 @@ tmp*
7878
/clients/python/artifacts/*
7979
/clients/python/client/build/*
8080
/clients/python/client/osparc/data/openapi.json
81-
/clients/python/test/e2e/pyproject.toml
81+
/clients/python/test/e2e/pytest.ini

‎clients/python/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,8 @@ e2e-shell: guard-GH_TOKEN ## shell for running e2e tests
151151
--entrypoint /bin/bash \
152152
-it \
153153
itisfoundation/osparc_python_tutorial_testing:v1 \
154-
-c "python -m venv /tmp/venv && source /tmp/venv/bin/activate && cd clients/python && make install-test && exec /bin/bash"
154+
-c "python -m venv /tmp/venv && source /tmp/venv/bin/activate && cd clients/python \
155+
&& make install-test && cd test/e2e && python ci/install_osparc_python_client.py && exec /bin/bash"
155156

156157
## DOCKER -------------------------------------------------------------------------------
157158

‎clients/python/requirements/test.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,5 @@ papermill
1111
pydantic
1212
pytest
1313
pytest-env
14-
toml
14+
pytest-html
1515
typer

‎clients/python/test/e2e/ci/_data_classes.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
import configparser
2+
from pathlib import Path
3+
from typing import Dict, Optional
4+
from urllib.parse import ParseResult, urlparse
5+
6+
from _utils import _PYTEST_INI
7+
from packaging.version import Version
8+
from pydantic import BaseModel, field_validator, model_validator
9+
10+
# Holds classes for passing data around between scripts.
11+
12+
13+
class ServerConfig(BaseModel):
14+
"""Holds data about server configuration"""
15+
16+
osparc_api_host: str
17+
osparc_api_key: str
18+
osparc_api_secret: str
19+
20+
@property
21+
def url(self) -> ParseResult:
22+
return urlparse(f"{self.osparc_api_host}")
23+
24+
@property
25+
def key(self) -> str:
26+
return self.osparc_api_key
27+
28+
@property
29+
def secret(self) -> str:
30+
return self.osparc_api_secret
31+
32+
33+
def is_empty(v):
34+
return v is None or v == ""
35+
36+
37+
class ClientConfig(BaseModel):
38+
"""Holds data about client configuration.
39+
This data should uniquely determine how to install client
40+
"""
41+
42+
osparc_client_version: Optional[str] = None
43+
osparc_client_repo: Optional[str] = None
44+
osparc_client_branch: Optional[str] = None
45+
osparc_client_workflow: Optional[str] = None
46+
osparc_client_runid: Optional[str] = None
47+
48+
@field_validator("osparc_client_version")
49+
def validate_client(cls, v):
50+
if (not is_empty(v)) and (not v == "latest"):
51+
try:
52+
_ = Version(v)
53+
except Exception:
54+
raise ValueError(f"Did not receive valid version: {v}")
55+
return v
56+
57+
@model_validator(mode="after")
58+
def check_consistency(self) -> "ClientConfig":
59+
msg: str = (
60+
f"Recieved osparc_client_version={self.osparc_client_version}, "
61+
f"osparc_client_repo={self.osparc_client_repo}"
62+
"and osparc_client_branch={self.osparc_client_branch}. "
63+
"Either a version or a repo, branch pair must be specified. Not both."
64+
)
65+
# check at least one is empty
66+
if not (
67+
is_empty(self.osparc_client_version)
68+
or (
69+
is_empty(self.osparc_client_repo)
70+
and is_empty(self.osparc_client_branch)
71+
)
72+
):
73+
raise ValueError(msg)
74+
# check not both empty
75+
if is_empty(self.osparc_client_version) and (
76+
is_empty(self.osparc_client_repo) and is_empty(self.osparc_client_branch)
77+
):
78+
raise ValueError(msg)
79+
if is_empty(self.osparc_client_version):
80+
if (
81+
is_empty(self.osparc_client_repo)
82+
or is_empty(self.osparc_client_branch)
83+
or is_empty(self.osparc_client_workflow)
84+
or is_empty(self.osparc_client_runid)
85+
):
86+
raise ValueError(msg)
87+
return self
88+
89+
@property
90+
def version(self) -> Optional[str]:
91+
return self.osparc_client_version
92+
93+
@property
94+
def repo(self) -> Optional[str]:
95+
return self.osparc_client_repo
96+
97+
@property
98+
def branch(self) -> Optional[str]:
99+
return self.osparc_client_branch
100+
101+
@property
102+
def workflow(self) -> Optional[str]:
103+
return self.osparc_client_workflow
104+
105+
@property
106+
def runid(self) -> Optional[str]:
107+
return self.osparc_client_runid
108+
109+
@property
110+
def compatibility_ref(self) -> str:
111+
"""Returns the reference for this client in the compatibility table"""
112+
if not is_empty(self.version):
113+
return "production"
114+
else:
115+
assert isinstance(self.branch, str)
116+
return self.branch
117+
118+
@property
119+
def client_ref(self) -> str:
120+
"""Returns a short hand reference for this client"""
121+
if not is_empty(self.version):
122+
assert isinstance(self.version, str)
123+
return self.version
124+
else:
125+
assert isinstance(self.branch, str)
126+
return self.branch
127+
128+
129+
class PytestConfig(BaseModel):
130+
"""Holds the pytest configuration
131+
N.B. paths are relative to clients/python/test/e2e
132+
"""
133+
134+
env: str
135+
required_plugins: str
136+
addopts: str
137+
138+
139+
class Artifacts(BaseModel):
140+
artifact_dir: Path
141+
result_data_frame: Path
142+
log_dir: Path
143+
144+
145+
class PytestIniFile(BaseModel):
146+
"""Model for validating the .ini file"""
147+
148+
pytest: PytestConfig
149+
client: ClientConfig
150+
server: ServerConfig
151+
artifacts: Artifacts
152+
153+
@classmethod
154+
def read(cls, pth: Path = _PYTEST_INI) -> "PytestIniFile":
155+
"""Read the pytest.ini file"""
156+
if not pth.is_file():
157+
raise ValueError(f"pth: {pth} must point to a pytest.ini file")
158+
obj = configparser.ConfigParser()
159+
obj.read(pth)
160+
config: Dict = {s: dict(obj.items(s)) for s in obj.sections()}
161+
return PytestIniFile(**config)
162+
163+
def generate(self, pth: Path = _PYTEST_INI) -> None:
164+
"""Generate the pytest.ini file"""
165+
pth.unlink(missing_ok=True)
166+
pth.parent.mkdir(exist_ok=True)
167+
config: configparser.ConfigParser = configparser.ConfigParser()
168+
for field_name in self.__fields__:
169+
model: BaseModel = getattr(self, field_name)
170+
config[field_name] = model.model_dump(exclude_none=True)
171+
with open(pth, "w") as f:
172+
config.write(f)

‎clients/python/test/e2e/ci/_utils.py‎

Lines changed: 19 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,25 @@
11
from enum import IntEnum
22
from pathlib import Path
3-
from typing import Optional
4-
from urllib.parse import ParseResult, urlparse
53

64
import pytest
7-
from packaging.version import Version
8-
from pydantic import BaseModel, field_validator, model_validator
5+
6+
# Paths -------------------------------------------------------
7+
8+
_E2E_DIR: Path = Path(__file__).parent.parent.resolve()
9+
_PYTHON_DIR: Path = _E2E_DIR.parent.parent
10+
_CI_DIR: Path = (_E2E_DIR / "ci").resolve()
11+
_PYTEST_INI: Path = (_E2E_DIR / "pytest.ini").resolve()
12+
_ARTIFACTS_DIR: Path = (_E2E_DIR.parent.parent / "artifacts" / "e2e").resolve()
13+
_COMPATIBILITY_JSON: Path = (
14+
_E2E_DIR / "data" / "server_client_compatibility.json"
15+
).resolve()
16+
17+
assert _COMPATIBILITY_JSON.is_file()
18+
19+
20+
def print_line():
21+
"""Print a line in log"""
22+
print(150 * "=")
923

1024

1125
# classed for handling errors ----------------------------------
@@ -22,7 +36,7 @@ class E2eExitCodes(IntEnum):
2236
"""
2337

2438
CI_SCRIPT_FAILURE = 100
25-
INVALID_CLIENT_VS_SERVER = 101
39+
INCOMPATIBLE_CLIENT_SERVER = 101
2640
INVALID_JSON_DATA = 102
2741

2842

@@ -32,142 +46,3 @@ class E2eExitCodes(IntEnum):
3246
)
3347
== set()
3448
)
35-
36-
# Data classes ----------------------------------------------
37-
38-
39-
class ServerConfig(BaseModel):
40-
"""Holds data about server configuration"""
41-
42-
OSPARC_API_HOST: str
43-
OSPARC_API_KEY: str
44-
OSPARC_API_SECRET: str
45-
46-
@field_validator("OSPARC_API_HOST")
47-
def check_url(cls, v):
48-
try:
49-
_ = urlparse(v)
50-
except Exception:
51-
raise ValueError("Could not parse 'OSPARC_API_HOST'. Received {v}.")
52-
return v
53-
54-
@property
55-
def url(self) -> ParseResult:
56-
return urlparse(self.OSPARC_API_HOST)
57-
58-
@property
59-
def key(self) -> str:
60-
return self.OSPARC_API_KEY
61-
62-
@property
63-
def secret(self) -> str:
64-
return self.OSPARC_API_SECRET
65-
66-
67-
def is_empty(v):
68-
return v is None or v == ""
69-
70-
71-
class ClientConfig(BaseModel):
72-
"""Holds data about client configuration.
73-
This data should uniquely determine how to install client
74-
"""
75-
76-
OSPARC_CLIENT_VERSION: Optional[str] = None
77-
OSPARC_CLIENT_REPO: Optional[str] = None
78-
OSPARC_CLIENT_BRANCH: Optional[str] = None
79-
OSPARC_CLIENT_WORKFLOW: Optional[str] = None
80-
OSPARC_CLIENT_RUNID: Optional[str] = None
81-
82-
@field_validator("OSPARC_CLIENT_VERSION")
83-
def validate_client(cls, v):
84-
if (not is_empty(v)) and (not v == "latest"):
85-
try:
86-
_ = Version(v)
87-
except Exception:
88-
raise ValueError(f"Did not receive valid version: {v}")
89-
return v
90-
91-
@model_validator(mode="after")
92-
def check_consistency(self) -> "ClientConfig":
93-
msg: str = (
94-
f"Recieved OSPARC_CLIENT_VERSION={self.OSPARC_CLIENT_VERSION}, "
95-
f"OSPARC_CLIENT_REPO={self.OSPARC_CLIENT_REPO}"
96-
"and OSPARC_CLIENT_BRANCH={self.OSPARC_CLIENT_BRANCH}. "
97-
"Either a version or a repo, branch pair must be specified. Not both."
98-
)
99-
# check at least one is empty
100-
if not (
101-
is_empty(self.OSPARC_CLIENT_VERSION)
102-
or (
103-
is_empty(self.OSPARC_CLIENT_REPO)
104-
and is_empty(self.OSPARC_CLIENT_BRANCH)
105-
)
106-
):
107-
raise ValueError(msg)
108-
# check not both empty
109-
if is_empty(self.OSPARC_CLIENT_VERSION) and (
110-
is_empty(self.OSPARC_CLIENT_REPO) and is_empty(self.OSPARC_CLIENT_BRANCH)
111-
):
112-
raise ValueError(msg)
113-
if is_empty(self.OSPARC_CLIENT_VERSION):
114-
if (
115-
is_empty(self.OSPARC_CLIENT_REPO)
116-
or is_empty(self.OSPARC_CLIENT_BRANCH)
117-
or is_empty(self.OSPARC_CLIENT_WORKFLOW)
118-
or is_empty(self.OSPARC_CLIENT_RUNID)
119-
):
120-
raise ValueError(msg)
121-
return self
122-
123-
@property
124-
def version(self) -> Optional[str]:
125-
return self.OSPARC_CLIENT_VERSION
126-
127-
@property
128-
def repo(self) -> Optional[str]:
129-
return self.OSPARC_CLIENT_REPO
130-
131-
@property
132-
def branch(self) -> Optional[str]:
133-
return self.OSPARC_CLIENT_BRANCH
134-
135-
@property
136-
def workflow(self) -> Optional[str]:
137-
return self.OSPARC_CLIENT_WORKFLOW
138-
139-
@property
140-
def runid(self) -> Optional[str]:
141-
return self.OSPARC_CLIENT_RUNID
142-
143-
@property
144-
def compatibility_ref(self) -> str:
145-
"""Returns the reference for this client in the compatibility table"""
146-
if not is_empty(self.version):
147-
return "production"
148-
else:
149-
assert isinstance(self.branch, str)
150-
return self.branch
151-
152-
@property
153-
def client_ref(self) -> str:
154-
"""Returns a short hand reference for this client"""
155-
if not is_empty(self.version):
156-
assert isinstance(self.version, str)
157-
return self.version
158-
else:
159-
assert isinstance(self.branch, str)
160-
return self.branch
161-
162-
163-
# Paths -------------------------------------------------------
164-
165-
_E2E_DIR: Path = Path(__file__).parent.parent.resolve()
166-
_CI_DIR: Path = (_E2E_DIR / "ci").resolve()
167-
_PYPROJECT_TOML: Path = (_E2E_DIR / "pyproject.toml").resolve()
168-
_ARTIFACTS_DIR: Path = (_E2E_DIR / ".." / ".." / "artifacts" / "e2e").resolve()
169-
_COMPATIBILITY_JSON: Path = (
170-
_E2E_DIR / "data" / "server_client_compatibility.json"
171-
).resolve()
172-
173-
assert _COMPATIBILITY_JSON.is_file()

0 commit comments

Comments
 (0)