Skip to content

Commit 9a64948

Browse files
authored
Refactoring webserver: async metrics handler, upgrades dependencies and fixes testing fixtures (#2343)
- CHANGES: makes metrics_handler implementation async by using a thread pool executor ot execute prometheos_client.generate_latest call - CHANGES: general upgrade of webserver dependencies: aiohttp and python-socketio cannot be upgraded to the latest. Needs further refactoring (see #2008 and notes in requirement files)! - FIXES: services/web/server/tests/integration/01/test_exporter.py was de-activated. - CHANGES: unified pylint testing by reusing pytest-simcore utils - CHANGES: moves socket-io fixtures to simcore-pytest
1 parent 2f08064 commit 9a64948

File tree

27 files changed

+557
-750
lines changed

27 files changed

+557
-750
lines changed
Lines changed: 7 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,19 @@
11
# pylint:disable=unused-variable
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
4-
import os
5-
import re
6-
from pathlib import Path
74

8-
import pytest
9-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
105

11-
12-
@pytest.fixture
13-
def pylintrc(osparc_simcore_root_dir):
14-
pylintrc = osparc_simcore_root_dir / ".pylintrc"
15-
assert pylintrc.exists()
16-
return pylintrc
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1710

1811

1912
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
2014
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2115

2216

2317
def test_no_pdbs_in_place(package_dir):
24-
# TODO: add also test_dir excluding this function!?
25-
# TODO: it can be commented!
26-
# TODO: add check on other undesired code strings?!
27-
MATCH = re.compile(r"pdb.set_trace()")
28-
EXCLUDE = ["__pycache__", ".git"]
29-
for root, dirs, files in os.walk(package_dir):
30-
for name in files:
31-
if name.endswith(".py"):
32-
pypth = Path(root) / name
33-
code = pypth.read_text()
34-
found = MATCH.findall(code)
35-
# TODO: should return line number
36-
assert not found, "pbd.set_trace found in %s" % pypth
37-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)

packages/pytest-simcore/src/pytest_simcore/websocket_client.py

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,32 @@
88
import pytest
99
import socketio
1010
from aiohttp import web
11+
from aiohttp.test_utils import TestClient
1112
from pytest_simcore.helpers.utils_assert import assert_status
1213
from yarl import URL
1314

1415

15-
@pytest.fixture
16-
def socketio_url(client) -> Callable:
17-
def create_url(client_override: Optional = None) -> str:
16+
@pytest.fixture()
17+
def client_session_id_factory() -> Callable[[], str]:
18+
def create() -> str:
19+
# NOTE:
20+
return str(uuid4())
21+
22+
return create
23+
24+
25+
@pytest.fixture()
26+
def socketio_url_factory(client) -> Callable:
27+
def create_url(client_override: Optional[TestClient] = None) -> str:
1828
SOCKET_IO_PATH = "/socket.io/"
1929
return str((client_override or client).make_url(SOCKET_IO_PATH))
2030

2131
yield create_url
2232

2333

24-
@pytest.fixture
25-
async def security_cookie_factory(client) -> Callable:
26-
async def creator(client_override: Optional = None) -> str:
34+
@pytest.fixture()
35+
async def security_cookie_factory(client: TestClient) -> Callable:
36+
async def creator(client_override: Optional[TestClient] = None) -> str:
2737
# get the cookie by calling the root entrypoint
2838
resp = await (client_override or client).get("/v0/")
2939
data, error = await assert_status(resp, web.HTTPOk)
@@ -40,24 +50,25 @@ async def creator(client_override: Optional = None) -> str:
4050
yield creator
4151

4252

43-
@pytest.fixture
44-
def client_session_id() -> str:
45-
return str(uuid4())
46-
47-
48-
@pytest.fixture
49-
async def socketio_client(
50-
socketio_url: Callable, security_cookie_factory: Callable
51-
) -> Callable:
53+
@pytest.fixture()
54+
async def socketio_client_factory(
55+
socketio_url_factory: Callable,
56+
security_cookie_factory: Callable,
57+
client_session_id_factory: Callable,
58+
) -> Callable[[str, Optional[TestClient]], socketio.AsyncClient]:
5259
clients = []
5360

5461
async def connect(
55-
client_session_id: str, client: Optional = None
62+
client_session_id: Optional[str] = None, client: Optional[TestClient] = None
5663
) -> socketio.AsyncClient:
64+
65+
if client_session_id is None:
66+
client_session_id = client_session_id_factory()
67+
5768
sio = socketio.AsyncClient(ssl_verify=False)
5869
# enginio 3.10.0 introduced ssl verification
5970
url = str(
60-
URL(socketio_url(client)).with_query(
71+
URL(socketio_url_factory(client)).with_query(
6172
{"client_session_id": client_session_id}
6273
)
6374
)
@@ -75,5 +86,6 @@ async def connect(
7586
yield connect
7687

7788
for sio in clients:
78-
await sio.disconnect()
89+
if sio.connected:
90+
await sio.disconnect()
7991
assert not sio.sid

packages/service-library/tests/conftest.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
import servicelib
1111
from servicelib.openapi import create_openapi_specs
1212

13+
pytest_plugins = [
14+
"pytest_simcore.repository_paths",
15+
]
16+
1317

1418
@pytest.fixture(scope="session")
1519
def here():

packages/service-library/tests/test_package.py

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,22 @@
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
44

5-
import os
6-
import re
7-
from pathlib import Path
85

9-
import pytest
10-
11-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1210
from servicelib.utils import is_osparc_repo_dir, search_osparc_repo_dir
1311

1412

15-
@pytest.fixture
16-
def pylintrc(osparc_simcore_root_dir):
17-
pylintrc = osparc_simcore_root_dir / ".pylintrc"
18-
assert pylintrc.exists()
19-
return pylintrc
20-
21-
2213
def test_run_pylint(pylintrc, package_dir):
14+
# fixtures in pytest_simcore.environs
2315
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2416

2517

2618
def test_no_pdbs_in_place(package_dir):
27-
# TODO: add also test_dir excluding this function!?
28-
# TODO: it can be commented!
29-
# TODO: add check on other undesired code strings?!
30-
MATCH = re.compile(r"pdb.set_trace()")
31-
EXCLUDE = ["__pycache__", ".git"]
32-
for root, dirs, files in os.walk(package_dir):
33-
for name in files:
34-
if name.endswith(".py"):
35-
pypth = Path(root) / name
36-
code = pypth.read_text()
37-
found = MATCH.findall(code)
38-
# TODO: should return line number
39-
assert not found, "pbd.set_trace found in %s" % pypth
40-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
19+
# fixtures in pytest_simcore.environs
20+
assert_no_pdb_in_code(code_dir=package_dir)
4121

4222

4323
def test_utils(osparc_simcore_root_dir, package_dir):

packages/simcore-sdk/tests/unit/test_package.py

Lines changed: 7 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,37 +2,18 @@
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
44

5-
import os
6-
import re
7-
from pathlib import Path
85

9-
import pytest
10-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
11-
12-
13-
@pytest.fixture
14-
def pylintrc(osparc_simcore_root_dir):
15-
pylintrc = osparc_simcore_root_dir / ".pylintrc"
16-
assert pylintrc.exists()
17-
return pylintrc
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1810

1911

2012
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
2114
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2215

2316

2417
def test_no_pdbs_in_place(package_dir):
25-
# TODO: add also test_dir excluding this function!?
26-
# TODO: it can be commented!
27-
# TODO: add check on other undesired code strings?!
28-
MATCH = re.compile(r"pdb.set_trace()")
29-
EXCLUDE = ["__pycache__", ".git"]
30-
for root, dirs, files in os.walk(package_dir):
31-
for name in files:
32-
if name.endswith(".py"):
33-
pypth = Path(root) / name
34-
code = pypth.read_text()
35-
found = MATCH.findall(code)
36-
# TODO: should return line number
37-
assert not found, "pbd.set_trace found in %s" % pypth
38-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)

services/api-server/tests/unit/test__code_syntax.py

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,18 @@
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
44

5-
import os
6-
import re
7-
from pathlib import Path
85

9-
import pytest
10-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
11-
12-
13-
@pytest.fixture
14-
def pylintrc(osparc_simcore_root_dir):
15-
pylintrc_path = osparc_simcore_root_dir / ".pylintrc"
16-
assert pylintrc_path.exists()
17-
return pylintrc_path
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1810

1911

2012
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
2114
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2215

2316

2417
def test_no_pdbs_in_place(package_dir):
25-
MATCH = re.compile(r"pdb.set_trace()")
26-
EXCLUDE = ["__pycache__", ".git"]
27-
for root, dirs, files in os.walk(package_dir):
28-
for name in files:
29-
if name.endswith(".py"):
30-
pypth = Path(root) / name
31-
code = pypth.read_text()
32-
found = MATCH.findall(code)
33-
assert not found, "pbd.set_trace found in %s" % pypth
34-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)

services/catalog/tests/unit/test__code_syntax.py

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,18 @@
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
44

5-
import os
6-
import re
7-
from pathlib import Path
85

9-
import pytest
10-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
11-
12-
13-
@pytest.fixture
14-
def pylintrc(osparc_simcore_root_dir):
15-
pylintrc_path = osparc_simcore_root_dir / ".pylintrc"
16-
assert pylintrc_path.exists()
17-
return pylintrc_path
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1810

1911

2012
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
2114
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2215

2316

2417
def test_no_pdbs_in_place(package_dir):
25-
MATCH = re.compile(r"pdb.set_trace()")
26-
EXCLUDE = ["__pycache__", ".git"]
27-
for root, dirs, files in os.walk(package_dir):
28-
for name in files:
29-
if name.endswith(".py"):
30-
pypth = Path(root) / name
31-
code = pypth.read_text()
32-
found = MATCH.findall(code)
33-
assert not found, "pbd.set_trace found in %s" % pypth
34-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)
Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,19 @@
11
# pylint:disable=unused-variable
22
# pylint:disable=unused-argument
33
# pylint:disable=redefined-outer-name
4-
import os
5-
import re
6-
from pathlib import Path
74

8-
import pytest
9-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
105

11-
12-
@pytest.fixture
13-
def pylintrc(osparc_simcore_root_dir):
14-
pylintrc_path = osparc_simcore_root_dir / ".pylintrc"
15-
assert pylintrc_path.exists()
16-
return pylintrc_path
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1710

1811

1912
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
2014
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
2115

2216

2317
def test_no_pdbs_in_place(package_dir):
24-
MATCH = re.compile(r"pdb.set_trace()")
25-
EXCLUDE = ["__pycache__", ".git"]
26-
for root, dirs, files in os.walk(package_dir):
27-
for name in files:
28-
if name.endswith(".py"):
29-
pypth = Path(root) / name
30-
code = pypth.read_text()
31-
found = MATCH.findall(code)
32-
assert not found, "pbd.set_trace found in %s" % pypth
33-
dirs[:] = [d for d in dirs if d not in EXCLUDE]
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)

services/storage/tests/conftest.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
)
4040

4141
pytest_plugins = [
42+
"pytest_simcore.repository_paths",
4243
"tests.fixtures.data_models",
4344
]
4445

services/storage/tests/test_package.py

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,17 @@
33
# pylint:disable=redefined-outer-name
44

55

6-
import pytest
7-
from pytest_simcore.helpers.utils_pylint import assert_pylint_is_passing
8-
9-
10-
@pytest.fixture
11-
def pylintrc(osparc_simcore_root_dir):
12-
pylintrc = osparc_simcore_root_dir / ".pylintrc"
13-
assert pylintrc.exists()
14-
return pylintrc
6+
from pytest_simcore.helpers.utils_pylint import (
7+
assert_no_pdb_in_code,
8+
assert_pylint_is_passing,
9+
)
1510

1611

1712
def test_run_pylint(pylintrc, package_dir):
13+
# fixtures in pytest_simcore.environs
1814
assert_pylint_is_passing(pylintrc=pylintrc, package_dir=package_dir)
15+
16+
17+
def test_no_pdbs_in_place(package_dir):
18+
# fixtures in pytest_simcore.environs
19+
assert_no_pdb_in_code(code_dir=package_dir)

0 commit comments

Comments
 (0)