Skip to content

Commit e6c6b45

Browse files
GitHKAndrei Neagu
GitHK
and
Andrei Neagu
authored
🚑️ fixes issue with env vars being stored as dict or list (#6052)
Co-authored-by: Andrei Neagu <[email protected]>
1 parent d080407 commit e6c6b45

File tree

4 files changed

+213
-8
lines changed

4 files changed

+213
-8
lines changed

‎packages/service-library/src/servicelib/async_utils.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,24 @@ class QueueElement:
4242
_sequential_jobs_contexts: dict[str, Context] = {}
4343

4444

45-
async def cancel_sequential_workers() -> None:
46-
"""Signals all workers to close thus avoiding errors on shutdown"""
47-
for context in _sequential_jobs_contexts.values():
45+
async def _safe_cancel(context: Context) -> None:
46+
try:
4847
await context.in_queue.put(None)
4948
if context.task is not None:
5049
context.task.cancel()
5150
with suppress(asyncio.CancelledError):
5251
await context.task
52+
except RuntimeError as e:
53+
if "Event loop is closed" in f"{e}":
54+
logger.warning("event loop is closed and could not cancel %s", context)
55+
else:
56+
raise
57+
58+
59+
async def cancel_sequential_workers() -> None:
60+
"""Signals all workers to close thus avoiding errors on shutdown"""
61+
for context in _sequential_jobs_contexts.values():
62+
await _safe_cancel(context)
5363

5464
_sequential_jobs_contexts.clear()
5565
logger.info("All run_sequentially_in_context pending workers stopped")

‎services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/validation.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,20 @@ def _apply_templating_directives(
107107

108108

109109
def _merge_env_vars(
110-
compose_spec_env_vars: list[str], settings_env_vars: list[str]
110+
compose_spec_env_vars: list[str] | dict[str, str],
111+
settings_env_vars: list[str] | dict[str, str],
111112
) -> list[str]:
112113
def _gen_parts_env_vars(
113-
env_vars: list[str],
114+
env_vars: list[str] | dict[str, str],
114115
) -> Generator[tuple[str, str], None, None]:
115-
for env_var in env_vars:
116-
key, value = env_var.split("=")
117-
yield key, value
116+
assert isinstance(env_vars, list | dict) # nosec
117+
118+
if isinstance(env_vars, list):
119+
for env_var in env_vars:
120+
key, value = env_var.split("=")
121+
yield key, value
122+
else:
123+
yield from env_vars.items()
118124

119125
# pylint: disable=unnecessary-comprehension
120126
dict_spec_env_vars = {k: v for k, v in _gen_parts_env_vars(compose_spec_env_vars)}

‎services/dynamic-sidecar/tests/mocks/internet_blocked_spec.yaml‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
networks:
2+
dy-sidecar_a019b83f-7cce-46bf-90cf-d02f7f0f089a:
3+
driver: overlay
4+
external: true
5+
name: dy-sidecar_a019b83f-7cce-46bf-90cf-d02f7f0f089a
6+
egress-0:
7+
internal: true
8+
production-simcore_interactive_services_subnet:
9+
driver: overlay
10+
external: true
11+
name: production-simcore_interactive_services_subnet
12+
with-internet:
13+
internal: false
14+
services:
15+
egress-0:
16+
command: "fake"
17+
environment:
18+
OSPARC_NODE_ID: a019b83f-7cce-46bf-90cf-d02f7f0f089a
19+
OSPARC_STUDY_ID: ef60eaa6-3f52-11ef-9745-0242ac172e8c
20+
image: envoyproxy/envoy:v1.25-latest
21+
labels:
22+
- io.simcore.runtime.cpu-limit=0
23+
- io.simcore.runtime.memory-limit=0
24+
- io.simcore.runtime.node-id=a019b83f-7cce-46bf-90cf-d02f7f0f089a
25+
- io.simcore.runtime.product-name=s4llite
26+
- io.simcore.runtime.project-id=ef60eaa6-3f52-11ef-9745-0242ac172e8c
27+
- io.simcore.runtime.simcore-user-agent=undefined
28+
- io.simcore.runtime.swarm-stack-name=production-simcore
29+
- io.simcore.runtime.user-id=47568
30+
networks:
31+
egress-0:
32+
aliases:
33+
- ip-10-1-0-25.ec2.internal
34+
with-internet: null
35+
rt-web-lite:
36+
cpus: 0.5
37+
depends_on:
38+
- s4l-core
39+
environment:
40+
- DY_SIDECAR_PATH_INPUTS=/home/smu/work/inputs
41+
- DY_SIDECAR_PATH_OUTPUTS=/home/smu/work/outputs
42+
- DY_SIDECAR_STATE_PATHS=[""/home/smu/work/workspace""]
43+
- SIMCORE_NANO_CPUS_LIMIT=500000000
44+
- SIMCORE_MEMORY_BYTES_LIMIT=1073741824
45+
- OSPARC_STUDY_ID=ef60eaa6-3f52-11ef-9745-0242ac172e8c
46+
- OSPARC_NODE_ID=a019b83f-7cce-46bf-90cf-d02f7f0f089a
47+
image: simcore/services/dynamic/sim4life-lite:2.0.106
48+
init: true
49+
labels:
50+
- io.simcore.runtime.cpu-limit=0.5
51+
- io.simcore.runtime.memory-limit=1073741824
52+
- io.simcore.runtime.node-id=a019b83f-7cce-46bf-90cf-d02f7f0f089a
53+
- io.simcore.runtime.product-name=s4llite
54+
- io.simcore.runtime.project-id=ef60eaa6-3f52-11ef-9745-0242ac172e8c
55+
- io.simcore.runtime.simcore-user-agent=undefined
56+
- io.simcore.runtime.swarm-stack-name=production-simcore
57+
- io.simcore.runtime.user-id=47568
58+
mem_limit: 1073741824
59+
mem_reservation: 1073741824
60+
networks:
61+
dy-sidecar_a019b83f-7cce-46bf-90cf-d02f7f0f089a: null
62+
s4l-core:
63+
cpus: 3.5
64+
depends_on:
65+
- egress-0
66+
environment:
67+
- DISPLAY=
68+
- DY_SIDECAR_PATH_INPUTS=/home/smu/work/inputs
69+
- DY_SIDECAR_PATH_OUTPUTS=/home/smu/work/outputs
70+
- DY_SIDECAR_STATE_PATHS=[""/home/smu/work/workspace""]
71+
- SIMCORE_NANO_CPUS_LIMIT=3500000000
72+
- SIMCORE_MEMORY_BYTES_LIMIT=17179869184
73+
- OSPARC_STUDY_ID=ef60eaa6-3f52-11ef-9745-0242ac172e8c
74+
- OSPARC_NODE_ID=a019b83f-7cce-46bf-90cf-d02f7f0f089a
75+
image: simcore/services/dynamic/s4l-core-lite:2.0.106
76+
init: true
77+
labels:
78+
- io.simcore.runtime.cpu-limit=3.5
79+
- io.simcore.runtime.memory-limit=17179869184
80+
- io.simcore.runtime.node-id=a019b83f-7cce-46bf-90cf-d02f7f0f089a
81+
- io.simcore.runtime.product-name=s4llite
82+
- io.simcore.runtime.project-id=ef60eaa6-3f52-11ef-9745-0242ac172e8c
83+
- io.simcore.runtime.simcore-user-agent=undefined
84+
- io.simcore.runtime.swarm-stack-name=production-simcore
85+
- io.simcore.runtime.user-id=47568
86+
mem_limit: 17179869184
87+
mem_reservation: 17179869184
88+
networks:
89+
egress-0: null
90+
runtime: nvidia
91+
volumes:
92+
- /tmp/.X11-unix:/tmp/.X11-unix
93+
s4l-core-stream:
94+
cpus: 0.5
95+
environment:
96+
- DY_SIDECAR_PATH_INPUTS=/home/smu/work/inputs
97+
- DY_SIDECAR_PATH_OUTPUTS=/home/smu/work/outputs
98+
- DY_SIDECAR_STATE_PATHS=[""/home/smu/work/workspace""]
99+
- SIMCORE_NANO_CPUS_LIMIT=500000000
100+
- SIMCORE_MEMORY_BYTES_LIMIT=1073741824
101+
- OSPARC_STUDY_ID=ef60eaa6-3f52-11ef-9745-0242ac172e8c
102+
- OSPARC_NODE_ID=a019b83f-7cce-46bf-90cf-d02f7f0f089a
103+
image: simcore/services/dynamic/s4l-core-stream:2.0.106
104+
init: true
105+
labels:
106+
- io.simcore.runtime.cpu-limit=0.5
107+
- io.simcore.runtime.memory-limit=1073741824
108+
- io.simcore.runtime.node-id=a019b83f-7cce-46bf-90cf-d02f7f0f089a
109+
- io.simcore.runtime.product-name=s4llite
110+
- io.simcore.runtime.project-id=ef60eaa6-3f52-11ef-9745-0242ac172e8c
111+
- io.simcore.runtime.simcore-user-agent=undefined
112+
- io.simcore.runtime.swarm-stack-name=production-simcore
113+
- io.simcore.runtime.user-id=47568
114+
mem_limit: 1073741824
115+
mem_reservation: 1073741824
116+
networks:
117+
with-internet: null
118+
sym-server:
119+
cpus: 0.5
120+
environment:
121+
- DY_SIDECAR_PATH_INPUTS=/home/smu/work/inputs
122+
- DY_SIDECAR_PATH_OUTPUTS=/home/smu/work/outputs
123+
- DY_SIDECAR_STATE_PATHS=[""/home/smu/work/workspace""]
124+
- SIMCORE_NANO_CPUS_LIMIT=500000000
125+
- SIMCORE_MEMORY_BYTES_LIMIT=2147483648
126+
- OSPARC_STUDY_ID=ef60eaa6-3f52-11ef-9745-0242ac172e8c
127+
- OSPARC_NODE_ID=a019b83f-7cce-46bf-90cf-d02f7f0f089a
128+
image: simcore/services/dynamic/sym-server-dy:2.0.106
129+
init: true
130+
labels:
131+
- io.simcore.runtime.cpu-limit=0.5
132+
- io.simcore.runtime.memory-limit=2147483648
133+
- io.simcore.runtime.node-id=a019b83f-7cce-46bf-90cf-d02f7f0f089a
134+
- io.simcore.runtime.product-name=s4llite
135+
- io.simcore.runtime.project-id=ef60eaa6-3f52-11ef-9745-0242ac172e8c
136+
- io.simcore.runtime.simcore-user-agent=undefined
137+
- io.simcore.runtime.swarm-stack-name=production-simcore
138+
- io.simcore.runtime.user-id=47568
139+
mem_limit: 2147483648
140+
mem_reservation: 2147483648
141+
version: \'2.3\'

‎services/dynamic-sidecar/tests/unit/test_core_validation.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,20 @@
22
# pylint: disable=unused-argument
33

44
from inspect import signature
5+
from pathlib import Path
56

67
import pytest
8+
from fastapi import FastAPI
9+
from models_library.projects_nodes_io import NodeID
10+
from models_library.services_types import RunID
11+
from pytest_mock import MockerFixture
712
from servicelib.docker_constants import DEFAULT_USER_SERVICES_NETWORK_NAME
813
from simcore_service_dynamic_sidecar.core.validation import (
914
_connect_user_services,
1015
parse_compose_spec,
16+
validate_compose_spec,
1117
)
18+
from simcore_service_dynamic_sidecar.modules.mounted_fs import MountedVolumes
1219

1320

1421
@pytest.fixture
@@ -129,3 +136,44 @@ def test_inject_backend_networking(
129136
DEFAULT_USER_SERVICES_NETWORK_NAME
130137
in parsed_compose_spec["services"]["iseg-web"]["networks"]
131138
)
139+
140+
141+
@pytest.fixture
142+
def mock_get_volume_by_label(mocker: MockerFixture) -> None:
143+
mocker.patch(
144+
"simcore_service_dynamic_sidecar.modules.mounted_fs.get_volume_by_label",
145+
autospec=True,
146+
return_value={"Mountpoint": "/fake/mount"},
147+
)
148+
149+
150+
@pytest.fixture
151+
def no_internet_spec(project_tests_dir: Path) -> str:
152+
no_intenret_file = project_tests_dir / "mocks" / "internet_blocked_spec.yaml"
153+
return no_intenret_file.read_text()
154+
155+
156+
@pytest.fixture
157+
def fake_mounted_volumes() -> MountedVolumes:
158+
return MountedVolumes(
159+
run_id=RunID.create(),
160+
node_id=NodeID("a019b83f-7cce-46bf-90cf-d02f7f0f089a"),
161+
inputs_path=Path("/"),
162+
outputs_path=Path("/"),
163+
user_preferences_path=None,
164+
state_paths=[],
165+
state_exclude=set(),
166+
compose_namespace="",
167+
dy_volumes=Path("/"),
168+
)
169+
170+
171+
async def test_regression_validate_compose_spec(
172+
mock_get_volume_by_label: None,
173+
app: FastAPI,
174+
no_internet_spec: str,
175+
fake_mounted_volumes: MountedVolumes,
176+
):
177+
await validate_compose_spec(
178+
app.state.settings, no_internet_spec, fake_mounted_volumes
179+
)

0 commit comments

Comments
 (0)