Skip to content

Commit f07ecb2

Browse files
bisgaard-itisbisgaard-itis
committed
Merge branch 'master' into run-single-map-jobs-in-celery-worker
2 parents 621578e + d7ee297 commit f07ecb2

File tree

4 files changed

+124
-41
lines changed

4 files changed

+124
-41
lines changed

‎services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py‎

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -978,42 +978,47 @@ async def _timeout_if_waiting_for_cluster_too_long(
978978
comp_run: CompRunsAtDB,
979979
comp_tasks: dict[NodeIDStr, CompTaskAtDB],
980980
) -> dict[NodeIDStr, CompTaskAtDB]:
981-
if comp_run.result is RunningState.WAITING_FOR_CLUSTER:
982-
tasks_waiting_for_cluster = [
983-
t
984-
for t in comp_tasks.values()
985-
if t.state is RunningState.WAITING_FOR_CLUSTER
986-
]
987-
# get latest modified task
988-
latest_modified_of_all_tasks = max(
989-
tasks_waiting_for_cluster, key=lambda task: task.modified
990-
).modified
991-
992-
if (
993-
arrow.utcnow().datetime - latest_modified_of_all_tasks
994-
) > self.settings.COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_CLUSTER_TIMEOUT:
995-
await CompTasksRepository.instance(
996-
self.db_engine
997-
).update_project_tasks_state(
998-
project_id,
999-
comp_run.run_id,
1000-
[task.node_id for task in tasks_waiting_for_cluster],
1001-
RunningState.FAILED,
1002-
optional_progress=1.0,
1003-
optional_stopped=arrow.utcnow().datetime,
1004-
)
1005-
for task in tasks_waiting_for_cluster:
1006-
task.state = RunningState.FAILED
1007-
msg = user_message(
1008-
"The system has timed out while waiting for computational resources. Please try running your project again or contact oSparc support if this issue persists.",
1009-
_version=1,
1010-
)
1011-
_logger.error(msg)
1012-
await publish_project_log(
1013-
self.rabbitmq_client,
1014-
user_id,
1015-
project_id,
1016-
log=msg,
1017-
log_level=logging.ERROR,
1018-
)
981+
if comp_run.result is not RunningState.WAITING_FOR_CLUSTER:
982+
return comp_tasks
983+
984+
tasks_waiting_for_cluster = [
985+
t
986+
for t in comp_tasks.values()
987+
if t.state is RunningState.WAITING_FOR_CLUSTER
988+
]
989+
if not tasks_waiting_for_cluster:
990+
return comp_tasks
991+
992+
# get latest modified task
993+
latest_modified_of_all_tasks = max(
994+
tasks_waiting_for_cluster, key=lambda task: task.modified
995+
).modified
996+
997+
if (
998+
arrow.utcnow().datetime - latest_modified_of_all_tasks
999+
) > self.settings.COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_CLUSTER_TIMEOUT:
1000+
await CompTasksRepository.instance(
1001+
self.db_engine
1002+
).update_project_tasks_state(
1003+
project_id,
1004+
comp_run.run_id,
1005+
[task.node_id for task in tasks_waiting_for_cluster],
1006+
RunningState.FAILED,
1007+
optional_progress=1.0,
1008+
optional_stopped=arrow.utcnow().datetime,
1009+
)
1010+
for task in tasks_waiting_for_cluster:
1011+
task.state = RunningState.FAILED
1012+
msg = user_message(
1013+
"The system has timed out while waiting for computational resources. Please try running your project again or contact oSparc support if this issue persists.",
1014+
_version=1,
1015+
)
1016+
_logger.error(msg)
1017+
await publish_project_log(
1018+
self.rabbitmq_client,
1019+
user_id,
1020+
project_id,
1021+
log=msg,
1022+
log_level=logging.ERROR,
1023+
)
10191024
return comp_tasks

‎tests/e2e-playwright/Makefile‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,10 @@ SLEEPERS_INPUT_FILE := .e2e-playwright-sleepers-env.txt
139139
S4L_INPUT_FILE := .e2e-playwright-sim4life-env.txt
140140
JUPYTER_LAB_INPUT_FILE := .e2e-playwright-jupyterlab-env.txt
141141
CLASSIC_TIP_INPUT_FILE := .e2e-playwright-classictip-env.txt
142+
RSM_INPUT_FILE := .e2e-playwright-rsm-env.txt
142143

143144
# Prompt the user for input and store it into variables
144-
$(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE) $(S4L_INPUT_FILE):
145+
$(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE) $(S4L_INPUT_FILE) $(RSM_INPUT_FILE):
145146
@read -p "Enter your product URL: " PRODUCT_URL; \
146147
read -p "Is the product billable [y/n]: " BILLABLE; \
147148
read -p "Is the product lite [y/n]: " IS_LITE; \
@@ -189,6 +190,20 @@ $(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE) $(S4L
189190
elif [ "$@" = "$(SLEEPERS_INPUT_FILE)" ]; then \
190191
read -p "Enter the number of sleepers: " NUM_SLEEPERS; \
191192
echo "--num-sleepers=$$NUM_SLEEPERS" >> $@; \
193+
elif [ "$@" = "$(RSM_INPUT_FILE)" ]; then \
194+
read -p "Enter the service key (default to mmux-vite-app-sumo-write): " SERVICE_KEY; \
195+
if [ -z "$$SERVICE_KEY" ]; then \
196+
echo "No service key specified, using default."; \
197+
echo "--service-key=mmux-vite-app-sumo-write" >> $@; \
198+
else \
199+
echo "--service-key=$$SERVICE_KEY" >> $@; \
200+
fi; \
201+
read -p "Enter the service version (default to latest): " SERVICE_VERSION; \
202+
if [ -z "$$SERVICE_VERSION" ]; then \
203+
echo "No service version specified, using default."; \
204+
else \
205+
echo "--service-version=$$SERVICE_VERSION" >> $@; \
206+
fi; \
192207
fi
193208

194209
# Run the tests
@@ -202,7 +217,10 @@ test-jupyterlab-anywhere: _check_venv_active $(JUPYTER_LAB_INPUT_FILE) ## run ju
202217
@$(call run_test, $(JUPYTER_LAB_INPUT_FILE), tests/jupyterlabs/test_jupyterlab.py)
203218

204219
test-tip-anywhere: _check_venv_active $(CLASSIC_TIP_INPUT_FILE) ## run classic tip test and cache settings
205-
$(call run_test, $(CLASSIC_TIP_INPUT_FILE), tests/tip/test_ti_plan.py)
220+
@$(call run_test, $(CLASSIC_TIP_INPUT_FILE), tests/tip/test_ti_plan.py)
221+
222+
test-response-surface-modeling-anywhere: _check_venv_active $(RSM_INPUT_FILE) ## run response surface modeling test and cache settings
223+
@$(call run_test, $(RSM_INPUT_FILE), tests/metamodeling/test_response_surface_modeling.py)
206224

207225
# Define the common test running function
208226
define run_test

‎tests/e2e-playwright/tests/conftest.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,7 @@ def _(
749749
expected_states,
750750
True,
751751
None,
752-
service_version, # noqa: FBT003
752+
service_version,
753753
)
754754

755755
return _

‎tests/e2e-playwright/tests/metamodeling/test_response_surface_modeling.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import logging
2+
import re
3+
from collections.abc import Callable
4+
from typing import Any, Final
5+
6+
from playwright.sync_api import Page
7+
from pydantic import AnyUrl
8+
from pytest_simcore.helpers.logging_tools import log_context
9+
from pytest_simcore.helpers.playwright import (
10+
MINUTE,
11+
RobustWebSocket,
12+
ServiceType,
13+
wait_for_service_running,
14+
)
15+
16+
_WAITING_FOR_SERVICE_TO_START: Final[int] = 5 * MINUTE
17+
_DEFAULT_RESPONSE_TO_WAIT_FOR: Final[re.Pattern] = re.compile(
18+
r"/flask/list_function_job_collections_for_functionid"
19+
)
20+
21+
22+
def test_response_surface_modeling(
23+
page: Page,
24+
create_project_from_service_dashboard: Callable[
25+
[ServiceType, str, str | None, str | None], dict[str, Any]
26+
],
27+
log_in_and_out: RobustWebSocket,
28+
service_key: str,
29+
service_version: str | None,
30+
product_url: AnyUrl,
31+
is_service_legacy: bool,
32+
):
33+
with (
34+
log_context(
35+
logging.INFO,
36+
f"Waiting for {service_key} to be responsive (waiting for {_DEFAULT_RESPONSE_TO_WAIT_FOR})",
37+
),
38+
page.expect_response(
39+
_DEFAULT_RESPONSE_TO_WAIT_FOR, timeout=_WAITING_FOR_SERVICE_TO_START
40+
),
41+
):
42+
project_data = create_project_from_service_dashboard(
43+
ServiceType.DYNAMIC, service_key, None, service_version
44+
)
45+
assert "workbench" in project_data, "Expected workbench to be in project data!"
46+
assert isinstance(
47+
project_data["workbench"], dict
48+
), "Expected workbench to be a dict!"
49+
node_ids: list[str] = list(project_data["workbench"])
50+
assert len(node_ids) == 1, "Expected 1 node in the workbench!"
51+
52+
wait_for_service_running(
53+
page=page,
54+
node_id=node_ids[0],
55+
websocket=log_in_and_out,
56+
timeout=_WAITING_FOR_SERVICE_TO_START,
57+
press_start_button=False,
58+
product_url=product_url,
59+
is_service_legacy=is_service_legacy,
60+
)

0 commit comments

Comments
 (0)