Skip to content

Commit 81b6bd2

Browse files
authored
🎨E2E: improvements on ClassicTIP test (#5955)
1 parent e4f4980 commit 81b6bd2

File tree

16 files changed

+515
-320
lines changed

16 files changed

+515
-320
lines changed

.vscode/extensions.json

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55
"DevSoft.svg-viewer-vscode",
66
"eamodio.gitlens",
77
"exiasr.hadolint",
8-
"hediet.vscode-drawio",
98
"ms-azuretools.vscode-docker",
109
"ms-python.black-formatter",
1110
"ms-python.pylint",
1211
"ms-python.python",
13-
"ms-vscode.makefile-tools",
1412
"njpwerner.autodocstring",
1513
"samuelcolvin.jinjahtml",
1614
"timonwong.shellcheck",

packages/pytest-simcore/src/pytest_simcore/logging_utils.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,7 @@ def log_context(
124124
error_message = (
125125
f"{ctx_msg.raised} ({_timedelta_as_minute_second_ms(elapsed_time)})"
126126
)
127-
logger.log(
128-
logging.ERROR,
127+
logger.exception(
129128
error_message,
130129
*args,
131130
**kwargs,

packages/pytest-simcore/src/pytest_simcore/playwright_utils.py

Lines changed: 139 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,21 @@
1+
import contextlib
12
import json
23
import logging
4+
import re
5+
from collections import defaultdict
36
from contextlib import ExitStack
4-
from dataclasses import dataclass
7+
from dataclasses import dataclass, field
58
from enum import Enum, unique
69
from typing import Any, Final
710

8-
from playwright.sync_api import WebSocket
11+
from playwright.sync_api import FrameLocator, Page, Request, WebSocket, expect
912
from pytest_simcore.logging_utils import log_context
1013

1114
SECOND: Final[int] = 1000
1215
MINUTE: Final[int] = 60 * SECOND
16+
NODE_START_REQUEST_PATTERN: Final[re.Pattern[str]] = re.compile(
17+
r"/projects/[^/]+/nodes/[^:]+:start"
18+
)
1319

1420

1521
@unique
@@ -42,6 +48,28 @@ def is_running(self) -> bool:
4248
)
4349

4450

51+
@unique
52+
class NodeProgressType(str, Enum):
53+
# NOTE: this is a partial duplicate of models_library/rabbitmq_messages.py
54+
# It must remain as such until that module is pydantic V2 compatible
55+
CLUSTER_UP_SCALING = "CLUSTER_UP_SCALING"
56+
SERVICE_INPUTS_PULLING = "SERVICE_INPUTS_PULLING"
57+
SIDECARS_PULLING = "SIDECARS_PULLING"
58+
SERVICE_OUTPUTS_PULLING = "SERVICE_OUTPUTS_PULLING"
59+
SERVICE_STATE_PULLING = "SERVICE_STATE_PULLING"
60+
SERVICE_IMAGES_PULLING = "SERVICE_IMAGES_PULLING"
61+
62+
@classmethod
63+
def required_types_for_started_service(cls) -> set["NodeProgressType"]:
64+
return {
65+
NodeProgressType.SERVICE_INPUTS_PULLING,
66+
NodeProgressType.SIDECARS_PULLING,
67+
NodeProgressType.SERVICE_OUTPUTS_PULLING,
68+
NodeProgressType.SERVICE_STATE_PULLING,
69+
NodeProgressType.SERVICE_IMAGES_PULLING,
70+
}
71+
72+
4573
class ServiceType(str, Enum):
4674
DYNAMIC = "DYNAMIC"
4775
COMPUTATIONAL = "COMPUTATIONAL"
@@ -84,6 +112,28 @@ def retrieve_project_state_from_decoded_message(event: SocketIOEvent) -> Running
84112
return RunningState(event.obj["data"]["state"]["value"])
85113

86114

115+
@dataclass(frozen=True, slots=True, kw_only=True)
116+
class NodeProgressEvent:
117+
node_id: str
118+
progress_type: NodeProgressType
119+
current_progress: float
120+
total_progress: float
121+
122+
123+
def retrieve_node_progress_from_decoded_message(
124+
event: SocketIOEvent,
125+
) -> NodeProgressEvent:
126+
assert event.name == _OSparcMessages.NODE_PROGRESS.value
127+
assert "progress_type" in event.obj
128+
assert "progress_report" in event.obj
129+
return NodeProgressEvent(
130+
node_id=event.obj["node_id"],
131+
progress_type=NodeProgressType(event.obj["progress_type"]),
132+
current_progress=float(event.obj["progress_report"]["actual_value"]),
133+
total_progress=float(event.obj["progress_report"]["total"]),
134+
)
135+
136+
87137
@dataclass
88138
class SocketIOProjectClosedWaiter:
89139
def __call__(self, message: str) -> bool:
@@ -139,6 +189,44 @@ def __call__(self, message: str) -> None:
139189
print("WS Message:", decoded_message.name, decoded_message.obj)
140190

141191

192+
@dataclass
193+
class SocketIONodeProgressCompleteWaiter:
194+
node_id: str
195+
_current_progress: dict[NodeProgressType, float] = field(
196+
default_factory=defaultdict
197+
)
198+
199+
def __call__(self, message: str) -> bool:
200+
with log_context(logging.DEBUG, msg=f"handling websocket {message=}") as ctx:
201+
# socket.io encodes messages like so
202+
# https://stackoverflow.com/questions/24564877/what-do-these-numbers-mean-in-socket-io-payload
203+
if message.startswith(_SOCKETIO_MESSAGE_PREFIX):
204+
decoded_message = decode_socketio_42_message(message)
205+
if decoded_message.name == _OSparcMessages.NODE_PROGRESS.value:
206+
node_progress_event = retrieve_node_progress_from_decoded_message(
207+
decoded_message
208+
)
209+
if node_progress_event.node_id == self.node_id:
210+
self._current_progress[node_progress_event.progress_type] = (
211+
node_progress_event.current_progress
212+
/ node_progress_event.total_progress
213+
)
214+
ctx.logger.info(
215+
"current startup progress: %s",
216+
f"{json.dumps({k:round(v,1) for k,v in self._current_progress.items()})}",
217+
)
218+
219+
return all(
220+
progress_type in self._current_progress
221+
for progress_type in NodeProgressType.required_types_for_started_service()
222+
) and all(
223+
round(progress, 1) == 1.0
224+
for progress in self._current_progress.values()
225+
)
226+
227+
return False
228+
229+
142230
def wait_for_pipeline_state(
143231
current_state: RunningState,
144232
*,
@@ -187,3 +275,52 @@ def on_web_socket_default_handler(ws) -> None:
187275
ws.on("framesent", lambda payload: ctx.logger.info("⬇️ %s", payload))
188276
ws.on("framereceived", lambda payload: ctx.logger.info("⬆️ %s", payload))
189277
ws.on("close", lambda payload: stack.close()) # noqa: ARG005
278+
279+
280+
def _node_started_predicate(request: Request) -> bool:
281+
return bool(
282+
re.search(NODE_START_REQUEST_PATTERN, request.url)
283+
and request.method.upper() == "POST"
284+
)
285+
286+
287+
def _trigger_service_start_if_button_available(page: Page, node_id: str) -> None:
288+
# wait for the start button to auto-disappear if it is still around after the timeout, then we click it
289+
with log_context(logging.INFO, msg="trigger start button if needed") as ctx:
290+
start_button_locator = page.get_by_test_id(f"Start_{node_id}")
291+
with contextlib.suppress(AssertionError, TimeoutError):
292+
expect(start_button_locator).to_be_visible(timeout=5000)
293+
expect(start_button_locator).to_be_enabled(timeout=5000)
294+
with page.expect_request(_node_started_predicate):
295+
start_button_locator.click()
296+
ctx.logger.info("triggered start button")
297+
298+
299+
def wait_for_service_running(
300+
*,
301+
page: Page,
302+
node_id: str,
303+
websocket: WebSocket,
304+
timeout: int,
305+
) -> FrameLocator:
306+
"""NOTE: if the service was already started this will not work as some of the required websocket events will not be emitted again
307+
In which case this will need further adjutment"""
308+
309+
waiter = SocketIONodeProgressCompleteWaiter(node_id=node_id)
310+
with (
311+
log_context(logging.INFO, msg="Waiting for node to run"),
312+
websocket.expect_event("framereceived", waiter, timeout=timeout),
313+
):
314+
_trigger_service_start_if_button_available(page, node_id)
315+
return page.frame_locator(f'[osparc-test-id="iframe_{node_id}"]')
316+
317+
318+
def app_mode_trigger_next_app(page: Page) -> None:
319+
with (
320+
log_context(logging.INFO, msg="triggering next app"),
321+
page.expect_request(_node_started_predicate),
322+
):
323+
# Move to next step (this auto starts the next service)
324+
next_button_locator = page.get_by_test_id("AppMode_NextBtn")
325+
if next_button_locator.is_visible() and next_button_locator.is_enabled():
326+
page.get_by_test_id("AppMode_NextBtn").click()

scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ def main(
8181
if "license" in file_path.name:
8282
continue
8383
# very bad HACK
84+
rich.print(f"checking {file_path.name}")
8485
if (
8586
any(_ in f"{file_path}" for _ in ("sim4life.io", "osparc-master"))
8687
and "openssh" not in f"{file_path}"

scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
str
99
] = r"osparc-computational-cluster-{role}-{swarm_stack_name}-user_id:{user_id:d}-wallet_id:{wallet_id:d}"
1010
DEFAULT_DYNAMIC_EC2_FORMAT: Final[str] = r"osparc-dynamic-autoscaled-worker-{key_name}"
11-
DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile(r"osparc-{random_name}.pem")
11+
DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile(
12+
r"{base_name}-{random_name}.pem"
13+
)
1214

1315
MINUTE: Final[int] = 60
1416
HOUR: Final[int] = 60 * MINUTE

tests/e2e-playwright/.gitignore

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
test-results
22
assets
33
report.html
4-
.e2e-playwright-env.txt
5-
.e2e-playwright-jupyterlab-env.txt
4+
.e2e-playwright-*.txt
65
report.xml

tests/e2e-playwright/Makefile

Lines changed: 48 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ test-sleepers: _check_venv_active ## runs sleepers test on local deploy
9292
--product-url=http://$(get_my_ip):9081 \
9393
--autoregister \
9494
--tracing=retain-on-failure \
95-
$(CURDIR)/tests/sleepers/sleepers.py
95+
$(CURDIR)/tests/sleepers/test_sleepers.py
9696

9797

9898
.PHONY: test-sleepers-dev
@@ -104,63 +104,56 @@ test-sleepers-dev: _check_venv_active ## runs sleepers test on local deploy
104104
--product-url=http://$(get_my_ip):9081 \
105105
--headed \
106106
--autoregister \
107-
$(CURDIR)/tests/sleepers/sleepers.py
107+
$(CURDIR)/tests/sleepers/test_sleepers.py
108+
109+
110+
# Define the files where user input will be saved
111+
SLEEPERS_INPUT_FILE := .e2e-playwright-sleepers-env.txt
112+
JUPYTER_LAB_INPUT_FILE := .e2e-playwright-jupyterlab-env.txt
113+
CLASSIC_TIP_INPUT_FILE := .e2e-playwright-classictip-env.txt
114+
115+
# Prompt the user for input and store it into variables
116+
$(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE):
117+
@read -p "Enter your product URL: " PRODUCT_URL; \
118+
read -p "Is the product billable [y/n]: " BILLABLE; \
119+
read -p "Is the test running in autoscaled deployment [y/n]: " AUTOSCALED; \
120+
read -p "Enter your username: " USER_NAME; \
121+
read -s -p "Enter your password: " PASSWORD; echo ""; \
122+
echo "--product-url=$$PRODUCT_URL --user-name=$$USER_NAME --password=$$PASSWORD" > $@; \
123+
if [ "$$BILLABLE" = "y" ]; then \
124+
echo "--product-billable" >> $@; \
125+
fi; \
126+
if [ "$$AUTOSCALED" = "y" ]; then \
127+
echo "--autoscaled" >> $@; \
128+
fi; \
129+
if [ "$@" = "$(JUPYTER_LAB_INPUT_FILE)" ]; then \
130+
read -p "Enter the size of the large file (human readable form e.g. 3Gib): " LARGE_FILE_SIZE; \
131+
echo "--service-key=jupyter-math --large-file-size=$$LARGE_FILE_SIZE" >> $@; \
132+
elif [ "$@" = "$(SLEEPERS_INPUT_FILE)" ]; then \
133+
read -p "Enter the number of sleepers: " NUM_SLEEPERS; \
134+
echo "--num-sleepers=$$NUM_SLEEPERS" >> $@; \
135+
fi
108136

137+
# Run the tests
138+
test-sleepers-anywhere: _check_venv_active $(SLEEPERS_INPUT_FILE)
139+
@$(call run_test, $(SLEEPERS_INPUT_FILE), tests/sleepers/test_sleepers.py)
109140

110-
# Define the file where user input will be saved
111-
USER_INPUT_FILE := .e2e-playwright-env.txt
112-
$(USER_INPUT_FILE):## Prompt the user for input and store it into variables
113-
@read -p "Enter your product URL: " PRODUCT_URL; \
114-
read -p "Is the product billable [y/n]: " BILLABLE; \
115-
read -p "Enter your username: " USER_NAME; \
116-
read -s -p "Enter your password: " PASSWORD; echo ""; \
117-
read -p "Enter the number of sleepers: " NUM_SLEEPERS; \
118-
echo "$$PRODUCT_URL $$USER_NAME $$PASSWORD $$NUM_SLEEPERS $$BILLABLE" > $(USER_INPUT_FILE)
119-
120-
# Read user input from the file and run the test
121-
test-sleepers-anywhere: _check_venv_active $(USER_INPUT_FILE) ## test sleepers anywhere and keeps a cache as to where
122-
@IFS=' ' read -r PRODUCT_URL USER_NAME PASSWORD NUM_SLEEPERS BILLABLE < $(USER_INPUT_FILE); \
123-
BILLABLE_FLAG=""; \
124-
if [ "$$BILLABLE" = "y" ]; then \
125-
BILLABLE_FLAG="--product-billable"; \
126-
fi; \
127-
pytest -s tests/sleepers/sleepers.py \
128-
--color=yes \
129-
--product-url=$$PRODUCT_URL \
130-
--user-name=$$USER_NAME \
131-
--password=$$PASSWORD \
132-
--num-sleepers=$$NUM_SLEEPERS \
133-
$$BILLABLE_FLAG \
134-
--browser chromium \
135-
--headed
136-
137-
# Define the file where user input will be saved
138-
JUPYTER_USER_INPUT_FILE := .e2e-playwright-jupyterlab-env.txt
139-
$(JUPYTER_USER_INPUT_FILE): ## Prompt the user for input and store it into variables
140-
@read -p "Enter your product URL: " PRODUCT_URL; \
141-
read -p "Is the product billable [y/n]: " BILLABLE; \
142-
read -p "Enter your username: " USER_NAME; \
143-
read -s -p "Enter your password: " PASSWORD; echo ""; \
144-
read -p "Enter the size of the large file (human readable form e.g. 3Gib): " LARGE_FILE_SIZE; \
145-
echo "$$PRODUCT_URL $$USER_NAME $$PASSWORD $$LARGE_FILE_SIZE $$BILLABLE" > $(JUPYTER_USER_INPUT_FILE)
146-
147-
test-jupyterlab-anywhere: _check_venv_active $(JUPYTER_USER_INPUT_FILE) ## test jupyterlabs anywhere and keeps a cache as to where
148-
@IFS=' ' read -r PRODUCT_URL USER_NAME PASSWORD LARGE_FILE_SIZE BILLABLE < $(JUPYTER_USER_INPUT_FILE); \
149-
BILLABLE_FLAG=""; \
150-
if [ "$$BILLABLE" = "y" ]; then \
151-
BILLABLE_FLAG="--product-billable"; \
152-
fi; \
153-
pytest -s tests/jupyterlabs/ \
141+
test-jupyterlab-anywhere: _check_venv_active $(JUPYTER_LAB_INPUT_FILE)
142+
@$(call run_test, $(JUPYTER_LAB_INPUT_FILE), tests/jupyterlabs/test_jupyterlab.py)
143+
144+
test-tip-anywhere: _check_venv_active $(CLASSIC_TIP_INPUT_FILE)
145+
$(call run_test, $(CLASSIC_TIP_INPUT_FILE), tests/tip/test_ti_plan.py)
146+
147+
# Define the common test running function
148+
define run_test
149+
TEST_ARGS=$$(cat $1 | xargs); \
150+
echo $$TEST_ARGS; \
151+
pytest -s $2 \
154152
--color=yes \
155-
--product-url=$$PRODUCT_URL \
156-
--user-name=$$USER_NAME \
157-
--password=$$PASSWORD \
158-
--large-file-size=$$LARGE_FILE_SIZE \
159-
--service-key=jupyter-math \
160-
$$BILLABLE_FLAG \
161153
--browser chromium \
162-
--headed
154+
--headed \
155+
$$TEST_ARGS
156+
endef
163157

164158
clean:
165-
@rm -rf $(USER_INPUT_FILE)
166-
@rm -rf $(JUPYTER_USER_INPUT_FILE)
159+
@rm -rf $(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE)

0 commit comments

Comments
 (0)