Merge branch 'master' into is468/empty-trash

Author: pcrespov

packages/pytest-simcore/src/pytest_simcore/helpers/playwright.py

@@ -10,9 +10,8 @@
 from enum import Enum, unique
 from typing import Any, Final
 
-import httpx
 from playwright._impl._sync_base import EventContextManager
-from playwright.sync_api import FrameLocator, Page, Request
+from playwright.sync_api import APIRequestContext, FrameLocator, Page, Request
 from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 from playwright.sync_api import WebSocket
 from pydantic import AnyUrl
@@ -179,7 +178,7 @@ def _attempt_reconnect(self, logger: logging.Logger) -> None:
                 self.ws.expect_event(event, predicate)
 
         except Exception as e:  # pylint: disable=broad-except
-            logger.error("🚨 Failed to reconnect WebSocket: %s", e)
+            logger.exception("🚨 Failed to reconnect WebSocket: %s", e)
 
     def expect_event(
         self,
@@ -294,6 +293,7 @@ class SocketIONodeProgressCompleteWaiter:
     node_id: str
     logger: logging.Logger
     product_url: AnyUrl
+    api_request_context: APIRequestContext
     _current_progress: dict[NodeProgressType, float] = field(
         default_factory=defaultdict
     )
@@ -326,7 +326,7 @@ def __call__(self, message: str) -> bool:
                         self.logger.info(
                             "Current startup progress [expected number of node-progress-types=%d]: %s",
                             len(NodeProgressType.required_types_for_started_service()),
-                            f"{json.dumps({k:round(v,1) for k,v in self._current_progress.items()})}",
+                            f"{json.dumps({k: round(v, 2) for k, v in self._current_progress.items()})}",
                         )
 
                 return self.got_expected_node_progress_types() and all(
@@ -337,23 +337,25 @@ def __call__(self, message: str) -> bool:
         _current_timestamp = datetime.now(UTC)
         if _current_timestamp - self._last_poll_timestamp > timedelta(seconds=5):
             url = f"https://{self.node_id}.services.{self.get_partial_product_url()}"
-            response = httpx.get(url, timeout=10)
-            self.logger.info(
-                "Querying the service endpoint from the E2E test. Url: %s Response: %s TIP: %s",
+            response = self.api_request_context.get(url, timeout=1000)
+            level = logging.DEBUG
+            if (response.status >= 400) and (response.status not in (502, 503)):
+                level = logging.ERROR
+            self.logger.log(
+                level,
+                "Querying service endpoint in case we missed some websocket messages. Url: %s Response: '%s' TIP: %s",
                 url,
-                response,
+                f"{response.status}: {response.text()}",
                 (
-                    "Response 401 is OK. It means that service is ready."
-                    if response.status_code == 401
-                    else "We are emulating the frontend; a 500 response is acceptable if the service is not yet ready."
+                    "We are emulating the frontend; a 5XX response is acceptable if the service is not yet ready."
                 ),
             )
-            if response.status_code <= 401:
+
+            if response.status <= 400:
                 # NOTE: If the response status is less than 400, it means that the backend is ready (There are some services that respond with a 3XX)
-                # MD: for now I have included 401 - as this also means that backend is ready
                 if self.got_expected_node_progress_types():
                     self.logger.warning(
-                        "⚠️ Progress bar didn't receive 100 percent but service is already running: %s ⚠️",  # https://github.com/ITISFoundation/osparc-simcore/issues/6449
+                        "⚠️ Progress bar didn't receive 100 percent but service is already running: %s. TIP: we missed some websocket messages! ⚠️",  # https://github.com/ITISFoundation/osparc-simcore/issues/6449
                         self.get_current_progress(),
                     )
                 return True
@@ -408,8 +410,9 @@ def _node_started_predicate(request: Request) -> bool:
 
 
 def _trigger_service_start(page: Page, node_id: str) -> None:
-    with log_context(logging.INFO, msg="trigger start button"), page.expect_request(
-        _node_started_predicate, timeout=35 * SECOND
+    with (
+        log_context(logging.INFO, msg="trigger start button"),
+        page.expect_request(_node_started_predicate, timeout=35 * SECOND),
     ):
         page.get_by_test_id(f"Start_{node_id}").click()
 
@@ -433,12 +436,14 @@ def expected_service_running(
         logging.INFO, msg=f"Waiting for node to run. Timeout: {timeout}"
     ) as ctx:
         waiter = SocketIONodeProgressCompleteWaiter(
-            node_id=node_id, logger=ctx.logger, product_url=product_url
+            node_id=node_id,
+            logger=ctx.logger,
+            product_url=product_url,
+            api_request_context=page.request,
         )
         service_running = ServiceRunning(iframe_locator=None)
 
         try:
-
             with websocket.expect_event("framereceived", waiter, timeout=timeout):
                 if press_start_button:
                     _trigger_service_start(page, node_id)
@@ -475,7 +480,10 @@ def wait_for_service_running(
         logging.INFO, msg=f"Waiting for node to run. Timeout: {timeout}"
     ) as ctx:
         waiter = SocketIONodeProgressCompleteWaiter(
-            node_id=node_id, logger=ctx.logger, product_url=product_url
+            node_id=node_id,
+            logger=ctx.logger,
+            product_url=product_url,
+            api_request_context=page.request,
         )
         with websocket.expect_event("framereceived", waiter, timeout=timeout):
             if press_start_button:

packages/service-library/src/servicelib/archiving_utils/_interface_7zip.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import re
+import shlex
 from collections.abc import Awaitable, Callable
 from contextlib import AsyncExitStack
 from pathlib import Path
@@ -214,7 +215,7 @@ async def archive_dir(
             "-mta=off",  # Don't store file access time
         ]
     )
-    command = f"{_7ZIP_EXECUTABLE} {options} {destination} {dir_to_compress}/*"
+    command = f"{_7ZIP_EXECUTABLE} {options} {shlex.quote(f'{destination}')} {shlex.quote(f'{dir_to_compress}')}/*"
 
     folder_size_bytes = sum(
         file.stat().st_size for file in iter_files_to_compress(dir_to_compress)
@@ -295,7 +296,7 @@ async def unarchive_dir(
     # get archive information
     archive_info_parser = _7ZipArchiveInfoParser()
     list_output = await _run_cli_command(
-        f"{_7ZIP_EXECUTABLE} l {archive_to_extract}",
+        f"{_7ZIP_EXECUTABLE} l {shlex.quote(f'{archive_to_extract}')}",
         output_handler=archive_info_parser.parse_chunk,
     )
     file_names_in_archive = _extract_file_names_from_archive(list_output)
@@ -330,7 +331,7 @@ async def _decompressed_bytes(byte_progress: NonNegativeInt) -> None:
             ]
         )
         await _run_cli_command(
-            f"{_7ZIP_EXECUTABLE} {options} {archive_to_extract} -o{destination_folder}",
+            f"{_7ZIP_EXECUTABLE} {options} {shlex.quote(f'{archive_to_extract}')} -o{shlex.quote(f'{destination_folder}')}",
             output_handler=_7ZipProgressParser(_decompressed_bytes).parse_chunk,
         )
 

packages/service-library/tests/archiving_utils/test_archiving__interface_7zip.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 import pytest
+from helpers import print_tree
 from pydantic import NonNegativeInt
 from servicelib.archiving_utils._interface_7zip import (
     _7ZipProgressParser,
@@ -61,12 +62,19 @@ async def _progress_handler(byte_progress: NonNegativeInt) -> None:
     assert sum(detected_entries) == expected_size
 
 
+def _assert_same_folder_content(f1: Path, f2: Path) -> None:
+    in_f1 = {x.relative_to(f1) for x in f1.rglob("*")}
+    in_f2 = {x.relative_to(f2) for x in f2.rglob("*")}
+    assert in_f1 == in_f2
+
+
 @pytest.mark.parametrize("compress", [True, False])
 async def test_archive_unarchive(
     mixed_file_types: Path, archive_path: Path, unpacked_archive: Path, compress: bool
 ):
     await archive_dir(mixed_file_types, archive_path, compress=compress)
     await unarchive_dir(archive_path, unpacked_archive)
+    _assert_same_folder_content(mixed_file_types, unpacked_archive)
 
 
 @pytest.fixture
@@ -82,6 +90,7 @@ async def test_archive_unarchive_empty_folder(
 ):
     await archive_dir(empty_folder, archive_path, compress=compress)
     await unarchive_dir(archive_path, unpacked_archive)
+    _assert_same_folder_content(empty_folder, unpacked_archive)
 
 
 @pytest.mark.parametrize(
@@ -102,3 +111,27 @@ def test__extract_file_names_from_archive(
     archive_list_stdout_path.read_text()
     files = _extract_file_names_from_archive(archive_list_stdout_path.read_text())
     assert len(files) == expected_file_count
+
+
+@pytest.mark.parametrize("compress", [True, False])
+async def test_archive_unarchive_with_names_with_spaces(tmp_path: Path, compress: bool):
+    to_archive_path = tmp_path / "'source of files!a ads now strange'"
+    to_archive_path.mkdir(parents=True, exist_ok=True)
+    assert to_archive_path.exists()
+
+    # generate some content
+    for i in range(10):
+        (to_archive_path / f"f{i}.txt").write_text("*" * i)
+    print_tree(to_archive_path)
+
+    archive_path = tmp_path / "archived version herre!)!(/£)!'"
+    assert not archive_path.exists()
+
+    extracted_to_path = tmp_path / "this is where i want them to be extracted to''''"
+    extracted_to_path.mkdir(parents=True, exist_ok=True)
+    assert extracted_to_path.exists()
+
+    # source and destination all with spaces
+    await archive_dir(to_archive_path, archive_path, compress=compress)
+    await unarchive_dir(archive_path, extracted_to_path)
+    _assert_same_folder_content(to_archive_path, extracted_to_path)

services/static-webserver/client/source/class/osparc/desktop/organizations/MembersList.js

@@ -137,12 +137,14 @@ qx.Class.define("osparc.desktop.organizations.MembersList", {
             const usersStore = osparc.store.Users.getInstance();
             selectedMembers.forEach(selectedMemberGId => promises.push(usersStore.getUser(selectedMemberGId)));
             Promise.all(promises)
-              .then(users => {
-                users.forEach(user => this.__addMember(user.getUsername()));
-              })
-              .catch(err => {
-                console.error(err);
+              .then(values => {
+                values.forEach(user => {
+                  if (user) {
+                    this.__addMember(user.getUsername());
+                  }
+                });
               })
+              .catch(console.error)
               .finally(collaboratorsManager.close());
           } else {
             collaboratorsManager.close();

services/static-webserver/client/source/class/osparc/store/Users.js

@@ -28,7 +28,7 @@ qx.Class.define("osparc.store.Users", {
   },
 
   members: {
-    fetchUser: function(groupId) {
+    __fetchUser: function(groupId) {
       const params = {
         url: {
           gid: groupId
@@ -41,14 +41,22 @@ qx.Class.define("osparc.store.Users", {
         });
     },
 
-    getUser: function(groupId, fetchIfNotFound = true) {
+    getUser: async function(groupId, fetchIfNotFound = true) {
       const userFound = this.getUsers().find(user => user.getGroupId() === groupId);
       if (userFound) {
-        return new Promise(resolve => resolve(userFound));
-      } else if (fetchIfNotFound) {
-        return this.fetchUser(groupId);
+        return userFound;
       }
-      return new Promise(reject => reject());
+      if (fetchIfNotFound) {
+        try {
+          const user = await this.__fetchUser(groupId);
+          if (user) {
+            return user;
+          }
+        } catch (error) {
+          console.error(error);
+        }
+      }
+      return null;
     },
 
     addUser: function(userData) {

tests/e2e-playwright/requirements/_test.txt

@@ -1,31 +1,31 @@
 annotated-types==0.7.0
     # via pydantic
-anyio==4.6.2.post1
+anyio==4.8.0
     # via httpx
 arrow==1.3.0
     # via -r requirements/_test.in
-certifi==2024.8.30
+certifi==2024.12.14
     # via
     #   httpcore
     #   httpx
     #   requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.1
     # via requests
-dnspython==2.6.1
+dnspython==2.7.0
     # via email-validator
 docker==7.1.0
     # via -r requirements/_test.in
 email-validator==2.2.0
     # via pydantic
-faker==29.0.0
+faker==33.3.1
     # via -r requirements/_test.in
-greenlet==3.0.3
+greenlet==3.1.1
     # via playwright
 h11==0.14.0
     # via httpcore
 httpcore==1.0.7
     # via httpx
-httpx==0.27.2
+httpx==0.28.1
     # via -r requirements/_test.in
 idna==3.10
     # via
@@ -35,25 +35,25 @@ idna==3.10
     #   requests
 iniconfig==2.0.0
     # via pytest
-jinja2==3.1.4
+jinja2==3.1.5
     # via pytest-html
-markupsafe==2.1.5
+markupsafe==3.0.2
     # via jinja2
-packaging==24.1
+packaging==24.2
     # via
     #   pytest
     #   pytest-sugar
-playwright==1.47.0
+playwright==1.49.1
     # via pytest-playwright
 pluggy==1.5.0
     # via pytest
-pydantic==2.10.3
+pydantic==2.10.5
     # via -r requirements/_test.in
-pydantic-core==2.27.1
+pydantic-core==2.27.2
     # via pydantic
 pyee==12.0.0
     # via playwright
-pytest==8.3.3
+pytest==8.3.4
     # via
     #   pytest-base-url
     #   pytest-html
@@ -69,7 +69,7 @@ pytest-instafail==0.5.0
     # via -r requirements/_test.in
 pytest-metadata==3.1.1
     # via pytest-html
-pytest-playwright==0.5.2
+pytest-playwright==0.6.2
     # via -r requirements/_test.in
 pytest-runner==6.0.1
     # via -r requirements/_test.in
@@ -87,26 +87,26 @@ requests==2.32.3
     # via
     #   docker
     #   pytest-base-url
-six==1.16.0
+six==1.17.0
     # via python-dateutil
 sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
+    # via anyio
 tenacity==9.0.0
     # via -r requirements/_test.in
-termcolor==2.4.0
+termcolor==2.5.0
     # via pytest-sugar
 text-unidecode==1.3
     # via python-slugify
-types-python-dateutil==2.9.0.20240906
+types-python-dateutil==2.9.0.20241206
     # via arrow
 typing-extensions==4.12.2
     # via
+    #   anyio
+    #   faker
     #   pydantic
     #   pydantic-core
     #   pyee
-urllib3==2.2.3
+urllib3==2.3.0
     # via
     #   docker
     #   requests