Merge remote-tracking branch 'upstream/master' into pr-osparc-speedup-lrt-result

Author: Andrei Neagu

.github/instructions/general.instructions.md

@@ -0,0 +1,11 @@
+---
+applyTo: '**'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
+
+## General Guidelines
+
+1. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js.
+2. **Environment Variables**: Use [Environment Variables Guide](../../docs/env-vars.md) for configuration. Avoid hardcoding sensitive information.
+3. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. Be concise.
+4. **Code Reviews**: Participate in code reviews and provide constructive feedback.

.github/instructions/node.instructions.md

@@ -0,0 +1,11 @@
+---
+applyTo: '**/*.js'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
+
+## 🛠️Coding Instructions for Node.js in This Repository
+
+* Use ES6+ syntax and features.
+* Follow the `package.json` configuration for dependencies and scripts.
+* Use `eslint` for linting and `prettier` for code formatting.
+* Write modular and reusable code, adhering to the project's structure.

.github/copilot-instructions.md …thub/instructions/python.instructions.md.github/copilot-instructions.md renamed to .github/instructions/python.instructions.md .github/copilot-instructions.md renamed to .github/instructions/python.instructions.md

@@ -1,14 +1,7 @@
-# GitHub Copilot Instructions
-
-This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects.
-
-## General Guidelines
-
-1. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js.
-2. **Environment Variables**: Use [Environment Variables Guide](../docs/env-vars.md) for configuration. Avoid hardcoding sensitive information.
-3. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer.
-
 ---
+applyTo: '**/*.py'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
 
 ## 🛠️Coding Instructions for Python in This Repository
 
@@ -25,8 +18,8 @@ Follow these rules **strictly** when generating Python code:
 
 ### 3. **Code Style & Formatting**
 
-* Follow [Python Coding Conventions](../docs/coding-conventions.md) **strictly**.
-* Format code with `black`.
+* Follow [Python Coding Conventions](../../docs/coding-conventions.md) **strictly**.
+* Format code with `black` and `ruff`.
 * Lint code with `ruff` and `pylint`.
 
 ### 4. **Library Compatibility**
@@ -51,11 +44,6 @@ Ensure compatibility with the following library versions:
 * Prefer `json_dumps` / `json_loads` from `common_library.json_serialization` instead of the built-in `json.dumps` / `json.loads`.
 * When using Pydantic models, prefer methods like `model.model_dump_json()` for serialization.
 
----
-
-## 🛠️Coding Instructions for Node.js in This Repository
-
-* Use ES6+ syntax and features.
-* Follow the `package.json` configuration for dependencies and scripts.
-* Use `eslint` for linting and `prettier` for code formatting.
-* Write modular and reusable code, adhering to the project's structure.
+### 7. **Running tests**
+* Use `--keep-docker-up` flag when testing to keep docker containers up between sessions.
+* Always activate the python virtual environment before running pytest.

packages/aws-library/src/aws_library/s3/__init__.py

@@ -22,10 +22,10 @@
 )
 
 __all__: tuple[str, ...] = (
-    "CopiedBytesTransferredCallback",
-    "MultiPartUploadLinks",
     "PRESIGNED_LINK_MAX_SIZE",
     "S3_MAX_FILE_SIZE",
+    "CopiedBytesTransferredCallback",
+    "MultiPartUploadLinks",
     "S3AccessError",
     "S3BucketInvalidError",
     "S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
     "S3RuntimeError",
     "S3UploadNotFoundError",
     "SimcoreS3API",
-    "UploadedBytesTransferredCallback",
     "UploadID",
+    "UploadedBytesTransferredCallback",
 )
 
 # nopycln: file

packages/celery-library/tests/unit/test_tasks.py

@@ -210,6 +210,7 @@ async def test_listing_task_uuids_contains_submitted_task(
 
 async def test_filtering_listing_tasks(
     celery_task_manager: CeleryTaskManager,
+    with_celery_worker: WorkController,
 ):
     class MyFilter(TaskFilter):
         user_id: int
@@ -218,39 +219,46 @@ class MyFilter(TaskFilter):
 
     user_id = 42
     expected_task_uuids: set[TaskUUID] = set()
+    all_tasks: list[tuple[TaskUUID, MyFilter]] = []
+
+    try:
+        for _ in range(5):
+            task_filter = MyFilter(
+                user_id=user_id,
+                product_name=_faker.word(),
+                client_app=_faker.word(),
+            )
+            task_uuid = await celery_task_manager.submit_task(
+                TaskMetadata(
+                    name=dreamer_task.__name__,
+                ),
+                task_filter=task_filter,
+            )
+            expected_task_uuids.add(task_uuid)
+            all_tasks.append((task_uuid, task_filter))
+
+        for _ in range(3):
+            task_filter = MyFilter(
+                user_id=_faker.pyint(min_value=100, max_value=200),
+                product_name=_faker.word(),
+                client_app=_faker.word(),
+            )
+            task_uuid = await celery_task_manager.submit_task(
+                TaskMetadata(
+                    name=dreamer_task.__name__,
+                ),
+                task_filter=task_filter,
+            )
+            all_tasks.append((task_uuid, task_filter))
 
-    for _ in range(5):
-        task_filter = MyFilter(
+        search_filter = MyFilter(
             user_id=user_id,
-            product_name=_faker.word(),
-            client_app=_faker.word(),
-        )
-        task_uuid = await celery_task_manager.submit_task(
-            TaskMetadata(
-                name=dreamer_task.__name__,
-            ),
-            task_filter=task_filter,
+            product_name=Wildcard(),
+            client_app=Wildcard(),
         )
-        expected_task_uuids.add(task_uuid)
-
-    for _ in range(3):
-        task_filter = MyFilter(
-            user_id=_faker.pyint(min_value=100, max_value=200),
-            product_name=_faker.word(),
-            client_app=_faker.word(),
-        )
-        await celery_task_manager.submit_task(
-            TaskMetadata(
-                name=dreamer_task.__name__,
-            ),
-            task_filter=task_filter,
-        )
-
-    search_filter = MyFilter(
-        user_id=user_id,
-        product_name=Wildcard(),
-        client_app=Wildcard(),
-    )
-    tasks = await celery_task_manager.list_tasks(search_filter)
-    assert expected_task_uuids == {task.uuid for task in tasks}
-    await asyncio.sleep(5 * 60)
+        tasks = await celery_task_manager.list_tasks(search_filter)
+        assert expected_task_uuids == {task.uuid for task in tasks}
+    finally:
+        # clean up all tasks. this should ideally be done in the fixture
+        for task_uuid, task_filter in all_tasks:
+            await celery_task_manager.cancel_task(task_filter, task_uuid)

packages/common-library/src/common_library/async_tools.py

@@ -89,32 +89,37 @@ async def cancel_wait_task(
         TimeoutError: raised if cannot cancel the task.
         CancelledError: raised ONLY if owner is being cancelled.
     """
+    if task.done():
+        # nothing to do here
+        return
 
-    cancelling = task.cancel()
-    if not cancelling:
-        return  # task was alredy cancelled
-
-    assert task.cancelling()  # nosec
-    assert not task.cancelled()  # nosec
-
+    # mark for cancellation
+    task.cancel("cancel_wait_task was called to cancel this task")
     try:
-
+        _logger.debug("Cancelling task %s", task.get_name())
         await asyncio.shield(
             # NOTE shield ensures that cancellation of the caller function won't stop you
             # from observing the cancellation/finalization of task.
             asyncio.wait_for(task, timeout=max_delay)
         )
-
-    except asyncio.CancelledError:
-        if not task.cancelled():
-            # task owner function is being cancelled -> propagate cancellation
-            raise
-
-        # else: task cancellation is complete, we can safely ignore it
-        _logger.debug(
-            "Task %s cancellation is complete",
+    except TimeoutError:
+        _logger.exception(
+            "Timeout while cancelling task %s after %s seconds",
             task.get_name(),
+            max_delay,
         )
+        raise
+    except asyncio.CancelledError:
+        current_task = asyncio.current_task()
+        assert current_task is not None  # nosec
+        if current_task.cancelling() > 0:
+            # owner function is being cancelled -> propagate cancellation
+            raise
+    finally:
+        if not task.done():
+            _logger.error("Failed to cancel %s", task.get_name())
+        else:
+            _logger.debug("Task %s cancelled", task.get_name())
 
 
 def delayed_start(

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -126,7 +126,7 @@ class ProjectGet(OutputSchema):
 
     # display
     name: str
-    description: str
+    description: Annotated[str, BeforeValidator(none_to_empty_str_pre_validator)]
     thumbnail: HttpUrl | Literal[""]
 
     type: ProjectType
@@ -144,7 +144,7 @@ class ProjectGet(OutputSchema):
     trashed_at: datetime | None
     trashed_by: Annotated[
         GroupID | None, Field(description="The primary gid of the user who trashed")
-    ]
+    ] = None
 
     # labeling
     tags: list[int]
@@ -166,10 +166,6 @@ class ProjectGet(OutputSchema):
     workspace_id: WorkspaceID | None
     folder_id: FolderID | None
 
-    _empty_description = field_validator("description", mode="before")(
-        none_to_empty_str_pre_validator
-    )
-
     @staticmethod
     def _update_json_schema_extra(schema: JsonDict) -> None:
         schema.update(

packages/models-library/src/models_library/projects.py

@@ -10,6 +10,7 @@
 from common_library.basic_types import DEFAULT_FACTORY
 from pydantic import (
     BaseModel,
+    BeforeValidator,
     ConfigDict,
     Field,
     HttpUrl,
@@ -85,13 +86,17 @@ class BaseProjectModel(BaseModel):
     ]
     description: Annotated[
         str,
+        BeforeValidator(none_to_empty_str_pre_validator),
         Field(
             description="longer one-line description about the project",
             examples=["Dabbling in temporal transitions ..."],
         ),
     ]
     thumbnail: Annotated[
         HttpUrl | None,
+        BeforeValidator(
+            empty_str_to_none_pre_validator,
+        ),
         Field(
             description="url of the project thumbnail",
             examples=["https://placeimg.com/171/96/tech/grayscale/?0.jpg"],
@@ -104,15 +109,6 @@ class BaseProjectModel(BaseModel):
     # Pipeline of nodes (SEE projects_nodes.py)
     workbench: Annotated[NodesDict, Field(description="Project's pipeline")]
 
-    # validators
-    _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")(
-        empty_str_to_none_pre_validator
-    )
-
-    _none_description_is_empty = field_validator("description", mode="before")(
-        none_to_empty_str_pre_validator
-    )
-
 
 class ProjectAtDB(BaseProjectModel):
     # Model used to READ from database