Merge branch 'master' into pr/giancarloromeo/8141

Author: pcrespov

.github/instructions/general.instructions.md

@@ -0,0 +1,11 @@
+---
+applyTo: '**'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
+
+## General Guidelines
+
+1. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js.
+2. **Environment Variables**: Use [Environment Variables Guide](../../docs/env-vars.md) for configuration. Avoid hardcoding sensitive information.
+3. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer. Be concise.
+4. **Code Reviews**: Participate in code reviews and provide constructive feedback.

.github/instructions/node.instructions.md

@@ -0,0 +1,11 @@
+---
+applyTo: '**/*.js'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
+
+## 🛠️Coding Instructions for Node.js in This Repository
+
+* Use ES6+ syntax and features.
+* Follow the `package.json` configuration for dependencies and scripts.
+* Use `eslint` for linting and `prettier` for code formatting.
+* Write modular and reusable code, adhering to the project's structure.

.github/copilot-instructions.md renamed to .github/instructions/python.instructions.md

@@ -1,14 +1,7 @@
-# GitHub Copilot Instructions
-
-This document provides guidelines and best practices for using GitHub Copilot in the `osparc-simcore` repository and other Python and Node.js projects.
-
-## General Guidelines
-
-1. **Test-Driven Development**: Write unit tests for all new functions and features. Use `pytest` for Python and appropriate testing frameworks for Node.js.
-2. **Environment Variables**: Use [Environment Variables Guide](../docs/env-vars.md) for configuration. Avoid hardcoding sensitive information.
-3. **Documentation**: Prefer self-explanatory code; add documentation only if explicitly requested by the developer.
-
 ---
+applyTo: '**/*.py'
+---
+Provide project context and coding guidelines that AI should follow when generating code, answering questions, or reviewing changes.
 
 ## 🛠️Coding Instructions for Python in This Repository
 
@@ -25,8 +18,8 @@ Follow these rules **strictly** when generating Python code:
 
 ### 3. **Code Style & Formatting**
 
-* Follow [Python Coding Conventions](../docs/coding-conventions.md) **strictly**.
-* Format code with `black`.
+* Follow [Python Coding Conventions](../../docs/coding-conventions.md) **strictly**.
+* Format code with `black` and `ruff`.
 * Lint code with `ruff` and `pylint`.
 
 ### 4. **Library Compatibility**
@@ -51,11 +44,6 @@ Ensure compatibility with the following library versions:
 * Prefer `json_dumps` / `json_loads` from `common_library.json_serialization` instead of the built-in `json.dumps` / `json.loads`.
 * When using Pydantic models, prefer methods like `model.model_dump_json()` for serialization.
 
----
-
-## 🛠️Coding Instructions for Node.js in This Repository
-
-* Use ES6+ syntax and features.
-* Follow the `package.json` configuration for dependencies and scripts.
-* Use `eslint` for linting and `prettier` for code formatting.
-* Write modular and reusable code, adhering to the project's structure.
+### 7. **Running tests**
+* Use `--keep-docker-up` flag when testing to keep docker containers up between sessions.
+* Always activate the python virtual environment before running pytest.

packages/celery-library/src/celery_library/errors.py

@@ -26,5 +26,11 @@ def decode_celery_transferrable_error(error: TransferrableCeleryError) -> Except
     return result
 
 
+class TaskSubmissionError(OsparcErrorMixin, Exception):
+    msg_template = (
+        "Unable to submit task {task_name} with id '{task_id}' and params {task_params}"
+    )
+
+
 class TaskNotFoundError(OsparcErrorMixin, Exception):
     msg_template = "Task with id '{task_id}' was not found"

packages/celery-library/src/celery_library/task_manager.py

@@ -4,6 +4,7 @@
 from uuid import uuid4
 
 from celery import Celery  # type: ignore[import-untyped]
+from celery.exceptions import CeleryError  # type: ignore[import-untyped]
 from common_library.async_tools import make_async
 from models_library.progress_bar import ProgressReport
 from servicelib.celery.models import (
@@ -21,7 +22,7 @@
 from servicelib.logging_utils import log_context
 from settings_library.celery import CelerySettings
 
-from .errors import TaskNotFoundError
+from .errors import TaskNotFoundError, TaskSubmissionError
 
 _logger = logging.getLogger(__name__)
 
@@ -50,21 +51,38 @@ async def submit_task(
         ):
             task_uuid = uuid4()
             task_id = task_filter.create_task_id(task_uuid=task_uuid)
-            self._celery_app.send_task(
-                task_metadata.name,
-                task_id=task_id,
-                kwargs={"task_id": task_id} | task_params,
-                queue=task_metadata.queue.value,
-            )
 
             expiry = (
                 self._celery_settings.CELERY_EPHEMERAL_RESULT_EXPIRES
                 if task_metadata.ephemeral
                 else self._celery_settings.CELERY_RESULT_EXPIRES
             )
-            await self._task_info_store.create_task(
-                task_id, task_metadata, expiry=expiry
-            )
+
+            try:
+                await self._task_info_store.create_task(
+                    task_id, task_metadata, expiry=expiry
+                )
+                self._celery_app.send_task(
+                    task_metadata.name,
+                    task_id=task_id,
+                    kwargs={"task_id": task_id} | task_params,
+                    queue=task_metadata.queue.value,
+                )
+            except CeleryError as exc:
+                try:
+                    await self._task_info_store.remove_task(task_id)
+                except CeleryError:
+                    _logger.warning(
+                        "Unable to cleanup task '%s' during error handling",
+                        task_id,
+                        exc_info=True,
+                    )
+                raise TaskSubmissionError(
+                    task_name=task_metadata.name,
+                    task_id=task_id,
+                    task_params=task_params,
+                ) from exc
+
             return task_uuid
 
     async def cancel_task(self, task_filter: TaskFilter, task_uuid: TaskUUID) -> None:

packages/common-library/src/common_library/async_tools.py

@@ -89,32 +89,37 @@ async def cancel_wait_task(
         TimeoutError: raised if cannot cancel the task.
         CancelledError: raised ONLY if owner is being cancelled.
     """
+    if task.done():
+        # nothing to do here
+        return
 
-    cancelling = task.cancel()
-    if not cancelling:
-        return  # task was alredy cancelled
-
-    assert task.cancelling()  # nosec
-    assert not task.cancelled()  # nosec
-
+    # mark for cancellation
+    task.cancel("cancel_wait_task was called to cancel this task")
     try:
-
+        _logger.debug("Cancelling task %s", task.get_name())
         await asyncio.shield(
             # NOTE shield ensures that cancellation of the caller function won't stop you
             # from observing the cancellation/finalization of task.
             asyncio.wait_for(task, timeout=max_delay)
         )
-
-    except asyncio.CancelledError:
-        if not task.cancelled():
-            # task owner function is being cancelled -> propagate cancellation
-            raise
-
-        # else: task cancellation is complete, we can safely ignore it
-        _logger.debug(
-            "Task %s cancellation is complete",
+    except TimeoutError:
+        _logger.exception(
+            "Timeout while cancelling task %s after %s seconds",
             task.get_name(),
+            max_delay,
         )
+        raise
+    except asyncio.CancelledError:
+        current_task = asyncio.current_task()
+        assert current_task is not None  # nosec
+        if current_task.cancelling() > 0:
+            # owner function is being cancelled -> propagate cancellation
+            raise
+    finally:
+        if not task.done():
+            _logger.error("Failed to cancel %s", task.get_name())
+        else:
+            _logger.debug("Task %s cancelled", task.get_name())
 
 
 def delayed_start(

packages/postgres-database/src/simcore_postgres_database/migration/versions/7e92447558e0_update_api_keys_uniqueness_constraint.py

@@ -0,0 +1,37 @@
+"""Update api-keys uniqueness constraint
+
+Revision ID: 7e92447558e0
+Revises: 06eafd25d004
+Create Date: 2025-09-12 09:56:45.164921+00:00
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7e92447558e0"
+down_revision = "06eafd25d004"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint("display_name_userid_uniqueness", "api_keys", type_="unique")
+    op.create_unique_constraint(
+        "display_name_userid_product_name_uniqueness",
+        "api_keys",
+        ["display_name", "user_id", "product_name"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(
+        "display_name_userid_product_name_uniqueness", "api_keys", type_="unique"
+    )
+    op.create_unique_constraint(
+        "display_name_userid_uniqueness", "api_keys", ["display_name", "user_id"]
+    )
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/api_keys.py

@@ -75,7 +75,10 @@
         "If set to NULL then the key does not expire.",
     ),
     sa.UniqueConstraint(
-        "display_name", "user_id", name="display_name_userid_uniqueness"
+        "display_name",
+        "user_id",
+        "product_name",
+        name="display_name_userid_product_name_uniqueness",
     ),
 )
 

packages/pytest-simcore/src/pytest_simcore/helpers/long_running_tasks.py

@@ -0,0 +1,39 @@
+# pylint: disable=protected-access
+
+import pytest
+from fastapi import FastAPI
+from servicelib.long_running_tasks.errors import TaskNotFoundError
+from servicelib.long_running_tasks.manager import (
+    LongRunningManager,
+)
+from servicelib.long_running_tasks.models import TaskContext
+from servicelib.long_running_tasks.task import TaskId
+from tenacity import (
+    AsyncRetrying,
+    retry_if_not_exception_type,
+    stop_after_delay,
+    wait_fixed,
+)
+
+
+def get_fastapi_long_running_manager(app: FastAPI) -> LongRunningManager:
+    manager = app.state.long_running_manager
+    assert isinstance(manager, LongRunningManager)
+    return manager
+
+
+async def assert_task_is_no_longer_present(
+    manager: LongRunningManager, task_id: TaskId, task_context: TaskContext
+) -> None:
+    async for attempt in AsyncRetrying(
+        reraise=True,
+        wait=wait_fixed(0.1),
+        stop=stop_after_delay(60),
+        retry=retry_if_not_exception_type(TaskNotFoundError),
+    ):
+        with attempt:  # noqa: SIM117
+            with pytest.raises(TaskNotFoundError):
+                # use internals to detirmine when it's no longer here
+                await manager._tasks_manager._get_tracked_task(  # noqa: SLF001
+                    task_id, task_context
+                )

packages/service-library/setup.cfg

@@ -21,6 +21,7 @@ markers =
 	testit: "marks test to run during development"
 	performance_test: "performance test"
 	no_cleanup_check_rabbitmq_server_has_no_errors: "no check in rabbitmq logs"
+	heavy_load: "marks test as heavy load"
 
 [mypy]
 plugins =