Merge branch 'master' into improve-celery-task-error-messages-and-improve-taskid-name

Author: bisgaard-itis

.env-devel

@@ -56,6 +56,7 @@ CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_c
 CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest
 CLUSTERS_KEEPER_DASK_NPROCS=1
 CLUSTERS_KEEPER_DASK_NTHREADS=0
+CLUSTERS_KEEPER_DASK_NTHREADS_MULTIPLIER=1
 CLUSTERS_KEEPER_DASK_WORKER_SATURATION=inf
 CLUSTERS_KEEPER_EC2_ACCESS=null
 CLUSTERS_KEEPER_SSM_ACCESS=null

.github/instructions/python.instructions.md

@@ -35,6 +35,8 @@ Ensure compatibility with the following library versions:
 
 * Use `f-string` formatting for all string interpolation except for logging message strings.
 * Use **relative imports** within the same package/module.
+  - For imports within the same repository/project, always use relative imports (e.g., `from ..constants import APP_SETTINGS_KEY` instead of `from simcore_service_webserver.constants import APP_SETTINGS_KEY`)
+  - Use absolute imports only for external libraries and packages
 * Place **all imports at the top** of the file.
 * Document functions when the code is not self-explanatory or if asked explicitly.
 
@@ -44,6 +46,18 @@ Ensure compatibility with the following library versions:
 * Prefer `json_dumps` / `json_loads` from `common_library.json_serialization` instead of the built-in `json.dumps` / `json.loads`.
 * When using Pydantic models, prefer methods like `model.model_dump_json()` for serialization.
 
-### 7. **Running tests**
+### 7. **aiohttp Framework**
+
+* **Application Keys**: Always use `web.AppKey` for type-safe application storage instead of string keys
+  - Define keys with specific types: `APP_MY_KEY: Final = web.AppKey("APP_MY_KEY", MySpecificType)`
+  - Use precise types instead of generic `object` when the actual type is known
+  - Example: `APP_SETTINGS_KEY: Final = web.AppKey("APP_SETTINGS_KEY", ApplicationSettings)`
+  - Store and retrieve: `app[APP_MY_KEY] = value` and `data = app[APP_MY_KEY]`
+* **Request Keys**: Use `web.AppKey` for request storage as well for consistency and type safety
+* **Middleware**: Follow the repository's middleware patterns for cross-cutting concerns
+* **Error Handling**: Use the established exception handling decorators and patterns
+* **Route Definitions**: Use `web.RouteTableDef()` and organize routes logically within modules
+
+### 8. **Running tests**
 * Use `--keep-docker-up` flag when testing to keep docker containers up between sessions.
 * Always activate the python virtual environment before running pytest.

packages/celery-library/Makefile

@@ -27,6 +27,7 @@ tests: ## runs unit tests
 		--durations=10 \
 		--exitfirst \
 		--failed-first \
+		--keep-docker-up \
 		--pdb \
 		-vv \
 		$(CURDIR)/tests
@@ -41,6 +42,7 @@ tests-ci: ## runs unit tests
 		--cov-report=term-missing \
 		--cov-report=xml \
 		--junitxml=junit.xml -o junit_family=legacy \
+		--keep-docker-up \
 		--cov=celery_library \
 		--durations=10 \
 		--log-date-format="%Y-%m-%d %H:%M:%S" \

packages/pytest-simcore/src/pytest_simcore/redis_service.py

@@ -4,7 +4,6 @@
 
 import logging
 from collections.abc import AsyncIterator
-from datetime import timedelta
 
 import pytest
 import tenacity
@@ -116,14 +115,6 @@ async def wait_till_redis_responsive(redis_url: URL | str) -> None:
         await client.aclose(close_connection_pool=True)
 
 
-@pytest.fixture
-def mock_redis_socket_timeout(mocker: MockerFixture) -> None:
-    # lowered to allow CI to properly shutdown RedisClientSDK instances
-    mocker.patch(
-        "servicelib.redis._client.DEFAULT_SOCKET_TIMEOUT", timedelta(seconds=0.25)
-    )
-
-
 @pytest.fixture
 async def use_in_memory_redis(mocker: MockerFixture) -> RedisSettings:
     mocker.patch("redis.asyncio.from_url", FakeAsyncRedis)

packages/service-library/src/servicelib/aiohttp/aiopg_utils.py

@@ -1,20 +1,21 @@
-""" Holderplace for random helpers using aiopg
+"""Holderplace for random helpers using aiopg
 
-    - Drop here functions/constants that at that time does
-    not fit in any of the setups. Then, they can be moved and
-    refactor when new abstractions are used in place.
+- Drop here functions/constants that at that time does
+not fit in any of the setups. Then, they can be moved and
+refactor when new abstractions are used in place.
 
-    - aiopg is used as a client sdk to interact asynchronously with postgres service
+- aiopg is used as a client sdk to interact asynchronously with postgres service
 
-    SEE for aiopg: https://aiopg.readthedocs.io/en/stable/sa.html
-    SEE for underlying psycopg: http://initd.org/psycopg/docs/module.html
-    SEE for extra keywords: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
+SEE for aiopg: https://aiopg.readthedocs.io/en/stable/sa.html
+SEE for underlying psycopg: http://initd.org/psycopg/docs/module.html
+SEE for extra keywords: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS
 """
 
 # TODO: Towards implementing https://github.com/ITISFoundation/osparc-simcore/issues/1195
 # TODO: deprecate this module. Move utils into retry_policies, simcore_postgres_database.utils_aiopg
 
 import logging
+from typing import Final
 
 import sqlalchemy as sa
 from aiohttp import web
@@ -31,6 +32,8 @@
 
 log = logging.getLogger(__name__)
 
+APP_AIOPG_ENGINE_KEY: Final = web.AppKey("APP_AIOPG_ENGINE_KEY", Engine)
+
 
 async def raise_if_not_responsive(engine: Engine):
     async with engine.acquire() as conn:

packages/service-library/src/servicelib/aiohttp/application_keys.py

@@ -1,4 +1,4 @@
-""" Namespace to keep all application storage keys
+"""Namespace to keep all application storage keys
 
 Unique keys to identify stored data
 Naming convention accounts for the storage scope: application, request, response, configuration and/or resources
@@ -8,22 +8,25 @@
 
  See https://aiohttp.readthedocs.io/en/stable/web_advanced.html#data-sharing-aka-no-singletons-please
 """
+
 from typing import Final
 
-# REQUIREMENTS:
-# - guarantees all keys are unique
-# - one place for all common keys
-# - hierarchical classification
+from aiohttp import ClientSession, web
+
+# APPLICATION's CONTEXT KEYS
+
+# NOTE: use these keys to store/retrieve data from aiohttp.web.Application
+# SEE https://docs.aiohttp.org/en/stable/web_quickstart.html#aiohttp-web-app-key
 
 #
 # web.Application keys, i.e. app[APP_*_KEY]
 #
-APP_CONFIG_KEY: Final[str] = f"{__name__ }.config"
-APP_SETTINGS_KEY: Final[str] = f"{__name__ }.settings"
+APP_CONFIG_KEY = web.AppKey("APP_CONFIG_KEY", dict[str, object])
 
 APP_AIOPG_ENGINE_KEY: Final[str] = f"{__name__ }.aiopg_engine"
 
-APP_CLIENT_SESSION_KEY: Final[str] = f"{__name__ }.session"
+APP_CLIENT_SESSION_KEY: web.AppKey[ClientSession] = web.AppKey("APP_CLIENT_SESSION_KEY")
+
 
 APP_FIRE_AND_FORGET_TASKS_KEY: Final[str] = f"{__name__}.tasks"
 

packages/service-library/src/servicelib/aiohttp/application_setup.py

@@ -16,11 +16,11 @@
     TypedDict,
 )
 
-from .application_keys import APP_CONFIG_KEY, APP_SETTINGS_KEY
+from .application_keys import APP_CONFIG_KEY
 
 _logger = logging.getLogger(__name__)
 
-APP_SETUP_COMPLETED_KEY: Final[str] = f"{__name__ }.setup"
+APP_SETUP_COMPLETED_KEY: Final[web.AppKey] = web.AppKey("setup_completed", list[str])
 
 
 class _SetupFunc(Protocol):
@@ -115,13 +115,14 @@ def _get_app_settings_and_field_name(
     arg_settings_name: str | None,
     setup_func_name: str,
     logger: logging.Logger,
+    app_settings_key: web.AppKey,
 ) -> tuple[_ApplicationSettings | None, str | None]:
-    app_settings: _ApplicationSettings | None = app.get(APP_SETTINGS_KEY)
+    app_settings: _ApplicationSettings | None = app.get(app_settings_key)
     settings_field_name = arg_settings_name
 
     if app_settings:
         if not settings_field_name:
-            # FIXME: hard-coded WEBSERVER_ temporary
+            # NOTE: hard-coded WEBSERVER_ temporary
             settings_field_name = f"WEBSERVER_{arg_module_name.split('.')[-1].upper()}"
 
         logger.debug("Checking addon's %s ", f"{settings_field_name=}")
@@ -246,6 +247,7 @@ def app_module_setup(
     module_name: str,
     category: ModuleCategory,
     *,
+    app_settings_key: web.AppKey,
     settings_name: str | None = None,
     depends: list[str] | None = None,
     logger: logging.Logger = _logger,
@@ -336,6 +338,7 @@ def _wrapper(app: web.Application, *args, **kargs) -> bool:
                     settings_name,
                     setup_func.__name__,
                     logger,
+                    app_settings_key,
                 )
 
                 if (

packages/service-library/src/servicelib/aiohttp/client_session.py

@@ -1,5 +1,4 @@
 from collections.abc import AsyncGenerator
-from typing import cast
 
 from aiohttp import ClientSession, ClientTimeout, web
 from common_library.json_serialization import json_dumps
@@ -41,10 +40,11 @@ async def persistent_client_session(app: web.Application) -> AsyncGenerator[None
 def get_client_session(app: web.Application) -> ClientSession:
     """Refers to the one-and-only client in the app"""
     assert APP_CLIENT_SESSION_KEY in app  # nosec
-    return cast(ClientSession, app[APP_CLIENT_SESSION_KEY])
+    return app[APP_CLIENT_SESSION_KEY]
 
 
 __all__: tuple[str, ...] = (
+    "APP_CLIENT_SESSION_KEY",
     "get_client_session",
     "persistent_client_session",
 )

packages/service-library/src/servicelib/aiohttp/docker_utils.py

@@ -1,6 +1,9 @@
 import logging
+from typing import Final
 
+import aiodocker
 import aiohttp
+from aiohttp import web
 from models_library.docker import DockerGenericTag
 from pydantic import TypeAdapter, ValidationError
 from settings_library.docker_registry import RegistrySettings
@@ -18,6 +21,8 @@
 
 _logger = logging.getLogger(__name__)
 
+APP_DOCKER_ENGINE_KEY: Final = web.AppKey("APP_DOCKER_ENGINE_KEY", aiodocker.Docker)
+
 
 async def retrieve_image_layer_information(
     image: DockerGenericTag, registry_settings: RegistrySettings

packages/service-library/src/servicelib/aiohttp/long_running_tasks/server.py

@@ -6,9 +6,17 @@
 running task.
 """
 
+from typing import Final
+
+from aiohttp import web
+
 from ._manager import get_long_running_manager
 from ._server import setup, start_long_running_task
 
+APP_LONG_RUNNING_TASKS_KEY: Final = web.AppKey(
+    "APP_LONG_RUNNING_TASKS_KEY", dict[str, object]
+)
+
 __all__: tuple[str, ...] = (
     "get_long_running_manager",
     "setup",