Merge branch 'master' into extract-celery-code

Author: giancarloromeo

.vscode/launch.template.json

@@ -44,6 +44,28 @@
       "justMyCode": false
     },
     {
+      // This test adds --external-envfile and expects a file named ".secrets" in the workspace root.
+      "name": "Python: Test w/ repo.config",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "pytest",
+      "args": [
+        "--ff",
+        "--log-cli-level=INFO",
+        "--external-envfile=${workspaceFolder}/.secrets",
+        "--pdb",
+        "--setup-show",
+        "--durations=5",
+        "-s",
+        "-vv",
+        "${file}"
+      ],
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false
+    },
+    {
+      // This tests enables the httpx spy and dumps captures in a json. Mainly for api-server
       "name": "Python: Test-Httpx-Spy",
       "type": "debugpy",
       "request": "launch",

packages/pytest-simcore/src/pytest_simcore/environment_configs.py

@@ -3,6 +3,7 @@
 # pylint: disable=unused-variable
 
 
+import logging
 import re
 from pathlib import Path
 from typing import Any
@@ -12,6 +13,8 @@
 from .helpers.monkeypatch_envs import load_dotenv, setenvs_from_dict
 from .helpers.typing_env import EnvVarsDict
 
+_logger = logging.getLogger(__name__)
+
 
 def pytest_addoption(parser: pytest.Parser):
     simcore_group = parser.getgroup("simcore")
@@ -20,12 +23,17 @@ def pytest_addoption(parser: pytest.Parser):
         action="store",
         type=Path,
         default=None,
-        help="Path to an env file. Consider passing a link to repo configs, i.e. `ln -s /path/to/osparc-ops-config/repo.config`",
+        help="Path to an env file. Replaces .env-devel in the tests by an external envfile."
+        "e.g. consider "
+        " `ln -s /path/to/osparc-ops-config/repo.config .secrets` and then "
+        " `pytest --external-envfile=.secrets --pdb tests/unit/test_core_settings.py`",
     )
 
 
 @pytest.fixture(scope="session")
-def external_envfile_dict(request: pytest.FixtureRequest) -> EnvVarsDict:
+def external_envfile_dict(
+    request: pytest.FixtureRequest, osparc_simcore_root_dir: Path
+) -> EnvVarsDict:
     """
     If a file under test folder prefixed with `.env-secret` is present,
     then this fixture captures it.
@@ -35,19 +43,43 @@ def external_envfile_dict(request: pytest.FixtureRequest) -> EnvVarsDict:
     """
     envs = {}
     if envfile := request.config.getoption("--external-envfile"):
-        print("🚨 EXTERNAL `envfile` option detected. Loading", envfile, "...")
+        _logger.warning(
+            "🚨 EXTERNAL `envfile` option detected. Loading '%s' ...", envfile
+        )
 
         assert isinstance(envfile, Path)
         assert envfile.exists()
         assert envfile.is_file()
 
+        envfile = envfile.resolve()
+        osparc_simcore_root_dir = osparc_simcore_root_dir.resolve()
+
+        if osparc_simcore_root_dir in envfile.parents and not any(
+            term in envfile.name.lower() for term in ("ignore", "secret")
+        ):
+            _logger.warning(
+                "🚨 CAUTION: The external envfile '%s' may contain sensitive data and could be accidentally versioned. "
+                "To prevent this, include the words 'secret' or 'ignore' in the filename.",
+                envfile.name,
+            )
+
         envs = load_dotenv(envfile)
 
+    if envs:
+        response = input(
+            f"🚨 CAUTION: You are about to run tests using environment variables loaded from '{envfile}'.\n"
+            "This may cause tests to interact with or modify real external systems (e.g., production or staging environments).\n"
+            "Proceeding could result in data loss or unintended side effects.\n"
+            "Are you sure you want to continue? [y/N]: "
+        )
+        if response.strip().lower() not in ("y", "yes"):
+            pytest.exit("Aborted by user due to external envfile usage.")
+
     return envs
 
 
 @pytest.fixture(scope="session")
-def skip_if_external_envfile_dict(external_envfile_dict: EnvVarsDict) -> None:
+def skip_if_no_external_envfile(external_envfile_dict: EnvVarsDict) -> None:
     if not external_envfile_dict:
         pytest.skip(reason="Skipping test since external-envfile is not set")
 

packages/service-library/src/servicelib/aiohttp/monitoring.py

@@ -1,6 +1,5 @@
 """Enables monitoring of some quantities needed for diagnostics"""
 
-import asyncio
 import logging
 from collections.abc import Awaitable, Callable
 from time import perf_counter
@@ -61,9 +60,8 @@ async def middleware_handler(request: web.Request, handler: Handler):
         # See https://prometheus.io/docs/concepts/metric_types
 
         log_exception: BaseException | None = None
-        resp: web.StreamResponse = web.HTTPInternalServerError(
-            reason="Unexpected exception"
-        )
+        response: web.StreamResponse = web.HTTPInternalServerError()
+
         canonical_endpoint = request.path
         if request.match_info.route.resource:
             canonical_endpoint = request.match_info.route.resource.canonical
@@ -86,29 +84,25 @@ async def middleware_handler(request: web.Request, handler: Handler):
                 endpoint=canonical_endpoint,
                 user_agent=user_agent,
             ):
-                resp = await handler(request)
+                response = await handler(request)
 
             assert isinstance(  # nosec
-                resp, web.StreamResponse
+                response, web.StreamResponse
             ), "Forgot envelope middleware?"
 
         except web.HTTPServerError as exc:
-            resp = exc
+            response = exc
             log_exception = exc
-            raise resp from exc
+            raise
+
         except web.HTTPException as exc:
-            resp = exc
+            response = exc
             log_exception = None
-            raise resp from exc
-        except asyncio.CancelledError as exc:
-            resp = web.HTTPInternalServerError(text=f"{exc}")
-            log_exception = exc
-            raise resp from exc
+            raise
+
         except Exception as exc:  # pylint: disable=broad-except
-            resp = web.HTTPInternalServerError(text=f"{exc}")
-            resp.__cause__ = exc
             log_exception = exc
-            raise resp from exc
+            raise
 
         finally:
             response_latency_seconds = perf_counter() - start_time
@@ -118,13 +112,13 @@ async def middleware_handler(request: web.Request, handler: Handler):
                 method=request.method,
                 endpoint=canonical_endpoint,
                 user_agent=user_agent,
-                http_status=resp.status,
+                http_status=response.status,
                 response_latency_seconds=response_latency_seconds,
             )
 
             if exit_middleware_cb:
                 with log_catch(logger=log, reraise=False):
-                    await exit_middleware_cb(request, resp)
+                    await exit_middleware_cb(request, response)
 
             if log_exception:
                 log.error(
@@ -135,12 +129,12 @@ async def middleware_handler(request: web.Request, handler: Handler):
                     request.method,
                     request.path,
                     response_latency_seconds,
-                    resp.status,
+                    response.status,
                     exc_info=log_exception,
                     stack_info=True,
                 )
 
-        return resp
+        return response
 
     setattr(  # noqa: B010
         middleware_handler, "__middleware_name__", f"{__name__}.monitor_{app_name}"

packages/service-library/src/servicelib/aiohttp/profiler_middleware.py

@@ -13,7 +13,7 @@ async def profiling_middleware(request: Request, handler):
         try:
             if _profiler.is_running or (_profiler.last_session is not None):
                 raise HTTPInternalServerError(
-                    reason="Profiler is already running. Only a single request can be profiled at any given time.",
+                    text="Profiler is already running. Only a single request can be profiled at any given time.",
                     headers={},
                 )
             _profiler.reset()
@@ -24,7 +24,7 @@ async def profiling_middleware(request: Request, handler):
 
             if response.content_type != MIMETYPE_APPLICATION_JSON:
                 raise HTTPInternalServerError(
-                    reason=f"Profiling middleware is not compatible with {response.content_type=}",
+                    text=f"Profiling middleware is not compatible with {response.content_type=}",
                     headers={},
                 )
 

packages/service-library/src/servicelib/aiohttp/requests_validation.py

@@ -51,7 +51,7 @@ def handle_validation_as_http_error(
             }
             for e in err.errors()
         ]
-        reason_msg = error_msg_template.format(
+        user_error_message = error_msg_template.format(
             failed=", ".join(d["loc"] for d in details)
         )
 
@@ -80,15 +80,14 @@ def handle_validation_as_http_error(
             error_str = json_dumps(
                 {
                     "error": {
-                        "msg": reason_msg,
+                        "msg": user_error_message,
                         "resource": resource_name,  # optional
                         "details": details,  # optional
                     }
                 }
             )
 
         raise web.HTTPUnprocessableEntity(  # 422
-            reason=reason_msg,
             text=error_str,
             content_type=MIMETYPE_APPLICATION_JSON,
         ) from err