Merge branch 'master' into is1701/pg-readonly-user

Author: pcrespov

.github/workflows/ci-testing-deploy.yml

@@ -276,6 +276,18 @@ jobs:
       fail-fast: false
     name: "[build] docker images"
     steps:
+      - name: Remove unused software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          echo "Available storage after:"
+          sudo df -h
+          echo
       - uses: actions/checkout@v4
       - name: setup docker buildx
         id: buildx
@@ -2485,7 +2497,6 @@ jobs:
           name: ${{ github.job }}_tracing
           path: tests/e2e-playwright/test-results
 
-
   system-test-environment-setup:
     timeout-minutes: 30 # if this timeout gets too small, then split the tests
     name: "[sys] environment setup"

packages/models-library/Makefile

@@ -110,14 +110,15 @@ erd-ServiceInput.svg: _erdantic
 DOWNLOADED_TEST_DATA_DIR = "$(CURDIR)/tests/data/.downloaded-ignore"
 
 .PHONY: _httpx
-_httpx: _check_venv_active
+_ensure_httpx: _check_venv_active
 	# ensures requirements installed
 	@python3 -c "import httpx" 2>/dev/null || uv pip install httpx
 
-PHONY: pull_test_data
-pull_test_data: $(DOT_ENV_FILE) _httpx ## downloads tests data from registry (this can take some time!)
-	# downloading all metadata files
+PHONY: tests-data
+tests-data: $(DOT_ENV_FILE) _ensure_httpx ## downloads tests data from registry defined in .env (time-intensive!)
+	# Downloading all metadata files ...
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
 	python3 "$(PACKAGES_DIR)/pytest-simcore/src/pytest_simcore/helpers/docker_registry.py" $(DOWNLOADED_TEST_DATA_DIR)
+	@echo "Run now 'pytest -vv -m diagnostics tests'"

packages/models-library/src/models_library/errors_classes.py

@@ -30,3 +30,7 @@ def _get_full_class_name(cls) -> str:
             )
         ]
         return ".".join(reversed(relevant_classes))
+
+    def error_context(self):
+        """Returns context in which error occurred and stored within the exception"""
+        return dict(**self.__dict__)

packages/models-library/tests/test_errors_classes.py

@@ -148,3 +148,14 @@ class MyErrorAfter(OsparcErrorMixin, ValueError):
         msg_template = "{value} and {missing}"
 
     assert str(MyErrorAfter(value=42)) == "42 and 'missing=?'"
+
+
+def test_exception_context():
+    class MyError(OsparcErrorMixin, ValueError):
+        msg_template = "{value} and {missing}"
+
+    exc = MyError(value=42, missing="foo", extra="bar")
+    assert exc.error_context() == {"value": 42, "missing": "foo", "extra": "bar"}
+
+    exc = MyError(value=42)
+    assert exc.error_context() == {"value": 42}

packages/notifications-library/src/notifications_library/templates/on_account_form.email.content.html

@@ -4,7 +4,7 @@
 <p>Dear Support team
 
 <p>
-  We have received the following request form for an account in {{ product.display_name }} from {{ host }}
+  We have received the following request form for an account in {{ product.display_name }} from <b>{{ host }}</b>
 </p>
 
 <pre>

packages/notifications-library/src/notifications_library/templates/on_account_form.email.content.txt

@@ -1,6 +1,6 @@
 Dear Support team,
 
-We have received the following request form for an account in {{ product.display_name }} from {{ host }}:
+We have received the following request form for an account in {{ product.display_name }} from **{{ host }}**:
 
 {{ dumps(request_form) }}
 

packages/service-library/src/servicelib/aiohttp/rest_middlewares.py

@@ -12,8 +12,11 @@
 from aiohttp import web
 from aiohttp.web_request import Request
 from aiohttp.web_response import StreamResponse
+from models_library.errors_classes import OsparcErrorMixin
 from models_library.utils.json_serialization import json_dumps
+from servicelib.error_codes import create_error_code
 
+from ..logging_utils import create_troubleshotting_log_message, get_log_record_extra
 from ..mimetype_constants import MIMETYPE_APPLICATION_JSON
 from ..utils import is_production_environ
 from .rest_models import ErrorItemType, ErrorType, LogMessageType
@@ -28,6 +31,7 @@
 from .typing_extension import Handler, Middleware
 
 DEFAULT_API_VERSION = "v0"
+FMSG_INTERNAL_ERROR_USER_FRIENDLY = "We apologize for the inconvenience. Our team has recorded the issue and is working to resolve it as quickly as possible. Thank you for your patience [{}]"
 
 
 _logger = logging.getLogger(__name__)
@@ -40,29 +44,41 @@ def is_api_request(request: web.Request, api_version: str) -> bool:
 
 def error_middleware_factory(
     api_version: str,
-    log_exceptions: bool = True,
 ) -> Middleware:
     _is_prod: bool = is_production_environ()
 
     def _process_and_raise_unexpected_error(request: web.BaseRequest, err: Exception):
+
+        error_code = create_error_code(err)
+        error_context: dict[str, Any] = {
+            "request.remote": f"{request.remote}",
+            "request.method": f"{request.method}",
+            "request.path": f"{request.path}",
+        }
+        if isinstance(err, OsparcErrorMixin):
+            error_context.update(err.error_context())
+
+        frontend_msg = FMSG_INTERNAL_ERROR_USER_FRIENDLY.format(error_code)
+        log_msg = create_troubleshotting_log_message(
+            message_to_user=frontend_msg,
+            error=err,
+            error_code=error_code,
+            error_context=error_context,
+        )
+
         http_error = create_http_error(
             err,
-            "Unexpected Server error",
+            frontend_msg,
             web.HTTPInternalServerError,
             skip_internal_error_details=_is_prod,
         )
-
-        if log_exceptions:
-            _logger.error(
-                'Unexpected server error "%s" from access: %s "%s %s". Responding with status %s',
-                type(err),
-                request.remote,
-                request.method,
-                request.path,
-                http_error.status,
-                exc_info=err,
-                stack_info=True,
-            )
+        _logger.exception(
+            log_msg,
+            extra=get_log_record_extra(
+                error_code=error_code,
+                user_id=error_context.get("user_id"),
+            ),
+        )
         raise http_error
 
     @web.middleware

packages/service-library/src/servicelib/aiohttp/rest_responses.py

@@ -110,7 +110,7 @@ def create_http_error(
         error = ErrorType(
             errors=items,
             status=http_error_cls.status_code,
-            message=items[0].message if items else default_message,
+            message=default_message,
         )
 
     assert not http_error_cls.empty_body  # nosec

packages/service-library/src/servicelib/logging_utils.py

@@ -16,6 +16,9 @@
 from pathlib import Path
 from typing import Any, TypeAlias, TypedDict, TypeVar
 
+from models_library.utils.json_serialization import json_dumps
+
+from .error_codes import ErrorCodeStr
 from .utils_secrets import mask_sensitive_data
 
 _logger = logging.getLogger(__name__)
@@ -320,20 +323,58 @@ def log_catch(logger: logging.Logger, *, reraise: bool = True) -> Iterator[None]
 
 class LogExtra(TypedDict, total=False):
     log_uid: str
+    log_oec: str
 
 
 LogLevelInt: TypeAlias = int
 LogMessageStr: TypeAlias = str
 
 
-def get_log_record_extra(*, user_id: int | str | None = None) -> LogExtra | None:
+def get_log_record_extra(
+    *,
+    user_id: int | str | None = None,
+    error_code: str | None = None,
+) -> LogExtra | None:
     extra: LogExtra = {}
+
     if user_id:
         assert int(user_id) > 0  # nosec
         extra["log_uid"] = f"{user_id}"
+    if error_code:
+        extra["log_oec"] = error_code
+
     return extra or None
 
 
+def create_troubleshotting_log_message(
+    message_to_user: str,
+    error: BaseException,
+    error_code: ErrorCodeStr,
+    error_context: dict[str, Any] | None = None,
+    tip: str | None = None,
+) -> str:
+    """Create a formatted message for _logger.exception(...)
+
+    Arguments:
+        message_to_user -- A user-friendly message to be displayed on the front-end explaining the issue in simple terms.
+        error -- the instance of the handled exception
+        error_code -- A unique error code (e.g., OEC or osparc-specific) to identify the type or source of the error for easier tracking.
+        error_context -- Additional context surrounding the exception, such as environment variables or function-specific data. This can be derived from exc.error_context() (relevant when using the OsparcErrorMixin)
+        tip -- Helpful suggestions or possible solutions explaining why the error may have occurred and how it could potentially be resolved
+    """
+    debug_data = json_dumps(
+        {
+            "exception_details": f"{error}",
+            "error_code": error_code,
+            "context": error_context,
+            "tip": tip,
+        },
+        indent=1,
+    )
+
+    return f"{message_to_user}.\n{debug_data}"
+
+
 def _un_capitalize(s: str) -> str:
     return s[:1].lower() + s[1:] if s else ""
 

packages/service-library/tests/aiohttp/long_running_tasks/conftest.py

@@ -33,7 +33,8 @@ async def _string_list_task(
         await asyncio.sleep(sleep_time)
         task_progress.update(message="generated item", percent=index / num_strings)
         if fail:
-            raise RuntimeError("We were asked to fail!!")
+            msg = "We were asked to fail!!"
+            raise RuntimeError(msg)
 
     # NOTE: this code is used just for the sake of not returning the default 200
     return web.json_response(