♻️ Is638/dynamic sidecar (round 5), OECs, 🔨 OAS swagger with servers (ITISFoundation#3165)

Author: pcrespov

packages/models-library/src/models_library/basic_types.py

@@ -43,6 +43,10 @@ class BootModeEnum(str, Enum):
     PRODUCTION = "production"
     DEVELOPMENT = "development"
 
+    def is_devel_mode(self) -> bool:
+        """returns True if this boot mode is used for development"""
+        return self in (self.DEBUG, self.DEVELOPMENT, self.LOCAL)
+
 
 class BuildTargetEnum(str, Enum):
     """

packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py

@@ -1,13 +1,4 @@
-from typing import Optional
+EnvVarsDict = dict[str, str]
 
-EnvVarsDict = dict[str, Optional[str]]
-#
-# NOTE: that this means that env vars do not require a value. If that happens a None is assigned
-#   For instance, a valid env file is
-#
-#   NAME=foo
-#   INDEX=33
-#   ONLY_NAME=
-#
-# will return env: EnvVarsDict = {"NAME": "foo", "INDEX": 33, "ONLY_NAME": None}
-#
+
+# SEE packages/pytest-simcore/tests/test_helpers_utils_envs.py

packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py

@@ -1,33 +1,65 @@
 import os
+
+#
+# .env (dotenv) files (or envfile)
+#
 from io import StringIO
+from pathlib import Path
+from typing import Union
 
-from _pytest.monkeypatch import MonkeyPatch
-from dotenv import dotenv_values
+import dotenv
+from pytest import MonkeyPatch
 
 from .typing_env import EnvVarsDict
 
 
-def setenvs_as_envfile(monkeypatch: MonkeyPatch, envfile_text: str) -> EnvVarsDict:
-    envs = dotenv_values(stream=StringIO(envfile_text))
+#
+# monkeypatch using dict
+#
+def setenvs_from_dict(monkeypatch: MonkeyPatch, envs: EnvVarsDict):
     for key, value in envs.items():
+        assert value is not None  # None keys cannot be is defined w/o value
         monkeypatch.setenv(key, str(value))
+    return envs
+
+
+def load_dotenv(envfile_content_or_path: Union[Path, str], **options) -> EnvVarsDict:
+    """Convenient wrapper around dotenv.dotenv_values"""
+    kwargs = options.copy()
+    if isinstance(envfile_content_or_path, Path):
+        # path
+        kwargs["dotenv_path"] = envfile_content_or_path
+    else:
+        assert isinstance(envfile_content_or_path, str)
+        # content
+        kwargs["stream"] = StringIO(envfile_content_or_path)
+
+    return {k: v or "" for k, v in dotenv.dotenv_values(**kwargs).items()}
+
+
+#
+# monkeypath using envfiles ('.env' and also denoted as dotfiles)
+#
+
+
+def setenvs_from_envfile(
+    monkeypatch: MonkeyPatch, content_or_path: str, **dotenv_kwags
+) -> EnvVarsDict:
+    """Batch monkeypatch.setenv(...) on all env vars in an envfile"""
+    envs = load_dotenv(content_or_path, **dotenv_kwags)
+    setenvs_from_dict(monkeypatch, envs)
 
     assert all(env in os.environ for env in envs)
     return envs
 
 
-def delenvs_as_envfile(
-    monkeypatch: MonkeyPatch, envfile_text: str, raising: bool
+def delenvs_from_envfile(
+    monkeypatch: MonkeyPatch, content_or_path: str, raising: bool, **dotenv_kwags
 ) -> EnvVarsDict:
-    envs = dotenv_values(stream=StringIO(envfile_text))
+    """Batch monkeypatch.delenv(...) on all env vars in an envfile"""
+    envs = load_dotenv(content_or_path, **dotenv_kwags)
     for key in envs.keys():
         monkeypatch.delenv(key, raising=raising)
 
     assert all(env not in os.environ for env in envs)
     return envs
-
-
-def setenvs_from_dict(monkeypatch: MonkeyPatch, envs: EnvVarsDict):
-    for key, value in envs.items():
-        assert value is not None  # key is defined w/o value
-        monkeypatch.setenv(key, str(value))

packages/pytest-simcore/tests/test_helpers_utils_envs.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+from textwrap import dedent
+
+from pytest_simcore.helpers.utils_envs import EnvVarsDict, load_dotenv
+
+
+def test_load_envfile(tmp_path: Path):
+
+    envfile = tmp_path / ".env"
+    envfile.write_text(
+        dedent(
+            """
+            NAME=foo
+            INDEX=33
+            ONLY_NAME=
+            NULLED=null
+        """
+        )
+    )
+
+    envs: EnvVarsDict = load_dotenv(envfile, verbose=True)
+
+    assert {
+        "NAME": "foo",
+        "INDEX": "33",
+        "NULLED": "null",
+        "ONLY_NAME": "",
+    } == envs

packages/service-library/src/servicelib/error_codes.py

@@ -0,0 +1,31 @@
+""" osparc ERROR CODES (OEC)
+  Unique identifier of an exception instance
+  Intended to report a user about unexpected errors.
+    Unexpected exceptions can be traced by matching the
+    logged error code with that appeneded to the user-friendly message
+
+SEE test_error_codes for some use cases
+"""
+
+
+import re
+from typing import TYPE_CHECKING
+
+from pydantic.tools import parse_obj_as
+from pydantic.types import constr
+
+_LABEL = "OEC:{}"
+_PATTERN = r"OEC:\d+"
+
+if TYPE_CHECKING:
+    ErrorCodeStr = str
+else:
+    ErrorCodeStr = constr(strip_whitespace=True, regex=_PATTERN)
+
+
+def create_error_code(exception: Exception) -> ErrorCodeStr:
+    return parse_obj_as(ErrorCodeStr, _LABEL.format(id(exception)))
+
+
+def parse_error_code(obj) -> set[ErrorCodeStr]:
+    return set(re.findall(_PATTERN, f"{obj}"))

packages/service-library/src/servicelib/fastapi/openapi.py

@@ -3,13 +3,42 @@
 
 import types
 from types import FunctionType
-from typing import Any, Dict
+from typing import Any
 
 from fastapi.applications import FastAPI
 from fastapi.routing import APIRoute, APIRouter
 
 from ..functools_utils import copy_func
 
+# Some common values for FastAPI(... server=[ ... ]) parameter
+# It will be added to the OpenAPI Specs (OAS).
+_OAS_DEVELOPMENT_SERVER = {
+    "description": "Development server",
+    "url": "http://{host}:{port}",
+    "variables": {
+        "host": {"default": "127.0.0.1"},
+        "port": {"default": "8000"},
+    },
+}
+
+
+def get_common_oas_options(is_devel_mode: bool) -> dict[str, Any]:
+    """common OAS options for FastAPI constructor"""
+    servers = None
+    if is_devel_mode:
+        # NOTE: for security, only exposed in devel mode
+        # Make sure also that this is NOT used in edge services
+        # SEE https://sonarcloud.io/project/security_hotspots?id=ITISFoundation_osparc-simcore&pullRequest=3165&hotspots=AYHPqDfX5LRQZ1Ko6y4-
+        servers = [
+            _OAS_DEVELOPMENT_SERVER,
+        ]
+
+    return dict(
+        servers=servers,
+        docs_url="/dev/doc",
+        redoc_url=None,  # default disabled
+    )
+
 
 def redefine_operation_id_in_router(router: APIRouter, operation_id_prefix: str):
     """
@@ -42,14 +71,14 @@ def redefine_operation_id_in_router(router: APIRouter, operation_id_prefix: str)
 )
 
 
-def patch_openapi_specs(app_openapi: Dict[str, Any]):
+def patch_openapi_specs(app_openapi: dict[str, Any]):
     """Patches app.openapi with some fixes and osparc conventions
 
     Modifies fastapi auto-generated OAS to pass our openapi validation.
     """
 
     def _patch(node):
-        if isinstance(node, Dict):
+        if isinstance(node, dict):
             for key in list(node.keys()):
                 # SEE fastapi ISSUE: https://github.com/tiangolo/fastapi/issues/240 (test_openap.py::test_exclusive_min_openapi_issue )
                 # SEE openapi-standard: https://swagger.io/docs/specification/data-models/data-types/#range
@@ -80,7 +109,7 @@ def override_fastapi_openapi_method(app: FastAPI):
     # pylint: disable=protected-access
     app._original_openapi = types.MethodType(copy_func(app.openapi), app)  # type: ignore
 
-    def _custom_openapi_method(self: FastAPI) -> Dict:
+    def _custom_openapi_method(self: FastAPI) -> dict:
         """Overrides FastAPI.openapi member function
         returns OAS schema with vendor extensions
         """

packages/service-library/tests/test_error_codes.py

@@ -0,0 +1,54 @@
+# pylint: disable=broad-except
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+import logging
+
+from servicelib.error_codes import create_error_code, parse_error_code
+
+logger = logging.getLogger(__name__)
+
+
+def test_error_code_use_case(caplog):
+    """use case for error-codes"""
+    try:
+        raise RuntimeError("Something unexpected went wrong")
+    except Exception as err:
+        # 1. Unexpected ERROR
+
+        # 2. create error-code
+        error_code = create_error_code(err)
+
+        # 3. log all details in service
+        caplog.clear()
+
+        # Can add a formatter that prefix error-codes
+        syslog = logging.StreamHandler()
+        syslog.setFormatter(
+            logging.Formatter("%(asctime)s %(error_code)s : %(message)s")
+        )
+        logger.addHandler(syslog)
+
+        logger.error("Fake Unexpected error", extra={"error_code": error_code})
+
+        # logs something like E.g. 2022-07-06 14:31:13,432 OEC:140350117529856 : Fake Unexpected error
+        assert parse_error_code(
+            f"2022-07-06 14:31:13,432 {error_code} : Fake Unexpected error"
+        ) == {
+            error_code,
+        }
+
+        assert caplog.records[0].error_code == error_code
+        assert caplog.records[0]
+
+        logger.error("Fake without error_code")
+
+        # 4. inform user (e.g. with new error or sending message)
+        user_message = (
+            f"This is a user-friendly message to inform about an error. [{error_code}]"
+        )
+
+        assert parse_error_code(user_message) == {
+            error_code,
+        }