Replace trafaret by pydantic for services config (iteration 1/2) (#1890)

- storage service updated
   - EXTRA: adds .env-devel and Makefile recipe for local development
   - EXTRA: adds swagger UI at /dev/doc
- webserver keeps for the moment both
More details in PR#1890

Author: pcrespov

packages/models-library/src/models_library/basic_types.py

@@ -0,0 +1,42 @@
+from enum import Enum
+
+from pydantic import conint, constr, confloat
+
+PortInt = conint(gt=0, lt=65535)
+
+VersionTag = constr(regex=r"^v\d$")
+
+NonNegativeFloat = confloat(ge=0)  # NOTE: = 0.0 + PositiveFloat
+
+
+class LogLevel(str, Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+
+
+class BootModeEnum(str, Enum):
+    """
+    Values taken by SC_BOOT_MODE environment variable
+    set in Dockerfile and used during docker/boot.sh
+    """
+
+    DEFAULT = "default"
+    LOCAL = "local-development"
+    DEBUG = "debug-ptvsd"
+    PRODUCTION = "production"
+    DEVELOPMENT = "development"
+
+
+class BuildTargetEnum(str, Enum):
+    """
+    Values taken by SC_BUILD_TARGET environment variable
+    set in Dockerfile that defines the stage targeted in the
+    docker image build
+    """
+
+    BUILD = "build"
+    CACHE = "cache"
+    PRODUCTION = "production"
+    DEVELOPMENT = "development"

packages/models-library/src/models_library/settings/__init__.py

@@ -0,0 +1,3 @@
+"""
+    Common settings
+"""

packages/models-library/src/models_library/settings/application_bases.py

@@ -0,0 +1,53 @@
+"""
+     Bases to build ApplicationSettings
+
+     TODO: when py 3.7 the use https://pydantic-docs.helpmanual.io/usage/models/#generic-models
+"""
+import logging
+from typing import Optional
+from pydantic import Field, BaseSettings, validator, AnyHttpUrl
+from ..basic_types import PortInt, BootModeEnum, LogLevel, VersionTag
+
+
+class BaseFastApiAppSettings(BaseSettings):
+    # DOCKER
+    boot_mode: Optional[BootModeEnum] = Field(..., env="SC_BOOT_MODE")
+
+    # LOGGING
+    log_level: LogLevel = Field("DEBUG", env=["LOG_LEVEL", "LOGLEVEL"])
+
+    @property
+    def logging_level(self) -> int:
+        return getattr(logging, self.log_level)
+
+    # SERVER (see : https://www.uvicorn.org/settings/)
+    host: str = "0.0.0.0"  # nosec
+    port: PortInt = 8000
+    debug: bool = False  # If True, debug tracebacks should be returned on errors.
+
+    # DEBUGGING
+    remote_debug_port: PortInt = 3000
+
+
+class BaseAiohttpAppSettings(BaseFastApiAppSettings):
+    port: PortInt = 8080
+
+
+class BaseServiceAPISettings(BaseSettings):
+    host: str
+    port: PortInt
+    vtag: VersionTag = Field("v0", description="Service API's version tag")
+
+    url: Optional[AnyHttpUrl] = None
+
+    @validator("url", pre=True)
+    @classmethod
+    def autofill_url(cls, v, values):
+        if v is None:
+            return AnyHttpUrl.build(
+                scheme="http",
+                host=values["host"],
+                port=f"{values['port']}",
+                path=f"/{values['vtag']}",
+            )
+        return v

…els-library/src/models_library/celery.py …ry/src/models_library/settings/celery.pypackages/models-library/src/models_library/celery.py renamed to packages/models-library/src/models_library/settings/celery.py packages/models-library/src/models_library/celery.py renamed to packages/models-library/src/models_library/settings/celery.py

@@ -5,13 +5,16 @@
 
 class CeleryConfig(BaseSettings):
     @classmethod
-    def create_from_env(cls) -> "CeleryConfig":
+    def create_from_env(cls, **override) -> "CeleryConfig":
         # this calls trigger env parsers
-        return cls(rabbit=RabbitConfig(), redis=RedisConfig())
+        return cls(rabbit=RabbitConfig(), redis=RedisConfig(), **override)
 
+    # NOTE: Defaults are evaluated at import time, use create_from_env to build instances
     rabbit: RabbitConfig = RabbitConfig()
     redis: RedisConfig = RedisConfig()
+
     task_name: str = "simcore.comp.task"
+
     publication_timeout: PositiveInt = 60
 
     @property

packages/models-library/src/models_library/settings/postgres.py

@@ -0,0 +1,51 @@
+from typing import Optional
+
+from pydantic import BaseSettings, Field, PostgresDsn, SecretStr, conint, validator
+
+from ..basic_types import PortInt
+
+
+class PostgresSettings(BaseSettings):
+    dns: Optional[PostgresDsn] = None
+
+    # entrypoint
+    host: str
+    port: PortInt = 5432
+
+    # auth
+    user: str
+    password: SecretStr
+
+    # database
+    db: str
+
+    # pool connection limits
+    minsize: conint(ge=1) = 10
+    maxsize: conint(ge=1) = 10
+
+    dsn: Optional[PostgresDsn] = Field(None, description="Database Source Name")
+
+    @validator("maxsize")
+    @classmethod
+    def check_size(cls, v, values):
+        if not (values["minsize"] <= v):
+            raise ValueError(f"assert minsize={values['minsize']} <= maxsize={v}")
+        return v
+
+    @validator("dsn", pre=True)
+    @classmethod
+    def autofill_dsn(cls, v, values):
+        if v is None:
+            return PostgresDsn.build(
+                scheme="postgresql",
+                user=values["user"],
+                password=values["password"].get_secret_value(),
+                host=values["host"],
+                port=f"{values['port']}",
+                path=f"/{values['db']}",
+            )
+        return v
+
+    class Config:
+        case_sensitive = False
+        env_prefix = "POSTGRES_"

…els-library/src/models_library/rabbit.py …ry/src/models_library/settings/rabbit.pypackages/models-library/src/models_library/rabbit.py renamed to packages/models-library/src/models_library/settings/rabbit.py packages/models-library/src/models_library/rabbit.py renamed to packages/models-library/src/models_library/settings/rabbit.py

@@ -0,0 +1,13 @@
+from pydantic import BaseSettings
+
+
+class S3Config(BaseSettings):
+    endpoint: str = "minio:9000"
+    access_key: str = "12345678"
+    secret_key: str = "12345678"
+    bucket_name: str = "simcore"
+    secure: bool = False
+
+    class Config:
+        case_sensitive = False
+        env_prefix = "S3_"

…dels-library/src/models_library/redis.py …ary/src/models_library/settings/redis.pypackages/models-library/src/models_library/redis.py renamed to packages/models-library/src/models_library/settings/redis.py packages/models-library/src/models_library/redis.py renamed to packages/models-library/src/models_library/settings/redis.py

@@ -12,7 +12,7 @@
 import pytest
 import tenacity
 
-from models_library.rabbit import RabbitConfig
+from models_library.settings.rabbit import RabbitConfig
 
 from .helpers.utils_docker import get_service_published_port
 

packages/models-library/src/models_library/settings/s3.py

@@ -8,7 +8,7 @@
 import aioredis
 import pytest
 import tenacity
-from models_library.redis import RedisConfig
+from models_library.settings.redis import RedisConfig
 from servicelib.redis_utils import RedisRetryPolicyUponInitialization
 from yarl import URL