Merge remote-tracking branch 'origin/issue-150-config-default-values' into issue-150-refactor-broker-builder-classes

# Conflicts:
#	packages/core/minos-microservice-networks/minos/networks/brokers/subscribers/queued/queues/pg.py
#	packages/plugins/minos-broker-kafka/minos/plugins/kafka/publisher.py
#	packages/plugins/minos-broker-kafka/minos/plugins/kafka/subscriber.py

Author: Sergio García Prado

packages/core/minos-microservice-common/minos/common/config/abc.py

@@ -23,6 +23,7 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Optional,
     Union,
 )
 
@@ -43,13 +44,17 @@
         InjectableMixin,
     )
 
+sentinel = object()
+
 
 @Injectable("config")
 class Config(ABC):
     """Config base class."""
 
     __slots__ = ("_file_path", "_data", "_with_environment", "_parameterized")
 
+    DEFAULT_VALUES: dict[str, Any] = dict()
+
     def __init__(self, path: Union[Path, str], with_environment: bool = True, **kwargs):
         super().__init__()
         if isinstance(path, str):
@@ -282,7 +287,7 @@ def get_by_key(self, key: str) -> Any:
         :return: A value instance.
         """
 
-        def _fn(k: str, data: dict[str, Any], previous: str) -> Any:
+        def _fn(k: str, data: dict[str, Any], previous: str = "", default: Optional[Any] = sentinel) -> Any:
             current, _sep, following = k.partition(".")
             full = f"{previous}.{current}".lstrip(".")
 
@@ -293,20 +298,33 @@ def _fn(k: str, data: dict[str, Any], previous: str) -> Any:
                 with suppress(KeyError):
                     return os.environ[self._to_environment_variable(full)]
 
-            part = data[current]
-            if not following:
-                if not isinstance(part, dict):
-                    return part
+            if default is not sentinel and current in default:
+                default_part = default[current]
+            else:
+                default_part = sentinel
+
+            if current not in data and default_part is not sentinel:
+                part = default_part
+            else:
+                part = data[current]
+
+            if following:
+                return _fn(following, part, full, default_part)
+
+            if not isinstance(part, dict):
+                return part
 
-                result = dict()
-                for subpart in part:
-                    result[subpart] = _fn(subpart, part, full)
-                return result
+            keys = part.keys()
+            if isinstance(default_part, dict):
+                keys |= default_part.keys()
 
-            return _fn(following, part, full)
+            result = dict()
+            for subpart in keys:
+                result[subpart] = _fn(subpart, part, full, default_part)
+            return result
 
         try:
-            return _fn(key, self._data, str())
+            return _fn(key, self._data, default=self.DEFAULT_VALUES)
         except Exception:
             raise MinosConfigException(f"{key!r} field is not defined on the configuration!")
 

packages/core/minos-microservice-common/minos/common/config/v1.py

@@ -5,9 +5,6 @@
 from contextlib import (
     suppress,
 )
-from pathlib import (
-    Path,
-)
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -156,12 +153,14 @@ def _get_interface_http(self) -> dict[str, Any]:
         except Exception as exc:
             raise MinosConfigException(f"The 'http' interface is not available: {exc!r}")
 
+        try:
+            connector = self.get_by_key("rest")
+        except MinosConfigException:
+            connector = dict()
+
         return {
             "port": import_module(port),
-            "connector": {
-                "host": self.get_by_key("rest.host"),
-                "port": int(self.get_by_key("rest.port")),
-            },
+            "connector": connector,
         }
 
     def _get_interface_broker(self) -> dict[str, Any]:
@@ -170,18 +169,27 @@ def _get_interface_broker(self) -> dict[str, Any]:
         except Exception as exc:
             raise MinosConfigException(f"The 'broker' interface is not available: {exc!r}")
 
+        try:
+            common = self.get_by_key("broker")
+        except MinosConfigException:
+            common = dict()
+
+        try:
+            common["queue"] = self.get_by_key("broker.queue")
+            common["queue"].pop("database", None)
+            common["queue"].pop("port", None)
+            common["queue"].pop("host", None)
+            common["queue"].pop("port", None)
+            common["queue"].pop("user", None)
+            common["queue"].pop("password", None)
+        except MinosConfigException:
+            common["queue"] = dict()
+
         return {
             "port": import_module(port),
             "publisher": dict(),
             "subscriber": dict(),
-            "common": {
-                "host": self.get_by_key("broker.host"),
-                "port": int(self.get_by_key("broker.port")),
-                "queue": {
-                    "records": int(self.get_by_key("broker.queue.records")),
-                    "retry": int(self.get_by_key("broker.queue.retry")),
-                },
-            },
+            "common": common,
         }
 
     def _get_interface_periodic(self):
@@ -254,10 +262,8 @@ def _get_database_broker(self):
         return self._get_database_by_name("broker.queue")
 
     def _get_database_saga(self) -> dict[str, Any]:
-        raw = self.get_by_key("saga.storage.path")
-        return {
-            "path": Path(raw) if raw.startswith("/") else self.file_path.parent / raw,
-        }
+        raw = self.get_by_key("saga.storage")
+        return raw
 
     def _get_database_event(self) -> dict[str, Any]:
         return self._get_database_by_name("repository")
@@ -269,20 +275,15 @@ def _get_database_snapshot(self) -> dict[str, Any]:
         return self._get_database_by_name("snapshot")
 
     def _get_database_by_name(self, prefix: str):
-        return {
-            "database": self.get_by_key(f"{prefix}.database"),
-            "user": self.get_by_key(f"{prefix}.user"),
-            "password": self.get_by_key(f"{prefix}.password"),
-            "host": self.get_by_key(f"{prefix}.host"),
-            "port": int(self.get_by_key(f"{prefix}.port")),
-        }
+        data = self.get_by_key(prefix)
+        data.pop("records", None)
+        data.pop("retry", None)
+        return data
 
     def _get_discovery(self) -> dict[str, Any]:
-        return {
-            "client": self.get_type_by_key("discovery.client"),
-            "host": self.get_by_key("discovery.host"),
-            "port": self.get_by_key("discovery.port"),
-        }
+        data = self.get_by_key("discovery")
+        data["client"] = self.get_type_by_key("discovery.client")
+        return data
 
     def _to_parameterized_variable(self, key: str) -> str:
         return self._PARAMETERIZED_MAPPER[key]

packages/core/minos-microservice-common/minos/common/config/v2.py

@@ -8,9 +8,6 @@
 from copy import (
     deepcopy,
 )
-from pathlib import (
-    Path,
-)
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -99,15 +96,8 @@ def _get_injections(self) -> list[Union[InjectableMixin, type[InjectableMixin]]]
 
     def _get_databases(self) -> dict[str, dict[str, Any]]:
         data = deepcopy(self.get_by_key("databases"))
-
-        if "saga" in data:
-            if "path" in data["saga"]:
-                data["saga"]["path"] = self._str_to_path(data["saga"]["path"])
         return data
 
-    def _str_to_path(self, raw: str) -> Path:
-        return Path(raw) if raw.startswith("/") else self.file_path.parent / raw
-
     def _get_interfaces(self) -> dict[str, dict[str, Any]]:
         data = deepcopy(self.get_by_key("interfaces"))
 

packages/core/minos-microservice-common/minos/common/database/abc.py

@@ -32,17 +32,66 @@
 class PostgreSqlMinosDatabase(SetupMixin):
     """PostgreSql Minos Database base class."""
 
-    def __init__(self, host: str, port: int, database: str, user: str, password: str, *args, **kwargs):
+    def __init__(
+        self,
+        database: str,
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+        user: Optional[str] = None,
+        password: Optional[str] = None,
+        *args,
+        **kwargs,
+    ):
         super().__init__(*args, **kwargs)
-        self.host = host
-        self.port = port
-        self.database = database
-        self.user = user
-        self.password = password
+        self._database = database
+        self._host = host
+        self._port = port
+        self._user = user
+        self._password = password
 
         self._pool = None
         self._owned_pool = False
 
+    @property
+    def database(self) -> str:
+        """Get the database's database.
+
+        :return: A ``str`` value.
+        """
+        return self.pool.database
+
+    @property
+    def host(self) -> str:
+        """Get the database's host.
+
+        :return: A ``str`` value.
+        """
+        return self.pool.host
+
+    @property
+    def port(self) -> int:
+        """Get the database's port.
+
+        :return: An ``int`` value.
+        """
+        return self.pool.port
+
+    @property
+    def user(self) -> str:
+        """Get the database's user.
+
+        :return: A ``str`` value.
+        """
+        return self.pool.user
+
+    @property
+    def password(self) -> str:
+        """Get the database's password.
+
+        :return: A ``str`` value.
+        """
+        return self.pool.password
+
     async def _destroy(self) -> None:
         if self._owned_pool:
             await self._pool.destroy()
@@ -178,6 +227,6 @@ def _build_pool(self, pool: PostgreSqlPool = None) -> tuple[PostgreSqlPool, bool
             return pool, False
 
         pool = PostgreSqlPool(
-            host=self.host, port=self.port, database=self.database, user=self.user, password=self.password
+            host=self._host, port=self._port, database=self._database, user=self._user, password=self._password
         )
         return pool, True

packages/core/minos-microservice-common/minos/common/database/pools.py

@@ -40,11 +40,30 @@
 class PostgreSqlPool(Pool[ContextManager]):
     """Postgres Pool class."""
 
-    def __init__(self, host: str, port: int, database: str, user: str, password: str, *args, **kwargs):
+    def __init__(
+        self,
+        database: str,
+        host: Optional[str] = None,
+        port: Optional[int] = None,
+        user: Optional[str] = None,
+        password: Optional[str] = None,
+        *args,
+        **kwargs,
+    ):
         super().__init__(*args, **kwargs)
+
+        if host is None:
+            host = "localhost"
+        if port is None:
+            port = 5432
+        if user is None:
+            user = "postgres"
+        if password is None:
+            password = ""
+
+        self.database = database
         self.host = host
         self.port = port
-        self.database = database
         self.user = user
         self.password = password
 

packages/core/minos-microservice-common/minos/common/storage/lmdb.py

@@ -97,7 +97,9 @@ def _get_table(self, table: str):
             return self._tables[table]
 
     @classmethod
-    def build(cls, path: Union[str, Path], max_db: int = 10, map_size: int = int(1e9), **kwargs) -> MinosStorageLmdb:
+    def build(
+        cls, path: Optional[Union[str, Path]] = None, max_db: int = 10, map_size: int = int(1e9), **kwargs
+    ) -> MinosStorageLmdb:
         """Build a new instance.
 
         :param path: Path in which the database is stored.
@@ -106,6 +108,8 @@ def build(cls, path: Union[str, Path], max_db: int = 10, map_size: int = int(1e9
         :param kwargs: Additional named arguments.
         :return: A ``MinosStorageLmdb`` instance.
         """
+        if path is None:
+            path = ".lmdb"
 
         env: lmdb.Environment = lmdb.open(str(path), max_dbs=max_db, map_size=map_size)
         return cls(env, **kwargs)

packages/core/minos-microservice-common/tests/test_common/test_config/test_abc.py

@@ -25,6 +25,8 @@
 class _Config(Config):
     """For testing purposes."""
 
+    DEFAULT_VALUES = {"foo": {"bar": 56}, "saga": {"name": "foobar"}}
+
     # noinspection PyPropertyDefinition
     @property
     def _version(self) -> int:
@@ -88,6 +90,13 @@ def test_file_raises(self):
     def test_get_by_key(self):
         self.assertEqual("Order", self.config.get_by_key("service.name"))
 
+    def test_get_by_key_with_default_without_overlap(self):
+        self.assertEqual(56, self.config.get_by_key("foo.bar"))
+
+    def test_get_by_key_with_default_with_overlap(self):
+        expected = {"storage": {"path": "./order.lmdb"}, "name": "foobar"}
+        self.assertEqual(expected, self.config.get_by_key("saga"))
+
     def test_get_by_key_raises(self):
         with self.assertRaises(MinosConfigException):
             self.assertEqual("Order", self.config.get_by_key("something"))