♻️ Refactors functional services package (ITISFoundation#2962)

Author: pcrespov

packages/models-library/src/models_library/function_service/_utils.py

@@ -0,0 +1,5 @@
+""" Catalog of services implemented as code instead of in a container
+
+
+"""
+from .api import *

packages/models-library/src/models_library/function_service/constants.py

@@ -0,0 +1,21 @@
+from typing import Final
+
+# NOTE: due to legacy reasons, the name remains with 'frontend' in it but
+# it now refers to a more general group: function sections that contains front-end services as well
+FUNCTION_SERVICE_KEY_PREFIX: Final[str] = "simcore/services/frontend"
+
+
+def is_function_service(service_key: str) -> bool:
+    return service_key.startswith(f"{FUNCTION_SERVICE_KEY_PREFIX}/")
+
+
+def is_parameter_service(service_key: str) -> bool:
+    return service_key.startswith(f"{FUNCTION_SERVICE_KEY_PREFIX}/parameter/")
+
+
+def is_iterator_service(service_key: str) -> bool:
+    return service_key.startswith(f"{FUNCTION_SERVICE_KEY_PREFIX}/data-iterator/")
+
+
+def is_iterator_consumer_service(service_key: str) -> bool:
+    return service_key.startswith(f"{FUNCTION_SERVICE_KEY_PREFIX}/iterator-consumer/")

packages/models-library/src/models_library/function_services_catalog.py

@@ -0,0 +1,34 @@
+"""
+    Factory to build catalog of i/o metadata for functions implemented in the front-end
+
+    NOTE: These definitions are currently needed in the catalog and director2
+    services. Since it is static data, instead of making a call from
+    director2->catalog, it was decided to share as a library
+"""
+
+import logging
+
+from models_library.function_services_catalog.services import nodes_group
+
+from ._settings import FunctionServiceSettings
+from ._utils import FunctionServices
+from .services import (
+    demo_units,
+    file_picker,
+    iter_range,
+    iter_sensitivity,
+    parameters,
+    probes,
+)
+
+logger = logging.getLogger(__name__)
+
+
+catalog = FunctionServices(settings=FunctionServiceSettings())
+catalog.extend(demo_units.services)
+catalog.extend(file_picker.services)
+catalog.extend(iter_range.services)
+catalog.extend(iter_sensitivity.services)
+catalog.extend(nodes_group.services)
+catalog.extend(parameters.services)
+catalog.extend(probes.services)

packages/models-library/src/models_library/function_services_catalog/__init__.py

@@ -0,0 +1,26 @@
+import json
+import os
+
+from pydantic import BaseSettings
+
+# Expects env var: FUNCTION_SERVICES_AUTHORS='{"OM":{"name": ...}, "EN":{...} }'
+try:
+    AUTHORS = json.loads(os.environ.get("FUNCTION_SERVICES_AUTHORS", "{}"))
+except json.decoder.JSONDecodeError:
+    AUTHORS = {}
+
+
+class FunctionServiceSettings(BaseSettings):
+    CATALOG_DEV_FEATURES_ENABLED: bool = False
+    DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False
+    WEBSERVER_DEV_FEATURES_ENABLED: bool = False
+
+    def is_dev_feature_enabled(self) -> bool:
+        # NOTE that this is imported in these services
+        # This solution is not ideal but will suffice
+        # until function-services are moved to the database
+        return (
+            self.CATALOG_DEV_FEATURES_ENABLED
+            or self.DIRECTOR_V2_DEV_FEATURES_ENABLED
+            or self.WEBSERVER_DEV_FEATURES_ENABLED
+        )

packages/models-library/src/models_library/function_services_catalog/_key_labels.py

@@ -0,0 +1,113 @@
+import logging
+from dataclasses import dataclass
+from typing import Callable, Dict, Iterable, Optional, Tuple
+from urllib.parse import quote
+
+from models_library.basic_regex import VERSION_RE
+from models_library.services import SERVICE_KEY_RE
+from pydantic import constr
+
+from ..services import Author, ServiceDockerData
+from ._settings import AUTHORS, FunctionServiceSettings
+
+log = logging.getLogger(__name__)
+
+
+_DEFAULT = {
+    "name": "Unknown",
+    "email": "[email protected]",
+    "affiliation": "unknown",
+}
+EN = Author.parse_obj(AUTHORS.get("EN", _DEFAULT))
+OM = Author.parse_obj(AUTHORS.get("OM", _DEFAULT))
+PC = Author.parse_obj(AUTHORS.get("PC", _DEFAULT))
+
+
+def create_fake_thumbnail_url(label: str) -> str:
+    return f"https://fakeimg.pl/100x100/ff0000%2C128/000%2C255/?text={quote(label)}"
+
+
+ServiceKey = constr(regex=SERVICE_KEY_RE)
+ServiceVersion = constr(regex=VERSION_RE)
+
+
+@dataclass
+class _Record:
+    meta: ServiceDockerData
+    implementation: Optional[Callable] = None
+    is_under_development: bool = False
+
+
+class FunctionServices:
+    """Used to register a collection of function services"""
+
+    def __init__(self, settings: Optional[FunctionServiceSettings] = None):
+        self._functions: Dict[Tuple[ServiceKey, ServiceVersion], _Record] = {}
+        self.settings = settings
+
+    def add_function_service(
+        self,
+        meta: ServiceDockerData,
+        implementation: Optional[Callable] = None,
+        is_under_development: bool = False,
+    ):
+
+        if not isinstance(meta, ServiceDockerData):
+            raise ValueError(f"Expected ServiceDockerData, got {type(meta)}")
+
+        # ensure unique
+        if (meta.key, meta.version) in self._functions:
+            raise ValueError(f"{(meta.key, meta.version)} is already registered")
+
+        # TODO: ensure callable signature fits metadata
+
+        # register
+        self._functions[(meta.key, meta.version)] = _Record(
+            meta=meta,
+            implementation=implementation,
+            is_under_development=is_under_development,
+        )
+
+    def extend(self, other: "FunctionServices"):
+        # pylint: disable=protected-access
+        for f in other._functions.values():
+            self.add_function_service(f.meta, f.implementation, f.is_under_development)
+
+    def skip_dev(self):
+        skip = True
+        if self.settings:
+            skip = not self.settings.is_dev_feature_enabled()
+        return skip
+
+    def iter_items(self):
+        skip_dev = self.skip_dev()
+        for key, value in self._functions.items():
+            if value.is_under_development and skip_dev:
+                continue
+            yield key, value
+
+    def iter_metadata(self) -> Iterable[ServiceDockerData]:
+        for _, f in self.iter_items():
+            yield f.meta
+
+    def iter_services_key_version(self) -> Iterable[Tuple[ServiceKey, ServiceVersion]]:
+        for kv, f in self.iter_items():
+            assert kv == (f.meta.key, f.meta.version)  # nosec
+            yield kv
+
+    def get_implementation(
+        self, service_key: ServiceKey, service_version: ServiceVersion
+    ) -> Optional[Callable]:
+        # raises KeyError if not found
+        func = self._functions[(service_key, service_version)]
+        return func.implementation
+
+    def get_metadata(
+        self, service_key: ServiceKey, service_version: ServiceVersion
+    ) -> ServiceDockerData:
+        # raises KeyError if not found
+        func = self._functions[(service_key, service_version)]
+        return func.meta
+
+    def __len__(self):
+        return len(self._functions)

packages/models-library/src/models_library/function_services_catalog/_registry.py

@@ -0,0 +1,32 @@
+"""
+    Factory to build catalog of i/o metadata for functions implemented in the front-end
+
+    NOTE: These definitions are currently needed in the catalog and director2
+    services. Since it is static data, instead of making a call from
+    director2->catalog, it was decided to share as a library
+"""
+
+from typing import Iterator, Tuple
+
+from ..services import ServiceDockerData
+from ._key_labels import is_function_service, is_iterator_service
+from ._registry import catalog
+
+assert catalog  # nosec
+assert is_iterator_service  # nosec
+
+
+def iter_service_docker_data() -> Iterator[ServiceDockerData]:
+    for meta_obj in catalog.iter_metadata():
+        # NOTE: the originals are this way not modified from outside
+        copied_meta_obj = meta_obj.copy(deep=True)
+        assert is_function_service(copied_meta_obj.key)  # nosec
+        yield copied_meta_obj
+
+
+__all__: Tuple[str, ...] = (
+    "catalog",
+    "is_function_service",
+    "is_iterator_service",
+    "iter_service_docker_data",
+)