🐛 ✨ Is353/meta func command and fixes (ITISFoundation#2943)

Author: pcrespov

.vscode/settings.template.json

@@ -7,11 +7,13 @@
   "files.associations": {
     ".*rc": "ini",
     ".env*": "ini",
-    "Dockerfile*": "dockerfile",
-    "**/requirements/*.txt": "pip-requirements",
     "**/requirements/*.in": "pip-requirements",
+    "**/requirements/*.txt": "pip-requirements",
+    "*logs.txt": "log",
+    "*.logs*": "log",
     "*Makefile": "makefile",
-    "*.cwl": "yaml"
+    "docker-compose*.yml": "dockercompose",
+    "Dockerfile*": "dockerfile"
   },
   "files.eol": "\n",
   "files.exclude": {

api/specs/common/schemas/services.yaml

@@ -49,6 +49,14 @@ components:
             vcs_url:
               type: string
 
+        container_spec:
+          type: object
+          properties:
+            command:
+              type: array
+              items:
+                type: string
+
     ServiceExtrasEnveloped:
       type: object
       required:

packages/models-library/src/models_library/service_settings_labels.py

@@ -4,7 +4,7 @@
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Literal, Optional
 
 from pydantic import BaseModel, Extra, Field, Json, PrivateAttr, validator
 
@@ -16,10 +16,39 @@ class _BaseConfig:
     keep_untouched = (cached_property,)
 
 
+class ContainerSpec(BaseModel):
+    """Implements entries that can be overriden for https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate
+    request body: TaskTemplate -> ContainerSpec
+    """
+
+    command: list[str] = Field(
+        alias="Command",
+        description="Used to override the container's command",
+        # NOTE: currently constraint to our use cases. Might mitigate some security issues.
+        min_items=1,
+        max_items=2,
+    )
+
+    class Config(_BaseConfig):
+        schema_extra = {
+            "examples": [
+                {"Command": ["executable"]},
+                {"Command": ["executable", "subcommand"]},
+                {"Command": ["ofs", "linear-regression"]},
+            ]
+        }
+
+
 class SimcoreServiceSettingLabelEntry(BaseModel):
+    """These values are used to build the request body of https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate
+    Specifically the section under ``TaskTemplate``
+    """
+
     _destination_container: str = PrivateAttr()
     name: str = Field(..., description="The name of the service setting")
-    setting_type: str = Field(
+    setting_type: Literal[
+        "string", "int", "integer", "number", "object", "ContainerSpec", "Resources"
+    ] = Field(
         ...,
         description="The type of the service setting (follows Docker REST API naming scheme)",
         alias="type",
@@ -38,7 +67,13 @@ class Config(_BaseConfig):
                     "type": "string",
                     "value": ["node.platform.os == linux"],
                 },
-                # resources
+                # SEE service_settings_labels.py::ContainerSpec
+                {
+                    "name": "ContainerSpec",
+                    "type": "ContainerSpec",
+                    "value": {"Command": ["run"]},
+                },
+                # SEE service_resources.py::ResourceValue
                 {
                     "name": "Resources",
                     "type": "Resources",
@@ -83,12 +118,12 @@ class PathMappingsLabel(BaseModel):
         ...,
         description="folder path where the service is expected to provide all its outputs",
     )
-    state_paths: List[Path] = Field(
+    state_paths: list[Path] = Field(
         [],
         description="optional list of paths which contents need to be persisted",
     )
 
-    state_exclude: Optional[List[str]] = Field(
+    state_exclude: Optional[list[str]] = Field(
         None,
         description="optional list unix shell rules used to exclude files from the state",
     )

packages/models-library/src/models_library/services.py

@@ -3,6 +3,7 @@
 NOTE: to dump json-schema from CLI use
     python -c "from models_library.services import ServiceDockerData as cls; print(cls.schema_json(indent=2))" > services-schema.json
 """
+
 from enum import Enum
 from typing import Any, Optional, Union
 
@@ -22,7 +23,11 @@
 from .basic_regex import VERSION_RE
 from .boot_options import BootOption, BootOptions
 from .services_ui import Widget
-from .utils.json_schema import InvalidJsonSchema, jsonschema_validate_schema
+from .utils.json_schema import (
+    InvalidJsonSchema,
+    any_ref_key,
+    jsonschema_validate_schema,
+)
 
 # CONSTANTS -------------------------------------------
 
@@ -203,6 +208,11 @@ def check_valid_json_schema(cls, v):
         if v is not None:
             try:
                 jsonschema_validate_schema(schema=v)
+
+                if any_ref_key(v):
+                    # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3030
+                    raise ValueError("Schemas with $ref are still not supported")
+
             except InvalidJsonSchema as err:
                 failed_path = "->".join(map(str, err.path))
                 raise ValueError(

packages/models-library/src/models_library/utils/json_schema.py

@@ -8,6 +8,7 @@
 # SEE possible enhancements in https://github.com/ITISFoundation/osparc-simcore/issues/3008
 
 
+from collections.abc import Sequence
 from contextlib import suppress
 from copy import deepcopy
 from typing import Any, Dict, Tuple
@@ -82,9 +83,20 @@ def jsonschema_validate_schema(schema: Dict[str, Any]):
     return schema
 
 
+def any_ref_key(obj):
+    if isinstance(obj, dict):
+        return "$ref" in obj.keys() or any_ref_key(tuple(obj.values()))
+
+    if isinstance(obj, Sequence) and not isinstance(obj, str):
+        return any(any_ref_key(v) for v in obj)
+
+    return False
+
+
 __all__: Tuple[str, ...] = (
+    "any_ref_key",
     "InvalidJsonSchema",
-    "JsonSchemaValidationError",
-    "jsonschema_validate_schema",
     "jsonschema_validate_data",
+    "jsonschema_validate_schema",
+    "JsonSchemaValidationError",
 )

packages/models-library/tests/test__models_examples.py

@@ -0,0 +1,59 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
+import json
+from contextlib import suppress
+from importlib import import_module
+from inspect import getmembers, isclass
+from pathlib import Path
+from typing import Any, Iterable, Optional, Set, Tuple, Type
+
+import models_library
+import pytest
+from models_library.utils.misc import extract_examples
+from pydantic import BaseModel, NonNegativeInt
+from pydantic.json import pydantic_encoder
+
+
+def iter_model_cls_examples(
+    exclude: Optional[Set] = None,
+) -> Iterable[Tuple[str, Type[BaseModel], NonNegativeInt, Any]]:
+    def _is_model_cls(cls) -> bool:
+        with suppress(TypeError):
+            # NOTE: issubclass( dict[models_library.services.ConstrainedStrValue, models_library.services.ServiceInput] ) raises TypeError
+            return cls is not BaseModel and isclass(cls) and issubclass(cls, BaseModel)
+        return False
+
+    exclude = exclude or set()
+
+    for filepath in Path(models_library.__file__).resolve().parent.glob("*.py"):
+        if not filepath.name.startswith("_"):
+            mod = import_module(f"models_library.{filepath.stem}")
+            for name, model_cls in getmembers(mod, _is_model_cls):
+                if name in exclude:
+                    continue
+                # NOTE: this is part of utils.misc and is tested here
+                examples = extract_examples(model_cls)
+                for index, example in enumerate(examples):
+                    yield (name, model_cls, index, example)
+
+
+@pytest.mark.parametrize(
+    "class_name, model_cls, example_index, test_example", iter_model_cls_examples()
+)
+def test_all_module_model_examples(
+    class_name: str,
+    model_cls: Type[BaseModel],
+    example_index: NonNegativeInt,
+    test_example: Any,
+):
+    """Automatically collects all BaseModel subclasses having examples and tests them against schemas"""
+    print(
+        f"test {example_index=} for {class_name=}:\n",
+        json.dumps(test_example, default=pydantic_encoder, indent=2),
+        "---",
+    )
+    model_instance = model_cls.parse_obj(test_example)
+    assert isinstance(model_instance, model_cls)

packages/models-library/tests/test_utils_json_schema.py

@@ -6,9 +6,11 @@
 from collections import deque
 
 import pytest
+from faker import Faker
 from models_library.utils.json_schema import (
     InvalidJsonSchema,
     JsonSchemaValidationError,
+    any_ref_key,
     jsonschema_validate_data,
     jsonschema_validate_schema,
 )
@@ -117,3 +119,56 @@ def test_jsonschema_validate_data_succeed(valid_schema):
         "b": True,
         "s": "foo",
     }
+
+
+def test_resolve_content_schema(faker: Faker):
+    #
+    # https://python-jsonschema.readthedocs.io/en/stable/_modules/jsonschema/validators/#RefResolver.in_scope
+    #
+    import jsonschema
+    import jsonschema.validators
+
+    with pytest.raises(jsonschema.ValidationError):
+        jsonschema.validate(instance=[2, 3, 4], schema={"maxItems": 2})
+
+    schema_with_ref = {
+        "title": "Complex_value",
+        "$ref": "#/definitions/Complex",
+        "definitions": {
+            "Complex": {
+                "title": "Complex",
+                "type": "object",
+                "properties": {
+                    "real": {"title": "Real", "default": 0, "type": "number"},
+                    "imag": {"title": "Imag", "default": 0, "type": "number"},
+                },
+            }
+        },
+    }
+
+    assert any_ref_key(schema_with_ref)
+
+    resolver = jsonschema.RefResolver.from_schema(schema_with_ref)
+    assert resolver.resolution_scope == ""
+    assert resolver.base_uri == ""
+
+    ref, schema_resolved = resolver.resolve(schema_with_ref["$ref"])
+
+    assert ref == "#/definitions/Complex"
+    assert schema_resolved == {
+        "title": "Complex",
+        "type": "object",
+        "properties": {
+            "real": {"title": "Real", "default": 0, "type": "number"},
+            "imag": {"title": "Imag", "default": 0, "type": "number"},
+        },
+    }
+
+    assert not any_ref_key(schema_resolved)
+
+    validator = jsonschema.validators.validator_for(schema_with_ref)
+    validator.check_schema(schema_with_ref)
+
+    instance = {"real": faker.pyfloat()}
+    assert validator(schema_with_ref).is_valid(instance)
+    assert validator(schema_resolved).is_valid(instance)

packages/models-library/tests/test_utils_misc.py

@@ -29,6 +29,11 @@ def to_labels(
         else:
             label = _json_dumps({key: value}, sort_keys=False)
 
+        # NOTE: docker-compose env var interpolation gets confused with schema's '$ref' and
+        # will replace it '$ref' with an empty string.
+        if isinstance(label, str) and "$ref" in label:
+            label = label.replace("$ref", "$$ref")
+
         labels[f"{prefix_key}.{key}"] = label
 
     return labels