✨Adding boot-options support (ITISFoundation#2686)

* boot-options support

* fixed spec regeneration

* applying patch

* pylint

* fixed integration test

* added mising env var

* fixed comment

* codestyle

* using exclude_patterns

* rename

* adding missing_ok option

* fix message

* added more information

* refactor

* refactor for readability

* added reason for usage

* updated openapi.json

* revert change and add clarificatio for usage

* pylint

* refactor exceptions

* removed subprocess call

* renamed functions

* better 204 handling

* docstring update

* refactor directory-watcher

* fixed tests for directory watcher

* fixed API interface

* fixed inerface to be more stanrad looking

* fixed usage in dynamic_sidecar

* added mising requirement

* refactor examples

* updated all modules

* removed function

* removed defult, not needed

* revert change

* refactor

* added some explenation for the cehcks

* remoing fixture

* simpler debugging

* using tmp_dir

* removed unused

* refactor

* further refactors to simplify

* refactor naming

* removing checks and disabling tests

* properly fixed signature

* renamed

* more renaming

* fixing tests

* updated comment

* codestyle

* update comment and example

* pylint

* fixing failing dv2 tests

* fix pylint

* fix webserver06 tests

* added misisng bootOptions

* foxed integration test

* update templates

* proper service state management

* pylint

* fix error messages

* wrong indentation level

* fixed test to allow for ports pulling

* fix failing public-api tests

* adding more information for debugging

* fix schema

* fix pylint

* fixing template

* codeclimate fix: refactor code

* fix import

* renamed and fixed validators

* using renamed version

* moved boot_options to node from project

* reverted change

* reverted change on template

* reverted change

* reverted tenoakte

* revert tempaltes

* removing column

* remove migration

* reverted to original

* regenerated correct schemas

* renaming

* frotend uses boot_optiosn on node

* fixed boot mode

* codestyle

* rename constants

* refactor to use context manager

* using prpoer constant type

* refactor to make more readable

* replaced formatter with error

* codestyle

* imports and codestyle

* fix jslint

* rename function

* refactor locationa nd name

* rename boot_options to bootOptions

* reverting changes

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

api/specs/common/schemas/node-meta-v0.0.1-converted.yaml

@@ -246,11 +246,11 @@ properties:
           - type
         properties:
           displayOrder:
-            description: >-
-              DEPRECATED: new display order is taken from the item position.
-              This property will be removed.
-            deprecated: true
             type: number
+            description: use this to numerically sort the properties for display
+            example:
+              - 1
+              - -0.2
           label:
             type: string
             description: short name for the property
@@ -291,3 +291,45 @@ properties:
             description: Units of the output value, if a physical quantity
             type: string
     additionalProperties: true
+  boot-options:
+    title: Boot Options
+    description: >-
+      Service defined boot options. These get injected in the service as env
+      variables.
+    type: object
+    x-patternProperties:
+      ^[_a-zA-Z0-9]+$:
+        title: BootOptionMode
+        type: object
+        properties:
+          label:
+            title: Label
+            type: string
+          description:
+            title: Description
+            type: string
+          default:
+            title: Default
+            type: string
+          items:
+            title: Items
+            type: object
+            additionalProperties:
+              title: BootOptionItem
+              type: object
+              properties:
+                label:
+                  title: Label
+                  type: string
+                description:
+                  title: Description
+                  type: string
+              required:
+                - label
+                - description
+        required:
+          - label
+          - description
+          - default
+          - items
+    additionalProperties: true

api/specs/common/schemas/node-meta-v0.0.1.json

@@ -180,9 +180,9 @@
           ],
           "properties": {
             "displayOrder": {
-            "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.",
-            "deprecated": true,
-            "type": "number"
+              "description": "DEPRECATED: new display order is taken from the item position. This property will be removed.",
+              "deprecated": true,
+              "type": "number"
             },
             "label": {
               "type": "string",
@@ -409,6 +409,59 @@
           }
         }
       }
+    },
+    "boot-options": {
+      "title": "Boot Options",
+      "description": "Service defined boot options. These get injected in the service as env variables.",
+      "type": "object",
+      "patternProperties": {
+        "^[_a-zA-Z0-9]+$": {
+          "title": "BootOptionMode",
+          "type": "object",
+          "properties": {
+            "label": {
+              "title": "Label",
+              "type": "string"
+            },
+            "description": {
+              "title": "Description",
+              "type": "string"
+            },
+            "default": {
+              "title": "Default",
+              "type": "string"
+            },
+            "items": {
+              "title": "Items",
+              "type": "object",
+              "additionalProperties": {
+                "title": "BootOptionItem",
+                "type": "object",
+                "properties": {
+                  "label": {
+                    "title": "Label",
+                    "type": "string"
+                  },
+                  "description": {
+                    "title": "Description",
+                    "type": "string"
+                  }
+                },
+                "required": [
+                  "label",
+                  "description"
+                ]
+              }
+            }
+          },
+          "required": [
+            "label",
+            "description",
+            "default",
+            "items"
+          ]
+        }
+      }
     }
   }
-}
+}

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -322,6 +322,16 @@ properties:
                   - ABORTED
                 type: string
             additionalProperties: false
+          bootOptions:
+            title: Boot Options
+            description: >-
+              Some services provide alternative parameters to be injected at
+              boot time. The user selection should be stored here, and it will
+              overwrite the services's defaults.
+            type: object
+            patternProperties:
+              '[a-zA-Z][a-azA-Z0-9_]*':
+                type: string
     additionalProperties: true
   ui:
     type: object

api/specs/common/schemas/project-v0.0.1.json

@@ -438,6 +438,16 @@
                 }
               },
               "additionalProperties": false
+            },
+            "bootOptions": {
+              "title": "Boot Options",
+              "description": "Some services provide alternative parameters to be injected at boot time. The user selection should be stored here, and it will overwrite the services's defaults.",
+              "type": "object",
+              "patternProperties": {
+                "[a-zA-Z][a-azA-Z0-9_]*": {
+                  "type": "string"
+                }
+              }
             }
           }
         }
@@ -668,4 +678,4 @@
       "description": "Object containing Quality Assessment related data"
     }
   }
-}
+}

packages/models-library/src/models_library/boot_options.py

@@ -0,0 +1,68 @@
+from typing import Dict
+
+from pydantic import BaseModel, constr, validator
+from typing_extensions import TypedDict
+
+ENV_VAR_KEY_RE = r"[a-zA-Z][a-azA-Z0-9_]*"
+EnvVarKey = constr(regex=ENV_VAR_KEY_RE)
+
+
+class BootChoice(TypedDict):
+    label: str
+    description: str
+
+
+class BootOption(BaseModel):
+    label: str
+    description: str
+    default: str
+    items: Dict[str, BootChoice]
+
+    @validator("items")
+    @classmethod
+    def ensure_default_included(cls, v, values):
+        default = values["default"]
+        if default not in v:
+            raise ValueError(
+                f"Expected default={default} to be present a key of items={v}"
+            )
+        return v
+
+    class Config:
+        schema_extra = {
+            "examples": [
+                {
+                    "label": "Boot mode",
+                    "description": "Start it in web page mode",
+                    "default": "0",
+                    "items": {
+                        "0": {
+                            "label": "Non Voila",
+                            "description": "Tooltip for non Voila boot mode",
+                        },
+                        "1": {
+                            "label": "Voila",
+                            "description": "Tooltip for Voila boot mode",
+                        },
+                    },
+                },
+                {
+                    "label": "Application theme",
+                    "description": "Select a theme for the application",
+                    "default": "b",
+                    "items": {
+                        "a": {
+                            "label": "Clear",
+                            "description": "Using white background",
+                        },
+                        "b": {
+                            "label": "Dark",
+                            "description": "Using black and gray tones",
+                        },
+                    },
+                },
+            ]
+        }
+
+
+BootOptions = Dict[EnvVarKey, BootOption]

packages/models-library/src/models_library/projects_nodes.py

@@ -26,6 +26,7 @@
     PortLink,
     SimCoreFileLink,
 )
+from .boot_options import EnvVarKey
 from .projects_nodes_ui import Position
 from .projects_state import RunningState
 from .services import PROPERTY_KEY_RE, SERVICE_KEY_RE
@@ -168,6 +169,16 @@ class Node(BaseModel):
         default_factory=NodeState, description="The node's state object"
     )
 
+    boot_options: Optional[Dict[EnvVarKey, str]] = Field(
+        None,
+        alias="bootOptions",
+        description=(
+            "Some services provide alternative parameters to be injected at boot time. "
+            "The user selection should be stored here, and it will overwrite the "
+            "services's defaults."
+        ),
+    )
+
     @validator("thumbnail", pre=True)
     @classmethod
     def convert_empty_str_to_none(cls, v):

packages/models-library/src/models_library/service_settings_labels.py

@@ -1,6 +1,5 @@
 # pylint: disable=unsubscriptable-object
 
-
 import json
 from enum import Enum
 from functools import cached_property
@@ -97,12 +96,18 @@ class PathMappingsLabel(BaseModel):
         description="optional list of paths which contents need to be persisted",
     )
 
+    state_exclude: Optional[List[str]] = Field(
+        None,
+        description="optional list unix shell rules used to exclude files from the state",
+    )
+
     class Config(_BaseConfig):
         schema_extra = {
             "example": {
                 "outputs_path": "/tmp/outputs",  # nosec
                 "inputs_path": "/tmp/inputs",  # nosec
                 "state_paths": ["/tmp/save_1", "/tmp_save_2"],  # nosec
+                "state_exclude": ["/tmp/strip_me/*", "*.py"],  # nosec
             }
         }
 

packages/models-library/src/models_library/services.py

@@ -21,6 +21,7 @@
 from pydantic.types import PositiveInt
 
 from .basic_regex import VERSION_RE
+from .boot_options import BootOption, BootOptions
 
 # NOTE: needs to end with / !!
 SERVICE_KEY_RE = r"^(simcore)/(services)/(comp|dynamic|frontend)(/[\w/-]+)+$"
@@ -373,6 +374,12 @@ class ServiceDockerData(ServiceKeyVersion, ServiceCommonData):
         ..., description="definition of the outputs of this node"
     )
 
+    boot_options: Optional[BootOptions] = Field(
+        None,
+        alias="boot-options",
+        description="Service defined boot options. These get injected in the service as env variables.",
+    )
+
     class Config:
         description = "Description of a simcore node 'class' with input and output"
         extra = Extra.forbid
@@ -453,6 +460,18 @@ class Config:
                             "fileToKeyMap": {"output_data.zip": "output_1"},
                         }
                     },
+                    "boot-options": {
+                        "example_service_defined_boot_mode": BootOption.Config.schema_extra[
+                            "examples"
+                        ][
+                            0
+                        ],
+                        "example_service_defined_theme_selection": BootOption.Config.schema_extra[
+                            "examples"
+                        ][
+                            1
+                        ],
+                    },
                 },
             ]
         }

packages/models-library/tests/test_services.py

@@ -3,6 +3,7 @@
 # pylint:disable=redefined-outer-name
 
 import re
+from copy import deepcopy
 from pathlib import Path
 from pprint import pformat
 from typing import Any, Callable, Dict, List
@@ -14,6 +15,7 @@
     COMPUTATIONAL_SERVICE_KEY_FORMAT,
     DYNAMIC_SERVICE_KEY_FORMAT,
     SERVICE_KEY_RE,
+    BootOption,
     ServiceAccessRightsAtDB,
     ServiceCommonData,
     ServiceDockerData,
@@ -230,3 +232,15 @@ def _find_pattern_entry(obj: Dict[str, Any], key: str) -> Any:
     for x_path in json_schema_entry_paths:
         json_pattern = _find_pattern_entry(json_schema_config, x_path)
         assert json_pattern == python_regex_pattern
+
+
+def test_boot_option() -> None:
+    for example in BootOption.Config.schema_extra["examples"]:
+        assert BootOption(**example)
+
+
+def test_boot_option_wrong_default() -> None:
+    for example in [deepcopy(x) for x in BootOption.Config.schema_extra["examples"]]:
+        with pytest.raises(ValueError):
+            example["default"] = "__undefined__"
+            assert BootOption(**example)

packages/service-library/src/servicelib/aiohttp/jsonschema_validation.py

@@ -11,9 +11,11 @@ def validate_instance(instance: Dict, schema: Dict, *, log_errors=True):
         validate(instance, schema)
     except ValidationError:
         if log_errors:
-            log.exception("Node validation error:")
+            log.exception("%s\n%s\nNode validation error:", f"{instance}", f"{schema=}")
         raise
     except SchemaError:
         if log_errors:
-            log.exception("Schema validation error:")
+            log.exception(
+                "%s\n%s\nSchema valdation error:", f"{instance}", f"{schema=}"
+            )
         raise