✨ Is353/meta function nodes (⚠️ devops) (ITISFoundation#2756)

Author: pcrespov

.codeclimate.yml

@@ -80,4 +80,5 @@ exclude_patterns:
   - "**/pytest-simcore/"
   - "**/pytest_plugin/"
   - "**/sandbox/"
+  - packages/models-library/src/models_library/utils/fastapi_encoders.py
   - services/web/server/src/simcore_service_webserver/exporter/formatters/sds/xlsx/templates/code_description.py

.env-devel

@@ -28,6 +28,8 @@ DIRECTOR_V2_DEV_FEATURES_ENABLED=0
 
 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
 
+FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
+
 POSTGRES_DB=simcoredb
 POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_HOST=postgres

.github/CODEOWNERS

@@ -30,7 +30,7 @@ Makefile                                  @pcrespov, @sanderegg
 /services/storage/                        @mguidon, @pcrespov
 /services/static-webserver                @GitHK
 /services/web/client/                     @odeimaiz, @ignapas
-/services/web/server/                     @pcrespov, @sanderegg
+/services/web/server/                     @pcrespov, @sanderegg, @GitHK
 /tests/environment-setup/                 @pcrespov
 /tests/performance/                       @pcrespov
 /tests/public-api/                        @pcrespov

api/specs/common/schemas/node-meta-v0.0.1-converted.yaml

@@ -137,7 +137,7 @@ properties:
           type:
             type: string
             pattern: >-
-              ^(number|integer|boolean|string|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$
+              ^(number|integer|boolean|string|ref_contentSchema|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$
             description: >-
               data type expected on this input glob matching for data type is
               allowed
@@ -154,6 +154,12 @@ properties:
               - data:text/plain
               - data:application/hdf5
               - data:application/[email protected]
+          contentSchema:
+            title: Content Schema
+            description: >-
+              jsonschema of the content at this input/output. Required when
+              type='ref_contentSchema'
+            type: object
           fileToKeyMap:
             description: Place the data associated with the named keys in files
             type: object
@@ -263,7 +269,8 @@ properties:
               - Age in seconds since 1970
           type:
             type: string
-            pattern: ^(number|integer|boolean|string|data:[^/\s,]+/[^/\s,]+)$
+            pattern: >-
+              ^(number|integer|boolean|string|ref_contentSchema|data:[^/\s,]+/[^/\s,]+)$
             description: data type expected on this output
             example:
               - number
@@ -274,6 +281,12 @@ properties:
               - 'data:application/vnd.ms-excel '
               - data:text/plain
               - data:application/hdf5
+          contentSchema:
+            title: Content Schema
+            description: >-
+              jsonschema of this input/output. Required when
+              type='ref_contentSchema'
+            type: object
           fileToKeyMap:
             description: >-
               Place the data stored in the named files and store it in the

api/specs/common/schemas/node-meta-v0.0.1.json

@@ -200,7 +200,7 @@
             },
             "type": {
               "type": "string",
-              "pattern": "^(number|integer|boolean|string|data:([^/\\s,]+/[^/\\s,]+|\\[[^/\\s,]+/[^/\\s,]+(,[^/\\s]+/[^/,\\s]+)*\\]))$",
+              "pattern": "^(number|integer|boolean|string|ref_contentSchema|data:([^/\\s,]+/[^/\\s,]+|\\[[^/\\s,]+/[^/\\s,]+(,[^/\\s]+/[^/,\\s]+)*\\]))$",
               "description": "data type expected on this input glob matching for data type is allowed",
               "examples": [
                 "number",
@@ -216,6 +216,11 @@
                 "data:application/[email protected]"
               ]
             },
+            "contentSchema": {
+              "title": "Content Schema",
+              "description": "jsonschema of the content at this input/output. Required when type='ref_contentSchema'",
+              "type": "object"
+            },
             "fileToKeyMap": {
               "description": "Place the data associated with the named keys in files",
               "type": "object",
@@ -372,7 +377,7 @@
             },
             "type": {
               "type": "string",
-              "pattern": "^(number|integer|boolean|string|data:[^/\\s,]+/[^/\\s,]+)$",
+              "pattern": "^(number|integer|boolean|string|ref_contentSchema|data:[^/\\s,]+/[^/\\s,]+)$",
               "description": "data type expected on this output",
               "examples": [
                 "number",
@@ -385,6 +390,11 @@
                 "data:application/hdf5"
               ]
             },
+            "contentSchema": {
+              "title": "Content Schema",
+              "description": "jsonschema of this input/output. Required when type='ref_contentSchema'",
+              "type": "object"
+            },
             "fileToKeyMap": {
               "description": "Place the data stored in the named files and store it in the locations pointed to by the respective output key.",
               "type": "object",
@@ -464,4 +474,4 @@
       }
     }
   }
-}
+}

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -182,6 +182,8 @@ properties:
                         format: uri
                       label:
                         type: string
+                  - type: array
+                    items: {}
           inputAccess:
             description: map with key - access level pairs
             type: object
@@ -246,6 +248,8 @@ properties:
                         format: uri
                       label:
                         type: string
+                  - type: array
+                    items: {}
           outputNode:
             type: boolean
             deprecated: true

api/specs/common/schemas/project-v0.0.1.json

@@ -243,6 +243,10 @@
                           "type": "string"
                         }
                       }
+                    },
+                    {
+                      "type": "array",
+                      "items": {}
                     }
                   ]
                 }
@@ -338,6 +342,10 @@
                           "type": "string"
                         }
                       }
+                    },
+                    {
+                      "type": "array",
+                      "items": {}
                     }
                   ]
                 }
@@ -678,4 +686,4 @@
       "description": "Object containing Quality Assessment related data"
     }
   }
-}
+}

packages/README.md

@@ -12,17 +12,21 @@ Every folder contains a python package with the following properties:
   # creates and activates venv
   make .venv
   source .venv/bin/activate
-
   cd packages/my-package
-
-  # for development
+```
+then for development
+```cmd
   pip install -r requirements/dev.txt
-
-  # for CI, we freeze dependencies and install tests
-  pip install -r requirements/base.txt
-  pip install .[test]
+  pytest tests
+```
+or simply (does a bit more than above)
+```
+make install-dev
+make tests
 ```
 
+
+
 - it can be a package with or without with *entrypoints*.
   - a **library** has no entrypoints
     - a collection of functions/classes organized in namespaces
@@ -37,11 +41,3 @@ Every folder contains a python package with the following properties:
     # or as a lib
     >> import mylib
   ```
-
-
-## TODO
-
-  - [ ] cookiecutters to assist creation of modules w/ or w/o entrypoints
-
-
-[simcore]:https://github.com/pcrespov/osparc-simcore

packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py

@@ -1,7 +1,7 @@
 import json
 from contextlib import suppress
 from pathlib import Path
-from typing import Any, Dict, Optional, Union, cast
+from typing import Any, Dict, List, Optional, Union, cast
 
 from models_library.generics import DictModel
 from models_library.services import PROPERTY_KEY_RE
@@ -75,7 +75,16 @@ class Config:
 
 
 PortKey = constr(regex=PROPERTY_KEY_RE)
-PortValue = Union[StrictBool, StrictInt, StrictFloat, StrictStr, FileUrl, None]
+PortValue = Union[
+    StrictBool,
+    StrictInt,
+    StrictFloat,
+    StrictStr,
+    FileUrl,
+    List[Any],
+    Dict[str, Any],
+    None,
+]
 
 
 class TaskInputData(DictModel[PortKey, PortValue]):

packages/models-library/src/models_library/basic_regex.py

@@ -19,3 +19,9 @@
 
 
 VERSION_RE = r"^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$"
+
+
+# Regex to detect whether a string can be used as a variable identifier (see tests)
+#  - cannot start with spaces, _ (we only want public) or numbers
+# https://docs.python.org/3/reference/lexical_analysis.html#identifiers
+PUBLIC_VARIABLE_NAME_RE = r"^[^_\W0-9]\w*$"