Skip to content

Commit bfbd031

Browse files
pcrespovpcrespov
authored
✨🗑️♻️🐛 Adds parameters in catalog for meta-modeling (⚠️ devops) and extras (ITISFoundation#2432)
* Catalog defines parameter nodes: /parameter/number, /parameter/boolean and /parameter/integer * Deprecates displayOrder in input/outputs * Adds dev-feature flags in catalog and web-server * improves request error handling * moves part of project_handlers.py into project_handlers_nodes.py to make file smaller * EXTRA: bug synchronise_meta_data_table deletes table in chunks to improve PR ITISFoundation#2428 that currently fails for large tables (>200k entries)
1 parent ae8c6d1 commit bfbd031

File tree

31 files changed

+694
-380
lines changed

31 files changed

+694
-380
lines changed

‎.env-devel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ API_SERVER_DEV_FEATURES_ENABLED=0
1414
BF_API_KEY=none
1515
BF_API_SECRET=none
1616

17+
CATALOG_DEV_FEATURES_ENABLED=0
18+
1719
DASK_SCHEDULER_HOST=dask-scheduler
1820

1921
DIRECTOR_REGISTRY_CACHING_TTL=900

‎.gitignore‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,12 @@ ENV/
110110
# Dependency directories
111111
node_modules/
112112

113-
# IDEs config
114-
.vscode/launch.json
115-
.vscode/settings.json
113+
# IDEs config (except templates)
114+
.vscode/*
115+
!.vscode/*.template.json
116116

117-
# Manual overrides
117+
118+
# manual overrides
118119
services/docker-compose.override.yml
119120

120121
# swarm files

‎api/specs/common/schemas/node-meta-v0.0.1.json‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,12 +180,9 @@
180180
],
181181
"properties": {
182182
"displayOrder": {
183-
"type": "number",
184-
"description": "use this to numerically sort the properties for display",
185-
"examples": [
186-
1,
187-
-0.2
188-
]
183+
"description": "DEPRECATED: new display order is taken from the item position. This property will be removed.",
184+
"deprecated": true,
185+
"type": "number"
189186
},
190187
"label": {
191188
"type": "string",

‎packages/models-library/src/models_library/frontend_services_catalog.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
"""
2+
Factory to build catalog of i/o metadata for functions implemented in the front-end
3+
4+
NOTE: These definitions are currently needed in the catalog and director2
5+
services. Since it is static data, instead of making a call from
6+
director2->catalog, it was decided to share as a library
7+
"""
8+
9+
import functools
10+
from typing import Iterator
11+
12+
from .services import Author, ServiceDockerData, ServiceType
13+
14+
FRONTEND_SERVICE_KEY_PREFIX = "simcore/services/frontend"
15+
16+
OM = Author(name="Odei Maiz", email="[email protected]", affiliation="IT'IS")
17+
18+
19+
def _create_file_picker_service() -> ServiceDockerData:
20+
return ServiceDockerData(
21+
key=f"{FRONTEND_SERVICE_KEY_PREFIX}/file-picker",
22+
version="1.0.0",
23+
type=ServiceType.FRONTEND,
24+
name="File Picker",
25+
description="File Picker",
26+
authors=[
27+
OM,
28+
],
29+
contact=OM.email,
30+
inputs={},
31+
outputs={
32+
"outFile": {
33+
"displayOrder": 0,
34+
"label": "File",
35+
"description": "Chosen File",
36+
"type": "data:*/*",
37+
}
38+
},
39+
)
40+
41+
42+
def _create_node_group_service() -> ServiceDockerData:
43+
#
44+
# NOTE: DO not mistake with simcore/services/frontend/nodes-group/macros/
45+
# which needs to be redefined.
46+
#
47+
meta = ServiceDockerData(
48+
key=f"{FRONTEND_SERVICE_KEY_PREFIX}/nodes-group",
49+
version="1.0.0",
50+
type=ServiceType.FRONTEND,
51+
name="Group",
52+
description="Group of nodes",
53+
authors=[
54+
OM,
55+
],
56+
contact=OM.email,
57+
inputs={},
58+
outputs={
59+
"outFile": {
60+
"displayOrder": 0,
61+
"label": "File",
62+
"description": "Chosen File",
63+
"type": "data:*/*",
64+
}
65+
},
66+
)
67+
68+
assert meta.outputs is not None # nosec
69+
assert list(meta.outputs.keys()) == ["outFile"], "name used in front-end" # nosec
70+
return meta
71+
72+
73+
def _create_parameter(param_type: str) -> ServiceDockerData:
74+
"""
75+
Represents a parameter (e.g. x=5) in a study
76+
77+
This is a parametrized node (or param-node in short)
78+
"""
79+
assert param_type in ["number", "boolean", "integer"] # nosec
80+
81+
LABEL = f"{param_type.capitalize()} Parameter"
82+
DESCRIPTION = f"Parameter of type {param_type}"
83+
84+
meta = ServiceDockerData(
85+
key=f"{FRONTEND_SERVICE_KEY_PREFIX}/parameter/{param_type}",
86+
version="1.0.0",
87+
type=ServiceType.FRONTEND,
88+
name=LABEL,
89+
description=DESCRIPTION,
90+
authors=[
91+
OM,
92+
],
93+
contact=OM.email,
94+
inputs={},
95+
outputs={
96+
"out_1": {
97+
"label": LABEL,
98+
"description": DESCRIPTION,
99+
"type": param_type,
100+
}
101+
},
102+
)
103+
104+
assert meta.outputs is not None # nosec
105+
assert list(meta.outputs.keys()) == ["out_1"], "name used in front-end" # nosec
106+
return meta
107+
108+
109+
def is_frontend_service(service_key: str) -> bool:
110+
return service_key.startswith(f"{FRONTEND_SERVICE_KEY_PREFIX}/")
111+
112+
113+
def is_parameter_service(service_key: str) -> bool:
114+
return service_key.startswith(f"{FRONTEND_SERVICE_KEY_PREFIX}/parameter/")
115+
116+
117+
_FACTORY_FUNCTIONS = [_create_file_picker_service, _create_node_group_service,] + [
118+
functools.partial(_create_parameter, param_type=p)
119+
for p in ["number", "boolean", "integer"]
120+
]
121+
122+
123+
def iter_service_docker_data() -> Iterator[ServiceDockerData]:
124+
for create in _FACTORY_FUNCTIONS:
125+
model_instance = create()
126+
yield model_instance

‎packages/models-library/src/models_library/services.py‎

Lines changed: 102 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -136,12 +136,13 @@ class ServiceProperty(BaseModel):
136136
## management
137137

138138
### human readable descriptors
139-
display_order: float = Field(
140-
...,
139+
display_order: Optional[float] = Field(
140+
None,
141141
alias="displayOrder",
142-
description="use this to numerically sort the properties for display",
143-
examples=[1, -0.2],
142+
deprecated=True,
143+
description="DEPRECATED: new display order is taken from the item position. This will be removed.",
144144
)
145+
145146
label: str = Field(..., description="short name for the property", example="Age")
146147
description: str = Field(
147148
...,
@@ -150,7 +151,7 @@ class ServiceProperty(BaseModel):
150151
)
151152

152153
# mathematical and physics descriptors
153-
property_type: constr(regex=PROPERTY_TYPE_RE) = Field(
154+
property_type: str = Field(
154155
...,
155156
alias="type",
156157
description="data type expected on this input glob matching for data type is allowed",
@@ -167,6 +168,7 @@ class ServiceProperty(BaseModel):
167168
"data:application/hdf5",
168169
"data:application/[email protected]",
169170
],
171+
regex=PROPERTY_TYPE_RE,
170172
)
171173

172174
# value
@@ -211,7 +213,7 @@ class Config(ServiceProperty.Config):
211213
"description": "Files downloaded from service connected at the input",
212214
"type": "data:*/*",
213215
},
214-
# latest:
216+
# v2
215217
{
216218
"displayOrder": 2,
217219
"label": "Sleep Time",
@@ -221,6 +223,15 @@ class Config(ServiceProperty.Config):
221223
"unit": "second",
222224
"widget": {"type": "TextArea", "details": {"minHeight": 3}},
223225
},
226+
# latest:
227+
{
228+
"label": "Sleep Time",
229+
"description": "Time to wait before completion",
230+
"type": "number",
231+
"defaultValue": 0,
232+
"unit": "second",
233+
"widget": {"type": "TextArea", "details": {"minHeight": 3}},
234+
},
224235
],
225236
}
226237

@@ -235,21 +246,28 @@ class ServiceOutput(ServiceProperty):
235246
class Config(ServiceProperty.Config):
236247
schema_extra = {
237248
"examples": [
238-
# previously:
249+
# v1
239250
{
240251
"displayOrder": 2,
241252
"label": "Time Slept",
242253
"description": "Time the service waited before completion",
243254
"type": "number",
244255
},
245-
# latest:
256+
# v2
246257
{
247258
"displayOrder": 2,
248259
"label": "Time Slept",
249260
"description": "Time the service waited before completion",
250261
"type": "number",
251262
"unit": "second",
252263
},
264+
# latest:
265+
{
266+
"label": "Time Slept",
267+
"description": "Time the service waited before completion",
268+
"type": "number",
269+
"unit": "second",
270+
},
253271
]
254272
}
255273

@@ -297,6 +315,7 @@ class ServiceCommonData(BaseModel):
297315
)
298316

299317
@validator("thumbnail", pre=True, always=False)
318+
@classmethod
300319
def validate_thumbnail(cls, value): # pylint: disable=no-self-argument,no-self-use
301320
if value == "":
302321
return None
@@ -312,11 +331,11 @@ class ServiceDockerData(ServiceKeyVersion, ServiceCommonData):
312331
Static metadata for a service injected in the image labels
313332
"""
314333

315-
integration_version: Optional[constr(regex=VERSION_RE)] = Field(
334+
integration_version: Optional[str] = Field(
316335
None,
317336
alias="integration-version",
318337
description="integration version number",
319-
# regex=VERSION_RE,
338+
regex=VERSION_RE,
320339
examples=["1.0.0"],
321340
)
322341
service_type: ServiceType = Field(
@@ -346,44 +365,83 @@ class Config:
346365
extra = Extra.forbid
347366

348367
schema_extra = {
349-
"example": {
350-
"name": "oSparc Python Runner",
351-
"key": "simcore/services/comp/osparc-python-runner",
352-
"type": "computational",
353-
"integration-version": "1.0.0",
354-
"version": "1.7.0",
355-
"description": "oSparc Python Runner",
356-
"contact": "[email protected]",
357-
"authors": [
358-
{
359-
"name": "John Smith",
360-
"email": "[email protected]",
361-
"affiliation": "Company",
368+
"examples": [
369+
{
370+
"name": "oSparc Python Runner",
371+
"key": "simcore/services/comp/osparc-python-runner",
372+
"type": "computational",
373+
"integration-version": "1.0.0",
374+
"version": "1.7.0",
375+
"description": "oSparc Python Runner",
376+
"contact": "[email protected]",
377+
"authors": [
378+
{
379+
"name": "John Smith",
380+
"email": "[email protected]",
381+
"affiliation": "Company",
382+
},
383+
{
384+
"name": "Richard Brown",
385+
"email": "[email protected]",
386+
"affiliation": "University",
387+
},
388+
],
389+
"inputs": {
390+
"input_1": {
391+
"displayOrder": 1,
392+
"label": "Input data",
393+
"description": "Any code, requirements or data file",
394+
"type": "data:*/*",
395+
}
362396
},
363-
{
364-
"name": "Richard Brown",
365-
"email": "[email protected]",
366-
"affiliation": "University",
397+
"outputs": {
398+
"output_1": {
399+
"displayOrder": 1,
400+
"label": "Output data",
401+
"description": "All data produced by the script is zipped as output_data.zip",
402+
"type": "data:*/*",
403+
"fileToKeyMap": {"output_data.zip": "output_1"},
404+
}
367405
},
368-
],
369-
"inputs": {
370-
"input_1": {
371-
"displayOrder": 1,
372-
"label": "Input data",
373-
"description": "Any code, requirements or data file",
374-
"type": "data:*/*",
375-
}
376406
},
377-
"outputs": {
378-
"output_1": {
379-
"displayOrder": 1,
380-
"label": "Output data",
381-
"description": "All data produced by the script is zipped as output_data.zip",
382-
"type": "data:*/*",
383-
"fileToKeyMap": {"output_data.zip": "output_1"},
384-
}
407+
# latest
408+
{
409+
"name": "oSparc Python Runner",
410+
"key": "simcore/services/comp/osparc-python-runner",
411+
"type": "computational",
412+
"integration-version": "1.0.0",
413+
"version": "1.7.0",
414+
"description": "oSparc Python Runner",
415+
"contact": "[email protected]",
416+
"authors": [
417+
{
418+
"name": "John Smith",
419+
"email": "[email protected]",
420+
"affiliation": "Company",
421+
},
422+
{
423+
"name": "Richard Brown",
424+
"email": "[email protected]",
425+
"affiliation": "University",
426+
},
427+
],
428+
"inputs": {
429+
"input_1": {
430+
"label": "Input data",
431+
"description": "Any code, requirements or data file",
432+
"type": "data:*/*",
433+
}
434+
},
435+
"outputs": {
436+
"output_1": {
437+
"label": "Output data",
438+
"description": "All data produced by the script is zipped as output_data.zip",
439+
"type": "data:*/*",
440+
"fileToKeyMap": {"output_data.zip": "output_1"},
441+
}
442+
},
385443
},
386-
}
444+
]
387445
}
388446

389447

0 commit comments

Comments
 (0)