Skip to content

Commit 2df6b54

Browse files
bisgaard-itisbisgaard-itis
committed
Merge branch 'master' into 7635-add-exemplars-to-prometheus-metrics
2 parents 5030332 + 97fafc4 commit 2df6b54

File tree

28 files changed

+801
-471
lines changed

28 files changed

+801
-471
lines changed

‎packages/models-library/src/models_library/service_settings_labels.py‎

Lines changed: 380 additions & 275 deletions
Large diffs are not rendered by default.

‎packages/models-library/tests/test_service_settings_labels.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,11 @@ def test_simcore_service_labels(example: dict, items: int, uses_dynamic_sidecar:
7878

7979
def test_service_settings():
8080
simcore_settings_settings_label = SimcoreServiceSettingsLabel.model_validate(
81-
SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"]
81+
SimcoreServiceSettingLabelEntry.model_json_schema()["examples"]
8282
)
8383
assert simcore_settings_settings_label
8484
assert len(simcore_settings_settings_label) == len(
85-
SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"]
85+
SimcoreServiceSettingLabelEntry.model_json_schema()["examples"]
8686
)
8787
assert simcore_settings_settings_label[0]
8888

@@ -122,16 +122,14 @@ def test_raises_error_if_http_entrypoint_is_missing():
122122

123123

124124
def test_path_mappings_none_state_paths():
125-
sample_data = deepcopy(
126-
PathMappingsLabel.model_config["json_schema_extra"]["examples"][0]
127-
)
125+
sample_data = deepcopy(PathMappingsLabel.model_json_schema()["examples"][0])
128126
sample_data["state_paths"] = None
129127
with pytest.raises(ValidationError):
130128
PathMappingsLabel(**sample_data)
131129

132130

133131
def test_path_mappings_json_encoding():
134-
for example in PathMappingsLabel.model_config["json_schema_extra"]["examples"]:
132+
for example in PathMappingsLabel.model_json_schema()["examples"]:
135133
path_mappings = PathMappingsLabel.model_validate(example)
136134
print(path_mappings)
137135
assert (
@@ -607,7 +605,7 @@ def test_resolving_some_service_labels_at_load_time(
607605
def test_user_preferences_path_is_part_of_exiting_volume():
608606
labels_data = {
609607
"simcore.service.paths-mapping": json.dumps(
610-
PathMappingsLabel.model_config["json_schema_extra"]["examples"][0]
608+
PathMappingsLabel.model_json_schema()["examples"][0]
611609
),
612610
"simcore.service.user-preferences-path": json.dumps(
613611
"/tmp/outputs" # noqa: S108

‎packages/service-integration/tests/test_osparc_config.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ def test_load_from_labels(
6262
)
6363
with open(config_path, "w") as fh:
6464
data = json.loads(
65-
model.model_dump_json(exclude_unset=True, by_alias=True, exclude_none=True)
65+
model.model_dump_json(
66+
exclude_unset=True, by_alias=True, exclude_none=True
67+
)
6668
)
6769
yaml.safe_dump(data, fh, sort_keys=False)
6870

@@ -73,10 +75,10 @@ def test_load_from_labels(
7375

7476
@pytest.mark.parametrize(
7577
"example_data",
76-
SimcoreServiceSettingLabelEntry.model_config["json_schema_extra"]["examples"],
78+
SimcoreServiceSettingLabelEntry.model_json_schema()["examples"],
7779
)
7880
def test_settings_item_in_sync_with_service_settings_label(
79-
example_data: dict[str, Any]
81+
example_data: dict[str, Any],
8082
):
8183
print(pformat(example_data))
8284

‎packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py‎

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from models_library.projects import ProjectID
66
from models_library.projects_nodes_io import NodeID, StorageFileID
7+
from models_library.service_settings_labels import LegacyState
78
from models_library.users import UserID
89
from pydantic import TypeAdapter
910
from servicelib.archiving_utils import unarchive_dir
@@ -101,14 +102,16 @@ async def _pull_legacy_archive(
101102
*,
102103
io_log_redirect_cb: LogRedirectCB,
103104
progress_bar: ProgressBarData,
105+
legacy_destination_path: Path | None = None,
104106
) -> None:
105107
# NOTE: the legacy way of storing states was as zip archives
108+
archive_path = legacy_destination_path or destination_path
106109
async with progress_bar.sub_progress(
107-
steps=2, description=f"pulling {destination_path.name}"
110+
steps=2, description=f"pulling {archive_path.name}"
108111
) as sub_prog:
109112
with TemporaryDirectory() as tmp_dir_name:
110113
archive_file = Path(tmp_dir_name) / __get_s3_name(
111-
destination_path, is_archive=True
114+
archive_path, is_archive=True
112115
)
113116

114117
s3_object = __create_s3_object_key(project_id, node_uuid, archive_file)
@@ -124,7 +127,7 @@ async def _pull_legacy_archive(
124127
progress_bar=sub_prog,
125128
aws_s3_cli_settings=None,
126129
)
127-
_logger.info("completed pull of %s.", destination_path)
130+
_logger.info("completed pull of %s.", archive_path)
128131

129132
if io_log_redirect_cb:
130133
await io_log_redirect_cb(
@@ -194,6 +197,7 @@ async def push(
194197
exclude_patterns: set[str] | None = None,
195198
progress_bar: ProgressBarData,
196199
aws_s3_cli_settings: AwsS3CliSettings | None,
200+
legacy_state: LegacyState | None,
197201
) -> None:
198202
"""pushes and removes the legacy archive if present"""
199203

@@ -208,23 +212,39 @@ async def push(
208212
progress_bar=progress_bar,
209213
aws_s3_cli_settings=aws_s3_cli_settings,
210214
)
215+
211216
archive_exists = await _state_metadata_entry_exists(
212217
user_id=user_id,
213218
project_id=project_id,
214219
node_uuid=node_uuid,
215220
path=source_path,
216221
is_archive=True,
217222
)
223+
if archive_exists:
224+
with log_context(_logger, logging.INFO, "removing legacy archive"):
225+
await _delete_legacy_archive(
226+
project_id=project_id,
227+
node_uuid=node_uuid,
228+
path=source_path,
229+
)
218230

219-
if not archive_exists:
220-
return
221-
222-
with log_context(_logger, logging.INFO, "removing legacy data archive"):
223-
await _delete_legacy_archive(
231+
if legacy_state:
232+
legacy_archive_exists = await _state_metadata_entry_exists(
233+
user_id=user_id,
224234
project_id=project_id,
225235
node_uuid=node_uuid,
226-
path=source_path,
236+
path=legacy_state.old_state_path,
237+
is_archive=True,
227238
)
239+
if legacy_archive_exists:
240+
with log_context(
241+
_logger, logging.INFO, f"removing legacy archive in {legacy_state}"
242+
):
243+
await _delete_legacy_archive(
244+
project_id=project_id,
245+
node_uuid=node_uuid,
246+
path=legacy_state.old_state_path,
247+
)
228248

229249

230250
async def pull(
@@ -237,9 +257,41 @@ async def pull(
237257
r_clone_settings: RCloneSettings,
238258
progress_bar: ProgressBarData,
239259
aws_s3_cli_settings: AwsS3CliSettings | None,
260+
legacy_state: LegacyState | None,
240261
) -> None:
241262
"""restores the state folder"""
242263

264+
if legacy_state and legacy_state.new_state_path == destination_path:
265+
_logger.info(
266+
"trying to restore from legacy_state=%s, destination_path=%s",
267+
legacy_state,
268+
destination_path,
269+
)
270+
legacy_state_exists = await _state_metadata_entry_exists(
271+
user_id=user_id,
272+
project_id=project_id,
273+
node_uuid=node_uuid,
274+
path=legacy_state.old_state_path,
275+
is_archive=True,
276+
)
277+
_logger.info("legacy_state_exists=%s", legacy_state_exists)
278+
if legacy_state_exists:
279+
with log_context(
280+
_logger,
281+
logging.INFO,
282+
f"restoring data from legacy archive in {legacy_state}",
283+
):
284+
await _pull_legacy_archive(
285+
user_id=user_id,
286+
project_id=project_id,
287+
node_uuid=node_uuid,
288+
destination_path=legacy_state.new_state_path,
289+
io_log_redirect_cb=io_log_redirect_cb,
290+
progress_bar=progress_bar,
291+
legacy_destination_path=legacy_state.old_state_path,
292+
)
293+
return
294+
243295
state_archive_exists = await _state_metadata_entry_exists(
244296
user_id=user_id,
245297
project_id=project_id,
@@ -248,7 +300,7 @@ async def pull(
248300
is_archive=True,
249301
)
250302
if state_archive_exists:
251-
with log_context(_logger, logging.INFO, "restoring legacy data archive"):
303+
with log_context(_logger, logging.INFO, "restoring data from legacy archive"):
252304
await _pull_legacy_archive(
253305
user_id=user_id,
254306
project_id=project_id,

‎packages/simcore-sdk/tests/unit/test_node_data_data_manager.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ async def test_push_file(
172172
mock_filemanager.reset_mock()
173173

174174

175+
@pytest.mark.parametrize("create_legacy_archive", [False, True])
175176
async def test_pull_legacy_archive(
176177
user_id: int,
177178
project_id: ProjectID,
@@ -181,6 +182,7 @@ async def test_pull_legacy_archive(
181182
create_files: Callable[..., list[Path]],
182183
mock_io_log_redirect_cb: LogRedirectCB,
183184
faker: Faker,
185+
create_legacy_archive: bool,
184186
):
185187
assert tmpdir.exists()
186188
# create a folder to compress from
@@ -200,7 +202,13 @@ async def test_pull_legacy_archive(
200202
create_files(files_number, test_control_folder)
201203
compressed_file_name = test_compression_folder / test_folder.stem
202204
archive_file = make_archive(
203-
f"{compressed_file_name}", "zip", root_dir=test_control_folder
205+
(
206+
f"{compressed_file_name}_legacy"
207+
if create_legacy_archive
208+
else f"{compressed_file_name}"
209+
),
210+
"zip",
211+
root_dir=test_control_folder,
204212
)
205213
assert Path(archive_file).exists()
206214
# create mock downloaded folder
@@ -229,13 +237,16 @@ async def test_pull_legacy_archive(
229237
test_folder,
230238
io_log_redirect_cb=mock_io_log_redirect_cb,
231239
progress_bar=progress_bar,
240+
legacy_destination_path=(
241+
Path(f"{test_folder}_legacy") if create_legacy_archive else None
242+
),
232243
)
233244
assert progress_bar._current_steps == pytest.approx(1) # noqa: SLF001
234245
mock_temporary_directory.assert_called_once()
235246
mock_filemanager.download_path_from_s3.assert_called_once_with(
236247
user_id=user_id,
237248
local_path=test_compression_folder,
238-
s3_object=f"{project_id}/{node_uuid}/{test_folder.stem}.zip",
249+
s3_object=f"{project_id}/{node_uuid}/{f'{test_folder.stem}_legacy' if create_legacy_archive else test_folder.stem}.zip",
239250
store_id=SIMCORE_LOCATION,
240251
store_name=None,
241252
io_log_redirect_cb=mock_io_log_redirect_cb,

‎services/director-v2/openapi.json‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2291,6 +2291,26 @@
22912291
}
22922292
}
22932293
},
2294+
"LegacyState": {
2295+
"properties": {
2296+
"old_state_path": {
2297+
"type": "string",
2298+
"format": "path",
2299+
"title": "Old State Path"
2300+
},
2301+
"new_state_path": {
2302+
"type": "string",
2303+
"format": "path",
2304+
"title": "New State Path"
2305+
}
2306+
},
2307+
"type": "object",
2308+
"required": [
2309+
"old_state_path",
2310+
"new_state_path"
2311+
],
2312+
"title": "LegacyState"
2313+
},
22942314
"NATRule": {
22952315
"properties": {
22962316
"hostname": {
@@ -2449,6 +2469,17 @@
24492469
],
24502470
"title": "Volume Size Limits",
24512471
"description": "Apply volume size limits to entries in: `inputs_path`, `outputs_path` and `state_paths`. Limits must be parsable by Pydantic's ByteSize."
2472+
},
2473+
"legacy_state": {
2474+
"anyOf": [
2475+
{
2476+
"$ref": "#/components/schemas/LegacyState"
2477+
},
2478+
{
2479+
"type": "null"
2480+
}
2481+
],
2482+
"description": "if present, the service needs to first try to download the legacy statecoming from a different path."
24522483
}
24532484
},
24542485
"additionalProperties": false,

‎services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,7 @@ def _assemble_env_vars_for_boot_options(
426426
return SimcoreServiceSettingsLabel(
427427
root=[
428428
SimcoreServiceSettingLabelEntry(
429-
name="env", type="string", value=list(env_vars)
429+
name="env", setting_type="string", value=list(env_vars)
430430
)
431431
]
432432
)

‎services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,11 @@ def _get_environment_variables(
134134
"DY_SIDECAR_USER_SERVICES_HAVE_INTERNET_ACCESS": f"{allow_internet_access}",
135135
"DY_SIDECAR_SYSTEM_MONITOR_TELEMETRY_ENABLE": f"{telemetry_enabled}",
136136
"DY_SIDECAR_STATE_EXCLUDE": json_dumps(f"{x}" for x in state_exclude),
137+
"DY_SIDECAR_LEGACY_STATE": (
138+
"null"
139+
if scheduler_data.paths_mapping.legacy_state is None
140+
else scheduler_data.paths_mapping.legacy_state.model_dump_json()
141+
),
137142
"DY_SIDECAR_CALLBACKS_MAPPING": callbacks_mapping.model_dump_json(),
138143
"DY_SIDECAR_STATE_PATHS": json_dumps(
139144
f"{x}" for x in scheduler_data.paths_mapping.state_paths
@@ -451,18 +456,18 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa:
451456
scheduler_data.product_name is not None
452457
), "ONLY for legacy. This function should not be called with product_name==None" # nosec
453458

454-
standard_simcore_docker_labels: dict[
455-
DockerLabelKey, str
456-
] = StandardSimcoreDockerLabels(
457-
user_id=scheduler_data.user_id,
458-
project_id=scheduler_data.project_id,
459-
node_id=scheduler_data.node_uuid,
460-
product_name=scheduler_data.product_name,
461-
simcore_user_agent=scheduler_data.request_simcore_user_agent,
462-
swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME,
463-
memory_limit=ByteSize(0), # this should get overwritten
464-
cpu_limit=0, # this should get overwritten
465-
).to_simcore_runtime_docker_labels()
459+
standard_simcore_docker_labels: dict[DockerLabelKey, str] = (
460+
StandardSimcoreDockerLabels(
461+
user_id=scheduler_data.user_id,
462+
project_id=scheduler_data.project_id,
463+
node_id=scheduler_data.node_uuid,
464+
product_name=scheduler_data.product_name,
465+
simcore_user_agent=scheduler_data.request_simcore_user_agent,
466+
swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME,
467+
memory_limit=ByteSize(0), # this should get overwritten
468+
cpu_limit=0, # this should get overwritten
469+
).to_simcore_runtime_docker_labels()
470+
)
466471

467472
service_labels: dict[str, str] = (
468473
{
@@ -494,9 +499,7 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa:
494499
)
495500
)
496501

497-
placement_substitutions: dict[
498-
str, DockerPlacementConstraint
499-
] = (
502+
placement_substitutions: dict[str, DockerPlacementConstraint] = (
500503
placement_settings.DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS
501504
)
502505
for image_resources in scheduler_data.service_resources.values():

‎services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"DY_DOCKER_HUB_REGISTRY_SETTINGS",
2121
"DY_SIDECAR_AWS_S3_CLI_SETTINGS",
2222
"DY_SIDECAR_CALLBACKS_MAPPING",
23+
"DY_SIDECAR_LEGACY_STATE",
2324
"DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED",
2425
"DY_SIDECAR_NODE_ID",
2526
"DY_SIDECAR_PATH_INPUTS",

‎services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,7 @@ def expected_dynamic_sidecar_spec(
267267
"DY_SIDECAR_SERVICE_VERSION": "2.4.5",
268268
"DY_SIDECAR_PRODUCT_NAME": osparc_product_name,
269269
"DY_SIDECAR_USER_PREFERENCES_PATH": "None",
270+
"DY_SIDECAR_LEGACY_STATE": "null",
270271
"DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED": "True",
271272
"POSTGRES_DB": "test",
272273
"POSTGRES_HOST": "localhost",

0 commit comments

Comments
 (0)