🐛 Regression fix: computational services always restarting (ITISFoundation#3176)

Author: sanderegg

ci/github/unit-testing/service-library.bash

@@ -27,7 +27,6 @@ test_all() {
     --durations=10 \
     --log-date-format="%Y-%m-%d %H:%M:%S" \
     --log-format="%(asctime)s %(levelname)s %(message)s" \
-    --numprocesses=auto \
     --verbose \
     -m "not heavy_load" \
     packages/service-library/tests

packages/pytest-simcore/src/pytest_simcore/helpers/utils_dict.py

@@ -1,5 +1,6 @@
 """ Utils to operate with dicts """
 
+from copy import deepcopy
 from typing import Any, Mapping, Optional, Union
 
 ConfigDict = dict[str, Any]
@@ -13,24 +14,37 @@ def get_from_dict(obj: Mapping[str, Any], dotted_key: str, default=None) -> Any:
     return value.get(keys[-1], default)
 
 
-def copy_from_dict(data: dict[str, Any], *, include: Optional[Union[set, dict]] = None):
+def copy_from_dict_ex(data: dict[str, Any], exclude: set[str]) -> dict[str, Any]:
+    # NOTE: to be refactored by someone and merged with the next method
+    return {k: v for k, v in data.items() if k not in exclude}
+
+
+def copy_from_dict(
+    data: dict[str, Any],
+    *,
+    include: Optional[Union[set, dict]] = None,
+    deep: bool = False
+):
     #
     # Analogous to advanced includes from pydantic exports
     #   https://pydantic-docs.helpmanual.io/usage/exporting_models/#advanced-include-and-exclude
     #
 
     if include is None:
-        return data.copy()
+        return deepcopy(data) if deep else data.copy()
 
     if include == ...:
-        return data
+        return deepcopy(data) if deep else data.copy()
 
     if isinstance(include, set):
         return {key: data[key] for key in include}
 
     assert isinstance(include, dict)  # nosec
 
-    return {key: copy_from_dict(data[key], include=include[key]) for key in include}
+    return {
+        key: copy_from_dict(data[key], include=include[key], deep=deep)
+        for key in include
+    }
 
 
 def update_dict(obj: dict, **updates):

packages/service-library/src/servicelib/archiving_utils.py

@@ -225,7 +225,7 @@ def _add_to_archive(
             for file_to_add in _iter_files_to_compress(
                 dir_to_compress, exclude_patterns
             ):
-                progress_bar.set_description(f"{desc}/{file_to_add.name}")
+                progress_bar.set_description(f"{desc}/{file_to_add.name}\n")
                 file_name_in_archive = (
                     _strip_directory_from_path(file_to_add, dir_to_compress)
                     if store_relative_path

services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs.py

@@ -9,7 +9,6 @@
 import pytest
 import respx
 from fastapi import FastAPI
-from fastapi.encoders import jsonable_encoder
 from models_library.aiodocker_api import AioDockerServiceSpec
 from models_library.service_settings_labels import (
     SimcoreServiceLabels,
@@ -317,9 +316,8 @@ def expected_dynamic_sidecar_spec(run_id: UUID) -> dict[str, Any]:
     }
 
 
-# TESTS
-
-
+# NOTE: this test is flaky because of a set is serialized unsorted
+@pytest.mark.flaky(max_runs=3)
 def test_get_dynamic_proxy_spec(
     mocked_catalog_service_api: respx.MockRouter,
     minimal_app: FastAPI,
@@ -336,28 +334,32 @@ def test_get_dynamic_proxy_spec(
         dynamic_sidecar_settings.dict()
         == minimal_app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.dict()
     )
-
+    expected_dynamic_sidecar_spec_model = AioDockerServiceSpec.parse_obj(
+        expected_dynamic_sidecar_spec
+    )
+    assert expected_dynamic_sidecar_spec_model.TaskTemplate
+    assert expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec
+    assert expected_dynamic_sidecar_spec_model.TaskTemplate.ContainerSpec.Env
     for count in range(1, 11):  # loop to check it does not repeat copies
         print(f"{count:*^50}")
-        dynamic_sidecar_spec = get_dynamic_sidecar_spec(
+        dynamic_sidecar_spec: AioDockerServiceSpec = get_dynamic_sidecar_spec(
             scheduler_data=scheduler_data,
             dynamic_sidecar_settings=dynamic_sidecar_settings,
             dynamic_sidecar_network_id=dynamic_sidecar_network_id,
             swarm_network_id=swarm_network_id,
             settings=cast(SimcoreServiceSettingsLabel, simcore_service_labels.settings),
             app_settings=minimal_app.state.settings,
         )
-        assert dynamic_sidecar_spec
-        assert jsonable_encoder(dynamic_sidecar_spec) == jsonable_encoder(
-            AioDockerServiceSpec.parse_obj(expected_dynamic_sidecar_spec)
-        )
+
+        assert dynamic_sidecar_spec == expected_dynamic_sidecar_spec_model
         dynamic_sidecar_spec_accumulated = dynamic_sidecar_spec
-    assert jsonable_encoder(dynamic_sidecar_spec_accumulated) == jsonable_encoder(
-        AioDockerServiceSpec.parse_obj(expected_dynamic_sidecar_spec)
-    )
+    assert dynamic_sidecar_spec_accumulated is not None
+    assert dynamic_sidecar_spec_accumulated == expected_dynamic_sidecar_spec_model
     # TODO: finish test when working on https://github.com/ITISFoundation/osparc-simcore/issues/2454
 
 
+# NOTE: this test is flaky because of a set is serialized unsorted
+@pytest.mark.flaky(max_runs=3)
 async def test_merge_dynamic_sidecar_specs_with_user_specific_specs(
     mocked_catalog_service_api: respx.MockRouter,
     minimal_app: FastAPI,
@@ -379,9 +381,8 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs(
         app_settings=minimal_app.state.settings,
     )
     assert dynamic_sidecar_spec
-    assert (
-        jsonable_encoder(dynamic_sidecar_spec, by_alias=True, exclude_unset=True)
-        == expected_dynamic_sidecar_spec
+    assert dynamic_sidecar_spec == AioDockerServiceSpec.parse_obj(
+        expected_dynamic_sidecar_spec
     )
 
     catalog_client = CatalogClient.instance(minimal_app)

services/storage/tests/unit/test_handlers_simcore_s3.py

@@ -362,6 +362,7 @@ def mock_check_project_exists(mocker: MockerFixture):
     )
 
 
+@pytest.mark.flaky(max_runs=3)
 @pytest.mark.parametrize(
     "project",
     [pytest.param(prj, id=prj.name) for prj in _get_project_with_data()],