Merge branch 'master' into introduce-vip-models-pricing-2-part

matusdrobuliak66 · matusdrobuliak66 · commit 8c0099364952 · 2024-12-05T09:17:31.000+01:00
diff --git a/.env-devel b/.env-devel
@@ -126,6 +126,7 @@ DIRECTOR_V2_TRACING={}
 # DYNAMIC_SCHEDULER ----
 DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
+DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
 DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
@@ -211,7 +212,7 @@ RESOURCE_USAGE_TRACKER_S3=null
 RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 R_CLONE_OPTION_RETRIES=3
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_PROVIDER=MINIO
diff --git a/packages/service-library/src/servicelib/redis.py b/packages/service-library/src/servicelib/redis.py
@@ -17,6 +17,7 @@
 from redis.backoff import ExponentialBackoff
 from settings_library.redis import RedisDatabase, RedisSettings
 from tenacity import retry
+from yarl import URL
 
 from .background_task import periodic_task
 from .logging_utils import log_catch, log_context
@@ -94,7 +95,8 @@ def __post_init__(self):
     async def setup(self) -> None:
         if not await self.ping():
             await self.shutdown()
-            raise CouldNotConnectToRedisError(dsn=self.redis_dsn)
+            url_safe: URL = URL(self.redis_dsn).with_password("???")
+            raise CouldNotConnectToRedisError(dsn=f"{url_safe}")
 
         self._health_check_task = asyncio.create_task(
             self._check_health(),
diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py
@@ -26,6 +26,6 @@ class RCloneSettings(BaseCustomSettings):
     )
     # SEE https://rclone.org/docs/#buffer-size-size
     R_CLONE_OPTION_BUFFER_SIZE: str = Field(
-        default="0M",
+        default="16M",
         description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
     )
diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py
@@ -8,10 +8,9 @@
 from pathlib import Path
 from typing import Final
 
-from common_library.errors_classes import OsparcErrorMixin
-
 from aiocache import cached  # type: ignore[import-untyped]
 from aiofiles import tempfile
+from common_library.errors_classes import OsparcErrorMixin
 from models_library.basic_types import IDStr
 from pydantic import AnyUrl, BaseModel, ByteSize
 from servicelib.progress_bar import ProgressBarData
@@ -207,7 +206,6 @@ async def _sync_sources(
             f"{r_clone_settings.R_CLONE_OPTION_TRANSFERS}",
             # below two options reduce to a minimum the memory footprint
             # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-            "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
             "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
             r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE,
             "--use-json-log",
diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone_utils.py
@@ -52,14 +52,14 @@ class SyncProgressLogParser(BaseLogParser):
 
 
     This command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
+    rclone --buffer-size 0M --transfers 5 sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3 --progress
     generates this but the rclone modifies the terminal printed lines which python does not like so much
     Transferred:        4.666 GiB / 4.666 GiB, 100%, 530.870 MiB/s, ETA 0s
     Transferred:            4 / 4, 100%
     Elapsed time:         9.6s
 
     This other command:
-    rclone --use-mmap --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
+    rclone --buffer-size 0M --transfers 5 --use-json-log --stats-log-level INFO -v --stats 500ms sync mys3:simcore/5cfdef88-013b-11ef-910e-0242ac14003e/2d544003-9eb8-47e4-bcf7-95a8c31845f7/workspace ./tests3
     prints stuff such as:
     {"level":"info","msg":"Copied (new)","object":"README.ipynb","objectType":"*s3.Object","size":5123,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408277+00:00"}
     {"level":"info","msg":"Copied (new)","object":".hidden_do_not_remove","objectType":"*s3.Object","size":219,"source":"operations/copy.go:360","time":"2024-04-23T14:05:10.408246+00:00"}
diff --git a/services/agent/src/simcore_service_agent/services/backup.py b/services/agent/src/simcore_service_agent/services/backup.py
@@ -161,7 +161,6 @@ async def _store_in_s3(
         f"{settings.AGENT_VOLUMES_CLEANUP_PARALLELISM}",
         # below two options reduce to a minimum the memory footprint
         # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
-        "--use-mmap",  # docs https://rclone.org/docs/#use-mmap
         "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
         "0M",
         "--stats",
diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel
@@ -63,7 +63,7 @@ S3_BUCKET_NAME=simcore
 R_CLONE_PROVIDER=MINIO
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_OPTION_RETRIES=3
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 
 TRACING_OBSERVABILITY_BACKEND_ENDPOINT=http://jaeger:9411
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py
@@ -286,7 +286,7 @@ def expected_dynamic_sidecar_spec(
                     "RABBIT_PORT": "5672",
                     "RABBIT_USER": "admin",
                     "RABBIT_SECURE": "False",
-                    "R_CLONE_OPTION_BUFFER_SIZE": "0M",
+                    "R_CLONE_OPTION_BUFFER_SIZE": "16M",
                     "R_CLONE_OPTION_RETRIES": "3",
                     "R_CLONE_OPTION_TRANSFERS": "5",
                     "R_CLONE_PROVIDER": "MINIO",
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
@@ -562,6 +562,7 @@ services:
       REDIS_PASSWORD: ${REDIS_PASSWORD}
       DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
       DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
+      DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER}
       DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL}
       DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
       DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rpc/_services.py
@@ -12,9 +12,7 @@
     ServiceWasNotFoundError,
 )
 
-from ...core.settings import ApplicationSettings
-from ...services.director_v2 import DirectorV2Client
-from ...services.service_tracker import set_request_as_running, set_request_as_stopped
+from ...services import scheduler_interface
 
 router = RPCRouter()
 
@@ -23,23 +21,16 @@
 async def get_service_status(
     app: FastAPI, *, node_id: NodeID
 ) -> NodeGet | DynamicServiceGet | NodeGetIdle:
-    director_v2_client = DirectorV2Client.get_from_app_state(app)
-    response: NodeGet | DynamicServiceGet | NodeGetIdle = (
-        await director_v2_client.get_status(node_id)
-    )
-    return response
+    return await scheduler_interface.get_service_status(app, node_id=node_id)
 
 
 @router.expose()
 async def run_dynamic_service(
     app: FastAPI, *, dynamic_service_start: DynamicServiceStart
 ) -> NodeGet | DynamicServiceGet:
-    director_v2_client = DirectorV2Client.get_from_app_state(app)
-    response: NodeGet | DynamicServiceGet = (
-        await director_v2_client.run_dynamic_service(dynamic_service_start)
+    return await scheduler_interface.run_dynamic_service(
+        app, dynamic_service_start=dynamic_service_start
     )
-    await set_request_as_running(app, dynamic_service_start)
-    return response
 
 
 @router.expose(
@@ -51,12 +42,6 @@ async def run_dynamic_service(
 async def stop_dynamic_service(
     app: FastAPI, *, dynamic_service_stop: DynamicServiceStop
 ) -> None:
-    director_v2_client = DirectorV2Client.get_from_app_state(app)
-    settings: ApplicationSettings = app.state.settings
-    await director_v2_client.stop_dynamic_service(
-        node_id=dynamic_service_stop.node_id,
-        simcore_user_agent=dynamic_service_stop.simcore_user_agent,
-        save_state=dynamic_service_stop.save_state,
-        timeout=settings.DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT,
+    return await scheduler_interface.stop_dynamic_service(
+        app, dynamic_service_stop=dynamic_service_stop
     )
-    await set_request_as_stopped(app, dynamic_service_stop)
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py
@@ -68,6 +68,14 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
         ),
     )
 
+    DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: bool = Field(
+        default=False,
+        description=(
+            "this is a way to switch between different dynamic schedulers for the new style services"
+            # NOTE: this option should be removed when the scheduling will be done via this service
+        ),
+    )
+
     @field_validator("DYNAMIC_SCHEDULER_LOGLEVEL", mode="before")
     @classmethod
     def _validate_log_level(cls, value: str) -> str:
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/scheduler_interface.py
@@ -0,0 +1,60 @@
+from fastapi import FastAPI
+from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet
+from models_library.api_schemas_dynamic_scheduler.dynamic_services import (
+    DynamicServiceStart,
+    DynamicServiceStop,
+)
+from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle
+from models_library.projects_nodes_io import NodeID
+
+from ..core.settings import ApplicationSettings
+from .director_v2 import DirectorV2Client
+from .service_tracker import set_request_as_running, set_request_as_stopped
+
+
+async def get_service_status(
+    app: FastAPI, *, node_id: NodeID
+) -> NodeGet | DynamicServiceGet | NodeGetIdle:
+    settings: ApplicationSettings = app.state.settings
+    if settings.DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER:
+        raise NotImplementedError
+
+    director_v2_client = DirectorV2Client.get_from_app_state(app)
+    response: NodeGet | DynamicServiceGet | NodeGetIdle = (
+        await director_v2_client.get_status(node_id)
+    )
+    return response
+
+
+async def run_dynamic_service(
+    app: FastAPI, *, dynamic_service_start: DynamicServiceStart
+) -> NodeGet | DynamicServiceGet:
+    settings: ApplicationSettings = app.state.settings
+    if settings.DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER:
+        raise NotImplementedError
+
+    director_v2_client = DirectorV2Client.get_from_app_state(app)
+    response: NodeGet | DynamicServiceGet = (
+        await director_v2_client.run_dynamic_service(dynamic_service_start)
+    )
+
+    await set_request_as_running(app, dynamic_service_start)
+    return response
+
+
+async def stop_dynamic_service(
+    app: FastAPI, *, dynamic_service_stop: DynamicServiceStop
+) -> None:
+    settings: ApplicationSettings = app.state.settings
+    if settings.DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER:
+        raise NotImplementedError
+
+    director_v2_client = DirectorV2Client.get_from_app_state(app)
+    await director_v2_client.stop_dynamic_service(
+        node_id=dynamic_service_stop.node_id,
+        simcore_user_agent=dynamic_service_stop.simcore_user_agent,
+        save_state=dynamic_service_stop.save_state,
+        timeout=settings.DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT,
+    )
+
+    await set_request_as_stopped(app, dynamic_service_stop)
diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/status_monitor/_deferred_get_status.py
@@ -12,8 +12,7 @@
 from servicelib.deferred_tasks import BaseDeferredHandler, TaskUID
 from servicelib.deferred_tasks._base_deferred_handler import DeferredContext
 
-from .. import service_tracker
-from ..director_v2 import DirectorV2Client
+from .. import scheduler_interface, service_tracker
 from ..notifier import notify_service_status_change
 
 _logger = logging.getLogger(__name__)
@@ -47,9 +46,8 @@ async def run(
         app: FastAPI = context["app"]
         node_id: NodeID = context["node_id"]
 
-        director_v2_client: DirectorV2Client = DirectorV2Client.get_from_app_state(app)
         service_status: NodeGet | RunningDynamicServiceDetails | NodeGetIdle = (
-            await director_v2_client.get_status(node_id)
+            await scheduler_interface.get_service_status(app, node_id=node_id)
         )
         _logger.debug(
             "Service status type=%s, %s", type(service_status), service_status
diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py
@@ -20,6 +20,7 @@
 from models_library.users import UserID
 from pydantic import TypeAdapter
 from pytest_mock import MockerFixture
+from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
 from pytest_simcore.helpers.typing_env import EnvVarsDict
 from servicelib.rabbitmq import RabbitMQRPCClient, RPCServerError
 from servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler import services
@@ -133,12 +134,35 @@ def mock_director_v2_service_state(
         yield None
 
 
+@pytest.fixture(
+    params=[
+        False,
+        pytest.param(
+            True,
+            marks=pytest.mark.xfail(
+                reason="INTERNAL scheduler implementation is missing"
+            ),
+        ),
+    ]
+)
+def use_internal_scheduler(request: pytest.FixtureRequest) -> bool:
+    return request.param
+
+
 @pytest.fixture
 def app_environment(
     app_environment: EnvVarsDict,
     rabbit_service: RabbitSettings,
     redis_service: RedisSettings,
+    use_internal_scheduler: bool,
+    monkeypatch: pytest.MonkeyPatch,
 ) -> EnvVarsDict:
+    setenvs_from_dict(
+        monkeypatch,
+        {
+            "DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER": f"{use_internal_scheduler}",
+        },
+    )
     return app_environment
 
 
diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile
@@ -187,7 +187,7 @@ RUN chmod +x services/dynamic-sidecar/docker/*.sh
 HEALTHCHECK \
   --interval=10s \
   --timeout=5s \
-  --start-period=30s \
+  --start-period=180s \
   --start-interval=1s \
   --retries=5 \
   CMD ["python3", "services/dynamic-sidecar/docker/healthcheck.py", "http://localhost:8000/health"]
diff --git a/services/dynamic-sidecar/docker/entrypoint.sh b/services/dynamic-sidecar/docker/entrypoint.sh
@@ -93,12 +93,12 @@ fi
 # Change ownership of volumes mount directory
 # directories are empty at this point
 # each individual subdirectory is a unique volume
-chown --verbose --recursive "$SC_USER_NAME":"$GROUPNAME" "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
+chown --recursive "$SC_USER_NAME":"$GROUPNAME" "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 # Allow owner and group to edit write and execute
 # files from all the subdirectories
 # When the service access files downloaded by the dynamic-sidecar
 # it uses group permissions
-chmod --verbose --recursive 774 "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
+chmod --recursive 774 "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 
 echo "$INFO Starting $* ..."
 echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"
diff --git a/services/static-webserver/client/source/class/osparc/widget/PersistentIframe.js b/services/static-webserver/client/source/class/osparc/widget/PersistentIframe.js

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,6 @@ class RCloneSettings(BaseCustomSettings):`
`26`	`26`	`)`
`27`	`27`	`# SEE https://rclone.org/docs/#buffer-size-size`
`28`	`28`	`R_CLONE_OPTION_BUFFER_SIZE: str = Field(`
`29`		`- default="0M",`
	`29`	`+ default="16M",`
`30`	`30`	description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
`31`	`31`	`)`