✨ Adding rclount mount capability to dynamic-sidecar

.github/workflows/ci-testing-deploy.yml

@@ -1590,8 +1590,6 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           cache-dependency-glob: "**/director-v2/requirements/ci.txt"
-      - name: setup rclone docker volume plugin
-        run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
       - name: Download and load Docker images
         uses: ./.github/actions/download-load-docker-images
         with:

packages/settings-library/src/settings_library/r_clone.py

@@ -36,3 +36,29 @@ class RCloneSettings(BaseCustomSettings):
             description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
         ),
     ] = "16M"
+
+    R_CLONE_OPTION_CHECKERS: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--checkers X`: sets the number checkers",
+        ),
+    ] = 32
+
+    R_CLONE_S3_UPLOAD_CONCURRENCY: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--s3-upload-concurrency X`: sets the number of concurrent uploads to S3",
+        ),
+    ] = 8
+
+    R_CLONE_CHUNK_SIZE: Annotated[
+        str,
+        Field(description="`--s3-chunk-size X`: sets the chunk size for S3"),
+    ] = "128M"
+
+    R_CLONE_ORDER_BY: Annotated[
+        str,
+        Field(
+            description="`--order-by X`: sets the order of file upload, e.g., 'size,mixed'",
+        ),
+    ] = "size,mixed"

packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py

@@ -30,8 +30,7 @@
 _logger = logging.getLogger(__name__)
 
 
-class BaseRCloneError(OsparcErrorMixin, RuntimeError):
-    ...
+class BaseRCloneError(OsparcErrorMixin, RuntimeError): ...
 
 
 class RCloneFailedError(BaseRCloneError):
@@ -207,6 +206,16 @@ async def _sync_sources(
             # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
             "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
             r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE,
+            "--checkers",
+            f"{r_clone_settings.R_CLONE_OPTION_CHECKERS}",
+            "--s3-upload-concurrency",
+            f"{r_clone_settings.R_CLONE_S3_UPLOAD_CONCURRENCY}",
+            "--s3-chunk-size",
+            r_clone_settings.R_CLONE_CHUNK_SIZE,
+            # handles the order of file upload
+            "--order-by",
+            r_clone_settings.R_CLONE_ORDER_BY,
+            "--fast-list",
             "--use-json-log",
             # frequent polling for faster progress updates
             "--stats",

scripts/install_rclone.bash

@@ -9,7 +9,7 @@ set -o nounset  # abort on unbound variable
 set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
-R_CLONE_VERSION="1.63.1"
+R_CLONE_VERSION="1.70.3"
 TARGETARCH="${TARGETARCH:-amd64}"
 
 echo "platform ${TARGETARCH}"

services/agent/src/simcore_service_agent/services/backup.py

@@ -178,7 +178,7 @@ async def _store_in_s3(
         # below two options reduce to a minimum the memory footprint
         # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
         "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
-        "0M",
+        "16M",
         "--stats",
         "5s",
         "--stats-one-line",

services/director-v2/requirements/_test.in

@@ -10,7 +10,6 @@
 --constraint _base.txt
 
 aio_pika
-aioboto3
 alembic # migration due to pytest_simcore.postgres_service2
 asgi_lifespan
 async-asgi-testclient # replacement for fastapi.testclient.TestClient [see b) below]

services/director-v2/requirements/_test.txt

@@ -2,33 +2,10 @@ aio-pika==9.5.5
     # via
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
-aioboto3==14.3.0
-    # via -r requirements/_test.in
-aiobotocore==2.22.0
-    # via aioboto3
-aiofiles==24.1.0
-    # via
-    #   -c requirements/_base.txt
-    #   aioboto3
-aiohappyeyeballs==2.6.1
-    # via
-    #   -c requirements/_base.txt
-    #   aiohttp
-aiohttp==3.12.12
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -c requirements/_base.txt
-    #   aiobotocore
-aioitertools==0.12.0
-    # via aiobotocore
 aiormq==6.8.1
     # via
     #   -c requirements/_base.txt
     #   aio-pika
-aiosignal==1.3.2
-    # via
-    #   -c requirements/_base.txt
-    #   aiohttp
 alembic==1.15.2
     # via
     #   -c requirements/_base.txt
@@ -44,17 +21,9 @@ async-asgi-testclient==1.4.11
 attrs==25.3.0
     # via
     #   -c requirements/_base.txt
-    #   aiohttp
     #   pytest-docker
 bokeh==3.7.3
     # via dask
-boto3==1.37.3
-    # via aiobotocore
-botocore==1.37.3
-    # via
-    #   aiobotocore
-    #   boto3
-    #   s3transfer
 certifi==2025.4.26
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -103,11 +72,6 @@ fakeredis==2.30.3
     # via -r requirements/_test.in
 flaky==3.8.1
     # via -r requirements/_test.in
-frozenlist==1.6.0
-    # via
-    #   -c requirements/_base.txt
-    #   aiohttp
-    #   aiosignal
 fsspec==2025.3.2
     # via
     #   -c requirements/_base.txt
@@ -151,11 +115,6 @@ jinja2==3.1.6
     #   bokeh
     #   dask
     #   distributed
-jmespath==1.0.1
-    # via
-    #   aiobotocore
-    #   boto3
-    #   botocore
 locket==1.0.0
     # via
     #   -c requirements/_base.txt
@@ -180,8 +139,6 @@ msgpack==1.1.0
 multidict==6.4.4
     # via
     #   -c requirements/_base.txt
-    #   aiobotocore
-    #   aiohttp
     #   async-asgi-testclient
     #   yarl
 mypy==1.16.1
@@ -227,7 +184,6 @@ pprintpp==0.4.0
 propcache==0.3.1
     # via
     #   -c requirements/_base.txt
-    #   aiohttp
     #   yarl
 psutil==7.0.0
     # via
@@ -263,8 +219,6 @@ pytest-xdist==3.8.0
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/_base.txt
-    #   aiobotocore
-    #   botocore
     #   pandas
 pytz==2025.2
     # via pandas
@@ -287,8 +241,6 @@ requests==2.32.4
     #   docker
 respx==0.22.0
     # via -r requirements/_test.in
-s3transfer==0.11.3
-    # via boto3
 six==1.17.0
     # via
     #   -c requirements/_base.txt
@@ -348,21 +300,15 @@ urllib3==2.5.0
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
-    #   botocore
     #   distributed
     #   docker
     #   requests
-wrapt==1.17.2
-    # via
-    #   -c requirements/_base.txt
-    #   aiobotocore
 xyzservices==2025.4.0
     # via bokeh
 yarl==1.20.0
     # via
     #   -c requirements/_base.txt
     #   aio-pika
-    #   aiohttp
     #   aiormq
 zict==3.0.0
     # via

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -128,14 +128,6 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings):
     )
     DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False
 
-    DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: bool = Field(
-        default=False,
-        description=(
-            "Under development feature. If enabled state "
-            "is saved using rclone docker volumes."
-        ),
-    )
-
     # for passing self-signed certificate to spawned services
     DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID: str = Field(
         default="",

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -241,7 +241,6 @@ async def _get_mounts(
     scheduler_data: SchedulerData,
     dynamic_sidecar_settings: DynamicSidecarSettings,
     dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings,
-    app_settings: AppSettings,
     has_quota_support: bool,
     rpc_client: RabbitMQRPCClient,
     is_efs_enabled: bool,
@@ -321,19 +320,6 @@ async def _get_mounts(
                     storage_directory_name=_storage_directory_name,
                 )
             )
-        # for now only enable this with dev features enabled
-        elif app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
-            mounts.append(
-                DynamicSidecarVolumesPathsResolver.mount_r_clone(
-                    swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME,
-                    path=path_to_mount,
-                    node_uuid=scheduler_data.node_uuid,
-                    service_run_id=scheduler_data.run_id,
-                    project_id=scheduler_data.project_id,
-                    user_id=scheduler_data.user_id,
-                    r_clone_settings=dynamic_sidecar_settings.DYNAMIC_SIDECAR_R_CLONE_SETTINGS,
-                )
-            )
         else:
             mounts.append(
                 DynamicSidecarVolumesPathsResolver.mount_entry(
@@ -446,7 +432,6 @@ async def get_dynamic_sidecar_spec(  # pylint:disable=too-many-arguments# noqa:
         scheduler_data=scheduler_data,
         dynamic_services_scheduler_settings=dynamic_services_scheduler_settings,
         dynamic_sidecar_settings=dynamic_sidecar_settings,
-        app_settings=app_settings,
         has_quota_support=has_quota_support,
         rpc_client=rpc_client,
         is_efs_enabled=user_extra_properties.is_efs_enabled,

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py

@@ -356,16 +356,10 @@ async def attempt_pod_removal_and_data_saving(
 
         try:
             tasks = [
-                service_push_outputs(app, scheduler_data.node_uuid, sidecars_client)
+                service_push_outputs(app, scheduler_data.node_uuid, sidecars_client),
+                service_save_state(app, scheduler_data.node_uuid, sidecars_client),
             ]
 
-            # When enabled no longer uploads state via nodeports
-            # It uses rclone mounted volumes for this task.
-            if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
-                tasks.append(
-                    service_save_state(app, scheduler_data.node_uuid, sidecars_client)
-                )
-
             await logged_gather(*tasks, max_concurrency=2)
             scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True
 
@@ -478,7 +472,6 @@ async def wait_for_sidecar_api(app: FastAPI, scheduler_data: SchedulerData) -> N
 async def prepare_services_environment(
     app: FastAPI, scheduler_data: SchedulerData
 ) -> None:
-    app_settings: AppSettings = app.state.settings
     sidecars_client = await get_sidecars_client(app, scheduler_data.node_uuid)
     dynamic_sidecar_endpoint = scheduler_data.endpoint
 
@@ -547,11 +540,8 @@ async def _restore_service_state_with_metrics() -> None:
     tasks = [
         _pull_user_services_images_with_metrics(),
         _pull_output_ports_with_metrics(),
+        _restore_service_state_with_metrics(),
     ]
-    # When enabled no longer downloads state via nodeports
-    # S3 is used to store state paths
-    if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
-        tasks.append(_restore_service_state_with_metrics())
 
     await limited_gather(*tasks, limit=3)