Skip to content

Commit 55c4340

Browse files
author
Andrei Neagu
committed
removed unsued feature
1 parent 7407dd7 commit 55c4340

File tree

9 files changed

+35
-363
lines changed

9 files changed

+35
-363
lines changed

‎.github/workflows/ci-testing-deploy.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1590,8 +1590,6 @@ jobs:
15901590
with:
15911591
python-version: ${{ matrix.python }}
15921592
cache-dependency-glob: "**/director-v2/requirements/ci.txt"
1593-
- name: setup rclone docker volume plugin
1594-
run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash
15951593
- name: Download and load Docker images
15961594
uses: ./.github/actions/download-load-docker-images
15971595
with:

‎ci/github/helpers/install_rclone_docker_volume_plugin.bash‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎services/director-v2/requirements/_test.in‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
--constraint _base.txt
1111

1212
aio_pika
13-
aioboto3
1413
alembic # migration due to pytest_simcore.postgres_service2
1514
asgi_lifespan
1615
async-asgi-testclient # replacement for fastapi.testclient.TestClient [see b) below]

‎services/director-v2/requirements/_test.txt‎

Lines changed: 0 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,33 +2,10 @@ aio-pika==9.5.5
22
# via
33
# -c requirements/_base.txt
44
# -r requirements/_test.in
5-
aioboto3==14.3.0
6-
# via -r requirements/_test.in
7-
aiobotocore==2.22.0
8-
# via aioboto3
9-
aiofiles==24.1.0
10-
# via
11-
# -c requirements/_base.txt
12-
# aioboto3
13-
aiohappyeyeballs==2.6.1
14-
# via
15-
# -c requirements/_base.txt
16-
# aiohttp
17-
aiohttp==3.12.12
18-
# via
19-
# -c requirements/../../../requirements/constraints.txt
20-
# -c requirements/_base.txt
21-
# aiobotocore
22-
aioitertools==0.12.0
23-
# via aiobotocore
245
aiormq==6.8.1
256
# via
267
# -c requirements/_base.txt
278
# aio-pika
28-
aiosignal==1.3.2
29-
# via
30-
# -c requirements/_base.txt
31-
# aiohttp
329
alembic==1.15.2
3310
# via
3411
# -c requirements/_base.txt
@@ -44,17 +21,9 @@ async-asgi-testclient==1.4.11
4421
attrs==25.3.0
4522
# via
4623
# -c requirements/_base.txt
47-
# aiohttp
4824
# pytest-docker
4925
bokeh==3.7.3
5026
# via dask
51-
boto3==1.37.3
52-
# via aiobotocore
53-
botocore==1.37.3
54-
# via
55-
# aiobotocore
56-
# boto3
57-
# s3transfer
5827
certifi==2025.4.26
5928
# via
6029
# -c requirements/../../../requirements/constraints.txt
@@ -103,11 +72,6 @@ fakeredis==2.30.3
10372
# via -r requirements/_test.in
10473
flaky==3.8.1
10574
# via -r requirements/_test.in
106-
frozenlist==1.6.0
107-
# via
108-
# -c requirements/_base.txt
109-
# aiohttp
110-
# aiosignal
11175
fsspec==2025.3.2
11276
# via
11377
# -c requirements/_base.txt
@@ -151,11 +115,6 @@ jinja2==3.1.6
151115
# bokeh
152116
# dask
153117
# distributed
154-
jmespath==1.0.1
155-
# via
156-
# aiobotocore
157-
# boto3
158-
# botocore
159118
locket==1.0.0
160119
# via
161120
# -c requirements/_base.txt
@@ -180,8 +139,6 @@ msgpack==1.1.0
180139
multidict==6.4.4
181140
# via
182141
# -c requirements/_base.txt
183-
# aiobotocore
184-
# aiohttp
185142
# async-asgi-testclient
186143
# yarl
187144
mypy==1.16.1
@@ -227,7 +184,6 @@ pprintpp==0.4.0
227184
propcache==0.3.1
228185
# via
229186
# -c requirements/_base.txt
230-
# aiohttp
231187
# yarl
232188
psutil==7.0.0
233189
# via
@@ -263,8 +219,6 @@ pytest-xdist==3.8.0
263219
python-dateutil==2.9.0.post0
264220
# via
265221
# -c requirements/_base.txt
266-
# aiobotocore
267-
# botocore
268222
# pandas
269223
pytz==2025.2
270224
# via pandas
@@ -287,8 +241,6 @@ requests==2.32.4
287241
# docker
288242
respx==0.22.0
289243
# via -r requirements/_test.in
290-
s3transfer==0.11.3
291-
# via boto3
292244
six==1.17.0
293245
# via
294246
# -c requirements/_base.txt
@@ -348,21 +300,15 @@ urllib3==2.5.0
348300
# via
349301
# -c requirements/../../../requirements/constraints.txt
350302
# -c requirements/_base.txt
351-
# botocore
352303
# distributed
353304
# docker
354305
# requests
355-
wrapt==1.17.2
356-
# via
357-
# -c requirements/_base.txt
358-
# aiobotocore
359306
xyzservices==2025.4.0
360307
# via bokeh
361308
yarl==1.20.0
362309
# via
363310
# -c requirements/_base.txt
364311
# aio-pika
365-
# aiohttp
366312
# aiormq
367313
zict==3.0.0
368314
# via

‎services/director-v2/src/simcore_service_director_v2/core/settings.py‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -128,14 +128,6 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings):
128128
)
129129
DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False
130130

131-
DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: bool = Field(
132-
default=False,
133-
description=(
134-
"Under development feature. If enabled state "
135-
"is saved using rclone docker volumes."
136-
),
137-
)
138-
139131
# for passing self-signed certificate to spawned services
140132
DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID: str = Field(
141133
default="",

‎services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -321,19 +321,6 @@ async def _get_mounts(
321321
storage_directory_name=_storage_directory_name,
322322
)
323323
)
324-
# for now only enable this with dev features enabled
325-
elif app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
326-
mounts.append(
327-
DynamicSidecarVolumesPathsResolver.mount_r_clone(
328-
swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME,
329-
path=path_to_mount,
330-
node_uuid=scheduler_data.node_uuid,
331-
service_run_id=scheduler_data.run_id,
332-
project_id=scheduler_data.project_id,
333-
user_id=scheduler_data.user_id,
334-
r_clone_settings=dynamic_sidecar_settings.DYNAMIC_SIDECAR_R_CLONE_SETTINGS,
335-
)
336-
)
337324
else:
338325
mounts.append(
339326
DynamicSidecarVolumesPathsResolver.mount_entry(

‎services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -356,16 +356,10 @@ async def attempt_pod_removal_and_data_saving(
356356

357357
try:
358358
tasks = [
359-
service_push_outputs(app, scheduler_data.node_uuid, sidecars_client)
359+
service_push_outputs(app, scheduler_data.node_uuid, sidecars_client),
360+
service_save_state(app, scheduler_data.node_uuid, sidecars_client),
360361
]
361362

362-
# When enabled no longer uploads state via nodeports
363-
# It uses rclone mounted volumes for this task.
364-
if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
365-
tasks.append(
366-
service_save_state(app, scheduler_data.node_uuid, sidecars_client)
367-
)
368-
369363
await logged_gather(*tasks, max_concurrency=2)
370364
scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True
371365

@@ -547,11 +541,8 @@ async def _restore_service_state_with_metrics() -> None:
547541
tasks = [
548542
_pull_user_services_images_with_metrics(),
549543
_pull_output_ports_with_metrics(),
544+
_restore_service_state_with_metrics(),
550545
]
551-
# When enabled no longer downloads state via nodeports
552-
# S3 is used to store state paths
553-
if not app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED:
554-
tasks.append(_restore_service_state_with_metrics())
555546

556547
await limited_gather(*tasks, limit=3)
557548

‎services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py‎

Lines changed: 2 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import os
22
from pathlib import Path
3-
from typing import Any
3+
from typing import Any, Final
44

55
from models_library.api_schemas_directorv2.services import (
66
CHARS_IN_VOLUME_NAME_BEFORE_DIR_NAME,
@@ -20,72 +20,8 @@
2020
WRITE_SIZE,
2121
AwsEfsSettings,
2222
)
23-
from settings_library.r_clone import S3Provider
2423

25-
from ...core.dynamic_services_settings.sidecar import RCloneSettings
26-
from .errors import DynamicSidecarError
27-
28-
DY_SIDECAR_SHARED_STORE_PATH = Path("/shared-store")
29-
30-
31-
def _get_s3_volume_driver_config(
32-
r_clone_settings: RCloneSettings,
33-
project_id: ProjectID,
34-
node_uuid: NodeID,
35-
storage_directory_name: str,
36-
) -> dict[str, Any]:
37-
assert "/" not in storage_directory_name # nosec
38-
driver_config: dict[str, Any] = {
39-
"Name": "rclone",
40-
"Options": {
41-
"type": "s3",
42-
"s3-access_key_id": r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY,
43-
"s3-secret_access_key": r_clone_settings.R_CLONE_S3.S3_SECRET_KEY,
44-
"path": f"{r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME}/{project_id}/{node_uuid}/{storage_directory_name}",
45-
"allow-other": "true",
46-
"vfs-cache-mode": r_clone_settings.R_CLONE_VFS_CACHE_MODE.value,
47-
# Directly connected to how much time it takes for
48-
# files to appear on remote s3, please se discussion
49-
# SEE https://forum.rclone.org/t/file-added-to-s3-on-one-machine-not-visible-on-2nd-machine-unless-mount-is-restarted/20645
50-
# SEE https://rclone.org/commands/rclone_mount/#vfs-directory-cache
51-
"dir-cache-time": f"{r_clone_settings.R_CLONE_DIR_CACHE_TIME_SECONDS}s",
52-
"poll-interval": f"{r_clone_settings.R_CLONE_POLL_INTERVAL_SECONDS}s",
53-
},
54-
}
55-
if r_clone_settings.R_CLONE_S3.S3_ENDPOINT:
56-
driver_config["Options"][
57-
"s3-endpoint"
58-
] = r_clone_settings.R_CLONE_S3.S3_ENDPOINT
59-
60-
extra_options: dict[str, str] | None = None
61-
62-
if r_clone_settings.R_CLONE_PROVIDER == S3Provider.MINIO:
63-
extra_options = {
64-
"s3-provider": "Minio",
65-
"s3-region": "us-east-1",
66-
"s3-location_constraint": "",
67-
"s3-server_side_encryption": "",
68-
}
69-
elif r_clone_settings.R_CLONE_PROVIDER == S3Provider.CEPH:
70-
extra_options = {
71-
"s3-provider": "Ceph",
72-
"s3-acl": "private",
73-
}
74-
elif r_clone_settings.R_CLONE_PROVIDER == S3Provider.AWS:
75-
extra_options = {
76-
"s3-provider": "AWS",
77-
"s3-region": r_clone_settings.R_CLONE_S3.S3_REGION,
78-
"s3-acl": "private",
79-
}
80-
else:
81-
msg = f"Unexpected, all {S3Provider.__name__} should be covered"
82-
raise DynamicSidecarError(msg=msg)
83-
84-
assert extra_options is not None # nosec
85-
options: dict[str, Any] = driver_config["Options"]
86-
options.update(extra_options)
87-
88-
return driver_config
24+
DY_SIDECAR_SHARED_STORE_PATH: Final[Path] = Path("/shared-store")
8925

9026

9127
def _get_efs_volume_driver_config(
@@ -225,39 +161,6 @@ def mount_user_preferences(
225161
volume_size_limit="10M" if has_quota_support else None,
226162
)
227163

228-
@classmethod
229-
def mount_r_clone(
230-
cls,
231-
swarm_stack_name: str,
232-
path: Path,
233-
node_uuid: NodeID,
234-
service_run_id: ServiceRunID,
235-
project_id: ProjectID,
236-
user_id: UserID,
237-
r_clone_settings: RCloneSettings,
238-
) -> dict[str, Any]:
239-
return {
240-
"Source": cls.source(path, node_uuid, service_run_id),
241-
"Target": cls.target(path),
242-
"Type": "volume",
243-
"VolumeOptions": {
244-
"Labels": {
245-
"source": cls.source(path, node_uuid, service_run_id),
246-
"run_id": f"{service_run_id}",
247-
"node_uuid": f"{node_uuid}",
248-
"study_id": f"{project_id}",
249-
"user_id": f"{user_id}",
250-
"swarm_stack_name": swarm_stack_name,
251-
},
252-
"DriverConfig": _get_s3_volume_driver_config(
253-
r_clone_settings=r_clone_settings,
254-
project_id=project_id,
255-
node_uuid=node_uuid,
256-
storage_directory_name=cls.volume_name(path).strip("_"),
257-
),
258-
},
259-
}
260-
261164
@classmethod
262165
def mount_efs(
263166
cls,

0 commit comments

Comments
 (0)