Merge branch 'storage/batchDelete' of github.com:sanderegg/osparc-simcore into feature/batch-delete-files

Author: odeimaiz

packages/service-library/src/servicelib/fastapi/client_session.py

@@ -1,12 +1,20 @@
+import datetime
+
 import httpx
 from fastapi import FastAPI
 
 
-def setup_client_session(app: FastAPI, *, max_keepalive_connections: int = 20) -> None:
+def setup_client_session(
+    app: FastAPI,
+    *,
+    default_timeout: datetime.timedelta = datetime.timedelta(seconds=20),
+    max_keepalive_connections: int = 20
+) -> None:
     async def on_startup() -> None:
         session = httpx.AsyncClient(
             transport=httpx.AsyncHTTPTransport(http2=True),
             limits=httpx.Limits(max_keepalive_connections=max_keepalive_connections),
+            timeout=default_timeout.total_seconds(),
         )
         app.state.aiohttp_client_session = session
 

services/director/src/simcore_service_director/core/application.py

@@ -56,6 +56,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
     setup_client_session(
         app,
         max_keepalive_connections=settings.DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS,
+        default_timeout=settings.DIRECTOR_REGISTRY_CLIENT_TIMEOUT,
     )
     setup_registry(app)
 

services/director/src/simcore_service_director/core/settings.py

@@ -112,10 +112,21 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
         ),
     )
 
-    DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS: NonNegativeInt = 0
+    DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS: NonNegativeInt = 5
+    DIRECTOR_REGISTRY_CLIENT_TIMEOUT: datetime.timedelta = datetime.timedelta(
+        seconds=20
+    )
     DIRECTOR_REGISTRY_CLIENT_MAX_CONCURRENT_CALLS: PositiveInt = 20
     DIRECTOR_REGISTRY_CLIENT_MAX_NUMBER_OF_RETRIEVED_OBJECTS: PositiveInt = 30
 
+    @field_validator("DIRECTOR_REGISTRY_CLIENT_TIMEOUT")
+    @classmethod
+    def _check_positive(cls, value: datetime.timedelta) -> datetime.timedelta:
+        if value.total_seconds() < 0:
+            msg = "DIRECTOR_REGISTRY_CLIENT_TIMEOUT must be positive"
+            raise ValueError(msg)
+        return value
+
     @field_validator("DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS")
     @classmethod
     def _validate_substitutions(cls, v):

services/director/tests/unit/test_core_settings.py

@@ -4,7 +4,10 @@
 # pylint: disable=too-many-arguments
 
 
+import datetime
+
 import pytest
+from pydantic import ValidationError
 from pytest_simcore.helpers.monkeypatch_envs import (
     setenvs_from_dict,
     setenvs_from_envfile,
@@ -37,6 +40,16 @@ def test_valid_web_application_settings(app_environment: EnvVarsDict):
     )
 
 
+def test_invalid_client_timeout_raises(
+    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setenv(
+        "DIRECTOR_REGISTRY_CLIENT_TIMEOUT", f"{datetime.timedelta(seconds=-10)}"
+    )
+    with pytest.raises(ValidationError):
+        ApplicationSettings.create_from_envs()
+
+
 def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.delenv("DIRECTOR_DEFAULT_MAX_MEMORY", raising=False)
 

services/web/server/src/simcore_service_webserver/storage/_rest.py

@@ -182,10 +182,11 @@ def _create_data_response_from_async_job(
     request: web.Request,
     async_job: AsyncJobGet,
 ) -> web.Response:
+    async_job_id = f"{async_job.job_id}"
     return create_data_response(
         TaskGet(
-            task_id=f"{async_job.job_id}",
-            task_name=f"{async_job.job_id}",
+            task_id=async_job_id,
+            task_name=async_job_id,
             status_href=f"{request.url.with_path(str(request.app.router['get_async_job_status'].url_for(task_id=async_job_id)))}",
             abort_href=f"{request.url.with_path(str(request.app.router['abort_async_job'].url_for(task_id=async_job_id)))}",
             result_href=f"{request.url.with_path(str(request.app.router['get_async_job_result'].url_for(task_id=async_job_id)))}",
@@ -240,41 +241,6 @@ async def batch_delete_paths(request: web.Request):
     return _create_data_response_from_async_job(request, async_job)
 
 
-@routes.post(
-    f"{_storage_locations_prefix}/{{location_id}}/-/paths:batchDelete",
-    name="batch_delete_paths",
-)
-@login_required
-@permission_required("storage.files.*")
-async def batch_delete_paths(
-    request: web.Request,
-):
-    req_ctx = RequestContext.model_validate(request)
-    path_params = parse_request_path_parameters_as(StorageLocationPathParams, request)
-    body = await parse_request_body_as(BatchDeletePathsBodyParams, request)
-
-    rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app)
-    async_job, _ = await remote_delete_paths(
-        rabbitmq_rpc_client,
-        user_id=req_ctx.user_id,
-        product_name=req_ctx.product_name,
-        location_id=path_params.location_id,
-        paths=body.paths,
-    )
-
-    _job_id = f"{async_job.job_id}"
-    return create_data_response(
-        TaskGet(
-            task_id=_job_id,
-            task_name=_job_id,
-            status_href=f"{request.url.with_path(str(request.app.router['get_async_job_status'].url_for(task_id=_job_id)))}",
-            abort_href=f"{request.url.with_path(str(request.app.router['abort_async_job'].url_for(task_id=_job_id)))}",
-            result_href=f"{request.url.with_path(str(request.app.router['get_async_job_result'].url_for(task_id=_job_id)))}",
-        ),
-        status=status.HTTP_202_ACCEPTED,
-    )
-
-
 @routes.get(
     _storage_locations_prefix + "/{location_id}/datasets", name="list_datasets_metadata"
 )