many fixes

Author: sanderegg

services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py

@@ -369,7 +369,6 @@ def mock_env(
                     "STORAGE_PORT": local_settings.STORAGE_PORT,
                 }
             ),
-            "STORAGE_ENDPOINT": "storage:8080",
             "DYNAMIC_SIDECAR_IMAGE": image_name,
             "DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS": "{}",
             "TRAEFIK_SIMCORE_ZONE": "test_traefik_zone",

services/docker-compose.yml

@@ -1146,6 +1146,7 @@ services:
       STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL}
       STORAGE_MONITORING_ENABLED: 1
       STORAGE_PROFILING: ${STORAGE_PROFILING}
+      STORAGE_PORT: ${STORAGE_PORT}
       TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
       TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
     networks:

services/storage/docker/boot.sh

@@ -49,6 +49,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
     cd services/storage/src/simcore_service_storage && \
     python -m debugpy --listen 0.0.0.0:${STORAGE_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
+      --port ${STORAGE_PORT} \
       --reload \
       $reload_dir_packages
       --reload-dir . \

services/storage/src/simcore_service_storage/core/application.py

@@ -42,6 +42,7 @@
     "aiormq",
     "botocore",
     "httpcore",
+    "urllib3",
     "werkzeug",
 )
 _logger = logging.getLogger(__name__)

services/storage/src/simcore_service_storage/dsm_cleaner.py

@@ -26,6 +26,7 @@
 from fastapi import FastAPI
 from servicelib.async_utils import cancel_wait_task
 from servicelib.background_task_utils import exclusive_periodic
+from servicelib.logging_utils import log_context
 
 from .core.settings import get_application_settings
 from .dsm import get_dsm_provider
@@ -38,12 +39,12 @@
 
 
 async def dsm_cleaner_task(app: FastAPI) -> None:
-    _logger.info("starting dsm cleaner task...")
-    dsm = get_dsm_provider(app)
-    simcore_s3_dsm: SimcoreS3DataManager = cast(
-        SimcoreS3DataManager, dsm.get(SimcoreS3DataManager.get_location_id())
-    )
-    await simcore_s3_dsm.clean_expired_uploads()
+    with log_context(_logger, logging.INFO, "dsm cleaner task"):
+        dsm = get_dsm_provider(app)
+        simcore_s3_dsm: SimcoreS3DataManager = cast(
+            SimcoreS3DataManager, dsm.get(SimcoreS3DataManager.get_location_id())
+        )
+        await simcore_s3_dsm.clean_expired_uploads()
 
 
 def setup_dsm_cleaner(app: FastAPI) -> None: