🐛On-demand computational backend failing (#7735)

Author: sanderegg

packages/service-library/src/servicelib/aiohttp/rest_responses.py

@@ -8,10 +8,10 @@
 from common_library.error_codes import ErrorCodeStr
 from common_library.json_serialization import json_dumps
 from models_library.rest_error import ErrorGet, ErrorItemType
-from servicelib.rest_constants import RESPONSE_MODEL_POLICY
 
 from ..aiohttp.status import HTTP_200_OK
 from ..mimetype_constants import MIMETYPE_APPLICATION_JSON
+from ..rest_constants import RESPONSE_MODEL_POLICY
 from ..rest_responses import is_enveloped
 from ..status_codes_utils import get_code_description
 
@@ -54,7 +54,7 @@ def create_http_error(
     http_error_cls: type[HTTPError] = web.HTTPInternalServerError,
     *,
     skip_internal_error_details: bool = False,
-    error_code: ErrorCodeStr | None = None
+    error_code: ErrorCodeStr | None = None,
 ) -> HTTPError:
     """
     - Response body conforms OAS schema model

services/agent/docker/boot.sh

@@ -32,7 +32,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 #
@@ -48,7 +52,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
 
   exec sh -c "
     cd services/agent/src/simcore_service_agent && \
-    python -m debugpy --listen 0.0.0.0:${AGENT_SERVER_REMOTE_DEBUG_PORT} -m uvicorn main:the_app \
+    python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${AGENT_SERVER_REMOTE_DEBUG_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
       --port 8000 \
       --reload \

services/api-server/docker/boot.sh

@@ -27,7 +27,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 # RUNNING application ----------------------------------------
@@ -40,7 +44,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
 
   exec sh -c "
     cd services/api-server/src/simcore_service_api_server && \
-    python -m debugpy --listen 0.0.0.0:${API_SERVER_REMOTE_DEBUG_PORT} -m uvicorn main:the_app \
+    python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${API_SERVER_REMOTE_DEBUG_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
       --reload \
       $reload_dir_packages

services/autoscaling/docker/boot.sh

@@ -31,7 +31,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 #
@@ -47,7 +51,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
 
   exec sh -c "
     cd services/autoscaling/src/simcore_service_autoscaling && \
-    python -m debugpy --listen 0.0.0.0:${AUTOSCALING_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
+    python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${AUTOSCALING_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
       --reload \
       $reload_dir_packages

services/catalog/docker/boot.sh

@@ -27,7 +27,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 # RUNNING application ----------------------------------------
@@ -40,7 +44,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
 
   exec sh -c "
     cd services/catalog/src/simcore_service_catalog && \
-    python -m debugpy --listen 0.0.0.0:${CATALOG_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
+    python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${CATALOG_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
       --reload \
       $reload_dir_packages

services/clusters-keeper/docker/boot.sh

@@ -32,7 +32,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 #
@@ -48,7 +52,7 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
 
   exec sh -c "
     cd services/clusters-keeper/src/simcore_service_clusters_keeper && \
-    python -m debugpy --listen 0.0.0.0:${CLUSTERS_KEEPER_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
+    python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${CLUSTERS_KEEPER_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \
       --host 0.0.0.0 \
       --reload \
       $reload_dir_packages

services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py

@@ -23,11 +23,27 @@
 from ..rpc.rpc_routes import setup_rpc_routes
 from .settings import ApplicationSettings
 
-logger = logging.getLogger(__name__)
+_LOG_LEVEL_STEP = logging.CRITICAL - logging.ERROR
+_NOISY_LOGGERS = (
+    "aiobotocore",
+    "aio_pika",
+    "aiormq",
+    "botocore",
+    "werkzeug",
+)
+
+_logger = logging.getLogger(__name__)
 
 
 def create_app(settings: ApplicationSettings) -> FastAPI:
-    logger.info("app settings: %s", settings.model_dump_json(indent=1))
+    # keep mostly quiet noisy loggers
+    quiet_level: int = max(
+        min(logging.root.level + _LOG_LEVEL_STEP, logging.CRITICAL), logging.WARNING
+    )
+    for name in _NOISY_LOGGERS:
+        logging.getLogger(name).setLevel(quiet_level)
+
+    _logger.info("app settings: %s", settings.model_dump_json(indent=1))
 
     app = FastAPI(
         debug=settings.CLUSTERS_KEEPER_DEBUG,

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -3,6 +3,8 @@ services:
     image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG}
     init: true
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
+    volumes:
+      - computational_shared_data:${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data}
     networks:
       - cluster
     environment:
@@ -15,6 +17,9 @@ services:
       DASK_WORKER_SATURATION: ${DASK_WORKER_SATURATION}
       LOG_LEVEL: ${LOG_LEVEL}
 
+      SIDECAR_COMP_SERVICES_SHARED_FOLDER: /home/scu/computational_shared_data
+      SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: computational_shared_data
+
     ports:
       - 8786:8786 # dask-scheduler access
       - 8787:8787 # dashboard
@@ -110,7 +115,7 @@ services:
       EC2_INSTANCES_TIME_BEFORE_DRAINING: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING}
       EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}
       LOG_FORMAT_LOCAL_DEV_ENABLED: 1
-      LOG_LEVEL: ${LOG_LEVEL:-WARNING}
+      LOG_LEVEL: ${LOG_LEVEL}
       REDIS_HOST: redis
       REDIS_PORT: 6379
     volumes:

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py

@@ -7,10 +7,9 @@
 
 
 def get_scheduler_url(ec2_instance: EC2InstanceData) -> AnyUrl:
-    url: AnyUrl = TypeAdapter(AnyUrl).validate_python(
+    return TypeAdapter(AnyUrl).validate_python(
         f"tls://{ec2_instance.aws_private_dns}:8786"
     )
-    return url
 
 
 def get_scheduler_auth(app: FastAPI) -> ClusterAuthentication:

services/dask-sidecar/docker/boot.sh

@@ -33,7 +33,11 @@ fi
 
 if [ "${SC_BOOT_MODE}" = "debug" ]; then
   # NOTE: production does NOT pre-installs debugpy
-  uv pip install debugpy
+  if command -v uv >/dev/null 2>&1; then
+    uv pip install debugpy
+  else
+    pip install debugpy
+  fi
 fi
 
 # RUNNING application ----------------------------------------