♻️ Is638/dynamic sidecar refactors tests (round 3) (ITISFoundation#3154)

Author: pcrespov

services/dynamic-sidecar/.env-devel

@@ -5,7 +5,7 @@
 
 # environs in Dockerfile ----------------
 SC_BOOT_MODE=local-development
-
+DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR="/tmp/dy-volumes"
 
 # service specific required vars
 DYNAMIC_SIDECAR_COMPOSE_NAMESPACE=dev-namespace
@@ -28,4 +28,4 @@ S3_ENDPOINT=MINIO
 S3_ACCESS_KEY=mocked
 S3_SECRET_KEY=mocked
 S3_BUCKET_NAME=mocked
-R_CLONE_PROVIDER=MINIO
+R_CLONE_PROVIDER=MINIO

services/dynamic-sidecar/Dockerfile

@@ -45,7 +45,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 # directory where dynamic-sidecar stores creates and shares
 # volumes between itself and the spawned containers
-ENV DY_VOLUMES="/dy-volumes"
+ENV DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR="/dy-volumes"
 
 # rclone installation
 ARG R_CLONE_VERSION="1.58.0"
@@ -70,7 +70,7 @@ RUN apt-get update &&\
 # NOTE: python virtualenv is used here such that installed
 # packages may be moved to production image easily by copying the venv
 RUN python -m venv ${VIRTUAL_ENV}
-RUN mkdir -p ${DY_VOLUMES}
+RUN mkdir -p ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}
 
 RUN pip install --upgrade --no-cache-dir \
   pip~=22.0  \
@@ -124,7 +124,7 @@ WORKDIR /home/scu
 
 # Starting from clean base image, copies pre-installed virtualenv from prod-only-deps
 COPY --chown=scu:scu --from=prod-only-deps  ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-COPY --chown=scu:scu --from=prod-only-deps  ${DY_VOLUMES} ${DY_VOLUMES}
+COPY --chown=scu:scu --from=prod-only-deps  ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR} ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}
 
 # Copies booting scripts
 COPY --chown=scu:scu services/dynamic-sidecar/docker services/dynamic-sidecar/docker
@@ -157,7 +157,7 @@ ENV SC_BUILD_TARGET=development
 WORKDIR /devel
 
 RUN chown -R scu:scu ${VIRTUAL_ENV}
-RUN chown -R scu:scu ${DY_VOLUMES}
+RUN chown -R scu:scu ${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}
 
 EXPOSE 8000
 EXPOSE 3000

services/dynamic-sidecar/docker/entrypoint.sh

@@ -98,16 +98,16 @@ fi
 # Change ownership of volumes mount directory
 # directories are empty at this point
 # each individual subdirectory is a unique volume
-chown --verbose --recursive "$SC_USER_NAME":"$GROUPNAME" "${DY_VOLUMES}"
+chown --verbose --recursive "$SC_USER_NAME":"$GROUPNAME" "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 # Allow owner and group to edit write and execute
 # files from all the subdirectories
 # When the service access files downloaded by the dynamic-sidecar
 # it uses group permissions
-chmod --verbose --recursive 774 "${DY_VOLUMES}"
+chmod --verbose --recursive 774 "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}"
 
 echo "$INFO Starting $* ..."
 echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"
 echo "  local dir : $(ls -al)"
-echo "  volumes dir : $(ls -al "${DY_VOLUMES}")"
+echo "  volumes dir : $(ls -al "${DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR}")"
 
 exec gosu "$SC_USER_NAME" "$@"

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/cli.py

@@ -3,7 +3,7 @@
 
 import typer
 from settings_library.utils_cli import create_settings_command
-from simcore_service_dynamic_sidecar.core.application import create_basic_app
+from simcore_service_dynamic_sidecar.core.application import create_base_app
 
 from ._meta import PROJECT_NAME
 from .core.settings import DynamicSidecarSettings
@@ -18,7 +18,7 @@
 @main.command()
 def openapi():
     """Prints OpenAPI specifications in json format"""
-    app = create_basic_app()
+    app = create_base_app()
     typer.secho(json.dumps(app.openapi(), indent=2))
 
 

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py

@@ -9,7 +9,7 @@
 from ..models.domains.shared_store import SharedStore
 from ..models.schemas.application_health import ApplicationHealth
 from ..modules.directory_watcher import setup_directory_watcher
-from ..modules.mounted_fs import setup_mounted_fs
+from ..modules.mounted_fs import MountedVolumes, setup_mounted_fs
 from .docker_logs import setup_background_log_fetcher
 from .error_handlers import http_error_handler, node_not_found_error_handler
 from .errors import BaseDynamicSidecarError
@@ -45,7 +45,7 @@ def setup_logger(settings: DynamicSidecarSettings):
     logging.root.setLevel(settings.log_level)
 
 
-def create_basic_app() -> FastAPI:
+def create_base_app() -> FastAPI:
     # settings
     settings = DynamicSidecarSettings.create_from_envs()
     setup_logger(settings)
@@ -71,7 +71,7 @@ def create_app():
     needed in other requests and used to share data.
     """
 
-    app = create_basic_app()
+    app = create_base_app()
 
     # MODULES SETUP --------------
 
@@ -114,3 +114,23 @@ async def _on_shutdown() -> None:
     app.add_event_handler("shutdown", _on_shutdown)
 
     return app
+
+
+class AppState:
+    def __init__(self, app: FastAPI):
+        self._app = app
+
+    @property
+    def settings(self) -> DynamicSidecarSettings:
+        assert isinstance(self._app.state.settings, DynamicSidecarSettings)  # nosec
+        return self._app.state.settings
+
+    @property
+    def mounted_volumes(self) -> MountedVolumes:
+        assert isinstance(self._app.state.mounted_volumes, MountedVolumes)  # nosec
+        return self._app.state.mounted_volumes
+
+    @property
+    def shared_store(self) -> SharedStore:
+        assert isinstance(self._app.state.shared_store, SharedStore)  # nosec
+        return self._app.state.shared_store

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py

@@ -23,16 +23,24 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
         description="boot mode helps determine if in development mode or normal operation",
     )
 
+    DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR: Path = Field(
+        ...,
+        description="Base directory where dynamic-sidecar stores creates "
+        "and shares volumes between itself and the spawned containers. "
+        "It is used as a mount directory for the director-v2."
+        "Sidecar must have r/w permissions in this folder.",
+    )
+
     # LOGGING
-    LOG_LEVEL: str = Field("WARNING")
+    LOG_LEVEL: str = Field(default="WARNING")
 
     # SERVICE SERVER (see : https://www.uvicorn.org/settings/)
     DYNAMIC_SIDECAR_HOST: str = Field(
-        "0.0.0.0",  # nosec
+        default="0.0.0.0",  # nosec
         description="host where to bind the application on which to serve",
     )
     DYNAMIC_SIDECAR_PORT: PortInt = Field(
-        8000, description="port where the server will be currently serving"
+        default=8000, description="port where the server will be currently serving"
     )
 
     DYNAMIC_SIDECAR_COMPOSE_NAMESPACE: str = Field(
@@ -44,28 +52,29 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
     )
 
     DYNAMIC_SIDECAR_MAX_COMBINED_CONTAINER_NAME_LENGTH: PositiveInt = Field(
-        63, description="the container name which will be used as hostname"
+        default=63, description="the container name which will be used as hostname"
     )
 
     DYNAMIC_SIDECAR_STOP_AND_REMOVE_TIMEOUT: PositiveInt = Field(
-        5,
+        default=5,
         description=(
             "When receiving SIGTERM the process has 10 seconds to cleanup its children "
             "forcing our children to stop in 5 seconds in all cases"
         ),
     )
 
     DEBUG: bool = Field(
-        False,
+        default=False,
         description="If set to True the application will boot into debug mode",
     )
 
     DYNAMIC_SIDECAR_REMOTE_DEBUG_PORT: PortInt = Field(
-        3000, description="ptsvd remote debugger starting port"
+        default=3000, description="ptsvd remote debugger starting port"
     )
 
     DYNAMIC_SIDECAR_DOCKER_COMPOSE_DOWN_TIMEOUT: PositiveInt = Field(
-        15, description="used during shutdown when containers swapend will be removed"
+        default=15,
+        description="used during shutdown when containers swapend will be removed",
     )
 
     DY_SIDECAR_PATH_INPUTS: Path = Field(

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/domains/shared_store.py

@@ -5,8 +5,9 @@
 
 class SharedStore(BaseModel):
     compose_spec: Optional[str] = Field(
-        None, description="stores the stringified compose spec"
+        default=None, description="stores the stringified compose spec"
     )
     container_names: list[str] = Field(
-        [], description="stores the container names from the compose_spec"
+        default_factory=list,
+        description="stores the container names from the compose_spec",
     )

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py

@@ -5,8 +5,8 @@
 
 class ApplicationHealth(BaseModel):
     is_healthy: bool = Field(
-        True, description="returns True if the service sis running correctly"
+        default=True, description="returns True if the service sis running correctly"
     )
     error_message: Optional[str] = Field(
-        None, description="in case of error this gets set"
+        default=None, description="in case of error this gets set"
     )

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/mounted_fs.py

@@ -9,8 +9,6 @@
 
 from ..core.docker_utils import get_volume_by_label
 
-DY_VOLUMES = Path("/dy-volumes")
-
 
 def _ensure_path(path: Path) -> Path:
     path.mkdir(parents=True, exist_ok=True)
@@ -41,12 +39,14 @@ def __init__(
         state_paths: list[Path],
         state_exclude: list[str],
         compose_namespace: str,
+        dy_volumes: Path,
     ) -> None:
         self.inputs_path: Path = inputs_path
         self.outputs_path: Path = outputs_path
         self.state_paths: list[Path] = state_paths
         self.state_exclude: list[str] = state_exclude
         self.compose_namespace = compose_namespace
+        self._dy_volumes = dy_volumes
 
         self._ensure_directories()
 
@@ -65,15 +65,15 @@ def volume_name_state_paths(self) -> Generator[str, None, None]:
 
     @cached_property
     def disk_inputs_path(self) -> Path:
-        return _ensure_path(DY_VOLUMES / self.inputs_path.relative_to("/"))
+        return _ensure_path(self._dy_volumes / self.inputs_path.relative_to("/"))
 
     @cached_property
     def disk_outputs_path(self) -> Path:
-        return _ensure_path(DY_VOLUMES / self.outputs_path.relative_to("/"))
+        return _ensure_path(self._dy_volumes / self.outputs_path.relative_to("/"))
 
     def disk_state_paths(self) -> Iterator[Path]:
         for state_path in self.state_paths:
-            yield _ensure_path(DY_VOLUMES / state_path.relative_to("/"))
+            yield _ensure_path(self._dy_volumes / state_path.relative_to("/"))
 
     def all_disk_paths(self) -> Iterator[Path]:
         # PC: keeps iterator to follow same style as disk_state_paths but IMO it is overreaching
@@ -86,7 +86,7 @@ def _ensure_directories(self) -> None:
         Creates the directories on its file system,
         these will be mounted elsewere.
         """
-        _ensure_path(DY_VOLUMES)
+        _ensure_path(self._dy_volumes)
         self.disk_inputs_path  # pylint:disable= pointless-statement
         self.disk_outputs_path  # pylint:disable= pointless-statement
         set(self.disk_state_paths())
@@ -129,6 +129,7 @@ def setup_mounted_fs(app: FastAPI) -> MountedVolumes:
         state_paths=settings.DY_SIDECAR_STATE_PATHS,
         state_exclude=settings.DY_SIDECAR_STATE_EXCLUDE,
         compose_namespace=settings.DYNAMIC_SIDECAR_COMPOSE_NAMESPACE,
+        dy_volumes=settings.DYNAMIC_SIDECAR_DY_VOLUMES_MOUNT_DIR,
     )
 
     return app.state.mounted_volumes