efs improvements

Author: matusdrobuliak66

packages/postgres-database/src/simcore_postgres_database/migration/versions/ea3952fe5a0e_add_enable_efs_to_group_extra_properties.py

@@ -0,0 +1,32 @@
+"""add `enable_efs` to group extra properties
+
+Revision ID: ea3952fe5a0e
+Revises: 8a742f3efdd9
+Create Date: 2024-10-07 06:24:42.464942+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ea3952fe5a0e"
+down_revision = "8a742f3efdd9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "groups_extra_properties",
+        sa.Column(
+            "enable_efs", sa.Boolean(), server_default=sa.text("false"), nullable=False
+        ),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("groups_extra_properties", "enable_efs")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/groups_extra_properties.py

@@ -63,6 +63,13 @@
         server_default=sa.sql.expression.false(),
         doc="If true, will send telemetry for new style dynamic services to frontend",
     ),
+    sa.Column(
+        "enable_efs",
+        sa.Boolean(),
+        nullable=False,
+        server_default=sa.sql.expression.false(),
+        doc="If true, will mount efs distributed file system when dynamic services starts",
+    ),
     sa.UniqueConstraint(
         "group_id", "product_name", name="group_id_product_name_uniqueness"
     ),

packages/postgres-database/src/simcore_postgres_database/utils_groups_extra_properties.py

@@ -32,6 +32,7 @@ class GroupExtraProperties(FromRowMixin):
     enable_telemetry: bool
     created: datetime.datetime
     modified: datetime.datetime
+    enable_efs: bool
 
 
 async def _list_table_entries_ordered_by_group_type(

services/director-v2/src/simcore_service_director_v2/modules/db/repositories/groups_extra_properties.py

@@ -1,3 +1,4 @@
+from pydantic import BaseModel
 from simcore_postgres_database.utils_groups_extra_properties import (
     GroupExtraProperties,
     GroupExtraPropertiesRepo,
@@ -6,6 +7,12 @@
 from ._base import BaseRepository
 
 
+class UserExtraProperties(BaseModel):
+    is_internet_enabled: bool
+    is_telemetry_enabled: bool
+    is_efs_enabled: bool
+
+
 class GroupsExtraPropertiesRepository(BaseRepository):
     async def _get_aggregated_properties_for_user(
         self,
@@ -31,3 +38,15 @@ async def is_telemetry_enabled(self, *, user_id: int, product_name: str) -> bool
         )
         telemetry_enabled: bool = group_extra_properties.enable_telemetry
         return telemetry_enabled
+
+    async def get_user_extra_properties(
+        self, *, user_id: int, product_name: str
+    ) -> UserExtraProperties:
+        group_extra_properties = await self._get_aggregated_properties_for_user(
+            user_id=user_id, product_name=product_name
+        )
+        return UserExtraProperties(
+            is_internet_enabled=group_extra_properties.internet_access,
+            is_telemetry_enabled=group_extra_properties.enable_telemetry,
+            is_efs_enabled=group_extra_properties.enable_efs,
+        )

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -30,6 +30,7 @@
 from ....core.dynamic_services_settings.sidecar import DynamicSidecarSettings
 from ....core.settings import AppSettings
 from ....models.dynamic_services_scheduler import SchedulerData
+from ....modules.db.repositories.groups_extra_properties import UserExtraProperties
 from .._namespace import get_compose_namespace
 from ..volumes import DynamicSidecarVolumesPathsResolver
 from ._constants import DOCKER_CONTAINER_SPEC_RESTART_POLICY_DEFAULTS
@@ -220,6 +221,7 @@ async def _get_mounts(
     app_settings: AppSettings,
     has_quota_support: bool,
     rpc_client: RabbitMQRPCClient,
+    is_efs_enabled: bool,
 ) -> list[dict[str, Any]]:
     mounts: list[dict[str, Any]] = [
         # docker socket needed to use the docker api
@@ -270,18 +272,9 @@ async def _get_mounts(
             )
         )
 
-    # We check whether user has access to EFS feature
-    use_efs = False
-    efs_settings = dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS
-    if (
-        efs_settings
-        and scheduler_data.user_id in efs_settings.EFS_ONLY_ENABLED_FOR_USERIDS
-    ):
-        use_efs = True
-
     # state paths now get mounted via different driver and are synced to s3 automatically
     for path_to_mount in scheduler_data.paths_mapping.state_paths:
-        if use_efs:
+        if is_efs_enabled:
             assert dynamic_sidecar_settings.DYNAMIC_SIDECAR_EFS_SETTINGS  # nosec
 
             _storage_directory_name = DynamicSidecarVolumesPathsResolver.volume_name(
@@ -414,7 +407,7 @@ async def get_dynamic_sidecar_spec(  # pylint:disable=too-many-arguments# noqa:
     allow_internet_access: bool,
     hardware_info: HardwareInfo | None,
     metrics_collection_allowed: bool,
-    telemetry_enabled: bool,
+    user_extra_properties: UserExtraProperties,
     rpc_client: RabbitMQRPCClient,
 ) -> AioDockerServiceSpec:
     """
@@ -434,6 +427,7 @@ async def get_dynamic_sidecar_spec(  # pylint:disable=too-many-arguments# noqa:
         app_settings=app_settings,
         has_quota_support=has_quota_support,
         rpc_client=rpc_client,
+        is_efs_enabled=user_extra_properties.is_efs_enabled,
     )
 
     ports = _get_ports(
@@ -514,7 +508,7 @@ async def get_dynamic_sidecar_spec(  # pylint:disable=too-many-arguments# noqa:
                     app_settings,
                     allow_internet_access=allow_internet_access,
                     metrics_collection_allowed=metrics_collection_allowed,
-                    telemetry_enabled=telemetry_enabled,
+                    telemetry_enabled=user_extra_properties.is_telemetry_enabled,
                 ),
                 "Hosts": [],
                 "Image": dynamic_sidecar_settings.DYNAMIC_SIDECAR_IMAGE,

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py

@@ -218,7 +218,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
         scheduler_data.run_id = RunID.create()
 
         # telemetry configuration
-        is_telemetry_enabled = await groups_extra_properties.is_telemetry_enabled(
+        user_extra_properties = await groups_extra_properties.get_user_extra_properties(
             user_id=scheduler_data.user_id, product_name=scheduler_data.product_name
         )
 
@@ -237,7 +237,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
             has_quota_support=dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS,
             allow_internet_access=allow_internet_access,
             metrics_collection_allowed=metrics_collection_allowed,
-            telemetry_enabled=is_telemetry_enabled,
+            user_extra_properties=user_extra_properties,
             rpc_client=rpc_client,
         )
 

services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py

@@ -38,6 +38,9 @@
 )
 from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData
 from simcore_service_director_v2.modules.catalog import CatalogClient
+from simcore_service_director_v2.modules.db.repositories.groups_extra_properties import (
+    UserExtraProperties,
+)
 from simcore_service_director_v2.modules.dynamic_sidecar.docker_service_specs import (
     get_dynamic_sidecar_spec,
 )
@@ -453,7 +456,11 @@ async def test_get_dynamic_proxy_spec(
             has_quota_support=False,
             allow_internet_access=False,
             metrics_collection_allowed=True,
-            telemetry_enabled=True,
+            user_extra_properties=UserExtraProperties(
+                is_internet_enabled=True,
+                is_telemetry_enabled=False,
+                is_efs_enabled=False,
+            ),
             rpc_client=Mock(),
         )
 
@@ -548,7 +555,11 @@ async def test_merge_dynamic_sidecar_specs_with_user_specific_specs(
         has_quota_support=False,
         allow_internet_access=False,
         metrics_collection_allowed=True,
-        telemetry_enabled=True,
+        user_extra_properties=UserExtraProperties(
+            is_internet_enabled=True,
+            is_telemetry_enabled=False,
+            is_efs_enabled=False,
+        ),
         rpc_client=Mock(),
     )
     assert dynamic_sidecar_spec