🎨 EFS Guardian adding data removal background task (#6562)

Author: matusdrobuliak66

packages/models-library/src/models_library/projects_state.py

@@ -52,6 +52,7 @@ class ProjectStatus(str, Enum):
     EXPORTING = "EXPORTING"
     OPENING = "OPENING"
     OPENED = "OPENED"
+    MAINTAINING = "MAINTAINING"
 
 
 class ProjectLocked(BaseModel):

packages/postgres-database/src/simcore_postgres_database/utils_projects.py

@@ -0,0 +1,41 @@
+import uuid
+from datetime import datetime, timezone
+
+import sqlalchemy as sa
+from pydantic import parse_obj_as
+from pydantic.errors import PydanticErrorMixin
+from sqlalchemy.ext.asyncio import AsyncConnection
+
+from .models.projects import projects
+from .utils_repos import transaction_context
+
+
+class DBBaseProjectError(PydanticErrorMixin, Exception):
+    msg_template: str = "Project utils unexpected error"
+
+
+class DBProjectNotFoundError(DBBaseProjectError):
+    msg_template: str = "Project project_uuid={project_uuid!r} not found"
+
+
+class ProjectsRepo:
+    def __init__(self, engine):
+        self.engine = engine
+
+    async def get_project_last_change_date(
+        self,
+        project_uuid: uuid.UUID,
+        *,
+        connection: AsyncConnection | None = None,
+    ) -> datetime:
+        async with transaction_context(self.engine, connection) as conn:
+            get_stmt = sa.select(projects.c.last_change_date).where(
+                projects.c.uuid == f"{project_uuid}"
+            )
+
+            result = await conn.execute(get_stmt)
+            row = result.first()
+            if row is None:
+                raise DBProjectNotFoundError(project_uuid=project_uuid)
+            date = parse_obj_as(datetime, row[0])
+            return date.replace(tzinfo=timezone.utc)

packages/postgres-database/tests/conftest.py

@@ -206,7 +206,7 @@ async def connection(aiopg_engine: Engine) -> AsyncIterator[SAConnection]:
 
 
 @pytest.fixture
-async def asyncpg_engine(
+async def asyncpg_engine(  # <-- WE SHOULD USE THIS ONE
     is_pdb_enabled: bool,
     pg_sa_engine: sa.engine.Engine,
     make_asyncpg_engine: Callable[[bool], AsyncEngine],

packages/postgres-database/tests/test_utils_projects.py

@@ -0,0 +1,67 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+# pylint: disable=too-many-arguments
+import uuid
+from collections.abc import Awaitable, Callable
+from datetime import datetime
+from typing import Any, AsyncIterator
+
+import pytest
+import sqlalchemy
+from aiopg.sa.connection import SAConnection
+from aiopg.sa.result import RowProxy
+from faker import Faker
+from simcore_postgres_database.models.projects import projects
+from simcore_postgres_database.utils_projects import (
+    DBProjectNotFoundError,
+    ProjectsRepo,
+)
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+
+async def _delete_project(connection: SAConnection, project_uuid: uuid.UUID) -> None:
+    result = await connection.execute(
+        sqlalchemy.delete(projects).where(projects.c.uuid == f"{project_uuid}")
+    )
+    assert result.rowcount == 1
+
+
+@pytest.fixture
+async def registered_user(
+    connection: SAConnection,
+    create_fake_user: Callable[..., Awaitable[RowProxy]],
+) -> RowProxy:
+    user = await create_fake_user(connection)
+    assert user
+    return user
+
+
+@pytest.fixture
+async def registered_project(
+    connection: SAConnection,
+    registered_user: RowProxy,
+    create_fake_project: Callable[..., Awaitable[RowProxy]],
+) -> AsyncIterator[dict[str, Any]]:
+    project = await create_fake_project(connection, registered_user)
+    assert project
+
+    yield dict(project)
+
+    await _delete_project(connection, project["uuid"])
+
+
+async def test_get_project_last_change_date(
+    asyncpg_engine: AsyncEngine, registered_project: dict, faker: Faker
+):
+    projects_repo = ProjectsRepo(asyncpg_engine)
+
+    project_last_change_date = await projects_repo.get_project_last_change_date(
+        project_uuid=registered_project["uuid"]
+    )
+    assert isinstance(project_last_change_date, datetime)
+
+    with pytest.raises(DBProjectNotFoundError):
+        await projects_repo.get_project_last_change_date(
+            project_uuid=faker.uuid4()  # <-- Non existing uuid in DB
+        )

packages/pytest-simcore/src/pytest_simcore/faker_projects_data.py

@@ -9,11 +9,15 @@
 """
 
 
+from typing import Any
+
 import pytest
 from faker import Faker
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
+from models_library.users import UserID
 from pydantic import parse_obj_as
+from pytest_simcore.helpers.faker_factories import random_project
 
 _MESSAGE = (
     "If set, it overrides the fake value of `{}` fixture."
@@ -43,3 +47,12 @@ def project_id(faker: Faker, request: pytest.FixtureRequest) -> ProjectID:
 @pytest.fixture
 def node_id(faker: Faker) -> NodeID:
     return parse_obj_as(NodeID, faker.uuid4())
+
+
+@pytest.fixture
+def project_data(
+    faker: Faker,
+    project_id: ProjectID,
+    user_id: UserID,
+) -> dict[str, Any]:
+    return random_project(fake=faker, uuid=f"{project_id}", prj_owner=user_id)

packages/service-library/src/servicelib/project_lock.py

@@ -0,0 +1,80 @@
+import datetime
+import logging
+from asyncio.log import logger
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Final
+
+import redis
+import redis.exceptions
+from models_library.projects import ProjectID
+from models_library.projects_access import Owner
+from models_library.projects_state import ProjectLocked, ProjectStatus
+from redis.asyncio.lock import Lock
+
+from .background_task import periodic_task
+from .logging_utils import log_context
+
+_logger = logging.getLogger(__name__)
+
+PROJECT_REDIS_LOCK_KEY: str = "project_lock:{}"
+PROJECT_LOCK_TIMEOUT: Final[datetime.timedelta] = datetime.timedelta(seconds=10)
+ProjectLock = Lock
+
+ProjectLockError = redis.exceptions.LockError
+
+
+async def _auto_extend_project_lock(project_lock: Lock) -> None:
+    # NOTE: the background task already catches anything that might raise here
+    await project_lock.reacquire()
+
+
+@asynccontextmanager
+async def lock_project(
+    redis_lock: Lock,
+    project_uuid: str | ProjectID,
+    status: ProjectStatus,
+    owner: Owner | None = None,
+) -> AsyncIterator[None]:
+    """Context manager to lock and unlock a project by user_id
+
+    Raises:
+        ProjectLockError: if project is already locked
+    """
+
+    try:
+        if not await redis_lock.acquire(
+            blocking=False,
+            token=ProjectLocked(
+                value=True,
+                owner=owner,
+                status=status,
+            ).json(),
+        ):
+            msg = f"Lock for project {project_uuid!r} owner {owner!r} could not be acquired"
+            raise ProjectLockError(msg)
+
+        with log_context(
+            _logger,
+            logging.DEBUG,
+            msg=f"with lock for {owner=}:{project_uuid=}:{status=}",
+        ):
+            async with periodic_task(
+                _auto_extend_project_lock,
+                interval=0.6 * PROJECT_LOCK_TIMEOUT,
+                task_name=f"{PROJECT_REDIS_LOCK_KEY.format(project_uuid)}_lock_auto_extend",
+                project_lock=redis_lock,
+            ):
+                yield
+
+    finally:
+        # let's ensure we release that stuff
+        try:
+            if await redis_lock.owned():
+                await redis_lock.release()
+        except (redis.exceptions.LockError, redis.exceptions.LockNotOwnedError) as exc:
+            logger.warning(
+                "releasing %s unexpectedly raised an exception: %s",
+                f"{redis_lock=!r}",
+                f"{exc}",
+            )

packages/service-library/tests/test_project_lock.py

@@ -0,0 +1 @@
+# NOTE: Tested in osparc-simcore/services/web/server/tests/unit/with_dbs/02/test_project_lock.py

services/director-v2/.env-devel

@@ -31,7 +31,7 @@ DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 
 LOG_LEVEL=DEBUG
 
-POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
+POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_USER=test
 POSTGRES_PASSWORD=test
 POSTGRES_DB=test

services/docker-compose.yml

@@ -413,6 +413,11 @@ services:
       REDIS_PORT: ${REDIS_PORT}
       REDIS_SECURE: ${REDIS_SECURE}
       REDIS_USER: ${REDIS_USER}
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_HOST: ${POSTGRES_HOST}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_PORT: ${POSTGRES_PORT}
+      POSTGRES_USER: ${POSTGRES_USER}
       SC_USER_ID: ${SC_USER_ID}
       SC_USER_NAME: ${SC_USER_NAME}
       EFS_USER_ID: ${EFS_USER_ID}

services/efs-guardian/requirements/_base.in

@@ -9,6 +9,7 @@
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in
 --requirement ../../../packages/aws-library/requirements/_base.in
+--requirement ../../../packages/postgres-database/requirements/_base.in
 # service-library[fastapi]
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_fastapi.in