Merge branch 'master' into enh/handle-non-existing-services

Author: odeimaiz

.github/workflows/ci-testing-deploy.yml

@@ -2656,7 +2656,7 @@ jobs:
     if: ${{ always() }}
     needs:
       [
-        system-test-e2e,
+        # system-test-e2e, NOTE: not required, until Odei will have a look
         system-test-e2e-playwright,
         system-test-environment-setup,
         system-test-public-api,

services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py

@@ -28,7 +28,6 @@
 from servicelib.redis import get_project_locked_state
 
 from ..._meta import API_VTAG as VTAG
-from ...catalog import catalog_service
 from ...login.decorators import login_required
 from ...redis import get_redis_lock_manager_client_sdk
 from ...resource_manager.user_sessions import PROJECT_ID_KEY, managed_resource
@@ -39,7 +38,6 @@
 from .. import _crud_api_create, _crud_api_read, _projects_service
 from .._permalink_service import update_or_pop_permalink_in_project
 from ..models import ProjectDict
-from ..utils import are_project_services_available, get_project_unavailable_services
 from . import _rest_utils
 from ._rest_exceptions import handle_plugin_requests_exceptions
 from ._rest_schemas import (
@@ -271,10 +269,6 @@ async def get_project(request: web.Request):
     req_ctx = AuthenticatedRequestContext.model_validate(request)
     path_params = parse_request_path_parameters_as(ProjectPathParams, request)
 
-    user_available_services = await catalog_service.get_services_for_user_in_product(
-        request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name
-    )
-
     project = await _projects_service.get_project_for_user(
         request.app,
         project_uuid=f"{path_params.project_id}",
@@ -283,21 +277,6 @@ async def get_project(request: web.Request):
         include_trashed_by_primary_gid=True,
     )
 
-    if not are_project_services_available(project, user_available_services):
-        unavilable_services = get_project_unavailable_services(
-            project, user_available_services
-        )
-        formatted_services = ", ".join(
-            f"{service}:{version}" for service, version in unavilable_services
-        )
-        # TODO: lack of permissions should be notified with https://httpstatuses.com/403 web.HTTPForbidden
-        raise web.HTTPNotFound(
-            reason=(
-                f"Project '{path_params.project_id}' uses unavailable services. Please ask "
-                f"for permission for the following services {formatted_services}"
-            )
-        )
-
     # Adds permalink
     await update_or_pop_permalink_in_project(request, project)
 

services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py

@@ -22,7 +22,6 @@
 )
 from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB
 
-from ..catalog import catalog_service
 from ..folders import _folders_repository
 from ..workspaces.api import check_user_workspace_access
 from . import _projects_service
@@ -120,10 +119,6 @@ async def list_projects(  # pylint: disable=too-many-arguments
 ) -> tuple[list[ProjectDict], int]:
     db = ProjectDBAPI.get_from_app_context(app)
 
-    user_available_services = await catalog_service.get_services_for_user_in_product(
-        app, user_id=user_id, product_name=product_name
-    )
-
     workspace_is_private = True
     if workspace_id:
         await check_user_workspace_access(
@@ -165,7 +160,6 @@ async def list_projects(  # pylint: disable=too-many-arguments
         filter_by_template_type=(
             ProjectTemplateTypeDB(template_type) if template_type else None
         ),
-        filter_by_services=user_available_services,
         filter_trashed=trashed,
         filter_hidden=show_hidden,
         # composed attrs
@@ -202,17 +196,12 @@ async def list_projects_full_depth(
 ) -> tuple[list[ProjectDict], int]:
     db = ProjectDBAPI.get_from_app_context(app)
 
-    user_available_services = await catalog_service.get_services_for_user_in_product(
-        app, user_id=user_id, product_name=product_name
-    )
-
     db_projects, db_project_types, total_number_projects = await db.list_projects_dicts(
         product_name=product_name,
         user_id=user_id,
         workspace_query=WorkspaceQuery(workspace_scope=WorkspaceScope.ALL),
         folder_query=FolderQuery(folder_scope=FolderScope.ALL),
         filter_trashed=trashed,
-        filter_by_services=user_available_services,
         filter_by_project_type=ProjectType.STANDARD,
         search_by_multi_columns=search_by_multi_columns,
         search_by_project_name=search_by_project_name,

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py

@@ -573,7 +573,6 @@ async def list_projects_dicts(  # pylint: disable=too-many-arguments,too-many-st
         # attribute filters
         filter_by_project_type: ProjectType | None = None,
         filter_by_template_type: ProjectTemplateType | None = None,
-        filter_by_services: list[dict] | None = None,
         filter_published: bool | None = None,
         filter_hidden: bool | None = False,
         filter_trashed: bool | None = False,
@@ -670,9 +669,7 @@ async def list_projects_dicts(  # pylint: disable=too-many-arguments,too-many-st
 
             prjs, prj_types = await self._execute_without_permission_check(
                 conn,
-                user_id=user_id,
                 select_projects_query=combined_query.offset(offset).limit(limit),
-                filter_by_services=filter_by_services,
             )
 
             return (

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py

@@ -12,11 +12,9 @@
 from models_library.projects import ProjectAtDB, ProjectID, ProjectTemplateType
 from models_library.projects_nodes import Node
 from models_library.projects_nodes_io import NodeIDStr
-from models_library.users import UserID
 from models_library.utils.change_case import camel_to_snake, snake_to_camel
 from pydantic import ValidationError
 from simcore_postgres_database.models.project_to_groups import project_to_groups
-from simcore_postgres_database.models.projects_to_products import projects_to_products
 from simcore_postgres_database.webserver_models import ProjectType, projects
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.sql.selectable import CompoundSelect, Select
@@ -32,7 +30,7 @@
     ProjectNotFoundError,
 )
 from .models import ProjectDict
-from .utils import are_project_services_available, find_changed_node_keys
+from .utils import find_changed_node_keys
 
 logger = logging.getLogger(__name__)
 
@@ -189,10 +187,8 @@ async def _upsert_tags_in_project(
     async def _execute_without_permission_check(
         self,
         conn: SAConnection,
-        user_id: UserID,
         *,
         select_projects_query: Select | CompoundSelect,
-        filter_by_services: list[dict] | None = None,
     ) -> tuple[list[dict[str, Any]], list[ProjectType]]:
         api_projects: list[dict] = []  # API model-compatible projects
         db_projects: list[dict] = []  # DB model-compatible projects
@@ -218,19 +214,6 @@ async def _execute_without_permission_check(
             prj: dict[str, Any] = dict(row.items())
             prj.pop("product_name", None)
 
-            if (
-                filter_by_services is not None
-                # This checks only old projects that are not in the projects_to_products table.
-                and row[projects_to_products.c.product_name] is None
-                and not are_project_services_available(prj, filter_by_services)
-            ):
-                logger.warning(
-                    "Project %s will not be listed for user %s since it has no access rights"
-                    " for one or more of the services that includes.",
-                    f"{row.id=}",
-                    f"{user_id=}",
-                )
-                continue
             db_projects.append(prj)
 
         # NOTE: DO NOT nest _get_tags_by_project in async loop above !!!

services/web/server/src/simcore_service_webserver/projects/utils.py

@@ -189,42 +189,6 @@ def is_graph_equal(
     return True
 
 
-def are_project_services_available(
-    project: dict[str, Any], available_services: list[dict[str, str]]
-) -> bool:
-    if not project["workbench"]:
-        # empty project
-        return True
-
-    # list services in project
-    needed_services = {
-        (srv["key"], srv["version"]) for _, srv in project["workbench"].items()
-    }
-
-    # list available services
-    available_services_set = {
-        (srv["key"], srv["version"]) for srv in available_services
-    }
-
-    return needed_services.issubset(available_services_set)
-
-
-def get_project_unavailable_services(
-    project: dict[str, Any], available_services: list[dict[str, Any]]
-) -> set[tuple[str, str]]:
-    # get project services
-    required: set[tuple[str, str]] = {
-        (s["key"], s["version"]) for _, s in project["workbench"].items()
-    }
-
-    # get available services
-    available: set[tuple[str, str]] = {
-        (s["key"], s["version"]) for s in available_services
-    }
-
-    return required - available
-
-
 def extract_dns_without_default_port(url: URL) -> str:
     port = "" if url.port == 80 else f":{url.port}"
     return f"{url.host}{port}"

services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py

@@ -97,13 +97,11 @@ async def test_copying_large_project_and_aborting_correctly_removes_new_project(
     standard_groups: list[dict[str, str]],
     user_project: dict[str, Any],
     expected: ExpectedResponse,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     slow_storage_subsystem_mock: MockedStorageSubsystem,
     project_db_cleaner: None,
     mocked_dynamic_services_interface: dict[str, MagicMock],
 ):
     assert client.app
-    catalog_subsystem_mock([user_project])
     # initiate a project copy that will last long (simulated by a long running storage)
     # POST /v0/projects
     create_url = client.app.router["create_project"].url_for()
@@ -150,13 +148,11 @@ async def test_copying_large_project_and_retrieving_copy_task(
     standard_groups: list[dict[str, str]],
     user_project: dict[str, Any],
     expected: ExpectedResponse,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     slow_storage_subsystem_mock: MockedStorageSubsystem,
     project_db_cleaner: None,
     mocked_dynamic_services_interface: dict[str, MagicMock],
 ):
     assert client.app
-    catalog_subsystem_mock([user_project])
 
     # initiate a project copy that will last long (simulated by a long running storage)
     # POST /v0/projects
@@ -198,13 +194,11 @@ async def test_creating_new_project_from_template_without_copying_data_creates_s
     standard_groups: list[dict[str, str]],
     template_project: dict[str, Any],
     expected: ExpectedResponse,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     slow_storage_subsystem_mock: MockedStorageSubsystem,
     project_db_cleaner: None,
     request_create_project: Callable[..., Awaitable[ProjectDict]],
 ):
     assert client.app
-    catalog_subsystem_mock([template_project])
     # create a project from another without copying data shall not call in the storage API
     # POST /v0/projects
     await request_create_project(
@@ -249,13 +243,11 @@ async def test_creating_new_project_as_template_without_copying_data_creates_ske
     standard_groups: list[dict[str, str]],
     user_project: dict[str, Any],
     expected: ExpectedResponse,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     slow_storage_subsystem_mock: MockedStorageSubsystem,
     project_db_cleaner: None,
     request_create_project: Callable[..., Awaitable[ProjectDict]],
 ):
     assert client.app
-    catalog_subsystem_mock([user_project])
     # create a project from another without copying data shall not call in the storage API
     # POST /v0/projects
     await request_create_project(

services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py

@@ -194,10 +194,8 @@ async def test_list_projects(
     user_project: dict[str, Any],
     template_project: dict[str, Any],
     expected: HTTPStatus,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     director_v2_service_mock: aioresponses,
 ):
-    catalog_subsystem_mock([user_project, template_project])
     data, *_ = await _list_and_assert_projects(client, expected)
 
     if data:
@@ -344,7 +342,6 @@ async def test_list_projects_with_innaccessible_services(
     user_project: dict[str, Any],
     template_project: dict[str, Any],
     expected: HTTPStatus,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     director_v2_service_mock: aioresponses,
     postgres_db: sa.engine.Engine,
     s4l_product_headers: dict[str, Any],
@@ -374,7 +371,6 @@ async def test_list_projects_with_innaccessible_services(
 
     # use-case 4: give user access to services
     # shall return the projects for any product
-    catalog_subsystem_mock([user_project, template_project])
     data, *_ = await _list_and_assert_projects(
         client, expected, headers=s4l_product_headers
     )
@@ -403,10 +399,7 @@ async def test_get_project(
     user_project: ProjectDict,
     template_project: ProjectDict,
     expected,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
 ):
-    catalog_subsystem_mock([user_project, template_project])
-
     # standard project
     await _assert_get_same_project(client, user_project, expected)
 
@@ -444,7 +437,6 @@ async def test_create_get_and_patch_project_ui_field(
     logged_user: UserInfoDict,
     primary_group: dict[str, str],
     request_create_project: Callable[..., Awaitable[ProjectDict]],
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     project_db_cleaner,
 ):
     assert client.app
@@ -462,8 +454,6 @@ async def test_create_get_and_patch_project_ui_field(
     )
     project_id = new_project["uuid"]
 
-    catalog_subsystem_mock([new_project])
-
     # Step 2: Get the project and check the ui.icon
     url = client.app.router["get_project"].url_for(project_id=project_id)
     resp = await client.get(f"{url}")
@@ -522,11 +512,9 @@ async def test_new_project_from_other_study(
     user_project: ProjectDict,
     expected: ExpectedResponse,
     storage_subsystem_mock,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     project_db_cleaner,
     request_create_project: Callable[..., Awaitable[ProjectDict]],
 ):
-    catalog_subsystem_mock([user_project])
     new_project = await request_create_project(
         client,
         expected.accepted,
@@ -608,7 +596,6 @@ async def test_new_template_from_project(
     user_project: dict[str, Any],
     expected: ExpectedResponse,
     storage_subsystem_mock: MockedStorageSubsystem,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     project_db_cleaner: None,
     request_create_project: Callable[..., Awaitable[ProjectDict]],
 ):
@@ -625,7 +612,6 @@ async def test_new_template_from_project(
 
     if new_template_prj:
         template_project = new_template_prj
-        catalog_subsystem_mock([template_project])
 
         templates, *_ = await _list_and_assert_projects(
             client, status.HTTP_200_OK, {"type": "template"}

services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__delete.py

@@ -61,7 +61,6 @@ async def test_delete_project(
     expected: ExpectedResponse,
     storage_subsystem_mock: MockedStorageSubsystem,
     mocked_dynamic_services_interface: dict[str, MagicMock],
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     fake_services: Callable[..., Awaitable[list[DynamicServiceGet]]],
     assert_get_same_project_caller: Callable,
     mock_dynamic_scheduler_rabbitmq: None,

services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py

@@ -133,7 +133,6 @@ async def test_list_projects_with_invalid_pagination_parameters(
     primary_group: dict[str, str],
     expected: ExpectedResponse,
     storage_subsystem_mock,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     director_v2_service_mock: aioresponses,
     project_db_cleaner,
     limit: int,
@@ -158,7 +157,6 @@ async def test_list_projects_with_pagination(
     primary_group: dict[str, str],
     expected: ExpectedResponse,
     storage_subsystem_mock,
-    catalog_subsystem_mock: Callable[[list[ProjectDict]], None],
     director_v2_service_mock: aioresponses,
     project_db_cleaner,
     limit: int,
@@ -175,7 +173,6 @@ async def test_list_projects_with_pagination(
         ]
     )
     if expected.created == status.HTTP_201_CREATED:
-        catalog_subsystem_mock(created_projects)
 
         assert len(created_projects) == NUM_PROJECTS
         NUMBER_OF_CALLS = ceil(NUM_PROJECTS / limit)