♻️ Refactors webserver module director_v2 and breaks dependency cycles (ITISFoundation#2567)

- CHANGES: Splits director-v2 app module into submodules for
- CHANGES: Splits project app module's handler into smaller submodules
- FIX: Replaces built-in TimeoutError by asyncio.TimeoutError. Those are differemt things.
- CHANGES: Cleanup simcore_service_webserver.utils
- FIX: Removed dependency cycles.

Author: pcrespov

services/web/server/src/simcore_service_webserver/__init__.py

@@ -1,7 +1,5 @@
 import warnings
 
-from ._meta import __version__
-
 #
 # NOTE: Some BaseSettings are using aliases (e.g. version for vtag) to facility construct
 #  pydantic settings from names defined in trafaret schemas for the config files

services/web/server/src/simcore_service_webserver/activity/__init__.py

@@ -1 +0,0 @@
-from .module_setup import setup_activity

services/web/server/src/simcore_service_webserver/activity/module_setup.py

@@ -2,9 +2,12 @@
 
 from aiohttp import web
 from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
-from servicelib.aiohttp.rest_routing import iter_path_operations, map_handlers_with_operations
+from servicelib.aiohttp.rest_routing import (
+    iter_path_operations,
+    map_handlers_with_operations,
+)
 
-from ..rest_config import APP_OPENAPI_SPECS_KEY
+from ..constants import APP_OPENAPI_SPECS_KEY
 from . import celery_client, handlers
 from .config import assert_valid_config
 

services/web/server/src/simcore_service_webserver/application.py

@@ -10,27 +10,27 @@
 from servicelib.rest_pagination_utils import monkey_patch_pydantic_url_regex
 
 from ._meta import WELCOME_MSG
-from .activity import setup_activity
+from .activity.module_setup import setup_activity
 from .catalog import setup_catalog
-from .clusters import setup_clusters
+from .clusters.module_setup import setup_clusters
 from .computation import setup_computation
 from .db import setup_db
 from .diagnostics import setup_diagnostics
-from .director import setup_director
+from .director.module_setup import setup_director
 from .director_v2 import setup_director_v2
 from .email import setup_email
-from .exporter import setup_exporter
+from .exporter.module_setup import setup_exporter
 from .groups import setup_groups
-from .login import setup_login
+from .login.module_setup import setup_login
 from .products import setup_products
-from .projects import setup_projects
+from .projects.module_setup import setup_projects
 from .publications import setup_publications
-from .resource_manager import setup_resource_manager
+from .resource_manager.module_setup import setup_resource_manager
 from .rest import setup_rest
 from .security import setup_security
 from .session import setup_session
 from .settings import setup_settings
-from .socketio import setup_socketio
+from .socketio.module_setup import setup_socketio
 from .statics import setup_statics
 from .storage import setup_storage
 from .studies_access import setup_studies_access

services/web/server/src/simcore_service_webserver/application_config.py

@@ -65,7 +65,7 @@ def create_schema() -> T.Dict:
                     ),
                 }
             ),
-            addon_section(tracing.tracing_section_name, optional=True): tracing.schema,
+            addon_section(tracing.CONFIG_SECTION_NAME, optional=True): tracing.schema,
             db_config.CONFIG_SECTION_NAME: db_config.schema,
             director_config.CONFIG_SECTION_NAME: director_config.schema,
             rest_config.CONFIG_SECTION_NAME: rest_config.schema,
@@ -82,20 +82,23 @@ def create_schema() -> T.Dict:
             activity_config.CONFIG_SECTION_NAME: activity_config.schema,
             resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
             # BELOW HERE minimal sections until more options are needed
-            addon_section("diagnostics", optional=True): minimal_addon_schema(),
-            addon_section("users", optional=True): minimal_addon_schema(),
-            addon_section("groups", optional=True): minimal_addon_schema(),
-            addon_section("tags", optional=True): minimal_addon_schema(),
-            addon_section("publications", optional=True): minimal_addon_schema(),
             addon_section("catalog", optional=True): catalog_config.schema,
-            addon_section("products", optional=True): minimal_addon_schema(),
+            addon_section("clusters", optional=True): minimal_addon_schema(),
             addon_section("computation", optional=True): minimal_addon_schema(),
+            addon_section("diagnostics", optional=True): minimal_addon_schema(),
             addon_section("director-v2", optional=True): minimal_addon_schema(),
+            addon_section("exporter", optional=True): minimal_addon_schema(),
+            addon_section("groups", optional=True): minimal_addon_schema(),
+            addon_section("meta", optional=True): minimal_addon_schema(),
+            addon_section("products", optional=True): minimal_addon_schema(),
+            addon_section("publications", optional=True): minimal_addon_schema(),
+            addon_section("security", optional=True): minimal_addon_schema(),
+            addon_section("statics", optional=True): minimal_addon_schema(),
             addon_section("studies_access", optional=True): minimal_addon_schema(),
             addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
-            addon_section("exporter", optional=True): minimal_addon_schema(),
+            addon_section("tags", optional=True): minimal_addon_schema(),
+            addon_section("users", optional=True): minimal_addon_schema(),
             addon_section("version_control", optional=True): minimal_addon_schema(),
-            addon_section("clusters", optional=True): minimal_addon_schema(),
         }
     )
 

services/web/server/src/simcore_service_webserver/catalog.py

@@ -6,7 +6,6 @@
 
 from aiohttp import web
 from aiohttp.web_routedef import RouteDef
-from servicelib.aiohttp.application_keys import APP_OPENAPI_SPECS_KEY
 from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
 from servicelib.aiohttp.rest_routing import iter_path_operations
 from yarl import URL
@@ -22,7 +21,7 @@
     KCATALOG_VERSION_PREFIX,
     assert_valid_config,
 )
-from .constants import RQ_PRODUCT_KEY, X_PRODUCT_NAME_HEADER
+from .constants import APP_OPENAPI_SPECS_KEY, RQ_PRODUCT_KEY, X_PRODUCT_NAME_HEADER
 from .login.decorators import RQT_USERID_KEY, login_required
 from .security_decorators import permission_required
 

services/web/server/src/simcore_service_webserver/clusters/__init__.py

@@ -1,3 +0,0 @@
-from .module_setup import setup_clusters
-
-__all__ = ["setup_clusters"]

services/web/server/src/simcore_service_webserver/constants.py

@@ -6,6 +6,7 @@
     APP_CONFIG_KEY,
     APP_DB_ENGINE_KEY,
     APP_JSONSCHEMA_SPECS_KEY,
+    APP_OPENAPI_SPECS_KEY,
 )
 from servicelib.request_keys import RQT_USERID_KEY
 

services/web/server/src/simcore_service_webserver/diagnostics_handlers.py

@@ -10,7 +10,7 @@
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.utils import logged_gather
 
-from . import catalog_client, db, director_v2, storage_api
+from . import catalog_client, db, director_v2_api, storage_api
 from ._meta import __version__, api_version, api_version_prefix, app_name
 from .diagnostics_core import HealthError, assert_healthy_app
 from .login.decorators import login_required
@@ -113,7 +113,7 @@ async def _check_storage():
 
     async def _check_director2():
         check.services["director_v2"] = {
-            "healthy": await director_v2.is_healthy(request.app)
+            "healthy": await director_v2_api.is_healthy(request.app)
         }
 
     async def _check_catalog():

services/web/server/src/simcore_service_webserver/director/__init__.py

@@ -1 +0,0 @@
-from .module_setup import setup_director