✨🐛Webserver: enable socketio horizontal scaling + ensure only 1 update goes through (⚠️ devops) (#4286)

Author: sanderegg

.env-wb-db-event-listener

@@ -0,0 +1,39 @@
+#
+# Explicit plugins DISABLED in the webserver to create a garbage-collector
+# Docs plugins config of services/web/server/src/simcore_service_webserver/application_settings.py
+#
+
+
+WEBSERVER_ACTIVITY=null
+WEBSERVER_CATALOG=null
+WEBSERVER_NOTIFICATIONS=0
+# WEBSERVER_DB_LISTENER=1 explicitely enabled in docker-compose
+WEBSERVER_DIAGNOSTICS=null
+#WEBSERVER_DIRECTOR_V2 from .env
+WEBSERVER_DIRECTOR=null
+WEBSERVER_EMAIL=null
+WEBSERVER_EXPORTER=null
+WEBSERVER_FRONTEND=null
+WEBSERVER_GARBAGE_COLLECTOR=null
+WEBSERVER_LOGIN=null
+WEBSERVER_PROJECTS=null
+#WEBSERVER_RABBITMQ from .env
+WEBSERVER_REDIS=null
+#WEBSERVER_REST needed for the healthcheck
+#WEBSERVER_RESOURCE_MANAGER from .env
+WEBSERVER_SCICRUNCH=null
+WEBSERVER_STATICWEB=null
+WEBSERVER_STORAGE=null
+WEBSERVER_STUDIES_DISPATCHER=null
+WEBSERVER_TRACING=null
+# --------
+WEBSERVER_CLUSTERS=0
+WEBSERVER_GROUPS=0
+WEBSERVER_META_MODELING=0
+WEBSERVER_PRODUCTS=0
+WEBSERVER_PUBLICATIONS=0
+#WEBSERVER_SOCKETIO=1 explicitely enabled in docker-compose
+WEBSERVER_STUDIES_DISPATCHER=null
+WEBSERVER_TAGS=0
+WEBSERVER_USERS=0
+WEBSERVER_VERSION_CONTROL=0

.env-wb-garbage-collector

@@ -7,13 +7,14 @@
 WEBSERVER_ACTIVITY=null
 WEBSERVER_CATALOG=null
 WEBSERVER_NOTIFICATIONS=0
+WEBSERVER_DB_LISTENER=0
 WEBSERVER_DIAGNOSTICS=null
 #WEBSERVER_DIRECTOR_V2 from .env
 WEBSERVER_DIRECTOR=null
 WEBSERVER_EMAIL=null
 WEBSERVER_EXPORTER=null
 WEBSERVER_FRONTEND=null
-#WEBSERVER_GARBAGE_COLLECTOR explicit in
+#WEBSERVER_GARBAGE_COLLECTOR explicitely enabled in docker-compose
 WEBSERVER_LOGIN=null
 WEBSERVER_PROJECTS=null
 #WEBSERVER_REDIS from .env

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -9,7 +9,6 @@
 import sqlalchemy as sa
 import tenacity
 from servicelib.json_serialization import json_dumps
-from sqlalchemy.orm import sessionmaker
 from tenacity.stop import stop_after_delay
 from tenacity.wait import wait_fixed
 
@@ -28,7 +27,9 @@ def execute_queries(
     with postgres_engine.connect() as con:
         for statement in sql_statements:
             try:
-                con.execution_options(autocommit=True).execute(statement)
+                with con.begin():
+                    con.execute(statement)
+
             except Exception as e:  # pylint: disable=broad-except
                 # when running tests initially the TEMPLATE_DB_TO_RESTORE dose not exist and will cause an error
                 # which can safely be ignored. The debug message is here to catch future errors which and
@@ -235,15 +236,3 @@ def postgres_host_config(
         "POSTGRES_ENDPOINT", f"{postgres_dsn['host']}:{postgres_dsn['port']}"
     )
     return postgres_dsn
-
-
-@pytest.fixture(scope="module")
-def postgres_session(postgres_db: sa.engine.Engine) -> Iterator[sa.orm.session.Session]:
-    from sqlalchemy.orm.session import Session
-
-    Session_cls = sessionmaker(postgres_db)
-    session: Session = Session_cls()
-
-    yield session
-
-    session.close()  # pylint: disable=no-member

packages/pytest-simcore/src/pytest_simcore/rabbit_service.py

@@ -12,6 +12,7 @@
 import pytest
 import tenacity
 from servicelib.rabbitmq import RabbitMQClient
+from settings_library.basic_types import PortInt
 from settings_library.rabbit import RabbitSettings
 from tenacity.before_sleep import before_sleep_log
 from tenacity.stop import stop_after_attempt
@@ -48,7 +49,7 @@ async def rabbit_settings(
         RABBIT_USER=testing_environ_vars["RABBIT_USER"],
         RABBIT_PASSWORD=testing_environ_vars["RABBIT_PASSWORD"],
         RABBIT_HOST=get_localhost_ip(),
-        RABBIT_PORT=int(port),
+        RABBIT_PORT=PortInt(port),
     )
 
     await wait_till_rabbit_responsive(settings.dsn)

packages/pytest-simcore/src/pytest_simcore/websocket_client.py

@@ -3,7 +3,7 @@
 # pylint:disable=redefined-outer-name
 
 import logging
-from typing import AsyncIterable, Awaitable, Callable, Optional
+from typing import AsyncIterable, Awaitable, Callable
 from uuid import uuid4
 
 import pytest
@@ -25,8 +25,8 @@ def _create() -> str:
 
 
 @pytest.fixture()
-def socketio_url_factory(client) -> Callable[[Optional[TestClient]], str]:
-    def _create(client_override: Optional[TestClient] = None) -> str:
+def socketio_url_factory(client) -> Callable[[TestClient | None], str]:
+    def _create(client_override: TestClient | None = None) -> str:
         SOCKET_IO_PATH = "/socket.io/"
         return str((client_override or client).make_url(SOCKET_IO_PATH))
 
@@ -36,8 +36,8 @@ def _create(client_override: Optional[TestClient] = None) -> str:
 @pytest.fixture()
 async def security_cookie_factory(
     client: TestClient,
-) -> Callable[[Optional[TestClient]], Awaitable[str]]:
-    async def _create(client_override: Optional[TestClient] = None) -> str:
+) -> Callable[[TestClient | None], Awaitable[str]]:
+    async def _create(client_override: TestClient | None = None) -> str:
         # get the cookie by calling the root entrypoint
         resp = await (client_override or client).get("/v0/")
         data, error = await assert_status(resp, web.HTTPOk)
@@ -60,14 +60,13 @@ async def socketio_client_factory(
     security_cookie_factory: Callable,
     client_session_id_factory: Callable,
 ) -> AsyncIterable[
-    Callable[[Optional[str], Optional[TestClient]], Awaitable[socketio.AsyncClient]]
+    Callable[[str | None, TestClient | None], Awaitable[socketio.AsyncClient]]
 ]:
     clients: list[socketio.AsyncClient] = []
 
     async def _connect(
-        client_session_id: Optional[str] = None, client: Optional[TestClient] = None
+        client_session_id: str | None = None, client: TestClient | None = None
     ) -> socketio.AsyncClient:
-
         if client_session_id is None:
             client_session_id = client_session_id_factory()
 
@@ -86,7 +85,7 @@ async def _connect(
             headers.update({"Cookie": cookie})
 
         print(f"--> Connecting socketio client to {url} ...")
-        await sio.connect(url, headers=headers)
+        await sio.connect(url, headers=headers, wait_timeout=10)
         assert sio.sid
         print("... connection done")
         clients.append(sio)

packages/settings-library/src/settings_library/rabbit.py

@@ -1,5 +1,6 @@
 from functools import cached_property
 
+from pydantic import parse_obj_as
 from pydantic.networks import AnyUrl
 from pydantic.types import SecretStr
 
@@ -14,7 +15,7 @@ class RabbitDsn(AnyUrl):
 class RabbitSettings(BaseCustomSettings):
     # host
     RABBIT_HOST: str
-    RABBIT_PORT: PortInt = 5672
+    RABBIT_PORT: PortInt = parse_obj_as(PortInt, 5672)
 
     # auth
     RABBIT_USER: str

services/docker-compose.devel.yml

@@ -86,6 +86,11 @@ services:
       SC_BOOT_MODE: debug-ptvsd
       WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-DEBUG}
 
+  wb-db-event-listener:
+    volumes: *webserver-volumes-dev
+    environment:
+      <<: *webserver-environment-dev
+
   wb-garbage-collector:
     volumes: *webserver-volumes-dev
     environment:

services/docker-compose.local.yml

@@ -84,6 +84,15 @@ services:
         - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.priority=3
         - traefik.http.routers.${SWARM_STACK_NAME}_webserver_local.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry
 
+  wb-db-event-listener:
+    environment:
+      SC_BOOT_MODE: ${SC_BOOT_MODE:-default}
+      WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-INFO}
+      REST_SWAGGER_API_DOC_ENABLED: 1
+    ports:
+      - "8080"
+      - "3013:3000"
+
   wb-garbage-collector:
     environment:
       SC_BOOT_MODE: ${SC_BOOT_MODE:-default}

services/docker-compose.yml

@@ -263,6 +263,7 @@ services:
       STORAGE_HOST: ${STORAGE_HOST:-storage}
       STORAGE_PORT: ${STORAGE_PORT:-8080}
       SWARM_STACK_NAME: ${SWARM_STACK_NAME:-simcore}
+      WEBSERVER_DB_LISTENER: 0
       WEBSERVER_GARBAGE_COLLECTOR: "null"
       WEBSERVER_LOGLEVEL: ${LOG_LEVEL:-WARNING}
     env_file:
@@ -297,6 +298,27 @@ services:
     networks:
       - default
       - interactive_services_subnet
+  wb-db-event-listener:
+    image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest}
+    init: true
+    hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
+    environment:
+      <<: *webserver-environment
+      WEBSERVER_DB_LISTENER: 1
+      WEBSERVER_SOCKETIO: 1
+    env_file:
+      - ../.env
+      - ../.env-wb-db-event-listener
+    deploy:
+      # NOTE: https://github.com/ITISFoundation/osparc-simcore/pull/4286
+      # NOTE: this MUSTN'T change, or weird things might happen
+      # this will stay until all legacy dynamic services are gone.
+      replicas: 1
+      placement:
+        constraints:
+          - node.platform.os == linux
+    networks:
+      - default
 
   wb-garbage-collector:
     image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest}

services/storage/Dockerfile

@@ -132,10 +132,10 @@ COPY --from=prod-only-deps --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 COPY --chown=scu:scu services/storage/docker services/storage/docker
 RUN chmod +x services/storage/docker/*.sh
 
-HEALTHCHECK --interval=30s \
-  --timeout=120s \
-  --start-period=30s \
-  --retries=3 \
+HEALTHCHECK --interval=10s \
+  --timeout=15s \
+  --start-period=5s \
+  --retries=5 \
   CMD ["python3", "/home/scu/services/storage/docker/healthcheck.py", "http://localhost:8080/v0/"]