From 945ca4fd245ca6e0aa5febbe84a9814cb6943c46 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 14 Oct 2025 11:47:57 +0200 Subject: [PATCH 01/37] introudce chatbot client skeleton --- .../chatbot/__init__.py | 8 ++ .../chatbot/_client.py | 108 ++++++++++++++++++ .../chatbot/plugin.py | 22 ++++ .../chatbot/settings.py | 17 +++ 4 files changed, 155 insertions(+) create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/__init__.py create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/_client.py create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/plugin.py create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/settings.py diff --git a/services/web/server/src/simcore_service_webserver/chatbot/__init__.py b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py new file mode 100644 index 00000000000..858a37c032f --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py @@ -0,0 +1,8 @@ +# mypy: disable-error-code=truthy-function +from ._client import ChatbotQuestionCreate, ChatbotRestClient, get_chatbot_rest_client + +__all__ = [ + "get_chatbot_rest_client", + "ChatbotQuestionCreate", + "ChatbotRestClient", +] diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py new file mode 100644 index 00000000000..d977684fe70 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -0,0 +1,108 @@ +"""Interface to communicate with Fogbugz API + +- Simple client to create cases in Fogbugz +""" + +import logging +from typing import Any, Final +from urllib.parse import urljoin + +import httpx +from aiohttp import web +from pydantic import AnyUrl, BaseModel, Field +from servicelib.aiohttp import status +from tenacity import ( + retry, + retry_if_exception_type, + retry_if_result, + stop_after_attempt, + wait_exponential, +) + +from .settings import get_plugin_settings + +_logger = logging.getLogger(__name__) + +_JSON_CONTENT_TYPE = "application/json" +_UNKNOWN_ERROR_MESSAGE = "Unknown error occurred" + + +class ChatbotQuestionCreate(BaseModel): + fogbugz_project_id: int = Field(description="Project ID in Fogbugz") + title: str = Field(description="Case title") + description: str = Field(description="Case description/first comment") + + +def _should_retry(response: httpx.Response | None) -> bool: + if response is None: + return True + return ( + response.status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR + or response.status_code == status.HTTP_429_TOO_MANY_REQUESTS + ) + + +class ChatbotRestClient: + def __init__(self, host: AnyUrl, port: int) -> None: + self._client = httpx.AsyncClient() + self.host = host + self.port = port + self._base_url = f"{self.host}:{self.port}" + + async def get_settings(self) -> dict[str, Any]: + """Fetches chatbot settings""" + url = urljoin(f"{self._base_url}", "/v1/chat/settings") + + @retry( + retry=( + retry_if_result(_should_retry) + | retry_if_exception_type( + ( + httpx.ConnectError, + httpx.TimeoutException, + httpx.NetworkError, + httpx.ProtocolError, + ) + ) + ), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=1, max=10), + reraise=True, + ) + async def _request() -> httpx.Response: + return await self._client.get(url) + + try: + response = await _request() + response.raise_for_status() + response_data: dict[str, Any] = response.json() + return response_data + except Exception: + _logger.error( # noqa: TRY400 + "Failed to fetch chatbot settings from %s", url + ) + raise + + async def __aenter__(self): + """Async context manager entry""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit - cleanup client""" + await self._client.aclose() + + +_APPKEY: Final = web.AppKey(ChatbotRestClient.__name__, ChatbotRestClient) + + +async def setup_chatbot_rest_client(app: web.Application) -> None: + settings = get_plugin_settings(app) + + client = ChatbotRestClient(host=settings.CHATBOT_HOST, port=settings.CHATBOT_PORT) + + app[_APPKEY] = client + + +def get_chatbot_rest_client(app: web.Application) -> ChatbotRestClient: + app_key: ChatbotRestClient = app[_APPKEY] + return app_key diff --git a/services/web/server/src/simcore_service_webserver/chatbot/plugin.py b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py new file mode 100644 index 00000000000..a1a6968874d --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py @@ -0,0 +1,22 @@ +"""tags management subsystem""" + +import logging + +from aiohttp import web + +from ..application_setup import ModuleCategory, app_setup_func +from ..products.plugin import setup_products +from ._client import setup_chatbot_rest_client + +_logger = logging.getLogger(__name__) + + +@app_setup_func( + __name__, + ModuleCategory.ADDON, + settings_name="WEBSERVER_CHATBOT", + logger=_logger, +) +def setup_chatbot(app: web.Application): + setup_products(app) + app.on_startup.append(setup_chatbot_rest_client) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py new file mode 100644 index 00000000000..a73cff705c2 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -0,0 +1,17 @@ +from aiohttp import web +from pydantic import AnyUrl +from settings_library.base import BaseCustomSettings + +from ..application_keys import APP_SETTINGS_APPKEY + + +class ChatbotSettings(BaseCustomSettings): + CHATBOT_HOST: AnyUrl + CHATBOT_PORT: int + + +def get_plugin_settings(app: web.Application) -> ChatbotSettings: + settings = app[APP_SETTINGS_APPKEY].WEBSERVER_CHATBOT + assert settings, "plugin.setup_chatbot not called?" # nosec + assert isinstance(settings, ChatbotSettings) # nosec + return settings From 39728283079e766632878f7cd426a1f05d271896 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 14 Oct 2025 17:18:13 +0200 Subject: [PATCH 02/37] introudce chatbot client skeleton --- .../simcore_service_webserver/application.py | 2 - .../application_settings.py | 8 ++ .../chatbot/__init__.py | 3 +- .../chatbot/_client.py | 87 +++++++++++++------ .../chatbot/settings.py | 17 +++- .../conversations/plugin.py | 2 + .../unit/with_dbs/04/test_chatbot_client.py | 74 ++++++++++++++++ 7 files changed, 158 insertions(+), 35 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index ba8e287880e..8e154029d67 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -36,7 +36,6 @@ from .dynamic_scheduler.plugin import setup_dynamic_scheduler from .email.plugin import setup_email from .exporter.plugin import setup_exporter -from .fogbugz.plugin import setup_fogbugz from .folders.plugin import setup_folders from .functions.plugin import setup_functions from .garbage_collector.plugin import setup_garbage_collector @@ -175,7 +174,6 @@ def create_application(tracing_config: TracingConfig) -> web.Application: setup_projects(app) # conversations - setup_fogbugz(app) # Needed for support conversations setup_conversations(app) # licenses diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 3d08171b8f9..3c7e38409da 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -32,6 +32,7 @@ from ._meta import API_VERSION, API_VTAG, APP_NAME from .application_keys import APP_SETTINGS_APPKEY from .catalog.settings import CatalogSettings +from .chatbot.settings import ChatbotSettings from .collaboration.settings import RealTimeCollaborationSettings from .diagnostics.settings import DiagnosticsSettings from .director_v2.settings import DirectorV2Settings @@ -260,6 +261,13 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] + WEBSERVER_CHATBOT: Annotated[ + ChatbotSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] + WEBSERVER_GARBAGE_COLLECTOR: Annotated[ GarbageCollectorSettings | None, Field( diff --git a/services/web/server/src/simcore_service_webserver/chatbot/__init__.py b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py index 858a37c032f..a51f5bd24b6 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/__init__.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py @@ -1,8 +1,7 @@ # mypy: disable-error-code=truthy-function -from ._client import ChatbotQuestionCreate, ChatbotRestClient, get_chatbot_rest_client +from ._client import ChatbotRestClient, get_chatbot_rest_client __all__ = [ "get_chatbot_rest_client", - "ChatbotQuestionCreate", "ChatbotRestClient", ] diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index d977684fe70..ca15342db8d 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -9,7 +9,7 @@ import httpx from aiohttp import web -from pydantic import AnyUrl, BaseModel, Field +from pydantic import BaseModel, Field from servicelib.aiohttp import status from tenacity import ( retry, @@ -23,14 +23,12 @@ _logger = logging.getLogger(__name__) + _JSON_CONTENT_TYPE = "application/json" -_UNKNOWN_ERROR_MESSAGE = "Unknown error occurred" -class ChatbotQuestionCreate(BaseModel): - fogbugz_project_id: int = Field(description="Project ID in Fogbugz") - title: str = Field(description="Case title") - description: str = Field(description="Case description/first comment") +class ChatResponse(BaseModel): + answer: str = Field(description="Answer from the chatbot") def _should_retry(response: httpx.Response | None) -> bool: @@ -42,33 +40,36 @@ def _should_retry(response: httpx.Response | None) -> bool: ) +def _chatbot_retry(): + """Retry configuration for chatbot API calls""" + return retry( + retry=( + retry_if_result(_should_retry) + | retry_if_exception_type( + ( + httpx.ConnectError, + httpx.TimeoutException, + httpx.NetworkError, + httpx.ProtocolError, + ) + ) + ), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=1, max=10), + reraise=True, + ) + + class ChatbotRestClient: - def __init__(self, host: AnyUrl, port: int) -> None: + def __init__(self, base_url: str) -> None: self._client = httpx.AsyncClient() - self.host = host - self.port = port - self._base_url = f"{self.host}:{self.port}" + self._base_url = base_url async def get_settings(self) -> dict[str, Any]: """Fetches chatbot settings""" url = urljoin(f"{self._base_url}", "/v1/chat/settings") - @retry( - retry=( - retry_if_result(_should_retry) - | retry_if_exception_type( - ( - httpx.ConnectError, - httpx.TimeoutException, - httpx.NetworkError, - httpx.ProtocolError, - ) - ) - ), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=1, max=10), - reraise=True, - ) + @_chatbot_retry() async def _request() -> httpx.Response: return await self._client.get(url) @@ -83,6 +84,36 @@ async def _request() -> httpx.Response: ) raise + async def ask_question(self, question: str) -> ChatResponse: + """Asks a question to the chatbot""" + url = urljoin(f"{self._base_url}", "/v1/chat") + + @_chatbot_retry() + async def _request() -> httpx.Response: + return await self._client.post( + url, + json={ + "question": question, + "llm": "gpt-3.5-turbo", + "embedding_model": "openai/text-embedding-3-large", + }, + headers={ + "Content-Type": _JSON_CONTENT_TYPE, + "Accept": _JSON_CONTENT_TYPE, + }, + ) + + try: + response = await _request() + response.raise_for_status() + response_data: dict[str, Any] = response.json() + return ChatResponse(**response_data) + except Exception: + _logger.error( # noqa: TRY400 + "Failed to ask question to chatbot at %s", url + ) + raise + async def __aenter__(self): """Async context manager entry""" return self @@ -96,9 +127,9 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def setup_chatbot_rest_client(app: web.Application) -> None: - settings = get_plugin_settings(app) + chatbot_settings = get_plugin_settings(app) - client = ChatbotRestClient(host=settings.CHATBOT_HOST, port=settings.CHATBOT_PORT) + client = ChatbotRestClient(base_url=chatbot_settings.base_url) app[_APPKEY] = client diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index a73cff705c2..de8977782f8 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -1,14 +1,25 @@ +from functools import cached_property + from aiohttp import web -from pydantic import AnyUrl from settings_library.base import BaseCustomSettings +from settings_library.utils_service import MixinServiceSettings, URLPart from ..application_keys import APP_SETTINGS_APPKEY -class ChatbotSettings(BaseCustomSettings): - CHATBOT_HOST: AnyUrl +class ChatbotSettings(BaseCustomSettings, MixinServiceSettings): + CHATBOT_HOST: str CHATBOT_PORT: int + @cached_property + def base_url(self) -> str: + # http://chatbot:8000/v1 + return self._compose_url( + prefix="CHATBOT", + port=URLPart.REQUIRED, + vtag=URLPart.EXCLUDE, + ) + def get_plugin_settings(app: web.Application) -> ChatbotSettings: settings = app[APP_SETTINGS_APPKEY].WEBSERVER_CHATBOT diff --git a/services/web/server/src/simcore_service_webserver/conversations/plugin.py b/services/web/server/src/simcore_service_webserver/conversations/plugin.py index 5d9fec4915c..8aebaeffe51 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/plugin.py +++ b/services/web/server/src/simcore_service_webserver/conversations/plugin.py @@ -6,6 +6,7 @@ from ..application_keys import APP_SETTINGS_APPKEY from ..application_setup import ModuleCategory, app_setup_func +from ..chatbot.plugin import setup_chatbot from ..fogbugz.plugin import setup_fogbugz from ._controller import _conversations_messages_rest, _conversations_rest @@ -23,6 +24,7 @@ def setup_conversations(app: web.Application): assert app[APP_SETTINGS_APPKEY].WEBSERVER_CONVERSATIONS # nosec setup_fogbugz(app) + setup_chatbot(app) app.router.add_routes(_conversations_rest.routes) app.router.add_routes(_conversations_messages_rest.routes) diff --git a/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py b/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py new file mode 100644 index 00000000000..4692dc9918f --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py @@ -0,0 +1,74 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + +from collections.abc import Iterator + +import httpx +import pytest +import respx +from aiohttp.test_utils import TestClient +from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_webserver.chatbot._client import ( + ChatResponse, + get_chatbot_rest_client, +) +from simcore_service_webserver.chatbot.settings import ChatbotSettings + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + mocker: MockerFixture, +): + return app_environment | setenvs_from_dict( + monkeypatch, + { + "CHATBOT_HOST": "chatbot", + "CHATBOT_PORT": "8000", + }, + ) + + +@pytest.fixture +def mocked_chatbot_api() -> Iterator[respx.MockRouter]: + _BASE_URL = "http://chatbot:8000" + + # Define responses in the order they will be called during the test + chatbot_answer_responses = [ + {"answer": "42"}, + ] + + with respx.mock(base_url=_BASE_URL) as mock: + # Create a side_effect that returns responses in sequence + mock.post(path="/v1/chat").mock( + side_effect=[ + httpx.Response(200, json=response) + for response in chatbot_answer_responses + ] + ) + yield mock + + +async def test_chatbot_client( + app_environment: EnvVarsDict, + client: TestClient, + mocked_chatbot_api: respx.MockRouter, +): + assert client.app + + settings = ChatbotSettings.create_from_envs() + assert settings.CHATBOT_HOST + assert settings.CHATBOT_PORT + + chatbot_client = get_chatbot_rest_client(client.app) + assert chatbot_client + + output = await chatbot_client.ask_question("What is the meaning of life?") + assert isinstance(output, ChatResponse) + assert output.answer == "42" From f5f8b9f96882427c787a8be763bd49c47553d8a5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 14 Oct 2025 17:20:24 +0200 Subject: [PATCH 03/37] fix --- .../server/src/simcore_service_webserver/chatbot/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index de8977782f8..40d34b6af99 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -13,7 +13,7 @@ class ChatbotSettings(BaseCustomSettings, MixinServiceSettings): @cached_property def base_url(self) -> str: - # http://chatbot:8000/v1 + # http://chatbot:8000 return self._compose_url( prefix="CHATBOT", port=URLPart.REQUIRED, From 81dcbe4986430a28e681e7afdcafa0d6cb57780f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 09:23:24 +0200 Subject: [PATCH 04/37] improve --- .../simcore_service_webserver/chatbot/_client.py | 14 +++++++++----- .../simcore_service_webserver/chatbot/settings.py | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index ca15342db8d..56559ef9a1a 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -19,7 +19,7 @@ wait_exponential, ) -from .settings import get_plugin_settings +from .settings import ChatbotSettings, get_plugin_settings _logger = logging.getLogger(__name__) @@ -61,9 +61,10 @@ def _chatbot_retry(): class ChatbotRestClient: - def __init__(self, base_url: str) -> None: + def __init__(self, base_url: str, chatbot_settings: ChatbotSettings) -> None: self._client = httpx.AsyncClient() self._base_url = base_url + self._chatbot_settings = chatbot_settings async def get_settings(self) -> dict[str, Any]: """Fetches chatbot settings""" @@ -94,8 +95,8 @@ async def _request() -> httpx.Response: url, json={ "question": question, - "llm": "gpt-3.5-turbo", - "embedding_model": "openai/text-embedding-3-large", + "llm": self._chatbot_settings.CHATBOT_LLM_MODEL, + "embedding_model": self._chatbot_settings.CHATBOT_EMBEDDING_MODEL, }, headers={ "Content-Type": _JSON_CONTENT_TYPE, @@ -129,7 +130,10 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): async def setup_chatbot_rest_client(app: web.Application) -> None: chatbot_settings = get_plugin_settings(app) - client = ChatbotRestClient(base_url=chatbot_settings.base_url) + client = ChatbotRestClient( + base_url=chatbot_settings.base_url, + chatbot_settings=chatbot_settings, + ) app[_APPKEY] = client diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index 40d34b6af99..6e818bf2f76 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -10,6 +10,8 @@ class ChatbotSettings(BaseCustomSettings, MixinServiceSettings): CHATBOT_HOST: str CHATBOT_PORT: int + CHATBOT_LLM_MODEL: str = "gpt-3.5-turbo" + CHATBOT_EMBEDDING_MODEL: str = "openai/text-embedding-3-large" @cached_property def base_url(self) -> str: From 910f649f415e6b79cf76bf4e262434b40fdc29d3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 09:30:45 +0200 Subject: [PATCH 05/37] modify docker compose --- .env-devel | 1 + services/docker-compose.yml | 4 ++++ .../application_settings.py | 13 ++++++------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.env-devel b/.env-devel index 9980683c063..45157d71c56 100644 --- a/.env-devel +++ b/.env-devel @@ -144,6 +144,7 @@ DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "unknown@osparc.io", "affiliation": "unknown"}}' +WEBSERVER_CHATBOT={} WEBSERVER_LICENSES={} WEBSERVER_FOGBUGZ={} LICENSES_ITIS_VIP_SYNCER_ENABLED=false diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 63c0ad5b48c..6d8e1c40c3c 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -713,6 +713,7 @@ services: PROMETHEUS_URL: ${WEBSERVER_PROMETHEUS_URL} WEBSERVER_CATALOG: ${WEBSERVER_CATALOG} + WEBSERVER_CHATBOT: ${WEBSERVER_CHATBOT} # WEBSERVER_CREDIT_COMPUTATION WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} @@ -923,6 +924,7 @@ services: WEBSERVER_ACTIVITY: ${WB_DB_EL_ACTIVITY} WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} + WEBSERVER_CHATBOT: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} @@ -1006,6 +1008,7 @@ services: WEBSERVER_ACTIVITY: ${WB_GC_ACTIVITY} WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_GC_CATALOG} + WEBSERVER_CHATBOT: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} @@ -1082,6 +1085,7 @@ services: WEBSERVER_ACTIVITY: "null" WEBSERVER_ANNOUNCEMENTS: 0 WEBSERVER_CATALOG: "null" + WEBSERVER_CHATBOT: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: 0 WEBSERVER_DIRECTOR_V2: "null" diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 3c7e38409da..44b3886712a 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -207,6 +207,12 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): description="catalog service client's plugin", ), ] + WEBSERVER_CHATBOT: Annotated[ + ChatbotSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] WEBSERVER_CELERY: Annotated[ CelerySettings | None, Field( @@ -261,13 +267,6 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] - WEBSERVER_CHATBOT: Annotated[ - ChatbotSettings | None, - Field( - json_schema_extra={"auto_default_from_env": True}, - ), - ] - WEBSERVER_GARBAGE_COLLECTOR: Annotated[ GarbageCollectorSettings | None, Field( From e18dcb9c1b6b767114b2854a7442a40e79d2601b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 09:42:29 +0200 Subject: [PATCH 06/37] review @GitHK --- .../src/simcore_service_webserver/chatbot/_client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index 56559ef9a1a..f66a120ac28 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -61,14 +61,13 @@ def _chatbot_retry(): class ChatbotRestClient: - def __init__(self, base_url: str, chatbot_settings: ChatbotSettings) -> None: + def __init__(self, chatbot_settings: ChatbotSettings) -> None: self._client = httpx.AsyncClient() - self._base_url = base_url self._chatbot_settings = chatbot_settings async def get_settings(self) -> dict[str, Any]: """Fetches chatbot settings""" - url = urljoin(f"{self._base_url}", "/v1/chat/settings") + url = urljoin(f"{self._chatbot_settings.base_url}", "/v1/chat/settings") @_chatbot_retry() async def _request() -> httpx.Response: @@ -87,7 +86,7 @@ async def _request() -> httpx.Response: async def ask_question(self, question: str) -> ChatResponse: """Asks a question to the chatbot""" - url = urljoin(f"{self._base_url}", "/v1/chat") + url = urljoin(f"{self._chatbot_settings.base_url}", "/v1/chat") @_chatbot_retry() async def _request() -> httpx.Response: @@ -131,7 +130,6 @@ async def setup_chatbot_rest_client(app: web.Application) -> None: chatbot_settings = get_plugin_settings(app) client = ChatbotRestClient( - base_url=chatbot_settings.base_url, chatbot_settings=chatbot_settings, ) From fdad989b5fadf9365a8f9c18e556c2fae60c2a78 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:06:19 +0200 Subject: [PATCH 07/37] review @pcrespov --- .../chatbot/_client.py | 17 ++++++++++++----- .../chatbot/settings.py | 6 +++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index f66a120ac28..872d58672f4 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -4,7 +4,7 @@ """ import logging -from typing import Any, Final +from typing import Annotated, Any, Final from urllib.parse import urljoin import httpx @@ -28,7 +28,7 @@ class ChatResponse(BaseModel): - answer: str = Field(description="Answer from the chatbot") + answer: Annotated[str, Field(description="Answer from the chatbot")] def _should_retry(response: httpx.Response | None) -> bool: @@ -76,8 +76,7 @@ async def _request() -> httpx.Response: try: response = await _request() response.raise_for_status() - response_data: dict[str, Any] = response.json() - return response_data + return response.json() except Exception: _logger.error( # noqa: TRY400 "Failed to fetch chatbot settings from %s", url @@ -86,7 +85,7 @@ async def _request() -> httpx.Response: async def ask_question(self, question: str) -> ChatResponse: """Asks a question to the chatbot""" - url = urljoin(f"{self._chatbot_settings.base_url}", "/v1/chat") + url = urljoin(self._chatbot_settings.base_url, "/v1/chat") @_chatbot_retry() async def _request() -> httpx.Response: @@ -135,6 +134,14 @@ async def setup_chatbot_rest_client(app: web.Application) -> None: app[_APPKEY] = client + # Add cleanup on app shutdown + async def cleanup_chatbot_client(app: web.Application) -> None: + client = app.get(_APPKEY) + if client: + await client._client.aclose() # noqa: SLF001 + + app.on_cleanup.append(cleanup_chatbot_client) + def get_chatbot_rest_client(app: web.Application) -> ChatbotRestClient: app_key: ChatbotRestClient = app[_APPKEY] diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index 6e818bf2f76..1b6c3e6ad10 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -1,6 +1,8 @@ from functools import cached_property from aiohttp import web +from models_library.basic_types import PortInt +from pydantic import ConfigDict from settings_library.base import BaseCustomSettings from settings_library.utils_service import MixinServiceSettings, URLPart @@ -8,8 +10,10 @@ class ChatbotSettings(BaseCustomSettings, MixinServiceSettings): + model_config = ConfigDict(str_strip_whitespace=True, str_min_length=1) + CHATBOT_HOST: str - CHATBOT_PORT: int + CHATBOT_PORT: PortInt CHATBOT_LLM_MODEL: str = "gpt-3.5-turbo" CHATBOT_EMBEDDING_MODEL: str = "openai/text-embedding-3-large" From 0cbd54e6905dd4fd5a997742e0aa0afc45c5db64 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:21:59 +0200 Subject: [PATCH 08/37] review @pcrespov --- .../src/simcore_service_webserver/chatbot/__init__.py | 7 ------- .../src/simcore_service_webserver/chatbot/_client.py | 11 +++-------- .../chatbot/chatbot_service.py | 7 +++++++ 3 files changed, 10 insertions(+), 15 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/chatbot_service.py diff --git a/services/web/server/src/simcore_service_webserver/chatbot/__init__.py b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py index a51f5bd24b6..e69de29bb2d 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/__init__.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/__init__.py @@ -1,7 +0,0 @@ -# mypy: disable-error-code=truthy-function -from ._client import ChatbotRestClient, get_chatbot_rest_client - -__all__ = [ - "get_chatbot_rest_client", - "ChatbotRestClient", -] diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index 872d58672f4..c684b67db7a 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -1,11 +1,5 @@ -"""Interface to communicate with Fogbugz API - -- Simple client to create cases in Fogbugz -""" - import logging from typing import Annotated, Any, Final -from urllib.parse import urljoin import httpx from aiohttp import web @@ -67,7 +61,7 @@ def __init__(self, chatbot_settings: ChatbotSettings) -> None: async def get_settings(self) -> dict[str, Any]: """Fetches chatbot settings""" - url = urljoin(f"{self._chatbot_settings.base_url}", "/v1/chat/settings") + url = httpx.URL(self._chatbot_settings.base_url).join("/v1/chat/settings") @_chatbot_retry() async def _request() -> httpx.Response: @@ -85,7 +79,7 @@ async def _request() -> httpx.Response: async def ask_question(self, question: str) -> ChatResponse: """Asks a question to the chatbot""" - url = urljoin(self._chatbot_settings.base_url, "/v1/chat") + url = httpx.URL(self._chatbot_settings.base_url).join("/v1/chat") @_chatbot_retry() async def _request() -> httpx.Response: @@ -146,3 +140,4 @@ async def cleanup_chatbot_client(app: web.Application) -> None: def get_chatbot_rest_client(app: web.Application) -> ChatbotRestClient: app_key: ChatbotRestClient = app[_APPKEY] return app_key + return app_key diff --git a/services/web/server/src/simcore_service_webserver/chatbot/chatbot_service.py b/services/web/server/src/simcore_service_webserver/chatbot/chatbot_service.py new file mode 100644 index 00000000000..a51f5bd24b6 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/chatbot_service.py @@ -0,0 +1,7 @@ +# mypy: disable-error-code=truthy-function +from ._client import ChatbotRestClient, get_chatbot_rest_client + +__all__ = [ + "get_chatbot_rest_client", + "ChatbotRestClient", +] From a8bb6670c5bf2ac65709cb3ecb3d826233b64b72 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:22:48 +0200 Subject: [PATCH 09/37] review @pcrespov --- .../web/server/src/simcore_service_webserver/chatbot/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/plugin.py b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py index a1a6968874d..c8c24006ddb 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/plugin.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py @@ -1,5 +1,3 @@ -"""tags management subsystem""" - import logging from aiohttp import web From 7397e6e1d22f46c53e1fdf2b5230fae8376fae21 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:25:07 +0200 Subject: [PATCH 10/37] fix --- .../web/server/src/simcore_service_webserver/chatbot/_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index c684b67db7a..9622c9e79b3 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -140,4 +140,3 @@ async def cleanup_chatbot_client(app: web.Application) -> None: def get_chatbot_rest_client(app: web.Application) -> ChatbotRestClient: app_key: ChatbotRestClient = app[_APPKEY] return app_key - return app_key From fa97066b8f95deecaa356f595a9b79e13d8103f0 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:50:02 +0200 Subject: [PATCH 11/37] fix --- .../server/src/simcore_service_webserver/chatbot/_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index 9622c9e79b3..904a22708ce 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -70,7 +70,8 @@ async def _request() -> httpx.Response: try: response = await _request() response.raise_for_status() - return response.json() + response_data: dict[str, Any] = response.json() + return response_data except Exception: _logger.error( # noqa: TRY400 "Failed to fetch chatbot settings from %s", url @@ -132,7 +133,7 @@ async def setup_chatbot_rest_client(app: web.Application) -> None: async def cleanup_chatbot_client(app: web.Application) -> None: client = app.get(_APPKEY) if client: - await client._client.aclose() # noqa: SLF001 + await client._client.aclose() # pylint: disable=protected-access # noqa: SLF001 app.on_cleanup.append(cleanup_chatbot_client) From 3bbc3f9e8452e719592a53473ea41e32fb1eba4f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:55:20 +0200 Subject: [PATCH 12/37] fix --- .../server/src/simcore_service_webserver/chatbot/settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index 1b6c3e6ad10..ad257207431 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -2,15 +2,14 @@ from aiohttp import web from models_library.basic_types import PortInt -from pydantic import ConfigDict -from settings_library.base import BaseCustomSettings +from settings_library.base import BaseCustomSettings, SettingsConfigDict from settings_library.utils_service import MixinServiceSettings, URLPart from ..application_keys import APP_SETTINGS_APPKEY class ChatbotSettings(BaseCustomSettings, MixinServiceSettings): - model_config = ConfigDict(str_strip_whitespace=True, str_min_length=1) + model_config = SettingsConfigDict(str_strip_whitespace=True, str_min_length=1) CHATBOT_HOST: str CHATBOT_PORT: PortInt From 6f6e332ca749075584e04d86c580e174cfedbd68 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 10:58:39 +0200 Subject: [PATCH 13/37] fix --- .../server/src/simcore_service_webserver/chatbot/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/settings.py b/services/web/server/src/simcore_service_webserver/chatbot/settings.py index ad257207431..e287f216157 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/settings.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/settings.py @@ -2,7 +2,8 @@ from aiohttp import web from models_library.basic_types import PortInt -from settings_library.base import BaseCustomSettings, SettingsConfigDict +from pydantic_settings import SettingsConfigDict +from settings_library.base import BaseCustomSettings from settings_library.utils_service import MixinServiceSettings, URLPart from ..application_keys import APP_SETTINGS_APPKEY From 82740f3b4e4832bb10f6f75ec8ba7f1e66b0bcd6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 15 Oct 2025 14:05:40 +0200 Subject: [PATCH 14/37] review @pcrespov --- .../chatbot/_client.py | 47 +++++++++---------- .../unit/with_dbs/04/test_chatbot_client.py | 2 - 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_client.py b/services/web/server/src/simcore_service_webserver/chatbot/_client.py index 904a22708ce..89eaeb55874 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_client.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_client.py @@ -5,6 +5,7 @@ from aiohttp import web from pydantic import BaseModel, Field from servicelib.aiohttp import status +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from tenacity import ( retry, retry_if_exception_type, @@ -18,9 +19,6 @@ _logger = logging.getLogger(__name__) -_JSON_CONTENT_TYPE = "application/json" - - class ChatResponse(BaseModel): answer: Annotated[str, Field(description="Answer from the chatbot")] @@ -34,24 +32,22 @@ def _should_retry(response: httpx.Response | None) -> bool: ) -def _chatbot_retry(): - """Retry configuration for chatbot API calls""" - return retry( - retry=( - retry_if_result(_should_retry) - | retry_if_exception_type( - ( - httpx.ConnectError, - httpx.TimeoutException, - httpx.NetworkError, - httpx.ProtocolError, - ) +_CHATBOT_RETRY = retry( + retry=( + retry_if_result(_should_retry) + | retry_if_exception_type( + ( + httpx.ConnectError, + httpx.TimeoutException, + httpx.NetworkError, + httpx.ProtocolError, ) - ), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=1, max=10), - reraise=True, - ) + ) + ), + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=1, max=10), + reraise=True, +) class ChatbotRestClient: @@ -63,7 +59,7 @@ async def get_settings(self) -> dict[str, Any]: """Fetches chatbot settings""" url = httpx.URL(self._chatbot_settings.base_url).join("/v1/chat/settings") - @_chatbot_retry() + @_CHATBOT_RETRY async def _request() -> httpx.Response: return await self._client.get(url) @@ -82,7 +78,7 @@ async def ask_question(self, question: str) -> ChatResponse: """Asks a question to the chatbot""" url = httpx.URL(self._chatbot_settings.base_url).join("/v1/chat") - @_chatbot_retry() + @_CHATBOT_RETRY async def _request() -> httpx.Response: return await self._client.post( url, @@ -92,16 +88,15 @@ async def _request() -> httpx.Response: "embedding_model": self._chatbot_settings.CHATBOT_EMBEDDING_MODEL, }, headers={ - "Content-Type": _JSON_CONTENT_TYPE, - "Accept": _JSON_CONTENT_TYPE, + "Content-Type": MIMETYPE_APPLICATION_JSON, + "Accept": MIMETYPE_APPLICATION_JSON, }, ) try: response = await _request() response.raise_for_status() - response_data: dict[str, Any] = response.json() - return ChatResponse(**response_data) + return ChatResponse.model_validate(response.json()) except Exception: _logger.error( # noqa: TRY400 "Failed to ask question to chatbot at %s", url diff --git a/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py b/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py index 4692dc9918f..365ef932f6e 100644 --- a/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py +++ b/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py @@ -10,7 +10,6 @@ import pytest import respx from aiohttp.test_utils import TestClient -from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_webserver.chatbot._client import ( @@ -24,7 +23,6 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict, - mocker: MockerFixture, ): return app_environment | setenvs_from_dict( monkeypatch, From 2625b5e03b3eb639a9f426bd36fc86c8674ef16b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Oct 2025 13:30:34 +0200 Subject: [PATCH 15/37] introduce AI answering support requests --- .../src/models_library/conversations.py | 5 +- .../src/models_library/rabbitmq_messages.py | 12 ++ ...c_add_chatbot_user_id_to_products_table.py | 42 +++++++ .../models/products.py | 14 +++ services/docker-compose.yml | 68 +++++++----- .../_process_chatbot_trigger_service.py | 105 ++++++++++++++++++ .../chatbot/plugin.py | 4 + .../_conversations_messages_rest.py | 21 ++-- .../_conversation_message_service.py | 64 +++++++++-- .../conversations/_conversation_service.py | 21 +++- .../conversations/conversations_service.py | 2 + .../products/_models.py | 3 + .../products/_repository.py | 1 + .../statics/_handlers.py | 2 + 14 files changed, 310 insertions(+), 54 deletions(-) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e1c7e416461c_add_chatbot_user_id_to_products_table.py create mode 100644 services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py diff --git a/packages/models-library/src/models_library/conversations.py b/packages/models-library/src/models_library/conversations.py index a0eb177b7f7..6856b3c2ce3 100644 --- a/packages/models-library/src/models_library/conversations.py +++ b/packages/models-library/src/models_library/conversations.py @@ -38,7 +38,10 @@ class ConversationMessageType(StrAutoEnum): # -IsSupportUser: TypeAlias = bool +class ConversationUserType(StrAutoEnum): + SUPPORT_USER = auto() + CHATBOT_USER = auto() + REGULAR_USER = auto() class ConversationGetDB(BaseModel): diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index 8ea54f2e9e5..34a297470ec 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -9,6 +9,7 @@ from common_library.basic_types import DEFAULT_FACTORY from pydantic import BaseModel, Field +from .conversations import ConversationGetDB, ConversationMessageID from .products import ProductName from .progress_bar import ProgressReport from .projects import ProjectID @@ -93,6 +94,17 @@ def routing_key(self) -> str | None: return None +class WebserverChatbotRabbitMessage(RabbitMessageBase): + channel_name: Literal["simcore.services.webserver-chatbot"] = ( + "simcore.services.webserver-chatbot" + ) + conversation: ConversationGetDB + last_message_id: ConversationMessageID + + def routing_key(self) -> str | None: + return None + + class ProgressType(StrAutoEnum): COMPUTATION_RUNNING = auto() # NOTE: this is the original only progress report diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e1c7e416461c_add_chatbot_user_id_to_products_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e1c7e416461c_add_chatbot_user_id_to_products_table.py new file mode 100644 index 00000000000..842e0386ce8 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e1c7e416461c_add_chatbot_user_id_to_products_table.py @@ -0,0 +1,42 @@ +"""add chatbot user id to products table + +Revision ID: e1c7e416461c +Revises: f641b3eacafd +Create Date: 2025-10-16 07:51:44.033767+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e1c7e416461c" +down_revision = "f641b3eacafd" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "products", sa.Column("support_chatbot_user_id", sa.BigInteger(), nullable=True) + ) + op.create_foreign_key( + "fk_products_support_chatbot_user_id", + "products", + "users", + ["support_chatbot_user_id"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "fk_products_support_chatbot_user_id", "products", type_="foreignkey" + ) + op.drop_column("products", "support_chatbot_user_id") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products.py b/packages/postgres-database/src/simcore_postgres_database/models/products.py index 414e7e4b2c0..bd105442800 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products.py @@ -19,6 +19,7 @@ from .base import metadata from .groups import groups from .jinja2_templates import jinja2_templates +from .users import users # NOTE: a default entry is created in the table Product # see packages/postgres-database/src/simcore_postgres_database/migration/versions/350103a7efbd_modified_products_table.py @@ -281,6 +282,19 @@ class ProductLoginSettingsDict(TypedDict, total=False): nullable=True, doc="Group associated to this product support", ), + sa.Column( + "support_chatbot_user_id", + sa.BigInteger, + sa.ForeignKey( + users.c.id, + name="fk_products_support_chatbot_user_id", + ondelete=RefActions.SET_NULL, + onupdate=RefActions.CASCADE, + ), + unique=False, + nullable=True, + doc="User associated to this product chatbot user", + ), sa.Column( "support_assigned_fogbugz_person_id", sa.BigInteger, diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 6d8e1c40c3c..1507e0645bc 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -24,7 +24,6 @@ x-tracing-open-telemetry: &tracing_open_telemetry_environments TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY: ${TRACING_OPENTELEMETRY_SAMPLING_PROBABILITY} - ## third-party party services x-postgres-settings: &postgres_settings POSTGRES_DB: ${POSTGRES_DB} @@ -66,7 +65,6 @@ x-s3-settings: &s3_settings S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - x-smtp-settings: &smtp_settings SMTP_HOST: ${SMTP_HOST} SMTP_PORT: ${SMTP_PORT} @@ -74,7 +72,6 @@ x-smtp-settings: &smtp_settings SMTP_PASSWORD: ${SMTP_PASSWORD} SMTP_PROTOCOL: ${SMTP_PROTOCOL} - ## simcore stack services x-catalog-settings: &catalog_settings @@ -108,7 +105,6 @@ x-invitations-settings: &invitations_settings INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} - services: api-server: image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} @@ -157,7 +153,6 @@ services: networks: &api_server_networks - default - api-worker: image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} init: true @@ -171,7 +166,6 @@ services: CELERY_QUEUES: "api_worker_queue" networks: *api_server_networks - autoscaling: image: ${DOCKER_REGISTRY:-itisfoundation}/autoscaling:${DOCKER_IMAGE_TAG:-latest} init: true @@ -430,7 +424,6 @@ services: RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - STORAGE_HOST: ${STORAGE_HOST} STORAGE_PORT: ${STORAGE_PORT} DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: ${DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH} @@ -559,7 +552,6 @@ services: RESOURCE_USAGE_TRACKER_TRACING: ${RESOURCE_USAGE_TRACKER_TRACING} RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_PORT} - dynamic-schdlr: image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} init: true @@ -594,8 +586,6 @@ services: DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} - - docker-api-proxy: image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} init: true @@ -714,6 +704,7 @@ services: WEBSERVER_CATALOG: ${WEBSERVER_CATALOG} WEBSERVER_CHATBOT: ${WEBSERVER_CHATBOT} + WEBSERVER_CONVERSATIONS: "true" # WEBSERVER_CREDIT_COMPUTATION WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} @@ -883,6 +874,9 @@ services: WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} WEBSERVER_RPC_NAMESPACE: ${WB_API_WEBSERVER_HOST} WEBSERVER_STATICWEB: "null" + WEBSERVER_CONVERSATIONS: "false" # override *webserver_environment + WEBSERVER_CHATBOT: "null" # override *webserver_environment + WEBSERVER_FOGBUGZ: "null" # override *webserver_environment # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: api @@ -925,6 +919,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} WEBSERVER_CHATBOT: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} @@ -954,7 +949,6 @@ services: WEBSERVER_USERS: ${WB_DB_EL_USERS} WEBSERVER_WALLETS: ${WB_DB_EL_WALLETS} - RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} deploy: @@ -984,7 +978,6 @@ services: GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc @@ -1009,6 +1002,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_GC_CATALOG} WEBSERVER_CHATBOT: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} @@ -1081,11 +1075,11 @@ services: SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE} SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY} - WEBSERVER_ACTIVITY: "null" WEBSERVER_ANNOUNCEMENTS: 0 WEBSERVER_CATALOG: "null" WEBSERVER_CHATBOT: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: 0 WEBSERVER_DIRECTOR_V2: "null" @@ -1223,7 +1217,6 @@ services: DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} DATCORE_ADAPTER_TRACING: ${DATCORE_ADAPTER_TRACING} - storage: image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} init: true @@ -1322,7 +1315,15 @@ services: - default - interactive_services_subnet healthcheck: - test: [ "CMD", "pg_isready", "--username", "${POSTGRES_USER}", "--dbname", "${POSTGRES_DB}" ] + test: + [ + "CMD", + "pg_isready", + "--username", + "${POSTGRES_USER}", + "--dbname", + "${POSTGRES_DB}", + ] interval: 5s retries: 5 # NOTES: this is not yet compatible with portainer deployment but could work also for other containers @@ -1336,19 +1337,24 @@ services: # - net.ipv4.tcp_keepalive_probes=9 # - net.ipv4.tcp_keepalive_time=600 # - command: - [ + command: [ "postgres", - "-c", "tcp_keepalives_idle=600", - "-c", "tcp_keepalives_interval=600", - "-c", "tcp_keepalives_count=5", - "-c", "max_connections=413", - "-c", "shared_buffers=256MB", + "-c", + "tcp_keepalives_idle=600", + "-c", + "tcp_keepalives_interval=600", + "-c", + "tcp_keepalives_count=5", + "-c", + "max_connections=413", + "-c", + "shared_buffers=256MB", # statement_timeout is set to 120 seconds (120_000 in ms), so that long running queries # are killed after 2 minutes. Since simcore services have timeout of 1 minute, so longer # queries will not be used. Setting >1 minutes to be safe # https://github.com/ITISFoundation/osparc-simcore/issues/7682#issuecomment-2923048445 - "-c", "statement_timeout=120000" + "-c", + "statement_timeout=120000", ] redis: @@ -1360,7 +1366,19 @@ services: # also aof (append only) is also enabled such that we get full durability at the expense # of backup size. The backup is written into /data. # https://redis.io/topics/persistence - [ "redis-server", "--save", "60 1", "--loglevel", "verbose", "--databases", "11", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}" ] + [ + "redis-server", + "--save", + "60 1", + "--loglevel", + "verbose", + "--databases", + "11", + "--appendonly", + "yes", + "--requirepass", + "${REDIS_PASSWORD}", + ] networks: - default - autoscaling_subnet @@ -1368,7 +1386,7 @@ services: volumes: - redis-data:/data healthcheck: - test: [ "CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping" ] + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] interval: 5s timeout: 30s retries: 50 diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py new file mode 100644 index 00000000000..8ae3942e865 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -0,0 +1,105 @@ +import functools +import logging +from collections.abc import AsyncIterator +from typing import Final + +from aiohttp import web +from models_library.basic_types import IDStr +from models_library.conversations import ConversationMessageType, ConversationUserType +from models_library.rabbitmq_messages import WebserverChatbotRabbitMessage +from models_library.rest_ordering import OrderBy, OrderDirection +from pydantic import TypeAdapter +from servicelib.logging_utils import log_catch, log_context +from servicelib.rabbitmq import RabbitMQClient +from yarl import URL + +from ..conversations import conversations_service +from ..rabbitmq import get_rabbitmq_client +from .chatbot_service import get_chatbot_rest_client + +_logger = logging.getLogger(__name__) + + +_RABBITMQ_WEBSERVER_CHATBOT_CONSUMER_APPKEY: Final = web.AppKey( + "RABBITMQ_WEBSERVER_CHATBOT_CONSUMER", str +) + +_CHATBOT_PROCESS_MESSAGE_TTL_IN_MS = 2 * 60 * 60 * 1000 # 2 hours + + +async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> bool: + rabbit_message = TypeAdapter(WebserverChatbotRabbitMessage).validate_json(data) + assert app # nosec + + # Get last 20 messages for the conversation ID + messages = await conversations_service.list_messages_for_conversation( + app=app, + conversation_id=rabbit_message.conversation.conversation_id, + offset=0, + limit=20, + order_by=OrderBy(field=IDStr("created"), direction=OrderDirection.DESC), + ) + _question_for_chatbot = "" + for msg in messages[1]: + _question_for_chatbot += f"{msg.content.strip()}\n" + + # Prepare them and talk to the chatbot service + chatbot_client = get_chatbot_rest_client(app) + chat_response = await chatbot_client.ask_question(_question_for_chatbot) + + # After I got respond, create a new support message in the conversation + await conversations_service.create_support_message( + app=app, + product_name=rabbit_message.conversation.product_name, + user_id=1, + conversation_user_type=ConversationUserType.CHATBOT_USER, + conversation=rabbit_message.conversation, + request_url=URL("http://dummy"), + request_host="dummy", + content=chat_response.answer, + type_=ConversationMessageType.MESSAGE, + ) + return True + + +async def _subscribe_to_rabbitmq(app) -> str: + with log_context( + _logger, + logging.INFO, + msg=f"Subscribing to {WebserverChatbotRabbitMessage.get_channel_name()} channel", + ): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + subscribed_queue, _ = await rabbit_client.subscribe( + WebserverChatbotRabbitMessage.get_channel_name(), + message_handler=functools.partial(_process_chatbot_trigger_message, app), + exclusive_queue=False, + message_ttl=_CHATBOT_PROCESS_MESSAGE_TTL_IN_MS, + ) + return subscribed_queue + + +async def _unsubscribe_from_rabbitmq(app) -> None: + with ( + log_context( + _logger, + logging.INFO, + msg=f"Unsubscribing from {WebserverChatbotRabbitMessage.get_channel_name()} channel", + ), + log_catch(_logger, reraise=False), + ): + rabbit_client: RabbitMQClient = get_rabbitmq_client(app) + if app[_RABBITMQ_WEBSERVER_CHATBOT_CONSUMER_APPKEY]: + await rabbit_client.unsubscribe( + app[_RABBITMQ_WEBSERVER_CHATBOT_CONSUMER_APPKEY] + ) + + +async def on_cleanup_ctx_rabbitmq_consumer( + app: web.Application, +) -> AsyncIterator[None]: + app[_RABBITMQ_WEBSERVER_CHATBOT_CONSUMER_APPKEY] = await _subscribe_to_rabbitmq(app) + + yield + + # cleanup + await _unsubscribe_from_rabbitmq(app) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/plugin.py b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py index c8c24006ddb..fbea507ec2a 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/plugin.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/plugin.py @@ -4,7 +4,9 @@ from ..application_setup import ModuleCategory, app_setup_func from ..products.plugin import setup_products +from ..rabbitmq import setup_rabbitmq from ._client import setup_chatbot_rest_client +from ._process_chatbot_trigger_service import on_cleanup_ctx_rabbitmq_consumer _logger = logging.getLogger(__name__) @@ -17,4 +19,6 @@ ) def setup_chatbot(app: web.Application): setup_products(app) + setup_rabbitmq(app) app.on_startup.append(setup_chatbot_rest_client) + app.cleanup_ctx.append(on_cleanup_ctx_rabbitmq_consumer) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py index ced1bfb1434..bc9f3619735 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py @@ -1,8 +1,6 @@ import logging -from typing import Any from aiohttp import web -from common_library.json_serialization import json_dumps from models_library.api_schemas_webserver.conversations import ( ConversationMessagePatch, ConversationMessageRestGet, @@ -18,7 +16,6 @@ PageQueryParameters, ) from models_library.rest_pagination_utils import paginate_data -from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import BaseModel, ConfigDict from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( @@ -58,10 +55,6 @@ class _ConversationMessageCreateBodyParams(BaseModel): model_config = ConfigDict(extra="forbid") -def _json_encoder_and_dumps(obj: Any, **kwargs): - return json_dumps(jsonable_encoder(obj), **kwargs) - - @routes.post( f"/{VTAG}/conversations/{{conversation_id}}/messages", name="create_conversation_message", @@ -83,18 +76,20 @@ async def create_conversation_message(request: web.Request): raise_unsupported_type(_conversation.type) # This function takes care of granting support user access to the message - _, is_support_user = await _conversation_service.get_support_conversation_for_user( - app=request.app, - user_id=req_ctx.user_id, - product_name=req_ctx.product_name, - conversation_id=path_params.conversation_id, + _, conversation_user_type = ( + await _conversation_service.get_support_conversation_for_user( + app=request.app, + user_id=req_ctx.user_id, + product_name=req_ctx.product_name, + conversation_id=path_params.conversation_id, + ) ) message = await _conversation_message_service.create_support_message( app=request.app, product_name=req_ctx.product_name, user_id=req_ctx.user_id, - is_support_user=is_support_user, + conversation_user_type=conversation_user_type, conversation=_conversation, request_url=request.url, request_host=request.host, diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index 99e463781a7..5c51d64368e 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -13,9 +13,11 @@ ConversationMessagePatchDB, ConversationMessageType, ConversationPatchDB, + ConversationUserType, ) from models_library.products import ProductName from models_library.projects import ProjectID +from models_library.rabbitmq_messages import WebserverChatbotRabbitMessage from models_library.rest_ordering import OrderBy, OrderDirection from models_library.rest_pagination import PageTotalCount from models_library.users import UserID @@ -25,6 +27,7 @@ from yarl import URL from ..products import products_service +from ..rabbitmq import get_rabbitmq_client from ..redis import get_redis_lock_manager_client_sdk from ..users import users_service from . import ( @@ -112,7 +115,7 @@ async def _create_support_message_with_first_check( *, product_name: ProductName, user_id: UserID, - is_support_user: bool, + conversation_user_type: ConversationUserType, conversation_id: ConversationID, # Creation attributes content: str, @@ -178,12 +181,20 @@ async def _create_support_message_and_check_if_it_is_first_message() -> ( ) # NOTE: Update conversation last modified (for frontend listing) and read states - if is_support_user: - _is_read_by_user = False - _is_read_by_support = True - else: - _is_read_by_user = True - _is_read_by_support = False + match conversation_user_type: + case ConversationUserType.REGULAR_USER: + _is_read_by_user = True + _is_read_by_support = False + case ConversationUserType.SUPPORT_USER: + _is_read_by_user = False + _is_read_by_support = True + case ConversationUserType.CHATBOT_USER: + _is_read_by_user = False + _is_read_by_support = False + case _: + msg = f"Unknown conversation user type: {conversation_user_type}" + raise ValueError(msg) + await _conversation_repository.update( app, conversation_id=conversation_id, @@ -196,12 +207,31 @@ async def _create_support_message_and_check_if_it_is_first_message() -> ( return message, is_first_message +async def _trigger_chatbot_processing( + app: web.Application, + conversation: ConversationGetDB, + last_message_id: ConversationMessageID, +) -> None: + """Triggers chatbot processing for a specific conversation.""" + rabbitmq_client = get_rabbitmq_client(app) + message = WebserverChatbotRabbitMessage( + conversation=conversation, + last_message_id=last_message_id, + ) + _logger.debug( + "Publishing chatbot processing message with conversation id %s and last message id %s.", + conversation.conversation_id, + last_message_id, + ) + await rabbitmq_client.publish(message.channel_name, message) + + async def create_support_message( app: web.Application, *, product_name: ProductName, user_id: UserID, - is_support_user: bool, + conversation_user_type: ConversationUserType, conversation: ConversationGetDB, request_url: URL, request_host: str, @@ -213,7 +243,7 @@ async def create_support_message( app=app, product_name=product_name, user_id=user_id, - is_support_user=is_support_user, + conversation_user_type=conversation_user_type, conversation_id=conversation.conversation_id, content=content, type_=type_, @@ -297,6 +327,17 @@ async def create_support_message( ) ) + if ( + product.support_chatbot_user_id + and conversation_user_type == ConversationUserType.CHATBOT_USER + ): + # If enabled, ask Chatbot to analyze the message history and respond + await _trigger_chatbot_processing( + app, + conversation=conversation, + last_message_id=message.message_id, + ) + return message @@ -412,13 +453,16 @@ async def list_messages_for_conversation( # pagination offset: int = 0, limit: int = 20, + # ordering + order_by: OrderBy | None = None, ) -> tuple[PageTotalCount, list[ConversationMessageGetDB]]: return await _conversation_message_repository.list_( app, conversation_id=conversation_id, offset=offset, limit=limit, - order_by=OrderBy( + order_by=order_by + or OrderBy( field=IDStr("created"), direction=OrderDirection.DESC ), # NOTE: Message should be ordered by creation date (latest first) ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_service.py index facb37be99d..1bd7da3fb11 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_service.py @@ -12,7 +12,7 @@ ConversationID, ConversationPatchDB, ConversationType, - IsSupportUser, + ConversationUserType, ) from models_library.products import ProductName from models_library.projects import ProjectID @@ -185,10 +185,21 @@ async def get_support_conversation_for_user( user_id: UserID, product_name: ProductName, conversation_id: ConversationID, -) -> tuple[ConversationGetDB, IsSupportUser]: +) -> tuple[ConversationGetDB, ConversationUserType]: # Check if user is part of support group (in that case he has access to all support conversations) product = products_service.get_product(app, product_name=product_name) _support_standard_group_id = product.support_standard_group_id + _chatbot_user_id = product.support_chatbot_user_id + + # Check if user is an AI bot + if _chatbot_user_id and user_id == _chatbot_user_id: + return ( + await get_conversation( + app, conversation_id=conversation_id, type_=ConversationType.SUPPORT + ), + ConversationUserType.CHATBOT_USER, + ) + if _support_standard_group_id is not None: _user_group_ids = await list_user_groups_ids_with_read_access( app, user_id=user_id @@ -199,7 +210,7 @@ async def get_support_conversation_for_user( await get_conversation( app, conversation_id=conversation_id, type_=ConversationType.SUPPORT ), - True, + ConversationUserType.SUPPORT_USER, ) _user_group_id = await users_service.get_user_primary_group_id(app, user_id=user_id) @@ -210,7 +221,7 @@ async def get_support_conversation_for_user( user_group_id=_user_group_id, type_=ConversationType.SUPPORT, ), - False, + ConversationUserType.REGULAR_USER, ) @@ -285,7 +296,7 @@ async def create_fogbugz_case_for_support_conversation( fogbugz_client = get_fogbugz_rest_client(app) fogbugz_case_data = FogbugzCaseCreate( fogbugz_project_id=product_support_assigned_fogbugz_project_id, - title=f"Request for Support on {host}", + title=f"Request for Support on {host} by {user['email']}", description=description, ) case_id = await fogbugz_client.create_case(fogbugz_case_data) diff --git a/services/web/server/src/simcore_service_webserver/conversations/conversations_service.py b/services/web/server/src/simcore_service_webserver/conversations/conversations_service.py index 61dcfe3330c..718679279b7 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/conversations_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/conversations_service.py @@ -1,6 +1,7 @@ # mypy: disable-error-code=truthy-function from ._conversation_message_service import ( create_message, + create_support_message, delete_message, get_message, list_messages_for_conversation, @@ -25,5 +26,6 @@ "list_messages_for_conversation", "update_conversation", "update_message", + "create_support_message", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/products/_models.py b/services/web/server/src/simcore_service_webserver/products/_models.py index 127533d16c7..3c794e682ec 100644 --- a/services/web/server/src/simcore_service_webserver/products/_models.py +++ b/services/web/server/src/simcore_service_webserver/products/_models.py @@ -143,6 +143,9 @@ class Product(BaseModel): support_standard_group_id: Annotated[ int | None, Field(description="Support standard group ID, None if disabled") ] = None + support_chatbot_user_id: Annotated[ + int | None, Field(description="Support chatbot user ID, None if disabled") + ] = None support_assigned_fogbugz_person_id: Annotated[ int | None, Field(description="Support assigned Fogbugz person ID, None if disabled"), diff --git a/services/web/server/src/simcore_service_webserver/products/_repository.py b/services/web/server/src/simcore_service_webserver/products/_repository.py index 64bf33af45e..d23dbca8a87 100644 --- a/services/web/server/src/simcore_service_webserver/products/_repository.py +++ b/services/web/server/src/simcore_service_webserver/products/_repository.py @@ -54,6 +54,7 @@ products.c.max_open_studies_per_user, products.c.group_id, products.c.support_standard_group_id, + products.c.support_chatbot_user_id, products.c.support_assigned_fogbugz_person_id, products.c.support_assigned_fogbugz_project_id, ] diff --git a/services/web/server/src/simcore_service_webserver/statics/_handlers.py b/services/web/server/src/simcore_service_webserver/statics/_handlers.py index 248d75dbf91..eaf09ad8999 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/statics/_handlers.py @@ -32,6 +32,8 @@ async def get_cached_frontend_index(request: web.Request): if product_name not in cached_index_per_product: raise web.HTTPNotFound(text=f"No index.html found for {product_name}") + # TODO: MD - cacheing product URL to APP state in memory + return web.Response( body=cached_index_per_product[product_name], content_type=MIMETYPE_TEXT_HTML ) From 972f42597d41c5848733f27e46e4422ec211a0e3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 16 Oct 2025 17:29:24 +0200 Subject: [PATCH 16/37] improve --- .../chatbot/_process_chatbot_trigger_service.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index 8ae3942e865..fb57fa8b379 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -14,6 +14,7 @@ from yarl import URL from ..conversations import conversations_service +from ..products import products_service from ..rabbitmq import get_rabbitmq_client from .chatbot_service import get_chatbot_rest_client @@ -31,6 +32,16 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> rabbit_message = TypeAdapter(WebserverChatbotRabbitMessage).validate_json(data) assert app # nosec + _product_name = rabbit_message.conversation.product_name + _product = products_service.get_product(app, product_name=_product_name) + + if _product.support_chatbot_user_id is None: + _logger.error( + "Product %s does not have support_chatbot_user_id configured, cannot process chatbot message. (This should not happen)", + _product_name, + ) + return True # return true to avoid re-processing + # Get last 20 messages for the conversation ID messages = await conversations_service.list_messages_for_conversation( app=app, @@ -51,7 +62,7 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> await conversations_service.create_support_message( app=app, product_name=rabbit_message.conversation.product_name, - user_id=1, + user_id=_product.support_chatbot_user_id, conversation_user_type=ConversationUserType.CHATBOT_USER, conversation=rabbit_message.conversation, request_url=URL("http://dummy"), From c31c94a7090af55a08434d4c3e66022385edf8f6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 10:25:17 +0200 Subject: [PATCH 17/37] introduce to store in app state the product url --- .../_process_chatbot_trigger_service.py | 4 ++-- .../_conversations_messages_rest.py | 1 - .../_conversation_message_service.py | 3 +-- .../products/_application_keys.py | 6 ++++- .../products/_models.py | 5 ++++ .../products/_service.py | 13 +++++++++-- .../products/_web_helpers.py | 23 +++++++++++++++++++ .../products/products_service.py | 2 ++ .../products/products_web.py | 5 +++- .../socketio/_handlers.py | 7 +++--- .../statics/_handlers.py | 1 + 11 files changed, 58 insertions(+), 12 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index fb57fa8b379..b3dae269233 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -59,14 +59,14 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> chat_response = await chatbot_client.ask_question(_question_for_chatbot) # After I got respond, create a new support message in the conversation + _product_url = products_service.get_product_url(app, product_name=_product_name) await conversations_service.create_support_message( app=app, product_name=rabbit_message.conversation.product_name, user_id=_product.support_chatbot_user_id, conversation_user_type=ConversationUserType.CHATBOT_USER, conversation=rabbit_message.conversation, - request_url=URL("http://dummy"), - request_host="dummy", + request_url=_product_url or URL("http://unknown"), content=chat_response.answer, type_=ConversationMessageType.MESSAGE, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py index bc9f3619735..af619195eb5 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py @@ -92,7 +92,6 @@ async def create_conversation_message(request: web.Request): conversation_user_type=conversation_user_type, conversation=_conversation, request_url=request.url, - request_host=request.host, content=body_params.content, type_=body_params.type, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index 5c51d64368e..9613c0730eb 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -234,7 +234,6 @@ async def create_support_message( conversation_user_type: ConversationUserType, conversation: ConversationGetDB, request_url: URL, - request_host: str, # Creation attributes content: str, type_: ConversationMessageType, @@ -279,7 +278,7 @@ async def create_support_message( user_id=user_id, message_content=message.content, conversation_url=_conversation_url, - host=request_host, + host=request_url.host or "unknown", product_support_assigned_fogbugz_project_id=product.support_assigned_fogbugz_project_id, fogbugz_url=str(fogbugz_settings_or_none.FOGBUGZ_URL), ) diff --git a/services/web/server/src/simcore_service_webserver/products/_application_keys.py b/services/web/server/src/simcore_service_webserver/products/_application_keys.py index febc0053f64..af2067a1da7 100644 --- a/services/web/server/src/simcore_service_webserver/products/_application_keys.py +++ b/services/web/server/src/simcore_service_webserver/products/_application_keys.py @@ -5,8 +5,12 @@ from aiohttp import web from models_library.products import ProductName -from ._models import Product +from ._models import Product, ProductBaseUrl PRODUCTS_APPKEY: Final = web.AppKey("PRODUCTS_APPKEY", dict[ProductName, Product]) +PRODUCTS_URL_MAPPING_APPKEY: Final = web.AppKey( + "PRODUCTS_URL_MAPPING_APPKEY", dict[ProductName, ProductBaseUrl] +) + DEFAULT_PRODUCT_APPKEY: Final = web.AppKey("DEFAULT_PRODUCT_APPKEY", ProductName) diff --git a/services/web/server/src/simcore_service_webserver/products/_models.py b/services/web/server/src/simcore_service_webserver/products/_models.py index 3c794e682ec..9f264cd7904 100644 --- a/services/web/server/src/simcore_service_webserver/products/_models.py +++ b/services/web/server/src/simcore_service_webserver/products/_models.py @@ -351,3 +351,8 @@ def get_template_name_for(self, filename: str) -> str | None: template_name_attribute: str = getattr(self, name) return template_name_attribute return None + + +class ProductBaseUrl(BaseModel): + schema: str + host: str diff --git a/services/web/server/src/simcore_service_webserver/products/_service.py b/services/web/server/src/simcore_service_webserver/products/_service.py index debdb47767e..e2c80fd58f0 100644 --- a/services/web/server/src/simcore_service_webserver/products/_service.py +++ b/services/web/server/src/simcore_service_webserver/products/_service.py @@ -4,11 +4,12 @@ from aiohttp import web from models_library.groups import GroupID from models_library.products import ProductName -from pydantic import ValidationError +from pydantic import HttpUrl, ValidationError from servicelib.exceptions import InvalidConfig from simcore_postgres_database.utils_products_prices import ProductPriceInfo +from yarl import URL -from ._application_keys import PRODUCTS_APPKEY +from ._application_keys import PRODUCTS_APPKEY, PRODUCTS_URL_MAPPING_APPKEY from ._models import CreditResult, Product, ProductStripeInfo from ._repository import ProductRepository from .errors import ( @@ -43,6 +44,14 @@ def get_product(app: web.Application, product_name: ProductName) -> Product: raise ProductNotFoundError(product_name=product_name) from exc +def get_product_url(app: web.Application, product_name: ProductName) -> URL: + try: + product_url: HttpUrl = app[PRODUCTS_URL_MAPPING_APPKEY][product_name] + return product_url + except KeyError as exc: + raise ProductNotFoundError(product_name=product_name) from exc + + def list_products(app: web.Application) -> list[Product]: products: list[Product] = list(app[PRODUCTS_APPKEY].values()) return products diff --git a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py index f70bc79ca97..8fe08d5eec7 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py @@ -1,4 +1,5 @@ import contextlib +import logging from pathlib import Path import aiofiles @@ -6,11 +7,15 @@ from models_library.products import ProductName from models_library.users import UserID from simcore_postgres_database.utils_products_prices import ProductPriceInfo +from simcore_service_webserver.products._models import ProductBaseUrl from .._resources import webserver_resources from ..constants import RQ_PRODUCT_KEY from ..groups import api as groups_service from . import _service +from ._application_keys import ( + PRODUCTS_URL_MAPPING_APPKEY, +) from ._web_events import PRODUCTS_TEMPLATES_DIR_APPKEY from .errors import ( FileTemplateNotFoundError, @@ -19,6 +24,8 @@ ) from .models import Product +_logger = logging.getLogger(__name__) + def get_product_name(request: web.Request) -> str: """Returns product name in request but might be undefined""" @@ -40,6 +47,22 @@ def get_current_product(request: web.Request) -> Product: return current_product +def set_product_url(request: web.Request, product_name: ProductName) -> None: + if ( + not request.app[PRODUCTS_URL_MAPPING_APPKEY].get(product_name) + and request.url.host + ): + request.app[PRODUCTS_URL_MAPPING_APPKEY][product_name] = ProductBaseUrl( + schema=request.url.scheme, host=request.url.host + ) + _logger.debug( + "Set product url for %s to %s://%s", + product_name, + request.url.scheme, + request.url.host, + ) + + async def is_user_in_product_support_group( request: web.Request, *, user_id: UserID ) -> bool: diff --git a/services/web/server/src/simcore_service_webserver/products/products_service.py b/services/web/server/src/simcore_service_webserver/products/products_service.py index 1fbc880d7e9..c7d55d91831 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_service.py +++ b/services/web/server/src/simcore_service_webserver/products/products_service.py @@ -3,6 +3,7 @@ get_product, get_product_stripe_info, get_product_ui, + get_product_url, is_product_billable, list_products, list_products_names, @@ -13,6 +14,7 @@ "get_product", "get_product_stripe_info", "get_product_ui", + "get_product_url", "is_product_billable", "list_products", "list_products_names", diff --git a/services/web/server/src/simcore_service_webserver/products/products_web.py b/services/web/server/src/simcore_service_webserver/products/products_web.py index 0cb372aacc3..35efe801d46 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_web.py +++ b/services/web/server/src/simcore_service_webserver/products/products_web.py @@ -1,18 +1,21 @@ -from ._application_keys import PRODUCTS_APPKEY +from ._application_keys import PRODUCTS_APPKEY, PRODUCTS_URL_MAPPING_APPKEY from ._web_helpers import ( get_current_product, get_current_product_credit_price_info, get_product_name, get_product_template_path, is_user_in_product_support_group, + set_product_url, ) __all__: tuple[str, ...] = ( "PRODUCTS_APPKEY", + "PRODUCTS_URL_MAPPING_APPKEY", "get_current_product", "get_current_product_credit_price_info", "get_product_name", "get_product_template_path", "is_user_in_product_support_group", + "set_product_url", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index c9ff4634148..bba0c27e941 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -140,9 +140,8 @@ async def connect( try: auth_user_handler = auth_user_factory(socket_id) - user_id, product_name, client_session_id = await auth_user_handler( - environ["aiohttp.request"] - ) + _request: web.Request = environ["aiohttp.request"] + user_id, product_name, client_session_id = await auth_user_handler(_request) _logger.info( "%s successfully connected with %s", f"{user_id=}", @@ -150,6 +149,8 @@ async def connect( extra=get_log_record_extra(user_id=user_id), ) + products_web.set_product_url(_request, product_name) + await _set_user_in_group_rooms(app, user_id, socket_id) await _set_user_in_project_rooms(app, user_id, client_session_id, socket_id) diff --git a/services/web/server/src/simcore_service_webserver/statics/_handlers.py b/services/web/server/src/simcore_service_webserver/statics/_handlers.py index eaf09ad8999..59046fd5c9f 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/statics/_handlers.py @@ -33,6 +33,7 @@ async def get_cached_frontend_index(request: web.Request): raise web.HTTPNotFound(text=f"No index.html found for {product_name}") # TODO: MD - cacheing product URL to APP state in memory + products_web.set return web.Response( body=cached_index_per_product[product_name], content_type=MIMETYPE_TEXT_HTML From 9ff225e4de07b05701dd97be8fde7eba4916dca5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 10:32:43 +0200 Subject: [PATCH 18/37] fix --- .../server/src/simcore_service_webserver/statics/_handlers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/statics/_handlers.py b/services/web/server/src/simcore_service_webserver/statics/_handlers.py index 59046fd5c9f..248d75dbf91 100644 --- a/services/web/server/src/simcore_service_webserver/statics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/statics/_handlers.py @@ -32,9 +32,6 @@ async def get_cached_frontend_index(request: web.Request): if product_name not in cached_index_per_product: raise web.HTTPNotFound(text=f"No index.html found for {product_name}") - # TODO: MD - cacheing product URL to APP state in memory - products_web.set - return web.Response( body=cached_index_per_product[product_name], content_type=MIMETYPE_TEXT_HTML ) From d05e7e2b1d9ecaa4ded324f4cc6a03420b66e674 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 10:41:53 +0200 Subject: [PATCH 19/37] fix --- .../_process_chatbot_trigger_service.py | 18 +++++++++++++++--- .../_conversations_messages_rest.py | 3 ++- .../_conversation_message_service.py | 8 ++++---- .../products/_models.py | 2 +- .../products/_service.py | 15 +++++++++------ .../products/_web_helpers.py | 2 +- .../products/products_service.py | 4 ++-- 7 files changed, 34 insertions(+), 18 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index b3dae269233..f8d34bb5871 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -11,7 +11,8 @@ from pydantic import TypeAdapter from servicelib.logging_utils import log_catch, log_context from servicelib.rabbitmq import RabbitMQClient -from yarl import URL +from simcore_service_webserver.products._models import ProductBaseUrl +from simcore_service_webserver.products.errors import ProductNotFoundError from ..conversations import conversations_service from ..products import products_service @@ -59,14 +60,25 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> chat_response = await chatbot_client.ask_question(_question_for_chatbot) # After I got respond, create a new support message in the conversation - _product_url = products_service.get_product_url(app, product_name=_product_name) + try: + _product_base_url = products_service.get_product_base_url( + app, product_name=_product_name + ) + except ProductNotFoundError: + _logger.warning( + "Product %s does not have base URL configured, cannot process chatbot message.", + _product_name, + ) + _product_base_url = ProductBaseUrl(scheme="http", host="unknown") + await conversations_service.create_support_message( app=app, product_name=rabbit_message.conversation.product_name, user_id=_product.support_chatbot_user_id, conversation_user_type=ConversationUserType.CHATBOT_USER, conversation=rabbit_message.conversation, - request_url=_product_url or URL("http://unknown"), + request_scheme=_product_base_url.scheme, + request_host=_product_base_url.host or "unknown", content=chat_response.answer, type_=ConversationMessageType.MESSAGE, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py index af619195eb5..c570f6976af 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py @@ -91,7 +91,8 @@ async def create_conversation_message(request: web.Request): user_id=req_ctx.user_id, conversation_user_type=conversation_user_type, conversation=_conversation, - request_url=request.url, + request_scheme=request.url.scheme, + request_host=request.url.host or "unknown", content=body_params.content, type_=body_params.type, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index 9613c0730eb..e61f438de31 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -24,7 +24,6 @@ from servicelib.redis import exclusive from simcore_service_webserver.application_keys import APP_SETTINGS_APPKEY from simcore_service_webserver.groups import api as group_service -from yarl import URL from ..products import products_service from ..rabbitmq import get_rabbitmq_client @@ -233,7 +232,8 @@ async def create_support_message( user_id: UserID, conversation_user_type: ConversationUserType, conversation: ConversationGetDB, - request_url: URL, + request_scheme: str, + request_host: str, # Creation attributes content: str, type_: ConversationMessageType, @@ -250,7 +250,7 @@ async def create_support_message( product = products_service.get_product(app, product_name=product_name) fogbugz_settings_or_none = app[APP_SETTINGS_APPKEY].WEBSERVER_FOGBUGZ - _conversation_url = f"{request_url.scheme}://{request_url.host}/#/conversation/{conversation.conversation_id}" + _conversation_url = f"{request_scheme}://{request_host}/#/conversation/{conversation.conversation_id}" if ( product.support_standard_group_id is None @@ -278,7 +278,7 @@ async def create_support_message( user_id=user_id, message_content=message.content, conversation_url=_conversation_url, - host=request_url.host or "unknown", + host=request_host, product_support_assigned_fogbugz_project_id=product.support_assigned_fogbugz_project_id, fogbugz_url=str(fogbugz_settings_or_none.FOGBUGZ_URL), ) diff --git a/services/web/server/src/simcore_service_webserver/products/_models.py b/services/web/server/src/simcore_service_webserver/products/_models.py index 9f264cd7904..4d2522e7557 100644 --- a/services/web/server/src/simcore_service_webserver/products/_models.py +++ b/services/web/server/src/simcore_service_webserver/products/_models.py @@ -354,5 +354,5 @@ def get_template_name_for(self, filename: str) -> str | None: class ProductBaseUrl(BaseModel): - schema: str + scheme: str host: str diff --git a/services/web/server/src/simcore_service_webserver/products/_service.py b/services/web/server/src/simcore_service_webserver/products/_service.py index e2c80fd58f0..99abca95adb 100644 --- a/services/web/server/src/simcore_service_webserver/products/_service.py +++ b/services/web/server/src/simcore_service_webserver/products/_service.py @@ -4,13 +4,12 @@ from aiohttp import web from models_library.groups import GroupID from models_library.products import ProductName -from pydantic import HttpUrl, ValidationError +from pydantic import ValidationError from servicelib.exceptions import InvalidConfig from simcore_postgres_database.utils_products_prices import ProductPriceInfo -from yarl import URL from ._application_keys import PRODUCTS_APPKEY, PRODUCTS_URL_MAPPING_APPKEY -from ._models import CreditResult, Product, ProductStripeInfo +from ._models import CreditResult, Product, ProductBaseUrl, ProductStripeInfo from ._repository import ProductRepository from .errors import ( BelowMinimumPaymentError, @@ -44,10 +43,14 @@ def get_product(app: web.Application, product_name: ProductName) -> Product: raise ProductNotFoundError(product_name=product_name) from exc -def get_product_url(app: web.Application, product_name: ProductName) -> URL: +def get_product_base_url( + app: web.Application, product_name: ProductName +) -> ProductBaseUrl: try: - product_url: HttpUrl = app[PRODUCTS_URL_MAPPING_APPKEY][product_name] - return product_url + product_base_url: ProductBaseUrl = app[PRODUCTS_URL_MAPPING_APPKEY][ + product_name + ] + return product_base_url except KeyError as exc: raise ProductNotFoundError(product_name=product_name) from exc diff --git a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py index 8fe08d5eec7..fae679dbdbb 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py @@ -53,7 +53,7 @@ def set_product_url(request: web.Request, product_name: ProductName) -> None: and request.url.host ): request.app[PRODUCTS_URL_MAPPING_APPKEY][product_name] = ProductBaseUrl( - schema=request.url.scheme, host=request.url.host + scheme=request.url.scheme, host=request.url.host ) _logger.debug( "Set product url for %s to %s://%s", diff --git a/services/web/server/src/simcore_service_webserver/products/products_service.py b/services/web/server/src/simcore_service_webserver/products/products_service.py index c7d55d91831..77463fcdc76 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_service.py +++ b/services/web/server/src/simcore_service_webserver/products/products_service.py @@ -1,9 +1,9 @@ from ._service import ( get_credit_amount, get_product, + get_product_base_url, get_product_stripe_info, get_product_ui, - get_product_url, is_product_billable, list_products, list_products_names, @@ -14,7 +14,7 @@ "get_product", "get_product_stripe_info", "get_product_ui", - "get_product_url", + "get_product_base_url", "is_product_billable", "list_products", "list_products_names", From 6a35bc31df16ec8ad2e768aa5bb987e1c07167f7 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 10:44:11 +0200 Subject: [PATCH 20/37] fix --- .../chatbot/_process_chatbot_trigger_service.py | 4 ++-- .../_controller/_conversations_messages_rest.py | 4 ++-- .../conversations/_conversation_message_service.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index f8d34bb5871..92377fa41d1 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -77,8 +77,8 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> user_id=_product.support_chatbot_user_id, conversation_user_type=ConversationUserType.CHATBOT_USER, conversation=rabbit_message.conversation, - request_scheme=_product_base_url.scheme, - request_host=_product_base_url.host or "unknown", + product_url_scheme=_product_base_url.scheme, + product_url_host=_product_base_url.host or "unknown", content=chat_response.answer, type_=ConversationMessageType.MESSAGE, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py index c570f6976af..23a2faa48d8 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py @@ -91,8 +91,8 @@ async def create_conversation_message(request: web.Request): user_id=req_ctx.user_id, conversation_user_type=conversation_user_type, conversation=_conversation, - request_scheme=request.url.scheme, - request_host=request.url.host or "unknown", + product_url_scheme=request.url.scheme, + product_url_host=request.url.host or "unknown", content=body_params.content, type_=body_params.type, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index e61f438de31..679b3361698 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -232,8 +232,8 @@ async def create_support_message( user_id: UserID, conversation_user_type: ConversationUserType, conversation: ConversationGetDB, - request_scheme: str, - request_host: str, + product_url_scheme: str, + product_url_host: str, # Creation attributes content: str, type_: ConversationMessageType, @@ -250,7 +250,7 @@ async def create_support_message( product = products_service.get_product(app, product_name=product_name) fogbugz_settings_or_none = app[APP_SETTINGS_APPKEY].WEBSERVER_FOGBUGZ - _conversation_url = f"{request_scheme}://{request_host}/#/conversation/{conversation.conversation_id}" + _conversation_url = f"{product_url_scheme}://{product_url_host}/#/conversation/{conversation.conversation_id}" if ( product.support_standard_group_id is None @@ -278,7 +278,7 @@ async def create_support_message( user_id=user_id, message_content=message.content, conversation_url=_conversation_url, - host=request_host, + host=product_url_host, product_support_assigned_fogbugz_project_id=product.support_assigned_fogbugz_project_id, fogbugz_url=str(fogbugz_settings_or_none.FOGBUGZ_URL), ) From d257e8714b4178534a2328b1f799427e04fc988d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 10:49:40 +0200 Subject: [PATCH 21/37] fix --- .../src/simcore_service_webserver/products/_web_helpers.py | 4 ++-- .../src/simcore_service_webserver/products/products_web.py | 4 ++-- .../src/simcore_service_webserver/socketio/_handlers.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py index fae679dbdbb..eed849f1313 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py @@ -7,7 +7,6 @@ from models_library.products import ProductName from models_library.users import UserID from simcore_postgres_database.utils_products_prices import ProductPriceInfo -from simcore_service_webserver.products._models import ProductBaseUrl from .._resources import webserver_resources from ..constants import RQ_PRODUCT_KEY @@ -16,6 +15,7 @@ from ._application_keys import ( PRODUCTS_URL_MAPPING_APPKEY, ) +from ._models import ProductBaseUrl from ._web_events import PRODUCTS_TEMPLATES_DIR_APPKEY from .errors import ( FileTemplateNotFoundError, @@ -47,7 +47,7 @@ def get_current_product(request: web.Request) -> Product: return current_product -def set_product_url(request: web.Request, product_name: ProductName) -> None: +def set_product_base_url(request: web.Request, product_name: ProductName) -> None: if ( not request.app[PRODUCTS_URL_MAPPING_APPKEY].get(product_name) and request.url.host diff --git a/services/web/server/src/simcore_service_webserver/products/products_web.py b/services/web/server/src/simcore_service_webserver/products/products_web.py index 35efe801d46..f624cb352f6 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_web.py +++ b/services/web/server/src/simcore_service_webserver/products/products_web.py @@ -5,7 +5,7 @@ get_product_name, get_product_template_path, is_user_in_product_support_group, - set_product_url, + set_product_base_url, ) __all__: tuple[str, ...] = ( @@ -16,6 +16,6 @@ "get_product_name", "get_product_template_path", "is_user_in_product_support_group", - "set_product_url", + "set_product_base_url", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index bba0c27e941..f419b19ed91 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -149,7 +149,7 @@ async def connect( extra=get_log_record_extra(user_id=user_id), ) - products_web.set_product_url(_request, product_name) + products_web.set_product_base_url(_request, product_name) await _set_user_in_group_rooms(app, user_id, socket_id) await _set_user_in_project_rooms(app, user_id, client_session_id, socket_id) From f543b97ab723050e06071597ec31ad576ea25f9d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 16:45:34 +0200 Subject: [PATCH 22/37] fix --- .../_process_chatbot_trigger_service.py | 18 +---------- .../_conversations_messages_rest.py | 2 -- .../_conversation_message_service.py | 8 ++--- .../products/_models.py | 6 ++++ .../products/_service.py | 16 ++-------- .../products/_web_helpers.py | 32 ++++++++----------- .../socketio/_handlers.py | 2 -- 7 files changed, 27 insertions(+), 57 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index 92377fa41d1..fd16072b13d 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -11,8 +11,6 @@ from pydantic import TypeAdapter from servicelib.logging_utils import log_catch, log_context from servicelib.rabbitmq import RabbitMQClient -from simcore_service_webserver.products._models import ProductBaseUrl -from simcore_service_webserver.products.errors import ProductNotFoundError from ..conversations import conversations_service from ..products import products_service @@ -55,30 +53,16 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> for msg in messages[1]: _question_for_chatbot += f"{msg.content.strip()}\n" - # Prepare them and talk to the chatbot service + # Talk to the chatbot service chatbot_client = get_chatbot_rest_client(app) chat_response = await chatbot_client.ask_question(_question_for_chatbot) - # After I got respond, create a new support message in the conversation - try: - _product_base_url = products_service.get_product_base_url( - app, product_name=_product_name - ) - except ProductNotFoundError: - _logger.warning( - "Product %s does not have base URL configured, cannot process chatbot message.", - _product_name, - ) - _product_base_url = ProductBaseUrl(scheme="http", host="unknown") - await conversations_service.create_support_message( app=app, product_name=rabbit_message.conversation.product_name, user_id=_product.support_chatbot_user_id, conversation_user_type=ConversationUserType.CHATBOT_USER, conversation=rabbit_message.conversation, - product_url_scheme=_product_base_url.scheme, - product_url_host=_product_base_url.host or "unknown", content=chat_response.answer, type_=ConversationMessageType.MESSAGE, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py index 23a2faa48d8..777c165eb3f 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_controller/_conversations_messages_rest.py @@ -91,8 +91,6 @@ async def create_conversation_message(request: web.Request): user_id=req_ctx.user_id, conversation_user_type=conversation_user_type, conversation=_conversation, - product_url_scheme=request.url.scheme, - product_url_host=request.url.host or "unknown", content=body_params.content, type_=body_params.type, ) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index 679b3361698..ae043bc67ab 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -232,8 +232,6 @@ async def create_support_message( user_id: UserID, conversation_user_type: ConversationUserType, conversation: ConversationGetDB, - product_url_scheme: str, - product_url_host: str, # Creation attributes content: str, type_: ConversationMessageType, @@ -250,7 +248,9 @@ async def create_support_message( product = products_service.get_product(app, product_name=product_name) fogbugz_settings_or_none = app[APP_SETTINGS_APPKEY].WEBSERVER_FOGBUGZ - _conversation_url = f"{product_url_scheme}://{product_url_host}/#/conversation/{conversation.conversation_id}" + _conversation_url = ( + f"{product.base_url}#/conversation/{conversation.conversation_id}" + ) if ( product.support_standard_group_id is None @@ -278,7 +278,7 @@ async def create_support_message( user_id=user_id, message_content=message.content, conversation_url=_conversation_url, - host=product_url_host, + host=product.base_url.host or "unknown", product_support_assigned_fogbugz_project_id=product.support_assigned_fogbugz_project_id, fogbugz_url=str(fogbugz_settings_or_none.FOGBUGZ_URL), ) diff --git a/services/web/server/src/simcore_service_webserver/products/_models.py b/services/web/server/src/simcore_service_webserver/products/_models.py index 4d2522e7557..645fb2f3f8c 100644 --- a/services/web/server/src/simcore_service_webserver/products/_models.py +++ b/services/web/server/src/simcore_service_webserver/products/_models.py @@ -17,6 +17,7 @@ BeforeValidator, ConfigDict, Field, + HttpUrl, PositiveInt, field_serializer, field_validator, @@ -87,6 +88,11 @@ class Product(BaseModel): Field(description="Host regex"), ] + base_url: Annotated[ + HttpUrl, + Field(description="Product Base URL"), + ] + support_email: Annotated[ LowerCaseEmailStr, Field( diff --git a/services/web/server/src/simcore_service_webserver/products/_service.py b/services/web/server/src/simcore_service_webserver/products/_service.py index 99abca95adb..debdb47767e 100644 --- a/services/web/server/src/simcore_service_webserver/products/_service.py +++ b/services/web/server/src/simcore_service_webserver/products/_service.py @@ -8,8 +8,8 @@ from servicelib.exceptions import InvalidConfig from simcore_postgres_database.utils_products_prices import ProductPriceInfo -from ._application_keys import PRODUCTS_APPKEY, PRODUCTS_URL_MAPPING_APPKEY -from ._models import CreditResult, Product, ProductBaseUrl, ProductStripeInfo +from ._application_keys import PRODUCTS_APPKEY +from ._models import CreditResult, Product, ProductStripeInfo from ._repository import ProductRepository from .errors import ( BelowMinimumPaymentError, @@ -43,18 +43,6 @@ def get_product(app: web.Application, product_name: ProductName) -> Product: raise ProductNotFoundError(product_name=product_name) from exc -def get_product_base_url( - app: web.Application, product_name: ProductName -) -> ProductBaseUrl: - try: - product_base_url: ProductBaseUrl = app[PRODUCTS_URL_MAPPING_APPKEY][ - product_name - ] - return product_base_url - except KeyError as exc: - raise ProductNotFoundError(product_name=product_name) from exc - - def list_products(app: web.Application) -> list[Product]: products: list[Product] = list(app[PRODUCTS_APPKEY].values()) return products diff --git a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py index eed849f1313..a0f79946b76 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py @@ -12,10 +12,6 @@ from ..constants import RQ_PRODUCT_KEY from ..groups import api as groups_service from . import _service -from ._application_keys import ( - PRODUCTS_URL_MAPPING_APPKEY, -) -from ._models import ProductBaseUrl from ._web_events import PRODUCTS_TEMPLATES_DIR_APPKEY from .errors import ( FileTemplateNotFoundError, @@ -47,20 +43,20 @@ def get_current_product(request: web.Request) -> Product: return current_product -def set_product_base_url(request: web.Request, product_name: ProductName) -> None: - if ( - not request.app[PRODUCTS_URL_MAPPING_APPKEY].get(product_name) - and request.url.host - ): - request.app[PRODUCTS_URL_MAPPING_APPKEY][product_name] = ProductBaseUrl( - scheme=request.url.scheme, host=request.url.host - ) - _logger.debug( - "Set product url for %s to %s://%s", - product_name, - request.url.scheme, - request.url.host, - ) +# def set_product_base_url(request: web.Request, product_name: ProductName) -> None: +# if ( +# not request.app[PRODUCTS_URL_MAPPING_APPKEY].get(product_name) +# and request.url.host +# ): +# request.app[PRODUCTS_URL_MAPPING_APPKEY][product_name] = ProductBaseUrl( +# scheme=request.url.scheme, host=request.url.host +# ) +# _logger.debug( +# "Set product url for %s to %s://%s", +# product_name, +# request.url.scheme, +# request.url.host, +# ) async def is_user_in_product_support_group( diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index f419b19ed91..ebb3649b70f 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -149,8 +149,6 @@ async def connect( extra=get_log_record_extra(user_id=user_id), ) - products_web.set_product_base_url(_request, product_name) - await _set_user_in_group_rooms(app, user_id, socket_id) await _set_user_in_project_rooms(app, user_id, client_session_id, socket_id) From c449228d6648871224233dee8650f9d22584f070 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 16:53:20 +0200 Subject: [PATCH 23/37] fix --- ...3501db935_add_base_url_to_product_table.py | 32 +++++++++++++++++++ .../models/products.py | 6 ++++ .../products/_repository.py | 1 + 3 files changed, 39 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ff13501db935_add_base_url_to_product_table.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ff13501db935_add_base_url_to_product_table.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ff13501db935_add_base_url_to_product_table.py new file mode 100644 index 00000000000..d22df4c348d --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ff13501db935_add_base_url_to_product_table.py @@ -0,0 +1,32 @@ +"""Add base_url to product table + +Revision ID: ff13501db935 +Revises: e1c7e416461c +Create Date: 2025-10-17 14:48:02.509847+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ff13501db935" +down_revision = "e1c7e416461c" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("products", sa.Column("base_url", sa.String(), nullable=True)) + # ### end Alembic commands ### + + op.execute("UPDATE products SET base_url = 'http://CHANGE_ME.localhost'") + + op.alter_column("products", "base_url", existing_type=sa.String(), nullable=True) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("products", "base_url") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/products.py b/packages/postgres-database/src/simcore_postgres_database/models/products.py index bd105442800..105e0f43f29 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/products.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/products.py @@ -154,6 +154,12 @@ class ProductLoginSettingsDict(TypedDict, total=False): nullable=False, doc="Regular expression that matches product hostname from an url string", ), + sa.Column( + "base_url", + sa.String, + nullable=False, + doc="Product base URL (scheme + host), ex. https://osparc.io", + ), # EMAILS -------------------- sa.Column( "support_email", diff --git a/services/web/server/src/simcore_service_webserver/products/_repository.py b/services/web/server/src/simcore_service_webserver/products/_repository.py index d23dbca8a87..c2ad406d5a5 100644 --- a/services/web/server/src/simcore_service_webserver/products/_repository.py +++ b/services/web/server/src/simcore_service_webserver/products/_repository.py @@ -42,6 +42,7 @@ products.c.display_name, products.c.short_name, products.c.host_regex, + products.c.base_url, products.c.support_email, products.c.product_owners_email, products.c.twilio_messaging_sid, From 18beecfb7365b25f8883efa6bc4ffc0342cd0452 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 16:55:44 +0200 Subject: [PATCH 24/37] fix --- .../src/simcore_service_webserver/products/products_service.py | 2 -- .../src/simcore_service_webserver/products/products_web.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/products/products_service.py b/services/web/server/src/simcore_service_webserver/products/products_service.py index 77463fcdc76..1fbc880d7e9 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_service.py +++ b/services/web/server/src/simcore_service_webserver/products/products_service.py @@ -1,7 +1,6 @@ from ._service import ( get_credit_amount, get_product, - get_product_base_url, get_product_stripe_info, get_product_ui, is_product_billable, @@ -14,7 +13,6 @@ "get_product", "get_product_stripe_info", "get_product_ui", - "get_product_base_url", "is_product_billable", "list_products", "list_products_names", diff --git a/services/web/server/src/simcore_service_webserver/products/products_web.py b/services/web/server/src/simcore_service_webserver/products/products_web.py index f624cb352f6..8af3ec60985 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_web.py +++ b/services/web/server/src/simcore_service_webserver/products/products_web.py @@ -5,7 +5,6 @@ get_product_name, get_product_template_path, is_user_in_product_support_group, - set_product_base_url, ) __all__: tuple[str, ...] = ( @@ -16,6 +15,5 @@ "get_product_name", "get_product_template_path", "is_user_in_product_support_group", - "set_product_base_url", ) # nopycln: file From 95e47bdd2f9a4b16e7641ccf23e3024ae1bedb63 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 17 Oct 2025 17:30:03 +0200 Subject: [PATCH 25/37] fix --- .../pytest-simcore/src/pytest_simcore/helpers/faker_factories.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py index fe43da30f15..1d78bd25601 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py @@ -306,6 +306,7 @@ def random_product( "display_name": suffix.capitalize().replace("_", " "), "short_name": suffix[:4], "host_regex": r"[a-zA-Z0-9]+\.com", + "base_url": f"https://{suffix}.com", "support_email": f"support@{suffix}.io", "product_owners_email": fake.random_element( elements=[f"product-owners@{suffix}.io", None] From e93a8b876513f6fd55277f037162c304a97ac5b0 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 19 Oct 2025 21:08:17 +0200 Subject: [PATCH 26/37] adding unit tests --- .../src/models_library/conversations.py | 24 +++++- .../_process_chatbot_trigger_service.py | 4 +- .../unit/with_dbs/04/chatbot/__init__.py | 0 .../conftest.py} | 38 ++------- .../04/chatbot/test_chatbot_client.py | 34 ++++++++ .../test_process_chatbot_trigger_message.py | 82 +++++++++++++++++++ 6 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/chatbot/__init__.py rename services/web/server/tests/unit/with_dbs/04/{test_chatbot_client.py => chatbot/conftest.py} (54%) create mode 100644 services/web/server/tests/unit/with_dbs/04/chatbot/test_chatbot_client.py create mode 100644 services/web/server/tests/unit/with_dbs/04/chatbot/test_process_chatbot_trigger_message.py diff --git a/packages/models-library/src/models_library/conversations.py b/packages/models-library/src/models_library/conversations.py index 6856b3c2ce3..60ec0be59ff 100644 --- a/packages/models-library/src/models_library/conversations.py +++ b/packages/models-library/src/models_library/conversations.py @@ -61,7 +61,29 @@ class ConversationGetDB(BaseModel): modified: datetime last_message_created_at: datetime - model_config = ConfigDict(from_attributes=True) + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ + "examples": [ + # Support message + { + "conversation_id": "42838344-03de-4ce2-8d93-589a5dcdfd05", + "product_name": "osparc", + "name": "test_conversation", + "project_uuid": "42838344-03de-4ce2-8d93-589a5dcdfd05", + "user_group_id": "789", + "type": ConversationType.SUPPORT, + "extra_context": {}, + "fogbugz_case_id": None, + "is_read_by_user": False, + "is_read_by_support": False, + "created": "2024-01-01T12:00:00", + "modified": "2024-01-01T12:00:00", + "last_message_created_at": "2024-01-01T12:00:00", + } + ] + }, + ) class ConversationMessageGetDB(BaseModel): diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index fd16072b13d..c462e46477e 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -42,7 +42,7 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> return True # return true to avoid re-processing # Get last 20 messages for the conversation ID - messages = await conversations_service.list_messages_for_conversation( + _, messages = await conversations_service.list_messages_for_conversation( app=app, conversation_id=rabbit_message.conversation.conversation_id, offset=0, @@ -50,7 +50,7 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> order_by=OrderBy(field=IDStr("created"), direction=OrderDirection.DESC), ) _question_for_chatbot = "" - for msg in messages[1]: + for msg in messages: _question_for_chatbot += f"{msg.content.strip()}\n" # Talk to the chatbot service diff --git a/services/web/server/tests/unit/with_dbs/04/chatbot/__init__.py b/services/web/server/tests/unit/with_dbs/04/chatbot/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py b/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py similarity index 54% rename from services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py rename to services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py index 365ef932f6e..2166df7bd44 100644 --- a/services/web/server/tests/unit/with_dbs/04/test_chatbot_client.py +++ b/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py @@ -1,22 +1,12 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable -# pylint: disable=too-many-arguments -# pylint: disable=too-many-statements - from collections.abc import Iterator import httpx import pytest import respx -from aiohttp.test_utils import TestClient +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from simcore_service_webserver.chatbot._client import ( - ChatResponse, - get_chatbot_rest_client, -) -from simcore_service_webserver.chatbot.settings import ChatbotSettings +from simcore_service_webserver.products import products_service @pytest.fixture @@ -53,20 +43,10 @@ def mocked_chatbot_api() -> Iterator[respx.MockRouter]: yield mock -async def test_chatbot_client( - app_environment: EnvVarsDict, - client: TestClient, - mocked_chatbot_api: respx.MockRouter, -): - assert client.app - - settings = ChatbotSettings.create_from_envs() - assert settings.CHATBOT_HOST - assert settings.CHATBOT_PORT - - chatbot_client = get_chatbot_rest_client(client.app) - assert chatbot_client - - output = await chatbot_client.ask_question("What is the meaning of life?") - assert isinstance(output, ChatResponse) - assert output.answer == "42" +@pytest.fixture +def mocked_get_current_product(mocker: MockerFixture) -> MockType: + mock = mocker.patch.object(products_service, "get_product") + mocked_product = mocker.Mock() + mocked_product.support_chatbot_user_id = 123 + mock.return_value = mocked_product + return mock diff --git a/services/web/server/tests/unit/with_dbs/04/chatbot/test_chatbot_client.py b/services/web/server/tests/unit/with_dbs/04/chatbot/test_chatbot_client.py new file mode 100644 index 00000000000..faa2c64fa7d --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/chatbot/test_chatbot_client.py @@ -0,0 +1,34 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + + +import respx +from aiohttp.test_utils import TestClient +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_webserver.chatbot._client import ( + ChatResponse, + get_chatbot_rest_client, +) +from simcore_service_webserver.chatbot.settings import ChatbotSettings + + +async def test_chatbot_client( + app_environment: EnvVarsDict, + client: TestClient, + mocked_chatbot_api: respx.MockRouter, +): + assert client.app + + settings = ChatbotSettings.create_from_envs() + assert settings.CHATBOT_HOST + assert settings.CHATBOT_PORT + + chatbot_client = get_chatbot_rest_client(client.app) + assert chatbot_client + + output = await chatbot_client.ask_question("What is the meaning of life?") + assert isinstance(output, ChatResponse) + assert output.answer == "42" diff --git a/services/web/server/tests/unit/with_dbs/04/chatbot/test_process_chatbot_trigger_message.py b/services/web/server/tests/unit/with_dbs/04/chatbot/test_process_chatbot_trigger_message.py new file mode 100644 index 00000000000..f1d4a46a36e --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/chatbot/test_process_chatbot_trigger_message.py @@ -0,0 +1,82 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + + +import pytest +import respx +from aiohttp.test_utils import TestClient +from models_library.conversations import ConversationGetDB +from models_library.rabbitmq_messages import WebserverChatbotRabbitMessage +from pytest_mock import MockerFixture, MockType +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_webserver.chatbot._process_chatbot_trigger_service import ( + _process_chatbot_trigger_message, +) +from simcore_service_webserver.conversations import conversations_service + + +@pytest.fixture +def mocked_conversations_service(mocker: MockerFixture) -> dict: + # Mock message objects with content attribute + mock_message_1 = mocker.Mock() + mock_message_1.content = "Hello, I need help with my simulation" + + mock_message_2 = mocker.Mock() + mock_message_2.content = "It's not working properly" + + mock_messages = [mock_message_1, mock_message_2] + + # Mock list_messages_for_conversation + list_messages_mock = mocker.patch.object( + conversations_service, "list_messages_for_conversation" + ) + list_messages_mock.return_value = (len(mock_messages), mock_messages) + + # Mock create_support_message + create_message_mock = mocker.patch.object( + conversations_service, "create_support_message" + ) + + return { + "list_messages": list_messages_mock, + "create_message": create_message_mock, + "mock_messages": mock_messages, + } + + +async def test_process_chatbot_trigger_message( + app_environment: EnvVarsDict, + client: TestClient, + mocked_get_current_product: MockType, + mocked_chatbot_api: respx.MockRouter, + mocked_conversations_service: dict, +): + assert client.app + + # Prepare message to bytes for processing + _conversation = ConversationGetDB.model_config["json_schema_extra"]["examples"][0] + _message = WebserverChatbotRabbitMessage( + conversation=_conversation, + last_message_id="42838344-03de-4ce2-8d93-589a5dcdfd05", + ) + assert _message + + message_bytes = _message.model_dump_json().encode() + + # This is the function under test + await _process_chatbot_trigger_message(app=client.app, data=message_bytes) + + # Assert that the necessary service calls were made + mocked_conversations_service["list_messages"].assert_called_once() + + assert mocked_chatbot_api.calls.call_count == 1 + _last_request_content = mocked_chatbot_api.calls.last.request.content.decode( + "utf-8" + ) + assert "Hello, I need help with my simulation" in _last_request_content + assert "It's not working properly" in _last_request_content + + mocked_conversations_service["create_message"].assert_called_once() From 55bee352b789679b5ba08c0be630c288293f12d5 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 19 Oct 2025 21:18:34 +0200 Subject: [PATCH 27/37] fixes --- .../_conversation_message_service.py | 7 ++++--- .../products/_application_keys.py | 6 +----- .../products/_web_helpers.py | 16 ---------------- .../products/products_web.py | 3 +-- .../socketio/_handlers.py | 5 +++-- .../tests/unit/with_dbs/04/chatbot/conftest.py | 6 ++++++ 6 files changed, 15 insertions(+), 28 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py index ae043bc67ab..83bf3ef80d2 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_service.py @@ -22,9 +22,9 @@ from models_library.rest_pagination import PageTotalCount from models_library.users import UserID from servicelib.redis import exclusive -from simcore_service_webserver.application_keys import APP_SETTINGS_APPKEY -from simcore_service_webserver.groups import api as group_service +from ..application_keys import APP_SETTINGS_APPKEY +from ..groups import api as group_service from ..products import products_service from ..rabbitmq import get_rabbitmq_client from ..redis import get_redis_lock_manager_client_sdk @@ -39,6 +39,7 @@ notify_conversation_message_deleted, notify_conversation_message_updated, ) +from .errors import ConversationError _logger = logging.getLogger(__name__) @@ -192,7 +193,7 @@ async def _create_support_message_and_check_if_it_is_first_message() -> ( _is_read_by_support = False case _: msg = f"Unknown conversation user type: {conversation_user_type}" - raise ValueError(msg) + raise ConversationError(msg) await _conversation_repository.update( app, diff --git a/services/web/server/src/simcore_service_webserver/products/_application_keys.py b/services/web/server/src/simcore_service_webserver/products/_application_keys.py index af2067a1da7..febc0053f64 100644 --- a/services/web/server/src/simcore_service_webserver/products/_application_keys.py +++ b/services/web/server/src/simcore_service_webserver/products/_application_keys.py @@ -5,12 +5,8 @@ from aiohttp import web from models_library.products import ProductName -from ._models import Product, ProductBaseUrl +from ._models import Product PRODUCTS_APPKEY: Final = web.AppKey("PRODUCTS_APPKEY", dict[ProductName, Product]) -PRODUCTS_URL_MAPPING_APPKEY: Final = web.AppKey( - "PRODUCTS_URL_MAPPING_APPKEY", dict[ProductName, ProductBaseUrl] -) - DEFAULT_PRODUCT_APPKEY: Final = web.AppKey("DEFAULT_PRODUCT_APPKEY", ProductName) diff --git a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py index a0f79946b76..370557fafee 100644 --- a/services/web/server/src/simcore_service_webserver/products/_web_helpers.py +++ b/services/web/server/src/simcore_service_webserver/products/_web_helpers.py @@ -43,22 +43,6 @@ def get_current_product(request: web.Request) -> Product: return current_product -# def set_product_base_url(request: web.Request, product_name: ProductName) -> None: -# if ( -# not request.app[PRODUCTS_URL_MAPPING_APPKEY].get(product_name) -# and request.url.host -# ): -# request.app[PRODUCTS_URL_MAPPING_APPKEY][product_name] = ProductBaseUrl( -# scheme=request.url.scheme, host=request.url.host -# ) -# _logger.debug( -# "Set product url for %s to %s://%s", -# product_name, -# request.url.scheme, -# request.url.host, -# ) - - async def is_user_in_product_support_group( request: web.Request, *, user_id: UserID ) -> bool: diff --git a/services/web/server/src/simcore_service_webserver/products/products_web.py b/services/web/server/src/simcore_service_webserver/products/products_web.py index 8af3ec60985..0cb372aacc3 100644 --- a/services/web/server/src/simcore_service_webserver/products/products_web.py +++ b/services/web/server/src/simcore_service_webserver/products/products_web.py @@ -1,4 +1,4 @@ -from ._application_keys import PRODUCTS_APPKEY, PRODUCTS_URL_MAPPING_APPKEY +from ._application_keys import PRODUCTS_APPKEY from ._web_helpers import ( get_current_product, get_current_product_credit_price_info, @@ -9,7 +9,6 @@ __all__: tuple[str, ...] = ( "PRODUCTS_APPKEY", - "PRODUCTS_URL_MAPPING_APPKEY", "get_current_product", "get_current_product_credit_price_info", "get_product_name", diff --git a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py index ebb3649b70f..c9ff4634148 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/socketio/_handlers.py @@ -140,8 +140,9 @@ async def connect( try: auth_user_handler = auth_user_factory(socket_id) - _request: web.Request = environ["aiohttp.request"] - user_id, product_name, client_session_id = await auth_user_handler(_request) + user_id, product_name, client_session_id = await auth_user_handler( + environ["aiohttp.request"] + ) _logger.info( "%s successfully connected with %s", f"{user_id=}", diff --git a/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py b/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py index 2166df7bd44..da21e144365 100644 --- a/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/chatbot/conftest.py @@ -1,3 +1,9 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + from collections.abc import Iterator import httpx From a777cf86aefb7736a23814dbb0ba908229597510 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 19 Oct 2025 21:29:33 +0200 Subject: [PATCH 28/37] fixes --- services/web/server/tests/unit/isolated/test_security_web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/web/server/tests/unit/isolated/test_security_web.py b/services/web/server/tests/unit/isolated/test_security_web.py index 166e0b02506..896f8d69e94 100644 --- a/services/web/server/tests/unit/isolated/test_security_web.py +++ b/services/web/server/tests/unit/isolated/test_security_web.py @@ -107,16 +107,19 @@ def app_products(expected_product_name: ProductName) -> OrderedDict[str, Product pp["tis"] = Product( name="tis", host_regex="tis", + base_url="https://tip.io", **column_defaults, ) pp["osparc"] = Product( name="osparc", host_regex="osparc", + base_url="https://osparc.io", **column_defaults, ) pp["s4l"] = Product( name="s4l", host_regex="s4l", + base_url="https://s4l.io", **column_defaults, ) From 030fb4e84217bc80c553034330433529b3f88aaa Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 19 Oct 2025 21:31:22 +0200 Subject: [PATCH 29/37] fixes --- packages/postgres-database/tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/postgres-database/tests/conftest.py b/packages/postgres-database/tests/conftest.py index 9016cff0d32..880a18d1fbf 100644 --- a/packages/postgres-database/tests/conftest.py +++ b/packages/postgres-database/tests/conftest.py @@ -354,7 +354,9 @@ async def _creator(product_name: str) -> Row: async with asyncpg_engine.begin() as connection: result = await connection.execute( sa.insert(products) - .values(name=product_name, host_regex=".*") + .values( + name=product_name, host_regex=".*", base_url="https://example.com" + ) .returning(sa.literal_column("*")) ) assert result From 8038645096e7e7a128d4ff1d15b29e0f66e9075a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 11:02:15 +0200 Subject: [PATCH 30/37] fixes --- .../server/src/simcore_service_webserver/products/_models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/products/_models.py b/services/web/server/src/simcore_service_webserver/products/_models.py index 645fb2f3f8c..8753d5a14f4 100644 --- a/services/web/server/src/simcore_service_webserver/products/_models.py +++ b/services/web/server/src/simcore_service_webserver/products/_models.py @@ -218,6 +218,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: # fake mandatory "name": "osparc", "host_regex": r"([\.-]{0,1}osparc[\.-])", + "base_url": "https://osparc.io", "twilio_messaging_sid": "1" * 34, "registration_email_template": "osparc_registration_email", "login_settings": { @@ -238,6 +239,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "display_name": "TI PT", "short_name": "TIPI", "host_regex": r"(^tis[\.-])|(^ti-solutions\.)|(^ti-plan\.)", + "base_url": "https://tip.io", "support_email": "support@foo.com", "manual_url": "https://foo.com", "issues_login_url": None, @@ -253,6 +255,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "display_name": "o²S²PARC FOO", "short_name": "osparcf", "host_regex": "([\\.-]{0,1}osparcf[\\.-])", + "base_url": "https://osparc.io", "support_email": "foo@osparcf.io", "vendor": { "url": "https://acme.com", From 3d580a3b8d76c74429b0e241338c9f3bbeeeade3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 11:04:20 +0200 Subject: [PATCH 31/37] fixes --- .../tests/test_utils_services_environments.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/postgres-database/tests/test_utils_services_environments.py b/packages/postgres-database/tests/test_utils_services_environments.py index 0a58704d3f8..a8fc5628e36 100644 --- a/packages/postgres-database/tests/test_utils_services_environments.py +++ b/packages/postgres-database/tests/test_utils_services_environments.py @@ -34,7 +34,9 @@ class ExpectedSecrets(NamedTuple): async def product_name(connection: SAConnection) -> str: a_product_name = "a_prod" await connection.execute( - products.insert().values(name=a_product_name, host_regex="") + products.insert().values( + name=a_product_name, host_regex="", base_url="http://example.com" + ) ) yield a_product_name await connection.execute(products.delete()) From f472fee5446a1c7fa7cae8b7f8a3a2aee8d4c630 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 11:06:22 +0200 Subject: [PATCH 32/37] fixes --- packages/postgres-database/tests/test_utils_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/postgres-database/tests/test_utils_services.py b/packages/postgres-database/tests/test_utils_services.py index 70b102fea70..7560b8f8071 100644 --- a/packages/postgres-database/tests/test_utils_services.py +++ b/packages/postgres-database/tests/test_utils_services.py @@ -151,6 +151,7 @@ def services_fixture(faker: Faker, pg_sa_engine: sa.engine.Engine) -> ServicesFi "display_name": "Product Osparc", "short_name": "osparc", "host_regex": r"^osparc.", + "base_url": "https://osparc.io", "priority": 0, } product_name = conn.execute( From d9df8ffc5cd5eda4c7bb219087db341ac0098a49 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 11:08:25 +0200 Subject: [PATCH 33/37] fixes --- .../tests/unit/with_dbs/02/test_projects_crud_handlers.py | 5 ++++- .../server/tests/unit/with_dbs/03/login/test_login_twofa.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 3fa5482fddf..3fd91466466 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -305,7 +305,10 @@ def s4l_products_db_name( with postgres_db.connect() as conn: conn.execute( products.insert().values( - name=s4l_product_name, host_regex="pytest", display_name="pytest" + name=s4l_product_name, + host_regex="pytest", + base_url="https://pytest.com", + display_name="pytest", ) ) diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_twofa.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_twofa.py index 832da7c5b9b..e934e2d0681 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_twofa.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_twofa.py @@ -405,6 +405,7 @@ async def test_send_email_code( name="osparc", display_name="The Foo Product", host_regex=re.compile(r".+"), + base_url="https://osparc.io", vendor={}, short_name="foo", support_email=support_email, From d00ed40be0f7b6002c64a438f1bbe72755e86399 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 11:40:37 +0200 Subject: [PATCH 34/37] fixes --- services/docker-compose.yml | 8 +++--- .../_process_chatbot_trigger_service.py | 25 ++++++++++++------- .../_conversation_message_repository.py | 10 ++++++-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 223277d9b1a..47f2eef98b5 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -877,7 +877,7 @@ services: WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} WEBSERVER_RPC_NAMESPACE: ${WB_API_WEBSERVER_HOST} WEBSERVER_STATICWEB: "null" - WEBSERVER_CONVERSATIONS: "false" # override *webserver_environment + WEBSERVER_CONVERSATIONS: "null" # override *webserver_environment WEBSERVER_CHATBOT: "null" # override *webserver_environment WEBSERVER_FOGBUGZ: "null" # override *webserver_environment @@ -922,7 +922,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "false" + WEBSERVER_CONVERSATIONS: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} @@ -1005,7 +1005,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_GC_CATALOG} WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "false" + WEBSERVER_CONVERSATIONS: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} @@ -1082,7 +1082,7 @@ services: WEBSERVER_ANNOUNCEMENTS: 0 WEBSERVER_CATALOG: "null" WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "false" + WEBSERVER_CONVERSATIONS: "null" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: 0 WEBSERVER_DIRECTOR_V2: "null" diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index c462e46477e..78314237c78 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -13,6 +13,7 @@ from servicelib.rabbitmq import RabbitMQClient from ..conversations import conversations_service +from ..conversations.errors import ConversationErrorNotFoundError from ..products import products_service from ..rabbitmq import get_rabbitmq_client from .chatbot_service import get_chatbot_rest_client @@ -57,15 +58,21 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> chatbot_client = get_chatbot_rest_client(app) chat_response = await chatbot_client.ask_question(_question_for_chatbot) - await conversations_service.create_support_message( - app=app, - product_name=rabbit_message.conversation.product_name, - user_id=_product.support_chatbot_user_id, - conversation_user_type=ConversationUserType.CHATBOT_USER, - conversation=rabbit_message.conversation, - content=chat_response.answer, - type_=ConversationMessageType.MESSAGE, - ) + try: + await conversations_service.create_support_message( + app=app, + product_name=rabbit_message.conversation.product_name, + user_id=_product.support_chatbot_user_id, + conversation_user_type=ConversationUserType.CHATBOT_USER, + conversation=rabbit_message.conversation, + content=chat_response.answer, + type_=ConversationMessageType.MESSAGE, + ) + except ConversationErrorNotFoundError: + _logger.debug( + "Can not create a support message as conversation %s was not found", + rabbit_message.conversation.conversation_id, + ) return True diff --git a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_repository.py b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_repository.py index dc43f38c74f..40e8537fce2 100644 --- a/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_repository.py +++ b/services/web/server/src/simcore_service_webserver/conversations/_conversation_message_repository.py @@ -24,7 +24,10 @@ from sqlalchemy.sql import select from ..db.plugin import get_asyncpg_engine -from .errors import ConversationMessageErrorNotFoundError +from .errors import ( + ConversationErrorNotFoundError, + ConversationMessageErrorNotFoundError, +) _logger = logging.getLogger(__name__) @@ -56,7 +59,10 @@ async def create( ) .returning(*_SELECTION_ARGS) ) - row = result.one() + row = result.one_or_none() + if row is None: + raise ConversationErrorNotFoundError(conversation_id=conversation_id) + return ConversationMessageGetDB.model_validate(row) From e69e74a175881129731d04f7d9419ddfef9bd023 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 12:15:19 +0200 Subject: [PATCH 35/37] fixes --- services/docker-compose.yml | 8 ++++---- .../chatbot/_process_chatbot_trigger_service.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 47f2eef98b5..223277d9b1a 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -877,7 +877,7 @@ services: WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} WEBSERVER_RPC_NAMESPACE: ${WB_API_WEBSERVER_HOST} WEBSERVER_STATICWEB: "null" - WEBSERVER_CONVERSATIONS: "null" # override *webserver_environment + WEBSERVER_CONVERSATIONS: "false" # override *webserver_environment WEBSERVER_CHATBOT: "null" # override *webserver_environment WEBSERVER_FOGBUGZ: "null" # override *webserver_environment @@ -922,7 +922,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} @@ -1005,7 +1005,7 @@ services: WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} WEBSERVER_CATALOG: ${WB_GC_CATALOG} WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} @@ -1082,7 +1082,7 @@ services: WEBSERVER_ANNOUNCEMENTS: 0 WEBSERVER_CATALOG: "null" WEBSERVER_CHATBOT: "null" - WEBSERVER_CONVERSATIONS: "null" + WEBSERVER_CONVERSATIONS: "false" WEBSERVER_CELERY: "null" WEBSERVER_DB_LISTENER: 0 WEBSERVER_DIRECTOR_V2: "null" diff --git a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py index 78314237c78..0d17c69ebe0 100644 --- a/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py +++ b/services/web/server/src/simcore_service_webserver/chatbot/_process_chatbot_trigger_service.py @@ -51,8 +51,16 @@ async def _process_chatbot_trigger_message(app: web.Application, data: bytes) -> order_by=OrderBy(field=IDStr("created"), direction=OrderDirection.DESC), ) _question_for_chatbot = "" - for msg in messages: - _question_for_chatbot += f"{msg.content.strip()}\n" + for inx, msg in enumerate(messages): + if inx == 0: + # Make last message stand out as the question + _question_for_chatbot += ( + "User last message: \n" + f"{msg.content.strip()} \n\n" + "Previous messages in the conversation: \n" + ) + else: + _question_for_chatbot += f"{msg.content.strip()}\n" # Talk to the chatbot service chatbot_client = get_chatbot_rest_client(app) From b0c54fa28940c0d1f350b7d7ca71a480a8328fe3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 15:31:48 +0200 Subject: [PATCH 36/37] fixes --- .../tests/unit/with_dbs/04/products/test_products_repository.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/04/products/test_products_repository.py b/services/web/server/tests/unit/with_dbs/04/products/test_products_repository.py index ed4550eee6d..06d095c4e0b 100644 --- a/services/web/server/tests/unit/with_dbs/04/products/test_products_repository.py +++ b/services/web/server/tests/unit/with_dbs/04/products/test_products_repository.py @@ -46,6 +46,7 @@ def products_raw_data() -> dict[ProductName, dict[str, Any]]: "display_name": "COMPLETE example", "short_name": "dummy", "host_regex": r"([\.-]{0,1}dummy[\.-])", + "base_url": "http://example.com", "support_email": "foo@osparc.io", "twilio_messaging_sid": None, "vendor": Vendor( @@ -83,6 +84,7 @@ def products_raw_data() -> dict[ProductName, dict[str, Any]]: "display_name": "MINIMAL example", "short_name": "dummy", "host_regex": "([\\.-]{0,1}osparc[\\.-])", + "base_url": "http://example.com", "support_email": "support@osparc.io", } From 1c561e1a25a5f597304c0181e0a4b2381baba9ca Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 20 Oct 2025 15:33:08 +0200 Subject: [PATCH 37/37] fixes --- .../postgres-database/tests/products/test_models_products.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/postgres-database/tests/products/test_models_products.py b/packages/postgres-database/tests/products/test_models_products.py index 1f34fab7aa4..4bab5fb0c43 100644 --- a/packages/postgres-database/tests/products/test_models_products.py +++ b/packages/postgres-database/tests/products/test_models_products.py @@ -67,11 +67,13 @@ async def test_jinja2_templates_table( { "name": "osparc", "host_regex": r"^osparc.", + "base_url": "https://osparc.io", "registration_email_template": registration_email_template, }, { "name": "s4l", "host_regex": r"(^s4l[\.-])|(^sim4life\.)", + "base_url": "https://sim4life.info", "short_name": "s4l web", "registration_email_template": registration_email_template, }, @@ -79,6 +81,7 @@ async def test_jinja2_templates_table( "name": "tis", "short_name": "TIP", "host_regex": r"(^ti.[\.-])|(^ti-solution\.)", + "base_url": "https://tis.com", }, ]: # aiopg doesn't support executemany!! @@ -133,6 +136,7 @@ async def test_insert_select_product( "display_name": "o²S²PARC", "short_name": "osparc", "host_regex": r"([\.-]{0,1}osparc[\.-])", + "base_url": "https://osparc.io", "support_email": "foo@osparc.io", "twilio_messaging_sid": None, "vendor": Vendor(