diff --git a/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py b/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py index bfcf16b84378..9e5056f67bc5 100644 --- a/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py +++ b/packages/service-library/src/servicelib/aiohttp/db_asyncpg_engine.py @@ -38,23 +38,22 @@ def get_async_engine(app: web.Application) -> AsyncEngine: return engine -async def connect_to_db(app: web.Application, settings: PostgresSettings) -> None: +async def connect_to_db( + app: web.Application, settings: PostgresSettings, application_name: str +) -> None: """ - db services up, data migrated and ready to use - sets an engine in app state (use `get_async_engine(app)` to retrieve) """ - if settings.POSTGRES_CLIENT_NAME: - settings = settings.model_copy( - update={"POSTGRES_CLIENT_NAME": settings.POSTGRES_CLIENT_NAME + "-asyncpg"} - ) - with log_context( _logger, logging.INFO, "Connecting app[APP_DB_ASYNC_ENGINE_KEY] to postgres with %s", f"{settings=}", ): - engine = await create_async_engine_and_database_ready(settings) + engine = await create_async_engine_and_database_ready( + settings, application_name + ) _set_async_engine_to_app_state(app, engine) _logger.info( diff --git a/packages/service-library/src/servicelib/db_asyncpg_utils.py b/packages/service-library/src/servicelib/db_asyncpg_utils.py index ead458f39011..0b35b3227238 100644 --- a/packages/service-library/src/servicelib/db_asyncpg_utils.py +++ b/packages/service-library/src/servicelib/db_asyncpg_utils.py @@ -18,7 +18,7 @@ @retry(**PostgresRetryPolicyUponInitialization(_logger).kwargs) async def create_async_engine_and_database_ready( - settings: PostgresSettings, + settings: PostgresSettings, application_name: str ) -> AsyncEngine: """ - creates asyncio engine @@ -31,15 +31,11 @@ async def create_async_engine_and_database_ready( ) server_settings = { - "jit": "off" - } # see https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#disabling-the-postgresql-jit-to-improve-enum-datatype-handling - if settings.POSTGRES_CLIENT_NAME: - assert isinstance(settings.POSTGRES_CLIENT_NAME, str) # nosec - server_settings.update( - { - "application_name": settings.POSTGRES_CLIENT_NAME, - } - ) + "jit": "off", + "application_name": settings.client_name( + f"{application_name}", suffix="asyncpg" + ), + } engine = create_async_engine( settings.dsn_with_async_sqlalchemy, @@ -75,7 +71,7 @@ async def check_postgres_liveness(engine: AsyncEngine) -> LivenessResult: @contextlib.asynccontextmanager async def with_async_pg_engine( - settings: PostgresSettings, + settings: PostgresSettings, *, application_name: str ) -> AsyncIterator[AsyncEngine]: """ Creates an asyncpg engine and ensures it is properly closed after use. @@ -86,9 +82,11 @@ async def with_async_pg_engine( logging.DEBUG, f"connection to db {settings.dsn_with_async_sqlalchemy}", ): - server_settings = None - if settings.POSTGRES_CLIENT_NAME: - assert isinstance(settings.POSTGRES_CLIENT_NAME, str) + server_settings = { + "application_name": settings.client_name( + application_name, suffix="asyncpg" + ), + } engine = create_async_engine( settings.dsn_with_async_sqlalchemy, diff --git a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py index 8f472dc9b518..c089b81e034a 100644 --- a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py +++ b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py @@ -14,7 +14,9 @@ _logger = logging.getLogger(__name__) -async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: +async def connect_to_db( + app: FastAPI, settings: PostgresSettings, application_name: str +) -> None: warnings.warn( "The 'connect_to_db' function is deprecated and will be removed in a future release. " "Please use 'postgres_lifespan' instead for managing the database connection lifecycle.", @@ -27,7 +29,9 @@ async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: logging.DEBUG, f"Connecting and migraging {settings.dsn_with_async_sqlalchemy}", ): - engine = await create_async_engine_and_database_ready(settings) + engine = await create_async_engine_and_database_ready( + settings, application_name + ) app.state.engine = engine _logger.debug( diff --git a/packages/service-library/src/servicelib/fastapi/postgres_lifespan.py b/packages/service-library/src/servicelib/fastapi/postgres_lifespan.py index 319a7121896a..e532d4a435fc 100644 --- a/packages/service-library/src/servicelib/fastapi/postgres_lifespan.py +++ b/packages/service-library/src/servicelib/fastapi/postgres_lifespan.py @@ -28,7 +28,9 @@ def create_postgres_database_input_state(settings: PostgresSettings) -> State: return {PostgresLifespanState.POSTGRES_SETTINGS: settings} -async def postgres_database_lifespan(_: FastAPI, state: State) -> AsyncIterator[State]: +async def postgres_database_lifespan( + app: FastAPI, state: State +) -> AsyncIterator[State]: _lifespan_name = f"{__name__}.{postgres_database_lifespan.__name__}" @@ -43,7 +45,7 @@ async def postgres_database_lifespan(_: FastAPI, state: State) -> AsyncIterator[ # connect to database async_engine: AsyncEngine = await create_async_engine_and_database_ready( - settings + settings, app.title ) try: diff --git a/packages/service-library/tests/fastapi/test_postgres_lifespan.py b/packages/service-library/tests/fastapi/test_postgres_lifespan.py index 0c656c371876..07cc3077df1d 100644 --- a/packages/service-library/tests/fastapi/test_postgres_lifespan.py +++ b/packages/service-library/tests/fastapi/test_postgres_lifespan.py @@ -83,7 +83,6 @@ async def test_lifespan_postgres_database_in_an_app( mock_create_async_engine_and_database_ready: MockType, app_lifespan: LifespanManager, ): - app = FastAPI(lifespan=app_lifespan) async with ASGILifespanManager( @@ -93,7 +92,7 @@ async def test_lifespan_postgres_database_in_an_app( ) as asgi_manager: # Verify that the async engine was created mock_create_async_engine_and_database_ready.assert_called_once_with( - app.state.settings.CATALOG_POSTGRES + app.state.settings.CATALOG_POSTGRES, app.title ) # Verify that the async engine is in the lifespan manager state diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index 5777fdb275be..325a3288414e 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -82,19 +82,19 @@ def dsn_with_async_sqlalchemy(self) -> str: ) return f"{url}" - @cached_property - def dsn_with_query(self) -> str: + def dsn_with_query(self, application_name: str, *, suffix: str | None) -> str: """Some clients do not support queries in the dsn""" dsn = self.dsn - return self._update_query(dsn) + return self._update_query(dsn, application_name, suffix=suffix) + + def client_name(self, application_name: str, *, suffix: str | None) -> str: + return f"{application_name}{'-' if self.POSTGRES_CLIENT_NAME else ''}{self.POSTGRES_CLIENT_NAME or ''}{'-' + suffix if suffix else ''}" - def _update_query(self, uri: str) -> str: + def _update_query(self, uri: str, application_name: str, suffix: str | None) -> str: # SEE https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS - new_params: dict[str, str] = {} - if self.POSTGRES_CLIENT_NAME: - new_params = { - "application_name": self.POSTGRES_CLIENT_NAME, - } + new_params: dict[str, str] = { + "application_name": self.client_name(application_name, suffix=suffix), + } if new_params: parsed_uri = urlparse(uri) diff --git a/packages/settings-library/tests/test_postgres.py b/packages/settings-library/tests/test_postgres.py index 6c9067c2d6b1..bdc33901f6eb 100644 --- a/packages/settings-library/tests/test_postgres.py +++ b/packages/settings-library/tests/test_postgres.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import pytest +from faker import Faker from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.postgres import PostgresSettings @@ -24,7 +25,6 @@ def mock_environment(mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPa def test_cached_property_dsn(mock_environment: EnvVarsDict): - settings = PostgresSettings.create_from_envs() # all are upper-case @@ -36,14 +36,17 @@ def test_cached_property_dsn(mock_environment: EnvVarsDict): assert "dsn" not in settings.model_dump() -def test_dsn_with_query(mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): +def test_dsn_with_query( + mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker +): settings = PostgresSettings() assert settings.POSTGRES_CLIENT_NAME assert settings.dsn == "postgresql://foo:secret@localhost:5432/foodb" + app_name = faker.pystr() assert ( - settings.dsn_with_query - == "postgresql://foo:secret@localhost:5432/foodb?application_name=Some+%2643+funky+name" + settings.dsn_with_query(app_name, suffix="my-suffix") + == f"postgresql://foo:secret@localhost:5432/foodb?application_name={app_name}-Some+%2643+funky+name-my-suffix" ) with monkeypatch.context() as patch: @@ -51,7 +54,9 @@ def test_dsn_with_query(mock_environment: EnvVarsDict, monkeypatch: pytest.Monke settings = PostgresSettings() assert not settings.POSTGRES_CLIENT_NAME - assert settings.dsn == settings.dsn_with_query + assert f"{settings.dsn}?application_name=blah" == settings.dsn_with_query( + "blah", suffix=None + ) def test_dsn_with_async_sqlalchemy_has_query( diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py index db552f193b72..679c9645aea2 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py @@ -169,7 +169,7 @@ async def _state_metadata_entry_exists( async def _delete_legacy_archive( - project_id: ProjectID, node_uuid: NodeID, path: Path + project_id: ProjectID, node_uuid: NodeID, path: Path, *, application_name: str ) -> None: """removes the .zip state archive from storage""" s3_object = __create_s3_object_key( @@ -180,13 +180,15 @@ async def _delete_legacy_archive( # NOTE: if service is opened by a person which the users shared it with, # they will not have the permission to delete the node # Removing it via it's owner allows to always have access to the delete operation. - owner_id = await DBManager().get_project_owner_user_id(project_id) + owner_id = await DBManager( + application_name=application_name + ).get_project_owner_user_id(project_id) await filemanager.delete_file( user_id=owner_id, store_id=SIMCORE_LOCATION, s3_object=s3_object ) -async def push( +async def push( # pylint: disable=too-many-arguments user_id: UserID, project_id: ProjectID, node_uuid: NodeID, @@ -198,6 +200,7 @@ async def push( progress_bar: ProgressBarData, aws_s3_cli_settings: AwsS3CliSettings | None, legacy_state: LegacyState | None, + application_name: str, ) -> None: """pushes and removes the legacy archive if present""" @@ -226,6 +229,7 @@ async def push( project_id=project_id, node_uuid=node_uuid, path=source_path, + application_name=application_name, ) if legacy_state: @@ -244,6 +248,7 @@ async def push( project_id=project_id, node_uuid=node_uuid, path=legacy_state.old_state_path, + application_name=application_name, ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/dbmanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/dbmanager.py index 76d6d8222d31..21c0f0173b91 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/dbmanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/dbmanager.py @@ -82,22 +82,25 @@ async def _update_comp_run_snapshot_tasks_if_computational( class DBContextManager: - def __init__(self, db_engine: AsyncEngine | None = None) -> None: + def __init__( + self, db_engine: AsyncEngine | None = None, *, application_name: str + ) -> None: self._db_engine: AsyncEngine | None = db_engine self._db_engine_created: bool = False + self._application_name: str = application_name @staticmethod - async def _create_db_engine() -> AsyncEngine: + async def _create_db_engine(application_name: str) -> AsyncEngine: settings = NodePortsSettings.create_from_envs() engine = await create_async_engine_and_database_ready( - settings.POSTGRES_SETTINGS + settings.POSTGRES_SETTINGS, f"{application_name}-simcore-sdk" ) assert isinstance(engine, AsyncEngine) # nosec return engine async def __aenter__(self) -> AsyncEngine: if not self._db_engine: - self._db_engine = await self._create_db_engine() + self._db_engine = await self._create_db_engine(self._application_name) self._db_engine_created = True return self._db_engine @@ -107,8 +110,9 @@ async def __aexit__(self, exc_type, exc, tb) -> None: class DBManager: - def __init__(self, db_engine: AsyncEngine | None = None): + def __init__(self, db_engine: AsyncEngine | None = None, *, application_name: str): self._db_engine = db_engine + self._application_name = application_name async def write_ports_configuration( self, @@ -124,7 +128,9 @@ async def write_ports_configuration( node_configuration = json_loads(json_configuration) async with ( - DBContextManager(self._db_engine) as engine, + DBContextManager( + self._db_engine, application_name=self._application_name + ) as engine, engine.begin() as connection, ): # 1. Update comp_tasks table @@ -154,7 +160,9 @@ async def get_ports_configuration_from_node_uuid( "Getting ports configuration of node %s from comp_tasks table", node_uuid ) async with ( - DBContextManager(self._db_engine) as engine, + DBContextManager( + self._db_engine, application_name=self._application_name + ) as engine, engine.connect() as connection, ): node = await _get_node_from_db(project_id, node_uuid, connection) @@ -171,7 +179,9 @@ async def get_ports_configuration_from_node_uuid( async def get_project_owner_user_id(self, project_id: ProjectID) -> UserID: async with ( - DBContextManager(self._db_engine) as engine, + DBContextManager( + self._db_engine, application_name=self._application_name + ) as engine, engine.connect() as connection, ): prj_owner = await connection.scalar( diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py index 8874f98efe74..9da18133af8f 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -22,17 +22,11 @@ async def ports( project_id: ProjectIDStr, node_uuid: NodeIDStr, *, - db_manager: DBManager | None = None, + db_manager: DBManager, r_clone_settings: RCloneSettings | None = None, io_log_redirect_cb: LogRedirectCB | None = None, aws_s3_cli_settings: AwsS3CliSettings | None = None ) -> Nodeports: - log.debug("creating node_ports_v2 object using provided dbmanager: %s", db_manager) - # NOTE: warning every dbmanager create a new db engine! - if db_manager is None: # NOTE: keeps backwards compatibility - log.debug("no db manager provided, creating one...") - db_manager = DBManager() - return await load( db_manager=db_manager, user_id=user_id, diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index e419fbdba8f4..f8086084d748 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -10,6 +10,7 @@ import pytest import simcore_sdk +from faker import Faker from helpers.utils_port_v2 import CONSTANT_UUID from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.postgres_tools import PostgresTestConfig @@ -85,3 +86,8 @@ def constant_uuid4(mocker: MockerFixture) -> None: "simcore_sdk.node_ports_common.data_items_utils.uuid4", return_value=CONSTANT_UUID, ) + + +@pytest.fixture +def mock_app_name(faker: Faker) -> str: + return faker.pystr() diff --git a/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py b/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py index a25e95aa715f..f0ba46092397 100644 --- a/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py +++ b/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py @@ -292,6 +292,7 @@ async def test_delete_legacy_archive( project_id=project_id, node_uuid=node_uuid, path=content_path, + application_name=faker.pystr(), ) assert ( diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_common_dbmanager.py b/packages/simcore-sdk/tests/integration/test_node_ports_common_dbmanager.py index 4f53d8c775ae..cfd97db1c982 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_common_dbmanager.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_common_dbmanager.py @@ -21,8 +21,9 @@ async def test_db_manager_read_config( node_uuid: str, node_ports_config: None, default_configuration: dict, + mock_app_name: str, ): - db_manager = DBManager() + db_manager = DBManager(application_name=mock_app_name) ports_configuration_str = await db_manager.get_ports_configuration_from_node_uuid( project_id, node_uuid ) @@ -37,13 +38,14 @@ async def test_db_manager_write_config( node_ports_config: None, create_special_configuration: Callable[..., Awaitable[tuple[dict, str, str]]], default_configuration_file: Path, + mock_app_name: str, ): # create an empty config await create_special_configuration() # read the default config json_configuration = default_configuration_file.read_text() # write the default config to the database - db_manager = DBManager() + db_manager = DBManager(application_name=mock_app_name) await db_manager.write_ports_configuration( json_configuration, project_id, node_uuid ) diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py index 63a18114134b..9832f758a2ef 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py @@ -34,6 +34,7 @@ from servicelib.progress_bar import ProgressBarData from settings_library.r_clone import RCloneSettings from simcore_sdk import node_ports_v2 +from simcore_sdk.node_ports_common.dbmanager import DBManager from simcore_sdk.node_ports_common.exceptions import UnboundPortError from simcore_sdk.node_ports_v2 import exceptions from simcore_sdk.node_ports_v2.links import ItemConcreteValue, PortLink @@ -162,12 +163,18 @@ async def option_r_clone_settings( return None +@pytest.fixture +def default_db_manager(faker: Faker) -> DBManager: + return node_ports_v2.DBManager(application_name=f"pytest_{faker.pystr()}") + + async def test_default_configuration( user_id: int, project_id: str, node_uuid: NodeIDStr, default_configuration: dict[str, Any], option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): config_dict = default_configuration await check_config_valid( @@ -176,6 +183,7 @@ async def test_default_configuration( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ), config_dict, ) @@ -187,6 +195,7 @@ async def test_invalid_ports( node_uuid: NodeIDStr, create_special_configuration: Callable[..., Awaitable[tuple[dict, str, str]]], option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): config_dict, _, _ = await create_special_configuration() PORTS = await node_ports_v2.ports( @@ -194,6 +203,7 @@ async def test_invalid_ports( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) @@ -228,6 +238,7 @@ async def test_port_value_accessors( item_value: ItemConcreteValue, item_pytype: type, option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): # pylint: disable=W0613, W0621 item_key = TypeAdapter(ServicePortKey).validate_python("some_key") config_dict, _, _ = await create_special_configuration( @@ -240,6 +251,7 @@ async def test_port_value_accessors( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) @@ -279,6 +291,7 @@ async def test_port_file_accessors( option_r_clone_settings: RCloneSettings | None, request: pytest.FixtureRequest, constant_uuid4: None, + default_db_manager: DBManager, ): if item_value == "symlink_path": item_value = request.getfixturevalue("symlink_path") @@ -300,6 +313,7 @@ async def test_port_file_accessors( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) assert ( @@ -370,6 +384,7 @@ async def test_adding_new_ports( create_special_configuration: Callable[..., Awaitable[tuple[dict, str, str]]], postgres_db: sa.engine.Engine, option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): config_dict, project_id, node_uuid = await create_special_configuration() PORTS = await node_ports_v2.ports( @@ -377,6 +392,7 @@ async def test_adding_new_ports( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) @@ -421,6 +437,7 @@ async def test_removing_ports( create_special_configuration: Callable[..., Awaitable[tuple[dict, str, str]]], postgres_db: sa.engine.Engine, option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): config_dict, project_id, node_uuid = await create_special_configuration( inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], @@ -431,6 +448,7 @@ async def test_removing_ports( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) # let's remove the first input @@ -475,6 +493,7 @@ async def test_get_value_from_previous_node( item_value: ItemConcreteValue, item_pytype: type, option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ): config_dict, _, _ = await create_2nodes_configuration( prev_node_inputs=None, @@ -491,6 +510,7 @@ async def test_get_value_from_previous_node( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) @@ -526,6 +546,7 @@ async def test_get_file_from_previous_node( item_pytype: type, option_r_clone_settings: RCloneSettings | None, constant_uuid4: None, + default_db_manager: DBManager, ): config_dict, _, _ = await create_2nodes_configuration( prev_node_inputs=None, @@ -543,6 +564,7 @@ async def test_get_file_from_previous_node( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) file_path = await (await PORTS.inputs)[ @@ -585,6 +607,7 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( item_pytype: type, option_r_clone_settings: RCloneSettings | None, constant_uuid4: None, + default_db_manager: DBManager, ): config_dict, _, this_node_uuid = await create_2nodes_configuration( prev_node_inputs=None, @@ -600,6 +623,7 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) # add a filetokeymap @@ -649,6 +673,7 @@ async def test_file_mapping( option_r_clone_settings: RCloneSettings | None, create_valid_file_uuid: Callable[[str, Path], SimcoreS3FileID], constant_uuid4: None, + default_db_manager: DBManager, ): config_dict, project_id, node_uuid = await create_special_configuration( inputs=[("in_1", item_type, await create_store_link(item_value))], @@ -661,6 +686,7 @@ async def test_file_mapping( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) # add a filetokeymap @@ -740,6 +766,7 @@ async def test_regression_concurrent_port_update_fails( parallel_int_item_value: int, port_count: int, option_r_clone_settings: RCloneSettings | None, + default_db_manager: DBManager, ) -> None: """ when using `await PORTS.outputs` test will fail @@ -754,6 +781,7 @@ async def test_regression_concurrent_port_update_fails( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) @@ -831,6 +859,7 @@ async def test_batch_update_inputs_outputs( output_callbacks: _Callbacks, spy_outputs_callbaks: dict[str, AsyncMock], use_output_callbacks: bool, + default_db_manager: DBManager, ) -> None: callbacks = output_callbacks if use_output_callbacks else None @@ -845,6 +874,7 @@ async def test_batch_update_inputs_outputs( project_id=project_id, node_uuid=node_uuid, r_clone_settings=option_r_clone_settings, + db_manager=default_db_manager, ) await check_config_valid(PORTS, config_dict) diff --git a/packages/simcore-sdk/tests/unit/conftest.py b/packages/simcore-sdk/tests/unit/conftest.py index 34cd932081cf..527e02d10038 100644 --- a/packages/simcore-sdk/tests/unit/conftest.py +++ b/packages/simcore-sdk/tests/unit/conftest.py @@ -3,7 +3,7 @@ # pylint:disable=redefined-outer-name import json -from collections.abc import AsyncIterator, Callable +from collections.abc import Callable from random import randint from typing import Any from uuid import uuid4 @@ -32,7 +32,8 @@ async def mock_db_manager( monkeypatch, project_id: str, node_uuid: str, -) -> AsyncIterator[Callable]: + mock_app_name: str, +) -> Callable[[dict[str, Any]], DBManager]: def _mock_db_manager(port_cfg: dict[str, Any]) -> DBManager: async def mock_get_ports_configuration_from_node_uuid(*args, **kwargs) -> str: return json.dumps(port_cfg) @@ -55,7 +56,6 @@ async def mock_write_ports_configuration( mock_write_ports_configuration, ) - db_manager = DBManager() - return db_manager + return DBManager(application_name=mock_app_name) return _mock_db_manager diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py index 250f9d2599d4..abd91d7b5910 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_nodeports_v2.py @@ -3,8 +3,9 @@ # pylint:disable=redefined-outer-name # pylint:disable=protected-access +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any from unittest.mock import AsyncMock import pytest @@ -222,8 +223,7 @@ async def test_node_ports_v2_packages( node_uuid: str, ): db_manager = mock_db_manager(default_configuration) - node_ports = await ports(user_id, project_id, node_uuid) - node_ports = await ports(user_id, project_id, node_uuid, db_manager=db_manager) + await ports(user_id, project_id, node_uuid, db_manager=db_manager) @pytest.fixture diff --git a/services/api-server/src/simcore_service_api_server/clients/postgres.py b/services/api-server/src/simcore_service_api_server/clients/postgres.py index 4f337bd82d6d..0b8e9542e8d3 100644 --- a/services/api-server/src/simcore_service_api_server/clients/postgres.py +++ b/services/api-server/src/simcore_service_api_server/clients/postgres.py @@ -3,13 +3,13 @@ from servicelib.fastapi.lifespan_utils import LifespanOnStartupError from sqlalchemy.ext.asyncio import AsyncEngine +from .._meta import APP_NAME from ..core.settings import ApplicationSettings class PostgresNotConfiguredError(LifespanOnStartupError): msg_template = LifespanOnStartupError.msg_template + ( - "Postgres settings are not configured. " - "Please check your application settings. " + "Postgres settings are not configured. Please check your application settings. " ) @@ -30,7 +30,9 @@ async def _on_startup() -> None: settings=settings, ) - await connect_to_db(app, settings.API_SERVER_POSTGRES) + await connect_to_db( + app, settings.API_SERVER_POSTGRES, application_name=APP_NAME + ) assert app.state.engine # nosec assert isinstance(app.state.engine, AsyncEngine) # nosec diff --git a/services/api-server/src/simcore_service_api_server/services_http/solver_job_outputs.py b/services/api-server/src/simcore_service_api_server/services_http/solver_job_outputs.py index 4554bc0ccc35..f225747b4271 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/solver_job_outputs.py +++ b/services/api-server/src/simcore_service_api_server/services_http/solver_job_outputs.py @@ -1,13 +1,14 @@ import logging from typing import Any, TypeAlias -from models_library.projects import ProjectID, ProjectIDStr -from models_library.projects_nodes_io import BaseFileLink, NodeID, NodeIDStr +from models_library.projects import ProjectID +from models_library.projects_nodes_io import BaseFileLink, NodeID from pydantic import StrictBool, StrictFloat, StrictInt, TypeAdapter from simcore_sdk import node_ports_v2 from simcore_sdk.node_ports_v2 import DBManager, Nodeports from sqlalchemy.ext.asyncio import AsyncEngine +from .._meta import APP_NAME from ..exceptions.backend_errors import SolverOutputNotFoundError log = logging.getLogger(__name__) @@ -26,13 +27,13 @@ async def get_solver_output_results( """ # get the DB engine - db_manager = DBManager(db_engine=db_engine) + db_manager = DBManager(db_engine=db_engine, application_name=APP_NAME) try: solver: Nodeports = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(f"{project_uuid}"), - node_uuid=NodeIDStr(f"{node_uuid}"), + project_id=f"{project_uuid}", + node_uuid=f"{node_uuid}", db_manager=db_manager, ) solver_output_results: dict[str, Any] = {} diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/__init__.py b/services/director-v2/src/simcore_service_director_v2/modules/db/__init__.py index 34a955cdfc25..852042c299d6 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/__init__.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/__init__.py @@ -6,10 +6,12 @@ from servicelib.fastapi.db_asyncpg_engine import get_engine as get_db_engine from settings_library.postgres import PostgresSettings +from ..._meta import APP_NAME + def setup(app: FastAPI, settings: PostgresSettings) -> None: async def on_startup() -> None: - await connect_to_db(app, settings) + await connect_to_db(app, settings, application_name=APP_NAME) async def on_shutdown() -> None: await close_db_connection(app) diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index e170adf3e6ee..367e5302e0a2 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -22,7 +22,7 @@ from models_library.api_schemas_directorv2.services import NodeRequirements from models_library.docker import DockerLabelKey, StandardSimcoreDockerLabels from models_library.errors import ErrorDict -from models_library.projects import ProjectID, ProjectIDStr +from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKey, ServiceVersion from models_library.services_types import ServiceRunID @@ -40,6 +40,7 @@ from simcore_sdk.node_ports_v2.links import ItemValue as _NPItemValue from sqlalchemy.ext.asyncio import AsyncEngine +from .._meta import APP_NAME from ..constants import LOGS_FILE_NAME, UNDEFINED_API_BASE_URL, UNDEFINED_DOCKER_LABEL from ..core.errors import ( ComputationalBackendNotConnectedError, @@ -88,10 +89,10 @@ async def create_node_ports( :raises PortsValidationError: if any of the ports assigned values are invalid """ try: - db_manager = node_ports_v2.DBManager(db_engine) + db_manager = node_ports_v2.DBManager(db_engine, application_name=APP_NAME) return await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(f"{project_id}"), + project_id=f"{project_id}", node_uuid=TypeAdapter(NodeIDStr).validate_python(f"{node_id}"), db_manager=db_manager, ) diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index 52ee07ef3789..66d68768f059 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -34,7 +34,6 @@ NodesDict, ProjectAtDB, ProjectID, - ProjectIDStr, ) from models_library.projects_networks import ( PROJECT_NETWORK_PREFIX, @@ -70,6 +69,7 @@ from simcore_sdk.node_data import data_manager from simcore_sdk.node_ports_common.file_io_utils import LogRedirectCB from simcore_sdk.node_ports_v2 import DBManager, Nodeports, Port +from simcore_service_director_v2._meta import APP_NAME from simcore_service_director_v2.constants import DYNAMIC_SIDECAR_SERVICE_PREFIX from simcore_service_director_v2.core.dynamic_services_settings.sidecar import ( RCloneSettings, @@ -327,7 +327,7 @@ def workbench_dynamic_services( @pytest.fixture async def db_manager(sqlalchemy_async_engine: AsyncEngine) -> DBManager: - return DBManager(sqlalchemy_async_engine) + return DBManager(sqlalchemy_async_engine, application_name=APP_NAME) def _is_docker_r_clone_plugin_installed() -> bool: @@ -510,7 +510,7 @@ async def _get_mapped_nodeports_values( for node_uuid in workbench: PORTS: Nodeports = await node_ports_v2.ports( user_id=user_id, - project_id=ProjectIDStr(project_id), + project_id=project_id, node_uuid=TypeAdapter(NodeIDStr).validate_python(node_uuid), db_manager=db_manager, ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py index 2c9fb9d5c508..e825970ad840 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/_meta.py @@ -1,12 +1,12 @@ -""" Package Metadata - -""" +"""Package Metadata""" from importlib.metadata import distribution, version _current_distribution = distribution("simcore-service-dynamic-sidecar") + PROJECT_NAME: str = _current_distribution.metadata["Name"] +APP_NAME: str = _current_distribution.metadata["Name"] API_VERSION: str = version("simcore-service-dynamic-sidecar") MAJOR, MINOR, PATCH = API_VERSION.split(".") diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 46cf61b0ebd7..12a1b77a7259 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -17,7 +17,7 @@ ) from simcore_sdk.node_ports_common.exceptions import NodeNotFound -from .._meta import API_VERSION, API_VTAG, PROJECT_NAME, SUMMARY, __version__ +from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY, __version__ from ..api.rest import get_main_router from ..api.rpc.routes import setup_rpc_api_routes from ..models.schemas.application_health import ApplicationHealth @@ -135,7 +135,7 @@ def create_base_app() -> FastAPI: assert app_settings.SC_BOOT_MODE # nosec app = FastAPI( debug=app_settings.SC_BOOT_MODE.is_devel_mode(), - title=PROJECT_NAME, + title=APP_NAME, description=SUMMARY, version=API_VERSION, openapi_url=f"/api/{API_VTAG}/openapi.json", diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/database.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/database.py index a1ccfb9805c0..88f62dd6d649 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/database.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/database.py @@ -2,6 +2,7 @@ from servicelib.db_asyncpg_utils import check_postgres_liveness, with_async_pg_engine from settings_library.postgres import PostgresSettings +from .._meta import APP_NAME from ..core.settings import ApplicationSettings from .service_liveness import ( wait_for_service_liveness, @@ -17,7 +18,10 @@ async def wait_for_database_liveness(app: FastAPI) -> None: assert isinstance(app_settings, ApplicationSettings) # nosec postgres_settings = app_settings.POSTGRES_SETTINGS assert isinstance(postgres_settings, PostgresSettings) # nosec - async with with_async_pg_engine(postgres_settings) as engine: + async with with_async_pg_engine( + postgres_settings, + application_name=f"{APP_NAME}-{app_settings.DY_SIDECAR_NODE_ID}", + ) as engine: await wait_for_service_liveness( check_postgres_liveness, engine, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 2e326a11258d..254ef2968b32 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -23,6 +23,7 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential +from .._meta import APP_NAME from ..core.docker_compose_utils import ( docker_compose_create, docker_compose_down, @@ -168,17 +169,19 @@ async def task_create_service_containers( assert shared_store.compose_spec # nosec - async with event_propagation_disabled(app), _reset_on_error( - shared_store - ), ProgressBarData( - num_steps=4, - progress_report_cb=functools.partial( - post_progress_message, - app, - ProgressType.SERVICE_CONTAINERS_STARTING, - ), - description="starting software", - ) as progress_bar: + async with ( + event_propagation_disabled(app), + _reset_on_error(shared_store), + ProgressBarData( + num_steps=4, + progress_report_cb=functools.partial( + post_progress_message, + app, + ProgressType.SERVICE_CONTAINERS_STARTING, + ), + description="starting software", + ) as progress_bar, + ): with log_context(_logger, logging.INFO, "load user services preferences"): if user_services_preferences.is_feature_enabled(app): await user_services_preferences.load_user_services_preferences(app) @@ -433,6 +436,7 @@ async def _save_state_folder( progress_bar=progress_bar, aws_s3_cli_settings=settings.DY_SIDECAR_AWS_S3_CLI_SETTINGS, legacy_state=_get_legacy_state_with_dy_volumes_path(settings), + application_name=f"{APP_NAME}-{settings.DY_SIDECAR_NODE_ID}", ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py index 9d84890dcea0..a65d69eb4912 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py @@ -15,7 +15,6 @@ from aiofiles.os import remove from aiofiles.tempfile import TemporaryDirectory as AioTemporaryDirectory from common_library.json_serialization import json_loads -from models_library.projects import ProjectIDStr from models_library.projects_nodes_io import NodeIDStr from models_library.services_types import ServicePortKey from pydantic import ByteSize, TypeAdapter @@ -29,10 +28,11 @@ from simcore_sdk.node_ports_common.file_io_utils import LogRedirectCB from simcore_sdk.node_ports_v2 import Port from simcore_sdk.node_ports_v2.links import ItemConcreteValue -from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports, OutputsCallbacks +from simcore_sdk.node_ports_v2.nodeports_v2 import OutputsCallbacks from simcore_sdk.node_ports_v2.port import SetKWargs from simcore_sdk.node_ports_v2.port_utils import is_file_type +from .._meta import APP_NAME from ..core.settings import ApplicationSettings, get_settings from ..modules.notifications import PortNotifier @@ -98,15 +98,19 @@ async def upload_outputs( # pylint:disable=too-many-statements # noqa: PLR0915 start_time = time.perf_counter() settings: ApplicationSettings = get_settings() - PORTS: Nodeports = await node_ports_v2.ports( + db_manager = node_ports_v2.DBManager( + application_name=f"{APP_NAME}-{settings.DY_SIDECAR_NODE_ID}" + ) + ports = await node_ports_v2.ports( user_id=settings.DY_SIDECAR_USER_ID, - project_id=ProjectIDStr(settings.DY_SIDECAR_PROJECT_ID), + project_id=f"{settings.DY_SIDECAR_PROJECT_ID}", node_uuid=TypeAdapter(NodeIDStr).validate_python( f"{settings.DY_SIDECAR_NODE_ID}" ), r_clone_settings=None, io_log_redirect_cb=io_log_redirect_cb, aws_s3_cli_settings=None, + db_manager=db_manager, ) # let's gather the tasks @@ -116,7 +120,7 @@ async def upload_outputs( # pylint:disable=too-many-statements # noqa: PLR0915 archiving_tasks: deque[Coroutine[None, None, None]] = deque() ports_to_set: list[Port] = [ port_value - for port_value in (await PORTS.outputs).values() + for port_value in (await ports.outputs).values() if (not port_keys) or (port_value.key in port_keys) ] @@ -220,7 +224,7 @@ async def _archive_dir_notified( if archiving_tasks: await limited_gather(*archiving_tasks, limit=4) - await PORTS.set_multiple( + await ports.set_multiple( ports_values, progress_bar=sub_progress, outputs_callbacks=OutputCallbacksWrapper(port_notifier), @@ -327,21 +331,25 @@ async def download_target_ports( start_time = time.perf_counter() settings: ApplicationSettings = get_settings() - PORTS: Nodeports = await node_ports_v2.ports( + db_manager = node_ports_v2.DBManager( + application_name=f"{APP_NAME}-{settings.DY_SIDECAR_NODE_ID}" + ) + ports = await node_ports_v2.ports( user_id=settings.DY_SIDECAR_USER_ID, - project_id=ProjectIDStr(settings.DY_SIDECAR_PROJECT_ID), + project_id=f"{settings.DY_SIDECAR_PROJECT_ID}", node_uuid=TypeAdapter(NodeIDStr).validate_python( f"{settings.DY_SIDECAR_NODE_ID}" ), r_clone_settings=None, io_log_redirect_cb=io_log_redirect_cb, aws_s3_cli_settings=None, + db_manager=db_manager, ) # let's gather all the data ports_to_get: list[Port] = [ port_value - for port_value in (await getattr(PORTS, port_type_name.value)).values() + for port_value in (await getattr(ports, port_type_name.value)).values() if (not port_keys) or (port_value.key in port_keys) ] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py index 3942e23b1845..e2d566f470b8 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_db.py @@ -32,6 +32,7 @@ async def save_preferences( user_preferences_path: Path, user_id: UserID, product_name: ProductName, + application_name: str, ): preference_class = get_model_class(service_key) @@ -40,7 +41,10 @@ async def save_preferences( service_key=service_key, service_version=service_version, value=dir_content ) - async with DBContextManager() as engine, engine.begin() as conn: + async with ( + DBContextManager(application_name=application_name) as engine, + engine.begin() as conn, + ): await UserServicesUserPreferencesRepo.save( conn, user_id=user_id, @@ -58,10 +62,14 @@ async def load_preferences( user_preferences_path: Path, user_id: UserID, product_name: ProductName, + application_name: str, ) -> None: preference_class = get_model_class(service_key) - async with DBContextManager() as engine, engine.connect() as conn: + async with ( + DBContextManager(application_name=application_name) as engine, + engine.connect() as conn, + ): payload = await UserServicesUserPreferencesRepo.load( conn, user_id=user_id, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_manager.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_manager.py index 3c9ede49dbce..2f4b5a0641ee 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_manager.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_manager.py @@ -19,6 +19,7 @@ class UserServicesPreferencesManager: service_version: ServiceVersion user_id: UserID product_name: ProductName + application_name: str _preferences_already_saved: bool = False async def load_preferences(self) -> None: @@ -28,6 +29,7 @@ async def load_preferences(self) -> None: service_version=self.service_version, user_id=self.user_id, product_name=self.product_name, + application_name=self.application_name, ) async def save_preferences(self) -> None: @@ -41,6 +43,7 @@ async def save_preferences(self) -> None: service_version=self.service_version, user_id=self.user_id, product_name=self.product_name, + application_name=self.application_name, ) self._preferences_already_saved = True diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_setup.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_setup.py index 83915fc151ad..0457a58e1e7a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_setup.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/user_services_preferences/_setup.py @@ -3,6 +3,7 @@ from fastapi import FastAPI from servicelib.logging_utils import log_context +from ..._meta import APP_NAME from ...core.settings import ApplicationSettings from ._manager import UserServicesPreferencesManager from ._utils import is_feature_enabled @@ -33,6 +34,7 @@ async def on_startup() -> None: service_version=settings.DY_SIDECAR_SERVICE_VERSION, user_id=settings.DY_SIDECAR_USER_ID, product_name=settings.DY_SIDECAR_PRODUCT_NAME, + application_name=f"{APP_NAME}-{settings.DY_SIDECAR_NODE_ID}", ) ) else: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/db.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/db.py index f5d5970216e0..e11e0edbfd6f 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/db.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/db.py @@ -1,10 +1,14 @@ from fastapi import FastAPI from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db +from ..._meta import APP_NAME + def setup(app: FastAPI): async def on_startup() -> None: - await connect_to_db(app, app.state.settings.EFS_GUARDIAN_POSTGRES) + await connect_to_db( + app, app.state.settings.EFS_GUARDIAN_POSTGRES, application_name=APP_NAME + ) async def on_shutdown() -> None: await close_db_connection(app) diff --git a/services/payments/src/simcore_service_payments/services/postgres.py b/services/payments/src/simcore_service_payments/services/postgres.py index fd84fba45ce7..7fecbb038ec2 100644 --- a/services/payments/src/simcore_service_payments/services/postgres.py +++ b/services/payments/src/simcore_service_payments/services/postgres.py @@ -2,6 +2,7 @@ from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db from sqlalchemy.ext.asyncio import AsyncEngine +from .._meta import APP_NAME from ..core.settings import ApplicationSettings @@ -16,7 +17,7 @@ def setup_postgres(app: FastAPI): async def _on_startup() -> None: settings: ApplicationSettings = app.state.settings - await connect_to_db(app, settings.PAYMENTS_POSTGRES) + await connect_to_db(app, settings.PAYMENTS_POSTGRES, application_name=APP_NAME) assert app.state.engine # nosec assert isinstance(app.state.engine, AsyncEngine) # nosec diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py index 1ccd94f436e6..243ca6ad4bf1 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/__init__.py @@ -4,6 +4,8 @@ from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db from servicelib.logging_utils import log_context +from ...._meta import APP_NAME + _logger = logging.getLogger(__name__) @@ -14,7 +16,11 @@ async def on_startup() -> None: logging.INFO, msg="RUT startup DB", ): - await connect_to_db(app, app.state.settings.RESOURCE_USAGE_TRACKER_POSTGRES) + await connect_to_db( + app, + app.state.settings.RESOURCE_USAGE_TRACKER_POSTGRES, + application_name=APP_NAME, + ) async def on_shutdown() -> None: with log_context( diff --git a/services/storage/src/simcore_service_storage/modules/db/__init__.py b/services/storage/src/simcore_service_storage/modules/db/__init__.py index 4fb5dacb2a7c..1b29632d6ed3 100644 --- a/services/storage/src/simcore_service_storage/modules/db/__init__.py +++ b/services/storage/src/simcore_service_storage/modules/db/__init__.py @@ -6,6 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine from tenacity import retry +from ..._meta import APP_NAME from ...core.settings import get_application_settings _logger = logging.getLogger(__name__) @@ -16,7 +17,9 @@ def setup_db(app: FastAPI) -> None: async def _on_startup() -> None: app_settings = get_application_settings(app) assert app_settings.STORAGE_POSTGRES is not None # nosec - await connect_to_db(app, app_settings.STORAGE_POSTGRES) + await connect_to_db( + app, app_settings.STORAGE_POSTGRES, application_name=APP_NAME + ) async def _on_shutdown() -> None: await close_db_connection(app) diff --git a/services/web/server/src/simcore_service_webserver/db/_aiopg.py b/services/web/server/src/simcore_service_webserver/db/_aiopg.py index 9d9feea1f807..62eb2b1e9952 100644 --- a/services/web/server/src/simcore_service_webserver/db/_aiopg.py +++ b/services/web/server/src/simcore_service_webserver/db/_aiopg.py @@ -24,20 +24,23 @@ ) from tenacity import retry +from .._meta import APP_NAME from .settings import PostgresSettings, get_plugin_settings _logger = logging.getLogger(__name__) @retry(**PostgresRetryPolicyUponInitialization(_logger).kwargs) -async def _ensure_pg_ready(settings: PostgresSettings) -> Engine: - engine: Engine = await create_engine( +async def _ensure_pg_ready( + settings: PostgresSettings, *, application_name: str +) -> Engine: + engine = await create_engine( settings.dsn, - application_name=settings.POSTGRES_CLIENT_NAME, + application_name=settings.client_name(f"{application_name}", suffix="aiopg"), minsize=settings.POSTGRES_MINSIZE, maxsize=settings.POSTGRES_MAXSIZE, ) - + assert isinstance(engine, Engine) # nosec try: await raise_if_migration_not_ready(engine) except (DBMigrationError, DBAPIError): @@ -48,7 +51,6 @@ async def _ensure_pg_ready(settings: PostgresSettings) -> Engine: async def postgres_cleanup_ctx(app: web.Application) -> AsyncIterator[None]: - settings = get_plugin_settings(app) with log_context( @@ -57,7 +59,7 @@ async def postgres_cleanup_ctx(app: web.Application) -> AsyncIterator[None]: "Connecting app[APP_AIOPG_ENGINE_KEY] to postgres with %s", f"{settings=}", ): - aiopg_engine = await _ensure_pg_ready(settings) + aiopg_engine = await _ensure_pg_ready(settings, application_name=APP_NAME) app[APP_AIOPG_ENGINE_KEY] = aiopg_engine _logger.info( diff --git a/services/web/server/src/simcore_service_webserver/db/_asyncpg.py b/services/web/server/src/simcore_service_webserver/db/_asyncpg.py index 03bac23ea2c2..51434bc50c6d 100644 --- a/services/web/server/src/simcore_service_webserver/db/_asyncpg.py +++ b/services/web/server/src/simcore_service_webserver/db/_asyncpg.py @@ -15,6 +15,7 @@ ) from sqlalchemy.ext.asyncio import AsyncEngine +from .._meta import APP_NAME from .settings import PostgresSettings, get_plugin_settings _logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ async def postgres_cleanup_ctx(app: web.Application) -> AsyncIterator[None]: settings: PostgresSettings = get_plugin_settings(app) - await connect_to_db(app, settings) + await connect_to_db(app, settings, application_name=APP_NAME) assert get_async_engine(app) # nosec assert isinstance(get_async_engine(app), AsyncEngine) # nosec diff --git a/services/web/server/src/simcore_service_webserver/login/plugin.py b/services/web/server/src/simcore_service_webserver/login/plugin.py index ae32d360d900..79f700510e94 100644 --- a/services/web/server/src/simcore_service_webserver/login/plugin.py +++ b/services/web/server/src/simcore_service_webserver/login/plugin.py @@ -12,6 +12,7 @@ from settings_library.email import SMTPSettings from settings_library.postgres import PostgresSettings +from .._meta import APP_NAME from ..constants import ( APP_PUBLIC_CONFIG_PER_PRODUCT, APP_SETTINGS_KEY, @@ -56,7 +57,7 @@ async def _setup_login_storage_ctx(app: web.Application): settings: PostgresSettings = get_db_plugin_settings(app) async with asyncpg.create_pool( - dsn=settings.dsn_with_query, + dsn=settings.dsn_with_query(f"{APP_NAME}-login", suffix="asyncpg"), min_size=settings.POSTGRES_MINSIZE, max_size=settings.POSTGRES_MAXSIZE, loop=asyncio.get_event_loop(),