diff --git a/.env-devel b/.env-devel index b82ba685e9b6..0e27dfde6f1a 100644 --- a/.env-devel +++ b/.env-devel @@ -194,13 +194,14 @@ PAYMENTS_USERNAME=admin PAYMENTS_TRACING={} POSTGRES_DB=simcoredb -POSTGRES_ENDPOINT=postgres:5432 POSTGRES_HOST=postgres POSTGRES_PASSWORD=adminadmin POSTGRES_PORT=5432 POSTGRES_USER=scu -POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199 +POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 POSTGRES_READONLY_PASSWORD=readonly POSTGRES_READONLY_USER=postgres_readonly diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index 19d9247e8eab..7814d413c07c 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -250,7 +250,6 @@ def postgres_env_vars_dict(postgres_dsn: PostgresTestConfig) -> EnvVarsDict: "POSTGRES_DB": postgres_dsn["database"], "POSTGRES_HOST": postgres_dsn["host"], "POSTGRES_PORT": f"{postgres_dsn['port']}", - "POSTGRES_ENDPOINT": f"{postgres_dsn['host']}:{postgres_dsn['port']}", } diff --git a/packages/service-library/src/servicelib/db_asyncpg_utils.py b/packages/service-library/src/servicelib/db_asyncpg_utils.py index 0b35b3227238..d72f77d95159 100644 --- a/packages/service-library/src/servicelib/db_asyncpg_utils.py +++ b/packages/service-library/src/servicelib/db_asyncpg_utils.py @@ -39,8 +39,8 @@ async def create_async_engine_and_database_ready( engine = create_async_engine( settings.dsn_with_async_sqlalchemy, - pool_size=settings.POSTGRES_MINSIZE, - max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE, + pool_size=settings.POSTGRES_MAX_POOLSIZE, + max_overflow=settings.POSTGRES_MAX_OVERFLOW, connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released @@ -90,8 +90,8 @@ async def with_async_pg_engine( engine = create_async_engine( settings.dsn_with_async_sqlalchemy, - pool_size=settings.POSTGRES_MINSIZE, - max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE, + pool_size=settings.POSTGRES_MAX_POOLSIZE, + max_overflow=settings.POSTGRES_MAX_OVERFLOW, connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index 90d456cbda07..64276b7fdce4 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -5,6 +5,7 @@ from pydantic import ( AliasChoices, Field, + NonNegativeInt, PostgresDsn, SecretStr, model_validator, @@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings): # pool connection limits POSTGRES_MINSIZE: Annotated[ - int, Field(description="Minimum number of connections in the pool", ge=2) - ] = 2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199 + int, + Field( + description="Minimum number of connections in the pool that are always created and kept", + ge=1, + ), + ] = 1 POSTGRES_MAXSIZE: Annotated[ - int, Field(description="Maximum number of connections in the pool", ge=2) + int, + Field( + description="Maximum number of connections in the pool that are kept", + ge=1, + ), ] = 50 + POSTGRES_MAX_POOLSIZE: Annotated[ + int, + Field( + description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand" + ), + ] = 10 + POSTGRES_MAX_OVERFLOW: Annotated[ + NonNegativeInt, Field(description="Maximal overflow connections") + ] = 20 POSTGRES_CLIENT_NAME: Annotated[ str | None, @@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "POSTGRES_USER": "usr", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "db", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "my_app", # first-choice "HOST": "should be ignored", "HOST_NAME": "should be ignored", @@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra) - model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra) diff --git a/packages/settings-library/tests/data/.env-compact b/packages/settings-library/tests/data/.env-compact index a3e835483443..b11273684dc2 100644 --- a/packages/settings-library/tests/data/.env-compact +++ b/packages/settings-library/tests/data/.env-compact @@ -3,4 +3,4 @@ APP_HOST=localhost APP_PORT=80 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}' -APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' +APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' diff --git a/packages/settings-library/tests/data/.env-granular b/packages/settings-library/tests/data/.env-granular index 333452ef1957..c27099b7b98b 100644 --- a/packages/settings-library/tests/data/.env-granular +++ b/packages/settings-library/tests/data/.env-granular @@ -12,9 +12,9 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb -# Minimum number of connections in the pool -POSTGRES_MINSIZE=2 -# Maximum number of connections in the pool +POSTGRES_MINSIZE=1 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 POSTGRES_MAXSIZE=50 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-mixed b/packages/settings-library/tests/data/.env-mixed index 3001f92335b9..4aed86d3b9b7 100644 --- a/packages/settings-library/tests/data/.env-mixed +++ b/packages/settings-library/tests/data/.env-mixed @@ -12,9 +12,9 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb -# Minimum number of connections in the pool -POSTGRES_MINSIZE=2 -# Maximum number of connections in the pool +POSTGRES_MINSIZE=1 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 POSTGRES_MAXSIZE=50 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-sample b/packages/settings-library/tests/data/.env-sample index f66bdf71c2c3..d6f115f76c23 100644 --- a/packages/settings-library/tests/data/.env-sample +++ b/packages/settings-library/tests/data/.env-sample @@ -8,9 +8,9 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=secret # Database name POSTGRES_DB=foodb -# Maximum number of connections in the pool -POSTGRES_MINSIZE=2 -# Minimum number of connections in the pool +POSTGRES_MINSIZE=1 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 POSTGRES_MAXSIZE=50 # --- APP_MODULE_FIELD --- diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index a4212bb6648e..641d1df62a3d 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -55,8 +55,10 @@ class _FakePostgresSettings(BaseCustomSettings): POSTGRES_PASSWORD: str POSTGRES_DB: str - POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2 - POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50 + POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1 + POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50 + POSTGRES_MAX_POOLSIZE: int = 10 + POSTGRES_MAX_OVERFLOW: Annotated[int, Field(ge=0)] = 20 POSTGRES_CLIENT_NAME: Annotated[ str | None, @@ -126,7 +128,6 @@ class S5(BaseCustomSettings): def test_parse_from_empty_envs( postgres_envvars_unset: None, model_classes_factory: Callable ): - S1, S2, S3, S4, S5 = model_classes_factory() with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info: @@ -160,7 +161,6 @@ def test_parse_from_individual_envs( monkeypatch: pytest.MonkeyPatch, model_classes_factory: Callable, ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -200,7 +200,9 @@ def test_parse_from_individual_envs( "POSTGRES_PASSWORD": "shh", "POSTGRES_DB": "db", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -215,7 +217,9 @@ def test_parse_from_individual_envs( "POSTGRES_PASSWORD": "shh", "POSTGRES_DB": "db", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -228,7 +232,6 @@ def test_parse_from_individual_envs( def test_parse_compact_env( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -262,7 +265,9 @@ def test_parse_compact_env( "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -342,7 +347,6 @@ def test_parse_compact_env( def test_parse_from_mixed_envs( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -372,7 +376,9 @@ def test_parse_from_mixed_envs( "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -472,7 +478,6 @@ def test_parse_from_mixed_envs( def test_toggle_plugin_1( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - *_, S4, S5 = model_classes_factory() # empty environ @@ -535,7 +540,6 @@ def test_toggle_plugin_3( def test_toggle_plugin_4( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - *_, S4, S5 = model_classes_factory() JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}' @@ -565,7 +569,6 @@ def test_toggle_plugin_4( ) with monkeypatch.context() as patch: - # Enables both but remove individuals setenvs_from_envfile( patch, diff --git a/packages/settings-library/tests/test_utils_cli.py b/packages/settings-library/tests/test_utils_cli.py index 9c855f236a48..a7194e1b4db0 100644 --- a/packages/settings-library/tests/test_utils_cli.py +++ b/packages/settings-library/tests/test_utils_cli.py @@ -74,8 +74,10 @@ def fake_granular_env_file_content() -> str: POSTGRES_USER=foo POSTGRES_PASSWORD=secret POSTGRES_DB=foodb - POSTGRES_MINSIZE=2 + POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 + POSTGRES_MAX_POOLSIZE=10 + POSTGRES_MAX_OVERFLOW=20 POSTGRES_CLIENT_NAME=None MODULE_VALUE=10 """ @@ -188,8 +190,10 @@ def test_cli_default_settings_envs( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, } @@ -219,8 +223,10 @@ def test_cli_compact_settings_envs( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, } @@ -244,7 +250,7 @@ def test_cli_compact_settings_envs( "APP_HOST": "localhost", "APP_PORT": "80", "APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}', - "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}', + "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50,"POSTGRES_MAX_POOLSIZE":10,"POSTGRES_MAX_OVERFLOW":20,"POSTGRES_CLIENT_NAME":null}', } settings_2 = fake_settings_class() @@ -261,7 +267,7 @@ def test_compact_format( APP_HOST=localhost APP_PORT=80 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}' - APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' + APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "None"}' """, ) @@ -292,10 +298,10 @@ def test_granular_format( POSTGRES_PASSWORD=secret # Database name POSTGRES_DB=foodb - # Minimum number of connections in the pool - POSTGRES_MINSIZE=2 - # Maximum number of connections in the pool + POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 + POSTGRES_MAX_POOLSIZE=10 + POSTGRES_MAX_OVERFLOW=20 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None """, @@ -313,8 +319,10 @@ def test_granular_format( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, ) diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py index 14190934aa19..c266cdd3cd52 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py @@ -18,8 +18,11 @@ async def db_engine( state: AppState, ) -> AsyncGenerator[AsyncEngine, Any]: async with contextlib.AsyncExitStack() as stack: - assert state.environment["POSTGRES_ENDPOINT"] # nosec - db_endpoint = state.environment["POSTGRES_ENDPOINT"] + assert state.environment["POSTGRES_HOST"] # nosec + assert state.environment["POSTGRES_PORT"] # nosec + db_endpoint = ( + f"{state.environment['POSTGRES_HOST']}:{state.environment['POSTGRES_PORT']}" + ) if state.main_bastion_host: assert state.ssh_key_path # nosec db_host, db_port = db_endpoint.split(":") diff --git a/services/api-server/tests/conftest.py b/services/api-server/tests/conftest.py index f0c05db2d1f7..0287fa38d5b0 100644 --- a/services/api-server/tests/conftest.py +++ b/services/api-server/tests/conftest.py @@ -72,8 +72,10 @@ def default_app_env_vars( env_vars["API_SERVER_DEV_FEATURES_ENABLED"] = "1" env_vars["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED"] = "1" env_vars["API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED"] = "0" - env_vars["POSTGRES_MINSIZE"] = "2" + env_vars["POSTGRES_MINSIZE"] = "1" env_vars["POSTGRES_MAXSIZE"] = "10" + env_vars["POSTGRES_MAX_POOLSIZE"] = "10" + env_vars["POSTGRES_MAX_OVERFLOW"] = "20" env_vars["API_SERVER_CELERY"] = "null" env_vars["API_SERVER_RABBITMQ"] = "null" diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index 33425caf3031..83b9a460ac08 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -29,7 +29,6 @@ DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' LOG_LEVEL=DEBUG -POSTGRES_ENDPOINT=postgres:5432 POSTGRES_USER=test POSTGRES_PASSWORD=test POSTGRES_DB=test diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index b32d01c6522f..3bbe927f20b9 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -152,7 +152,6 @@ def _get_environment_variables( "DYNAMIC_SIDECAR_LOG_LEVEL": app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_LOG_LEVEL, "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED": f"{app_settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED}", "POSTGRES_DB": f"{app_settings.POSTGRES.POSTGRES_DB}", - "POSTGRES_ENDPOINT": f"{app_settings.POSTGRES.POSTGRES_HOST}:{app_settings.POSTGRES.POSTGRES_PORT}", "POSTGRES_HOST": f"{app_settings.POSTGRES.POSTGRES_HOST}", "POSTGRES_PASSWORD": f"{app_settings.POSTGRES.POSTGRES_PASSWORD.get_secret_value()}", "POSTGRES_PORT": f"{app_settings.POSTGRES.POSTGRES_PORT}", @@ -458,9 +457,9 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: dynamic_sidecar_settings=dynamic_sidecar_settings, app_settings=app_settings ) - assert scheduler_data.product_name is not None, ( - "ONLY for legacy. This function should not be called with product_name==None" - ) # nosec + assert ( + scheduler_data.product_name is not None + ), "ONLY for legacy. This function should not be called with product_name==None" # nosec standard_simcore_docker_labels: dict[DockerLabelKey, str] = SimcoreContainerLabels( user_id=scheduler_data.user_id, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 01af42b5a6cc..d03cf3a929a7 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -41,7 +41,6 @@ "DYNAMIC_SIDECAR_TRACING", "NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS", "POSTGRES_DB", - "POSTGRES_ENDPOINT", "POSTGRES_HOST", "POSTGRES_PASSWORD", "POSTGRES_PORT", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 3029c2e70a34..99fd1525c20c 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -71,7 +71,6 @@ def mock_env( "DYNAMIC_SIDECAR_IMAGE": "local/dynamic-sidecar:MOCK", "LOG_LEVEL": "DEBUG", "POSTGRES_DB": "test", - "POSTGRES_ENDPOINT": "localhost:5432", "POSTGRES_HOST": "localhost", "POSTGRES_PASSWORD": "test", "POSTGRES_PORT": "5432", @@ -275,7 +274,6 @@ def expected_dynamic_sidecar_spec( "POSTGRES_PORT": "5432", "POSTGRES_USER": "test", "POSTGRES_PASSWORD": "test", - "POSTGRES_ENDPOINT": "localhost:5432", "RABBIT_HOST": "rabbit", "RABBIT_PASSWORD": "adminadmin", "RABBIT_PORT": "5672", diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 903149a8ee80..1fda3502b27e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -18,7 +18,16 @@ x-webserver-diagnostics: &webserver_diagnostics_environs DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY} DIAGNOSTICS_SLOW_DURATION_SECS: ${DIAGNOSTICS_SLOW_DURATION_SECS} - +x-postgres-settings: &postgres_settings + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} + POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE} + POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} services: api-server: @@ -26,7 +35,10 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: &api_server_environment - <<: *tracing_open_telemetry_environs + <<: + - *tracing_open_telemetry_environs + - *postgres_settings + API_SERVER_DEV_FEATURES_ENABLED: ${API_SERVER_DEV_FEATURES_ENABLED} API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} API_SERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} @@ -40,12 +52,6 @@ services: DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -185,7 +191,9 @@ services: init: true hostname: "cat-{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs CATALOG_BACKGROUND_TASK_REST_TIME: ${CATALOG_BACKGROUND_TASK_REST_TIME} CATALOG_DEV_FEATURES_ENABLED: ${CATALOG_DEV_FEATURES_ENABLED} CATALOG_LOGLEVEL: ${CATALOG_LOGLEVEL} @@ -198,11 +206,7 @@ services: DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -286,7 +290,9 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} @@ -299,12 +305,6 @@ services: DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} DIRECTOR_TRACING: ${DIRECTOR_TRACING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - REGISTRY_AUTH: ${REGISTRY_AUTH} REGISTRY_PATH: ${REGISTRY_PATH} REGISTRY_PW: ${REGISTRY_PW} @@ -333,7 +333,9 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs AWS_S3_CLI_S3: ${AWS_S3_CLI_S3} CATALOG_HOST: ${CATALOG_HOST} @@ -370,12 +372,6 @@ services: DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} MONITORING_ENABLED: ${MONITORING_ENABLED} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - R_CLONE_OPTION_BUFFER_SIZE: ${R_CLONE_OPTION_BUFFER_SIZE} R_CLONE_OPTION_RETRIES: ${R_CLONE_OPTION_RETRIES} R_CLONE_OPTION_TRANSFERS: ${R_CLONE_OPTION_TRANSFERS} @@ -447,7 +443,9 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} RABBIT_HOST: ${RABBIT_HOST} @@ -460,11 +458,6 @@ services: REDIS_PORT: ${REDIS_PORT} REDIS_SECURE: ${REDIS_SECURE} REDIS_USER: ${REDIS_USER} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} SC_USER_ID: ${SC_USER_ID} SC_USER_NAME: ${SC_USER_NAME} EFS_USER_ID: ${EFS_USER_ID} @@ -503,7 +496,9 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} @@ -524,11 +519,6 @@ services: PAYMENTS_SWAGGER_API_DOC_ENABLED: ${PAYMENTS_SWAGGER_API_DOC_ENABLED} PAYMENTS_TRACING: ${PAYMENTS_TRACING} PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -550,16 +540,13 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + PROMETHEUS_URL: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} PROMETHEUS_USERNAME: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} PROMETHEUS_PASSWORD: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} @@ -590,7 +577,9 @@ services: - default - docker-api-network environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs CATALOG_HOST: ${CATALOG_HOST} CATALOG_PORT: ${CATALOG_PORT} @@ -614,12 +603,6 @@ services: LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -708,6 +691,7 @@ services: hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: &webserver_environment <<: + - *postgres_settings - *tracing_open_telemetry_environs - *webserver_diagnostics_environs @@ -748,16 +732,6 @@ services: # WEBSERVER_CREDIT_COMPUTATION WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} - POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} - POSTGRES_USER: ${POSTGRES_USER} - # WEBSERVER_DIAGNOSTICS WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS} @@ -968,6 +942,8 @@ services: init: true hostname: "db-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: + <<: + - *postgres_settings WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} # NOTE: keep in sync with the prefix form the hostname @@ -976,14 +952,6 @@ services: WEBSERVER_HOST: ${WEBSERVER_HOST} WEBSERVER_PORT: ${WEBSERVER_PORT} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} @@ -1059,6 +1027,7 @@ services: hostname: "gc-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: <<: + - *postgres_settings - *tracing_open_telemetry_environs # WEBSERVER_DIRECTOR_V2 @@ -1073,13 +1042,8 @@ services: # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + + # WEBSERVER_RABBITMQ RABBIT_HOST: ${RABBIT_HOST} @@ -1159,8 +1123,9 @@ services: hostname: "auth-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: <<: - - *webserver_diagnostics_environs + - *postgres_settings - *tracing_open_telemetry_environs + - *webserver_diagnostics_environs APP_NAME: "simcore_service_wb_auth" WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY @@ -1171,13 +1136,6 @@ services: GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} # WEBSERVER_DIAGNOSTICS WEBSERVER_DIAGNOSTICS: ${WB_AUTH_DIAGNOSTICS} @@ -1273,27 +1231,21 @@ services: hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} - <<: *tracing_open_telemetry_environs - dask-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} init: true @@ -1356,15 +1308,12 @@ services: init: true hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" environment: &storage_environment + <<: + - *postgres_settings + - *tracing_open_telemetry_environs DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -1386,7 +1335,6 @@ services: STORAGE_PROFILING: ${STORAGE_PROFILING} STORAGE_PORT: ${STORAGE_PORT} STORAGE_TRACING: ${STORAGE_TRACING} - <<: *tracing_open_telemetry_environs networks: &storage_networks - default - interactive_services_subnet @@ -1444,12 +1392,7 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + <<: *postgres_settings networks: - default # actually needed for the postgres service only diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index ad4bca974e1a..4776a7feec44 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -8,9 +8,6 @@ from aiohttp import web from servicelib.aiohttp.application import create_safe_application -from simcore_service_webserver.collaboration.bootstrap import ( - setup_realtime_collaboration, -) from ._meta import ( WELCOME_AUTH_APP_MSG, @@ -24,6 +21,9 @@ from .api_keys.plugin import setup_api_keys from .application_settings import get_application_settings, setup_settings from .catalog.plugin import setup_catalog +from .collaboration.bootstrap import ( + setup_realtime_collaboration, +) from .conversations.plugin import setup_conversations from .db.plugin import setup_db from .db_listener.plugin import setup_db_listener diff --git a/services/web/server/src/simcore_service_webserver/application_settings_utils.py b/services/web/server/src/simcore_service_webserver/application_settings_utils.py index 4adf8936f944..00d322b2f6cf 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings_utils.py +++ b/services/web/server/src/simcore_service_webserver/application_settings_utils.py @@ -51,6 +51,12 @@ def convert_to_app_config(app_settings: ApplicationSettings) -> AppConfigDict: "host": getattr(app_settings.WEBSERVER_DB, "POSTGRES_HOST", None), "maxsize": getattr(app_settings.WEBSERVER_DB, "POSTGRES_MAXSIZE", None), "minsize": getattr(app_settings.WEBSERVER_DB, "POSTGRES_MINSIZE", None), + "maxpoolsize": getattr( + app_settings.WEBSERVER_DB, "POSTGRES_MAX_POOLSIZE", None + ), + "maxoverflow": getattr( + app_settings.WEBSERVER_DB, "POSTGRES_MAX_OVERFLOW", None + ), "password": getattr( app_settings.WEBSERVER_DB, "POSTGRES_PASSWORD", SecretStr("") ).get_secret_value(), diff --git a/services/web/server/tests/data/default_app_config-integration.yaml b/services/web/server/tests/data/default_app_config-integration.yaml index 86523a502212..2ccee731e919 100644 --- a/services/web/server/tests/data/default_app_config-integration.yaml +++ b/services/web/server/tests/data/default_app_config-integration.yaml @@ -24,7 +24,6 @@ db: enabled: true postgres: database: ${POSTGRES_DB} - endpoint: ${POSTGRES_ENDPOINT} host: ${POSTGRES_HOST} maxsize: 50 minsize: 2 diff --git a/services/web/server/tests/unit/isolated/test_activity.py b/services/web/server/tests/unit/isolated/test_activity.py index 28f9b7ef1755..2eaf7818f2ad 100644 --- a/services/web/server/tests/unit/isolated/test_activity.py +++ b/services/web/server/tests/unit/isolated/test_activity.py @@ -69,6 +69,8 @@ def app_environment( "POSTGRES_HOST": "postgres", "POSTGRES_MAXSIZE": "10", "POSTGRES_MINSIZE": "10", + "POSTGRES_MAX_POOLSIZE": "10", + "POSTGRES_MAX_OVERFLOW": "20", "POSTGRES_PASSWORD": "simcore", "POSTGRES_PORT": "5432", "POSTGRES_USER": "simcore",