From 6b0910e02640982016c869b71dfd1755ef2ed8b4 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 5 Sep 2025 15:04:26 +0200 Subject: [PATCH 1/9] pass all postgres ENVs to the services --- services/docker-compose.yml | 159 ++++++++++++------------------------ 1 file changed, 50 insertions(+), 109 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 903149a8ee80..ac48a730f645 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -18,7 +18,14 @@ x-webserver-diagnostics: &webserver_diagnostics_environs DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY} DIAGNOSTICS_SLOW_DURATION_SECS: ${DIAGNOSTICS_SLOW_DURATION_SECS} - +x-postgres-settings: &postgres_settings + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} + POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} services: api-server: @@ -26,7 +33,10 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: &api_server_environment - <<: *tracing_open_telemetry_environs + <<: + - *tracing_open_telemetry_environs + - *postgres_settings + API_SERVER_DEV_FEATURES_ENABLED: ${API_SERVER_DEV_FEATURES_ENABLED} API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} API_SERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} @@ -40,12 +50,6 @@ services: DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -185,7 +189,9 @@ services: init: true hostname: "cat-{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs CATALOG_BACKGROUND_TASK_REST_TIME: ${CATALOG_BACKGROUND_TASK_REST_TIME} CATALOG_DEV_FEATURES_ENABLED: ${CATALOG_DEV_FEATURES_ENABLED} CATALOG_LOGLEVEL: ${CATALOG_LOGLEVEL} @@ -198,11 +204,7 @@ services: DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -286,7 +288,9 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} @@ -299,12 +303,6 @@ services: DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} DIRECTOR_TRACING: ${DIRECTOR_TRACING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - REGISTRY_AUTH: ${REGISTRY_AUTH} REGISTRY_PATH: ${REGISTRY_PATH} REGISTRY_PW: ${REGISTRY_PW} @@ -333,7 +331,9 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs AWS_S3_CLI_S3: ${AWS_S3_CLI_S3} CATALOG_HOST: ${CATALOG_HOST} @@ -370,12 +370,6 @@ services: DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} MONITORING_ENABLED: ${MONITORING_ENABLED} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - R_CLONE_OPTION_BUFFER_SIZE: ${R_CLONE_OPTION_BUFFER_SIZE} R_CLONE_OPTION_RETRIES: ${R_CLONE_OPTION_RETRIES} R_CLONE_OPTION_TRANSFERS: ${R_CLONE_OPTION_TRANSFERS} @@ -447,7 +441,9 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} RABBIT_HOST: ${RABBIT_HOST} @@ -460,11 +456,6 @@ services: REDIS_PORT: ${REDIS_PORT} REDIS_SECURE: ${REDIS_SECURE} REDIS_USER: ${REDIS_USER} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} SC_USER_ID: ${SC_USER_ID} SC_USER_NAME: ${SC_USER_NAME} EFS_USER_ID: ${EFS_USER_ID} @@ -503,7 +494,9 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} @@ -524,11 +517,6 @@ services: PAYMENTS_SWAGGER_API_DOC_ENABLED: ${PAYMENTS_SWAGGER_API_DOC_ENABLED} PAYMENTS_TRACING: ${PAYMENTS_TRACING} PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -550,16 +538,13 @@ services: networks: - default environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + PROMETHEUS_URL: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} PROMETHEUS_USERNAME: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} PROMETHEUS_PASSWORD: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} @@ -590,7 +575,9 @@ services: - default - docker-api-network environment: - <<: *tracing_open_telemetry_environs + <<: + - *postgres_settings + - *tracing_open_telemetry_environs CATALOG_HOST: ${CATALOG_HOST} CATALOG_PORT: ${CATALOG_PORT} @@ -614,12 +601,6 @@ services: LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -708,6 +689,7 @@ services: hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: &webserver_environment <<: + - *postgres_settings - *tracing_open_telemetry_environs - *webserver_diagnostics_environs @@ -748,16 +730,6 @@ services: # WEBSERVER_CREDIT_COMPUTATION WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} - POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} - POSTGRES_USER: ${POSTGRES_USER} - # WEBSERVER_DIAGNOSTICS WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS} @@ -968,6 +940,8 @@ services: init: true hostname: "db-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: + <<: + - *postgres_settings WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} # NOTE: keep in sync with the prefix form the hostname @@ -976,14 +950,6 @@ services: WEBSERVER_HOST: ${WEBSERVER_HOST} WEBSERVER_PORT: ${WEBSERVER_PORT} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} @@ -1059,6 +1025,7 @@ services: hostname: "gc-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: <<: + - *postgres_settings - *tracing_open_telemetry_environs # WEBSERVER_DIRECTOR_V2 @@ -1073,13 +1040,8 @@ services: # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + + # WEBSERVER_RABBITMQ RABBIT_HOST: ${RABBIT_HOST} @@ -1159,8 +1121,9 @@ services: hostname: "auth-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 environment: <<: - - *webserver_diagnostics_environs + - *postgres_settings - *tracing_open_telemetry_environs + - *webserver_diagnostics_environs APP_NAME: "simcore_service_wb_auth" WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY @@ -1171,13 +1134,6 @@ services: GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - # WEBSERVER_DB - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} # WEBSERVER_DIAGNOSTICS WEBSERVER_DIAGNOSTICS: ${WB_AUTH_DIAGNOSTICS} @@ -1273,27 +1229,21 @@ services: hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} - <<: *tracing_open_telemetry_environs - dask-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} init: true @@ -1356,15 +1306,12 @@ services: init: true hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" environment: &storage_environment + <<: + - *postgres_settings + - *tracing_open_telemetry_environs DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} @@ -1386,7 +1333,6 @@ services: STORAGE_PROFILING: ${STORAGE_PROFILING} STORAGE_PORT: ${STORAGE_PORT} STORAGE_TRACING: ${STORAGE_TRACING} - <<: *tracing_open_telemetry_environs networks: &storage_networks - default - interactive_services_subnet @@ -1444,12 +1390,7 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" environment: - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} + <<: *postgres_settings networks: - default # actually needed for the postgres service only From 7b5db11b8697e9e58ce85193a560ad5ec84ab886 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 5 Sep 2025 15:05:53 +0200 Subject: [PATCH 2/9] adjust pool sizes --- .env-devel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env-devel b/.env-devel index b82ba685e9b6..0322d30be53b 100644 --- a/.env-devel +++ b/.env-devel @@ -199,8 +199,8 @@ POSTGRES_HOST=postgres POSTGRES_PASSWORD=adminadmin POSTGRES_PORT=5432 POSTGRES_USER=scu -POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199 -POSTGRES_MAXSIZE=50 +POSTGRES_MINSIZE=10 # NOTE: this is currently asyncpg pool size and aiopg min size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660 +POSTGRES_MAXSIZE=30 # NOTE: this is currently asyncpg 20 overflow size and aiopg max size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660 POSTGRES_READONLY_PASSWORD=readonly POSTGRES_READONLY_USER=postgres_readonly From ebddd63da3a7ac6f88cadda54f4b23d64cfc7fbc Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 5 Sep 2025 15:11:04 +0200 Subject: [PATCH 3/9] removed POSTGRES_ENDPOINT --- .env-devel | 1 - .../pytest-simcore/src/pytest_simcore/postgres_service.py | 1 - .../computational-clusters/autoscaled_monitor/db.py | 7 +++++-- services/director-v2/.env-devel | 1 - .../dynamic_sidecar/docker_service_specs/sidecar.py | 7 +++---- ...modules_dynamic_sidecar_docker_service_specs_sidecar.py | 1 - .../test_modules_dynamic_sidecar_docker_service_specs.py | 2 -- .../server/tests/data/default_app_config-integration.yaml | 1 - 8 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.env-devel b/.env-devel index 0322d30be53b..5ed77c83d46e 100644 --- a/.env-devel +++ b/.env-devel @@ -194,7 +194,6 @@ PAYMENTS_USERNAME=admin PAYMENTS_TRACING={} POSTGRES_DB=simcoredb -POSTGRES_ENDPOINT=postgres:5432 POSTGRES_HOST=postgres POSTGRES_PASSWORD=adminadmin POSTGRES_PORT=5432 diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py index 19d9247e8eab..7814d413c07c 100644 --- a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py @@ -250,7 +250,6 @@ def postgres_env_vars_dict(postgres_dsn: PostgresTestConfig) -> EnvVarsDict: "POSTGRES_DB": postgres_dsn["database"], "POSTGRES_HOST": postgres_dsn["host"], "POSTGRES_PORT": f"{postgres_dsn['port']}", - "POSTGRES_ENDPOINT": f"{postgres_dsn['host']}:{postgres_dsn['port']}", } diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py index 14190934aa19..c266cdd3cd52 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/db.py @@ -18,8 +18,11 @@ async def db_engine( state: AppState, ) -> AsyncGenerator[AsyncEngine, Any]: async with contextlib.AsyncExitStack() as stack: - assert state.environment["POSTGRES_ENDPOINT"] # nosec - db_endpoint = state.environment["POSTGRES_ENDPOINT"] + assert state.environment["POSTGRES_HOST"] # nosec + assert state.environment["POSTGRES_PORT"] # nosec + db_endpoint = ( + f"{state.environment['POSTGRES_HOST']}:{state.environment['POSTGRES_PORT']}" + ) if state.main_bastion_host: assert state.ssh_key_path # nosec db_host, db_port = db_endpoint.split(":") diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index 33425caf3031..83b9a460ac08 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -29,7 +29,6 @@ DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' LOG_LEVEL=DEBUG -POSTGRES_ENDPOINT=postgres:5432 POSTGRES_USER=test POSTGRES_PASSWORD=test POSTGRES_DB=test diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index b32d01c6522f..3bbe927f20b9 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -152,7 +152,6 @@ def _get_environment_variables( "DYNAMIC_SIDECAR_LOG_LEVEL": app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_LOG_LEVEL, "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED": f"{app_settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED}", "POSTGRES_DB": f"{app_settings.POSTGRES.POSTGRES_DB}", - "POSTGRES_ENDPOINT": f"{app_settings.POSTGRES.POSTGRES_HOST}:{app_settings.POSTGRES.POSTGRES_PORT}", "POSTGRES_HOST": f"{app_settings.POSTGRES.POSTGRES_HOST}", "POSTGRES_PASSWORD": f"{app_settings.POSTGRES.POSTGRES_PASSWORD.get_secret_value()}", "POSTGRES_PORT": f"{app_settings.POSTGRES.POSTGRES_PORT}", @@ -458,9 +457,9 @@ async def get_dynamic_sidecar_spec( # pylint:disable=too-many-arguments# noqa: dynamic_sidecar_settings=dynamic_sidecar_settings, app_settings=app_settings ) - assert scheduler_data.product_name is not None, ( - "ONLY for legacy. This function should not be called with product_name==None" - ) # nosec + assert ( + scheduler_data.product_name is not None + ), "ONLY for legacy. This function should not be called with product_name==None" # nosec standard_simcore_docker_labels: dict[DockerLabelKey, str] = SimcoreContainerLabels( user_id=scheduler_data.user_id, diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 01af42b5a6cc..d03cf3a929a7 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -41,7 +41,6 @@ "DYNAMIC_SIDECAR_TRACING", "NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS", "POSTGRES_DB", - "POSTGRES_ENDPOINT", "POSTGRES_HOST", "POSTGRES_PASSWORD", "POSTGRES_PORT", diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index 3029c2e70a34..99fd1525c20c 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -71,7 +71,6 @@ def mock_env( "DYNAMIC_SIDECAR_IMAGE": "local/dynamic-sidecar:MOCK", "LOG_LEVEL": "DEBUG", "POSTGRES_DB": "test", - "POSTGRES_ENDPOINT": "localhost:5432", "POSTGRES_HOST": "localhost", "POSTGRES_PASSWORD": "test", "POSTGRES_PORT": "5432", @@ -275,7 +274,6 @@ def expected_dynamic_sidecar_spec( "POSTGRES_PORT": "5432", "POSTGRES_USER": "test", "POSTGRES_PASSWORD": "test", - "POSTGRES_ENDPOINT": "localhost:5432", "RABBIT_HOST": "rabbit", "RABBIT_PASSWORD": "adminadmin", "RABBIT_PORT": "5672", diff --git a/services/web/server/tests/data/default_app_config-integration.yaml b/services/web/server/tests/data/default_app_config-integration.yaml index 86523a502212..2ccee731e919 100644 --- a/services/web/server/tests/data/default_app_config-integration.yaml +++ b/services/web/server/tests/data/default_app_config-integration.yaml @@ -24,7 +24,6 @@ db: enabled: true postgres: database: ${POSTGRES_DB} - endpoint: ${POSTGRES_ENDPOINT} host: ${POSTGRES_HOST} maxsize: 50 minsize: 2 From 33608bc95f5c743a4831cf24c7a9e77343c00c1f Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Fri, 5 Sep 2025 15:53:04 +0200 Subject: [PATCH 4/9] fix relative imports --- .../web/server/src/simcore_service_webserver/application.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index ad4bca974e1a..4776a7feec44 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -8,9 +8,6 @@ from aiohttp import web from servicelib.aiohttp.application import create_safe_application -from simcore_service_webserver.collaboration.bootstrap import ( - setup_realtime_collaboration, -) from ._meta import ( WELCOME_AUTH_APP_MSG, @@ -24,6 +21,9 @@ from .api_keys.plugin import setup_api_keys from .application_settings import get_application_settings, setup_settings from .catalog.plugin import setup_catalog +from .collaboration.bootstrap import ( + setup_realtime_collaboration, +) from .conversations.plugin import setup_conversations from .db.plugin import setup_db from .db_listener.plugin import setup_db_listener From 2050b0416081386defbc66fcc6efa1da6177f536 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 8 Sep 2025 08:41:43 +0200 Subject: [PATCH 5/9] added new envs --- .env-devel | 6 ++-- .../src/servicelib/db_asyncpg_utils.py | 13 ++++++--- .../src/settings_library/postgres.py | 29 +++++++++++++++---- .../settings-library/tests/data/.env-granular | 6 ++-- .../settings-library/tests/data/.env-mixed | 6 ++-- .../settings-library/tests/data/.env-sample | 6 ++-- .../tests/test_base_w_postgres.py | 16 +++++++--- .../settings-library/tests/test_utils_cli.py | 26 +++++++++++------ services/api-server/tests/conftest.py | 4 ++- services/docker-compose.yml | 2 ++ .../application_settings_utils.py | 6 ++++ .../tests/unit/isolated/test_activity.py | 2 ++ 12 files changed, 88 insertions(+), 34 deletions(-) diff --git a/.env-devel b/.env-devel index 5ed77c83d46e..0e27dfde6f1a 100644 --- a/.env-devel +++ b/.env-devel @@ -198,8 +198,10 @@ POSTGRES_HOST=postgres POSTGRES_PASSWORD=adminadmin POSTGRES_PORT=5432 POSTGRES_USER=scu -POSTGRES_MINSIZE=10 # NOTE: this is currently asyncpg pool size and aiopg min size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660 -POSTGRES_MAXSIZE=30 # NOTE: this is currently asyncpg 20 overflow size and aiopg max size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660 +POSTGRES_MINSIZE=1 +POSTGRES_MAXSIZE=50 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 POSTGRES_READONLY_PASSWORD=readonly POSTGRES_READONLY_USER=postgres_readonly diff --git a/packages/service-library/src/servicelib/db_asyncpg_utils.py b/packages/service-library/src/servicelib/db_asyncpg_utils.py index 0b35b3227238..9f79979ff15b 100644 --- a/packages/service-library/src/servicelib/db_asyncpg_utils.py +++ b/packages/service-library/src/servicelib/db_asyncpg_utils.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator from datetime import timedelta +import orjson from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult from settings_library.postgres import PostgresSettings from sqlalchemy.exc import SQLAlchemyError @@ -39,11 +40,13 @@ async def create_async_engine_and_database_ready( engine = create_async_engine( settings.dsn_with_async_sqlalchemy, - pool_size=settings.POSTGRES_MINSIZE, - max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE, + pool_size=settings.POSTGRES_MAX_POOLSIZE, + max_overflow=settings.POSTGRES_MAX_OVERFLOW, connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released + json_serializer=orjson.dumps, + json_deserializer=orjson.loads, ) try: @@ -90,11 +93,13 @@ async def with_async_pg_engine( engine = create_async_engine( settings.dsn_with_async_sqlalchemy, - pool_size=settings.POSTGRES_MINSIZE, - max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE, + pool_size=settings.POSTGRES_MAX_POOLSIZE, + max_overflow=settings.POSTGRES_MAX_OVERFLOW, connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released + json_serializer=orjson.dumps, + json_deserializer=orjson.loads, ) yield engine finally: diff --git a/packages/settings-library/src/settings_library/postgres.py b/packages/settings-library/src/settings_library/postgres.py index 90d456cbda07..64276b7fdce4 100644 --- a/packages/settings-library/src/settings_library/postgres.py +++ b/packages/settings-library/src/settings_library/postgres.py @@ -5,6 +5,7 @@ from pydantic import ( AliasChoices, Field, + NonNegativeInt, PostgresDsn, SecretStr, model_validator, @@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings): # pool connection limits POSTGRES_MINSIZE: Annotated[ - int, Field(description="Minimum number of connections in the pool", ge=2) - ] = 2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199 + int, + Field( + description="Minimum number of connections in the pool that are always created and kept", + ge=1, + ), + ] = 1 POSTGRES_MAXSIZE: Annotated[ - int, Field(description="Maximum number of connections in the pool", ge=2) + int, + Field( + description="Maximum number of connections in the pool that are kept", + ge=1, + ), ] = 50 + POSTGRES_MAX_POOLSIZE: Annotated[ + int, + Field( + description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand" + ), + ] = 10 + POSTGRES_MAX_OVERFLOW: Annotated[ + NonNegativeInt, Field(description="Maximal overflow connections") + ] = 20 POSTGRES_CLIENT_NAME: Annotated[ str | None, @@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "POSTGRES_USER": "usr", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "db", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "my_app", # first-choice "HOST": "should be ignored", "HOST_NAME": "should be ignored", @@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra) - model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra) diff --git a/packages/settings-library/tests/data/.env-granular b/packages/settings-library/tests/data/.env-granular index 333452ef1957..9fa892563173 100644 --- a/packages/settings-library/tests/data/.env-granular +++ b/packages/settings-library/tests/data/.env-granular @@ -12,9 +12,9 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb -# Minimum number of connections in the pool -POSTGRES_MINSIZE=2 -# Maximum number of connections in the pool +POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-mixed b/packages/settings-library/tests/data/.env-mixed index 3001f92335b9..094ae56915ed 100644 --- a/packages/settings-library/tests/data/.env-mixed +++ b/packages/settings-library/tests/data/.env-mixed @@ -12,9 +12,9 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb -# Minimum number of connections in the pool -POSTGRES_MINSIZE=2 -# Maximum number of connections in the pool +POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-sample b/packages/settings-library/tests/data/.env-sample index f66bdf71c2c3..5cb4d2d099cb 100644 --- a/packages/settings-library/tests/data/.env-sample +++ b/packages/settings-library/tests/data/.env-sample @@ -8,10 +8,10 @@ POSTGRES_USER=foo POSTGRES_PASSWORD=secret # Database name POSTGRES_DB=foodb -# Maximum number of connections in the pool -POSTGRES_MINSIZE=2 -# Minimum number of connections in the pool +POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 +POSTGRES_MAX_POOLSIZE=10 +POSTGRES_MAX_OVERFLOW=20 # --- APP_MODULE_FIELD --- # Some value for module 1 diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index a4212bb6648e..a4fc07a5d1ce 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -200,7 +200,9 @@ def test_parse_from_individual_envs( "POSTGRES_PASSWORD": "shh", "POSTGRES_DB": "db", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -215,7 +217,9 @@ def test_parse_from_individual_envs( "POSTGRES_PASSWORD": "shh", "POSTGRES_DB": "db", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -262,7 +266,9 @@ def test_parse_compact_env( "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } @@ -372,7 +378,9 @@ def test_parse_from_mixed_envs( "POSTGRES_PASSWORD": "shh2", "POSTGRES_DB": "db2", "POSTGRES_MAXSIZE": 50, - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, } } diff --git a/packages/settings-library/tests/test_utils_cli.py b/packages/settings-library/tests/test_utils_cli.py index 9c855f236a48..cfc38753edd8 100644 --- a/packages/settings-library/tests/test_utils_cli.py +++ b/packages/settings-library/tests/test_utils_cli.py @@ -74,8 +74,10 @@ def fake_granular_env_file_content() -> str: POSTGRES_USER=foo POSTGRES_PASSWORD=secret POSTGRES_DB=foodb - POSTGRES_MINSIZE=2 + POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 + POSTGRES_MAX_POOLSIZE=10 + POSTGRES_MAX_OVERFLOW=20 POSTGRES_CLIENT_NAME=None MODULE_VALUE=10 """ @@ -188,8 +190,10 @@ def test_cli_default_settings_envs( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, } @@ -219,8 +223,10 @@ def test_cli_compact_settings_envs( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, } @@ -244,7 +250,7 @@ def test_cli_compact_settings_envs( "APP_HOST": "localhost", "APP_PORT": "80", "APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}', - "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}', + "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME":null}', } settings_2 = fake_settings_class() @@ -261,7 +267,7 @@ def test_compact_format( APP_HOST=localhost APP_PORT=80 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}' - APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' + APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "None"}' """, ) @@ -292,10 +298,10 @@ def test_granular_format( POSTGRES_PASSWORD=secret # Database name POSTGRES_DB=foodb - # Minimum number of connections in the pool - POSTGRES_MINSIZE=2 - # Maximum number of connections in the pool + POSTGRES_MINSIZE=1 POSTGRES_MAXSIZE=50 + POSTGRES_MAX_POOLSIZE=10 + POSTGRES_MAX_OVERFLOW=20 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None """, @@ -313,8 +319,10 @@ def test_granular_format( "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", - "POSTGRES_MINSIZE": 2, + "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, + "POSTGRES_MAX_POOLSIZE": 10, + "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": None, }, ) diff --git a/services/api-server/tests/conftest.py b/services/api-server/tests/conftest.py index f0c05db2d1f7..0287fa38d5b0 100644 --- a/services/api-server/tests/conftest.py +++ b/services/api-server/tests/conftest.py @@ -72,8 +72,10 @@ def default_app_env_vars( env_vars["API_SERVER_DEV_FEATURES_ENABLED"] = "1" env_vars["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED"] = "1" env_vars["API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED"] = "0" - env_vars["POSTGRES_MINSIZE"] = "2" + env_vars["POSTGRES_MINSIZE"] = "1" env_vars["POSTGRES_MAXSIZE"] = "10" + env_vars["POSTGRES_MAX_POOLSIZE"] = "10" + env_vars["POSTGRES_MAX_OVERFLOW"] = "20" env_vars["API_SERVER_CELERY"] = "null" env_vars["API_SERVER_RABBITMQ"] = "null" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index ac48a730f645..fd2ea92b37a8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -24,6 +24,8 @@ x-postgres-settings: &postgres_settings POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE}, + POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW}, POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_USER: ${POSTGRES_USER} diff --git a/services/web/server/src/simcore_service_webserver/application_settings_utils.py b/services/web/server/src/simcore_service_webserver/application_settings_utils.py index 4adf8936f944..00d322b2f6cf 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings_utils.py +++ b/services/web/server/src/simcore_service_webserver/application_settings_utils.py @@ -51,6 +51,12 @@ def convert_to_app_config(app_settings: ApplicationSettings) -> AppConfigDict: "host": getattr(app_settings.WEBSERVER_DB, "POSTGRES_HOST", None), "maxsize": getattr(app_settings.WEBSERVER_DB, "POSTGRES_MAXSIZE", None), "minsize": getattr(app_settings.WEBSERVER_DB, "POSTGRES_MINSIZE", None), + "maxpoolsize": getattr( + app_settings.WEBSERVER_DB, "POSTGRES_MAX_POOLSIZE", None + ), + "maxoverflow": getattr( + app_settings.WEBSERVER_DB, "POSTGRES_MAX_OVERFLOW", None + ), "password": getattr( app_settings.WEBSERVER_DB, "POSTGRES_PASSWORD", SecretStr("") ).get_secret_value(), diff --git a/services/web/server/tests/unit/isolated/test_activity.py b/services/web/server/tests/unit/isolated/test_activity.py index 28f9b7ef1755..2eaf7818f2ad 100644 --- a/services/web/server/tests/unit/isolated/test_activity.py +++ b/services/web/server/tests/unit/isolated/test_activity.py @@ -69,6 +69,8 @@ def app_environment( "POSTGRES_HOST": "postgres", "POSTGRES_MAXSIZE": "10", "POSTGRES_MINSIZE": "10", + "POSTGRES_MAX_POOLSIZE": "10", + "POSTGRES_MAX_OVERFLOW": "20", "POSTGRES_PASSWORD": "simcore", "POSTGRES_PORT": "5432", "POSTGRES_USER": "simcore", From 4493d668621ec2ca18befb4b1f3175b7ecc8b52e Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 8 Sep 2025 08:50:16 +0200 Subject: [PATCH 6/9] removed orjson for now --- packages/service-library/src/servicelib/db_asyncpg_utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/service-library/src/servicelib/db_asyncpg_utils.py b/packages/service-library/src/servicelib/db_asyncpg_utils.py index 9f79979ff15b..d72f77d95159 100644 --- a/packages/service-library/src/servicelib/db_asyncpg_utils.py +++ b/packages/service-library/src/servicelib/db_asyncpg_utils.py @@ -4,7 +4,6 @@ from collections.abc import AsyncIterator from datetime import timedelta -import orjson from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult from settings_library.postgres import PostgresSettings from sqlalchemy.exc import SQLAlchemyError @@ -45,8 +44,6 @@ async def create_async_engine_and_database_ready( connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released - json_serializer=orjson.dumps, - json_deserializer=orjson.loads, ) try: @@ -98,8 +95,6 @@ async def with_async_pg_engine( connect_args={"server_settings": server_settings}, pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released - json_serializer=orjson.dumps, - json_deserializer=orjson.loads, ) yield engine finally: From 86176b6f56d3c30e62850b02f1ee0dd67179cfdb Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:28:39 +0200 Subject: [PATCH 7/9] @copilot review --- services/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index fd2ea92b37a8..1fda3502b27e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -24,8 +24,8 @@ x-postgres-settings: &postgres_settings POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE}, - POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW}, + POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE} + POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_USER: ${POSTGRES_USER} From f4ff7ac50fe300fd1e68003763f8e7d29cf3ce88 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:37:12 +0200 Subject: [PATCH 8/9] fixed dumb test --- packages/settings-library/tests/data/.env-compact | 2 +- packages/settings-library/tests/data/.env-granular | 2 +- packages/settings-library/tests/data/.env-mixed | 2 +- packages/settings-library/tests/data/.env-sample | 2 +- packages/settings-library/tests/test_utils_cli.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/settings-library/tests/data/.env-compact b/packages/settings-library/tests/data/.env-compact index a3e835483443..099dc82cfee2 100644 --- a/packages/settings-library/tests/data/.env-compact +++ b/packages/settings-library/tests/data/.env-compact @@ -3,4 +3,4 @@ APP_HOST=localhost APP_PORT=80 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}' -APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' +APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' diff --git a/packages/settings-library/tests/data/.env-granular b/packages/settings-library/tests/data/.env-granular index 9fa892563173..c27099b7b98b 100644 --- a/packages/settings-library/tests/data/.env-granular +++ b/packages/settings-library/tests/data/.env-granular @@ -13,8 +13,8 @@ POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb POSTGRES_MINSIZE=1 -POSTGRES_MAXSIZE=50 POSTGRES_MAX_POOLSIZE=10 POSTGRES_MAX_OVERFLOW=20 +POSTGRES_MAXSIZE=50 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-mixed b/packages/settings-library/tests/data/.env-mixed index 094ae56915ed..4aed86d3b9b7 100644 --- a/packages/settings-library/tests/data/.env-mixed +++ b/packages/settings-library/tests/data/.env-mixed @@ -13,8 +13,8 @@ POSTGRES_PASSWORD=********** # Database name POSTGRES_DB=foodb POSTGRES_MINSIZE=1 -POSTGRES_MAXSIZE=50 POSTGRES_MAX_POOLSIZE=10 POSTGRES_MAX_OVERFLOW=20 +POSTGRES_MAXSIZE=50 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux) POSTGRES_CLIENT_NAME=None diff --git a/packages/settings-library/tests/data/.env-sample b/packages/settings-library/tests/data/.env-sample index 5cb4d2d099cb..d6f115f76c23 100644 --- a/packages/settings-library/tests/data/.env-sample +++ b/packages/settings-library/tests/data/.env-sample @@ -9,9 +9,9 @@ POSTGRES_PASSWORD=secret # Database name POSTGRES_DB=foodb POSTGRES_MINSIZE=1 -POSTGRES_MAXSIZE=50 POSTGRES_MAX_POOLSIZE=10 POSTGRES_MAX_OVERFLOW=20 +POSTGRES_MAXSIZE=50 # --- APP_MODULE_FIELD --- # Some value for module 1 diff --git a/packages/settings-library/tests/test_utils_cli.py b/packages/settings-library/tests/test_utils_cli.py index cfc38753edd8..a7194e1b4db0 100644 --- a/packages/settings-library/tests/test_utils_cli.py +++ b/packages/settings-library/tests/test_utils_cli.py @@ -250,7 +250,7 @@ def test_cli_compact_settings_envs( "APP_HOST": "localhost", "APP_PORT": "80", "APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}', - "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME":null}', + "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50,"POSTGRES_MAX_POOLSIZE":10,"POSTGRES_MAX_OVERFLOW":20,"POSTGRES_CLIENT_NAME":null}', } settings_2 = fake_settings_class() From 1cad715a142d4e025796aae7d6c5edc2c8912b49 Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:13:57 +0200 Subject: [PATCH 9/9] missing --- packages/settings-library/tests/data/.env-compact | 2 +- .../settings-library/tests/test_base_w_postgres.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/settings-library/tests/data/.env-compact b/packages/settings-library/tests/data/.env-compact index 099dc82cfee2..b11273684dc2 100644 --- a/packages/settings-library/tests/data/.env-compact +++ b/packages/settings-library/tests/data/.env-compact @@ -3,4 +3,4 @@ APP_HOST=localhost APP_PORT=80 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}' -APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' +APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}' diff --git a/packages/settings-library/tests/test_base_w_postgres.py b/packages/settings-library/tests/test_base_w_postgres.py index a4fc07a5d1ce..641d1df62a3d 100644 --- a/packages/settings-library/tests/test_base_w_postgres.py +++ b/packages/settings-library/tests/test_base_w_postgres.py @@ -55,8 +55,10 @@ class _FakePostgresSettings(BaseCustomSettings): POSTGRES_PASSWORD: str POSTGRES_DB: str - POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2 - POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50 + POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1 + POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50 + POSTGRES_MAX_POOLSIZE: int = 10 + POSTGRES_MAX_OVERFLOW: Annotated[int, Field(ge=0)] = 20 POSTGRES_CLIENT_NAME: Annotated[ str | None, @@ -126,7 +128,6 @@ class S5(BaseCustomSettings): def test_parse_from_empty_envs( postgres_envvars_unset: None, model_classes_factory: Callable ): - S1, S2, S3, S4, S5 = model_classes_factory() with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info: @@ -160,7 +161,6 @@ def test_parse_from_individual_envs( monkeypatch: pytest.MonkeyPatch, model_classes_factory: Callable, ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -232,7 +232,6 @@ def test_parse_from_individual_envs( def test_parse_compact_env( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -348,7 +347,6 @@ def test_parse_compact_env( def test_parse_from_mixed_envs( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - S1, S2, S3, S4, S5 = model_classes_factory() # environment @@ -480,7 +478,6 @@ def test_parse_from_mixed_envs( def test_toggle_plugin_1( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - *_, S4, S5 = model_classes_factory() # empty environ @@ -543,7 +540,6 @@ def test_toggle_plugin_3( def test_toggle_plugin_4( postgres_envvars_unset: None, monkeypatch, model_classes_factory ): - *_, S4, S5 = model_classes_factory() JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}' @@ -573,7 +569,6 @@ def test_toggle_plugin_4( ) with monkeypatch.context() as patch: - # Enables both but remove individuals setenvs_from_envfile( patch,