🎨Maintenance & Performance: Properly setup asyncpg DB engine

.env-devel

@@ -194,13 +194,14 @@ PAYMENTS_USERNAME=admin
 PAYMENTS_TRACING={}
 
 POSTGRES_DB=simcoredb
-POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_HOST=postgres
 POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
-POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+POSTGRES_MINSIZE=1
 POSTGRES_MAXSIZE=50
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_READONLY_PASSWORD=readonly
 POSTGRES_READONLY_USER=postgres_readonly
 

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -250,7 +250,6 @@ def postgres_env_vars_dict(postgres_dsn: PostgresTestConfig) -> EnvVarsDict:
         "POSTGRES_DB": postgres_dsn["database"],
         "POSTGRES_HOST": postgres_dsn["host"],
         "POSTGRES_PORT": f"{postgres_dsn['port']}",
-        "POSTGRES_ENDPOINT": f"{postgres_dsn['host']}:{postgres_dsn['port']}",
     }
 
 

packages/service-library/src/servicelib/db_asyncpg_utils.py

@@ -39,8 +39,8 @@ async def create_async_engine_and_database_ready(
 
     engine = create_async_engine(
         settings.dsn_with_async_sqlalchemy,
-        pool_size=settings.POSTGRES_MINSIZE,
-        max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
+        pool_size=settings.POSTGRES_MAX_POOLSIZE,
+        max_overflow=settings.POSTGRES_MAX_OVERFLOW,
         connect_args={"server_settings": server_settings},
         pool_pre_ping=True,  # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
         future=True,  # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released
@@ -90,8 +90,8 @@ async def with_async_pg_engine(
 
             engine = create_async_engine(
                 settings.dsn_with_async_sqlalchemy,
-                pool_size=settings.POSTGRES_MINSIZE,
-                max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
+                pool_size=settings.POSTGRES_MAX_POOLSIZE,
+                max_overflow=settings.POSTGRES_MAX_OVERFLOW,
                 connect_args={"server_settings": server_settings},
                 pool_pre_ping=True,  # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
                 future=True,  # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released

packages/settings-library/src/settings_library/postgres.py

@@ -5,6 +5,7 @@
 from pydantic import (
     AliasChoices,
     Field,
+    NonNegativeInt,
     PostgresDsn,
     SecretStr,
     model_validator,
@@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings):
 
     # pool connection limits
     POSTGRES_MINSIZE: Annotated[
-        int, Field(description="Minimum number of connections in the pool", ge=2)
-    ] = 2  # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+        int,
+        Field(
+            description="Minimum number of connections in the pool that are always created and kept",
+            ge=1,
+        ),
+    ] = 1
     POSTGRES_MAXSIZE: Annotated[
-        int, Field(description="Maximum number of connections in the pool", ge=2)
+        int,
+        Field(
+            description="Maximum number of connections in the pool that are kept",
+            ge=1,
+        ),
     ] = 50
+    POSTGRES_MAX_POOLSIZE: Annotated[
+        int,
+        Field(
+            description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand"
+        ),
+    ] = 10
+    POSTGRES_MAX_OVERFLOW: Annotated[
+        NonNegativeInt, Field(description="Maximal overflow connections")
+    ] = 20
 
     POSTGRES_CLIENT_NAME: Annotated[
         str | None,
@@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "POSTGRES_USER": "usr",
                         "POSTGRES_PASSWORD": "secret",
                         "POSTGRES_DB": "db",
-                        "POSTGRES_MINSIZE": 2,
+                        "POSTGRES_MINSIZE": 1,
                         "POSTGRES_MAXSIZE": 50,
+                        "POSTGRES_MAX_POOLSIZE": 10,
+                        "POSTGRES_MAX_OVERFLOW": 20,
                         "POSTGRES_CLIENT_NAME": "my_app",  # first-choice
                         "HOST": "should be ignored",
                         "HOST_NAME": "should be ignored",
@@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
         )
 
     model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)
-    model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)

packages/settings-library/tests/data/.env-compact

@@ -3,4 +3,4 @@
 APP_HOST=localhost
 APP_PORT=80
 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
-APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
+APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'

packages/settings-library/tests/data/.env-granular

@@ -12,9 +12,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
-# Minimum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Maximum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
 POSTGRES_CLIENT_NAME=None

packages/settings-library/tests/data/.env-mixed

@@ -12,9 +12,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
-# Minimum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Maximum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
 POSTGRES_CLIENT_NAME=None

packages/settings-library/tests/data/.env-sample

@@ -8,9 +8,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=secret
 # Database name
 POSTGRES_DB=foodb
-# Maximum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Minimum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 
 # --- APP_MODULE_FIELD ---

packages/settings-library/tests/test_base_w_postgres.py

@@ -55,8 +55,10 @@ class _FakePostgresSettings(BaseCustomSettings):
             POSTGRES_PASSWORD: str
 
             POSTGRES_DB: str
-            POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2
-            POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50
+            POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1
+            POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50
+            POSTGRES_MAX_POOLSIZE: int = 10
+            POSTGRES_MAX_OVERFLOW: Annotated[int, Field(ge=0)] = 20
 
             POSTGRES_CLIENT_NAME: Annotated[
                 str | None,
@@ -126,7 +128,6 @@ class S5(BaseCustomSettings):
 def test_parse_from_empty_envs(
     postgres_envvars_unset: None, model_classes_factory: Callable
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info:
@@ -160,7 +161,6 @@ def test_parse_from_individual_envs(
     monkeypatch: pytest.MonkeyPatch,
     model_classes_factory: Callable,
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -200,7 +200,9 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 2,
+            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MAX_POOLSIZE": 10,
+            "POSTGRES_MAX_OVERFLOW": 20,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -215,7 +217,9 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 2,
+            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MAX_POOLSIZE": 10,
+            "POSTGRES_MAX_OVERFLOW": 20,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -228,7 +232,6 @@ def test_parse_from_individual_envs(
 def test_parse_compact_env(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -262,7 +265,9 @@ def test_parse_compact_env(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }
@@ -342,7 +347,6 @@ def test_parse_compact_env(
 def test_parse_from_mixed_envs(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -372,7 +376,9 @@ def test_parse_from_mixed_envs(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }
@@ -472,7 +478,6 @@ def test_parse_from_mixed_envs(
 def test_toggle_plugin_1(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     *_, S4, S5 = model_classes_factory()
 
     # empty environ
@@ -535,7 +540,6 @@ def test_toggle_plugin_3(
 def test_toggle_plugin_4(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     *_, S4, S5 = model_classes_factory()
     JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}'
 
@@ -565,7 +569,6 @@ def test_toggle_plugin_4(
         )
 
     with monkeypatch.context() as patch:
-
         # Enables both but remove individuals
         setenvs_from_envfile(
             patch,

packages/settings-library/tests/test_utils_cli.py

@@ -74,8 +74,10 @@ def fake_granular_env_file_content() -> str:
         POSTGRES_USER=foo
         POSTGRES_PASSWORD=secret
         POSTGRES_DB=foodb
-        POSTGRES_MINSIZE=2
+        POSTGRES_MINSIZE=1
         POSTGRES_MAXSIZE=50
+        POSTGRES_MAX_POOLSIZE=10
+        POSTGRES_MAX_OVERFLOW=20
         POSTGRES_CLIENT_NAME=None
         MODULE_VALUE=10
     """
@@ -188,8 +190,10 @@ def test_cli_default_settings_envs(
                 "POSTGRES_USER": "foo",
                 "POSTGRES_PASSWORD": "secret",
                 "POSTGRES_DB": "foodb",
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
                 "POSTGRES_MAXSIZE": 50,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             },
         }
@@ -219,8 +223,10 @@ def test_cli_compact_settings_envs(
                 "POSTGRES_USER": "foo",
                 "POSTGRES_PASSWORD": "secret",
                 "POSTGRES_DB": "foodb",
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
                 "POSTGRES_MAXSIZE": 50,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             },
         }
@@ -244,7 +250,7 @@ def test_cli_compact_settings_envs(
             "APP_HOST": "localhost",
             "APP_PORT": "80",
             "APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}',
-            "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}',
+            "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50,"POSTGRES_MAX_POOLSIZE":10,"POSTGRES_MAX_OVERFLOW":20,"POSTGRES_CLIENT_NAME":null}',
         }
 
         settings_2 = fake_settings_class()
@@ -261,7 +267,7 @@ def test_compact_format(
         APP_HOST=localhost
         APP_PORT=80
         APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
-        APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
+        APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "None"}'
         """,
     )
 
@@ -292,10 +298,10 @@ def test_granular_format(
     POSTGRES_PASSWORD=secret
     # Database name
     POSTGRES_DB=foodb
-    # Minimum number of connections in the pool
-    POSTGRES_MINSIZE=2
-    # Maximum number of connections in the pool
+    POSTGRES_MINSIZE=1
     POSTGRES_MAXSIZE=50
+    POSTGRES_MAX_POOLSIZE=10
+    POSTGRES_MAX_OVERFLOW=20
     # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
     POSTGRES_CLIENT_NAME=None
     """,
@@ -313,8 +319,10 @@ def test_granular_format(
             "POSTGRES_USER": "foo",
             "POSTGRES_PASSWORD": "secret",
             "POSTGRES_DB": "foodb",
-            "POSTGRES_MINSIZE": 2,
+            "POSTGRES_MINSIZE": 1,
             "POSTGRES_MAXSIZE": 50,
+            "POSTGRES_MAX_POOLSIZE": 10,
+            "POSTGRES_MAX_OVERFLOW": 20,
             "POSTGRES_CLIENT_NAME": None,
         },
     )

scripts/maintenance/computational-clusters/autoscaled_monitor/db.py

@@ -18,8 +18,11 @@ async def db_engine(
     state: AppState,
 ) -> AsyncGenerator[AsyncEngine, Any]:
     async with contextlib.AsyncExitStack() as stack:
-        assert state.environment["POSTGRES_ENDPOINT"]  # nosec
-        db_endpoint = state.environment["POSTGRES_ENDPOINT"]
+        assert state.environment["POSTGRES_HOST"]  # nosec
+        assert state.environment["POSTGRES_PORT"]  # nosec
+        db_endpoint = (
+            f"{state.environment['POSTGRES_HOST']}:{state.environment['POSTGRES_PORT']}"
+        )
         if state.main_bastion_host:
             assert state.ssh_key_path  # nosec
             db_host, db_port = db_endpoint.split(":")

services/api-server/tests/conftest.py

@@ -72,8 +72,10 @@ def default_app_env_vars(
     env_vars["API_SERVER_DEV_FEATURES_ENABLED"] = "1"
     env_vars["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED"] = "1"
     env_vars["API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED"] = "0"
-    env_vars["POSTGRES_MINSIZE"] = "2"
+    env_vars["POSTGRES_MINSIZE"] = "1"
     env_vars["POSTGRES_MAXSIZE"] = "10"
+    env_vars["POSTGRES_MAX_POOLSIZE"] = "10"
+    env_vars["POSTGRES_MAX_OVERFLOW"] = "20"
     env_vars["API_SERVER_CELERY"] = "null"
     env_vars["API_SERVER_RABBITMQ"] = "null"
 

services/director-v2/.env-devel

@@ -29,7 +29,6 @@ DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 
 LOG_LEVEL=DEBUG
 
-POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_USER=test
 POSTGRES_PASSWORD=test
 POSTGRES_DB=test

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -152,7 +152,6 @@ def _get_environment_variables(
         "DYNAMIC_SIDECAR_LOG_LEVEL": app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_LOG_LEVEL,
         "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED": f"{app_settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED}",
         "POSTGRES_DB": f"{app_settings.POSTGRES.POSTGRES_DB}",
-        "POSTGRES_ENDPOINT": f"{app_settings.POSTGRES.POSTGRES_HOST}:{app_settings.POSTGRES.POSTGRES_PORT}",
         "POSTGRES_HOST": f"{app_settings.POSTGRES.POSTGRES_HOST}",
         "POSTGRES_PASSWORD": f"{app_settings.POSTGRES.POSTGRES_PASSWORD.get_secret_value()}",
         "POSTGRES_PORT": f"{app_settings.POSTGRES.POSTGRES_PORT}",
@@ -458,9 +457,9 @@ async def get_dynamic_sidecar_spec(  # pylint:disable=too-many-arguments# noqa:
         dynamic_sidecar_settings=dynamic_sidecar_settings, app_settings=app_settings
     )
 
-    assert scheduler_data.product_name is not None, (
-        "ONLY for legacy. This function should not be called with product_name==None"
-    )  # nosec
+    assert (
+        scheduler_data.product_name is not None
+    ), "ONLY for legacy. This function should not be called with product_name==None"  # nosec
 
     standard_simcore_docker_labels: dict[DockerLabelKey, str] = SimcoreContainerLabels(
         user_id=scheduler_data.user_id,