🎨 Make POSTGRES_MINSIZE and POSTGRES_MAXSIZE configurable

.env-devel

@@ -194,7 +194,8 @@ POSTGRES_HOST=postgres
 POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
-
+POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+POSTGRES_MAXSIZE=50
 POSTGRES_READONLY_PASSWORD=readonly
 POSTGRES_READONLY_USER=postgres_readonly
 

packages/postgres-database/tests/conftest.py

@@ -81,7 +81,7 @@ def sync_engine(postgres_service: str) -> Iterable[sqlalchemy.engine.Engine]:
 def _make_asyncpg_engine(postgres_service: str) -> Callable[[bool], AsyncEngine]:
     # NOTE: users is responsible of `await engine.dispose()`
     dsn = postgres_service.replace("postgresql://", "postgresql+asyncpg://")
-    minsize = 1
+    minsize = 2
     maxsize = 50
 
     def _(echo: bool):

packages/service-library/src/servicelib/db_asyncpg_utils.py

@@ -30,12 +30,16 @@ async def create_async_engine_and_database_ready(
         raise_if_migration_not_ready,
     )
 
-    server_settings = None
+    server_settings = {
+        "jit": "off"
+    }  # see https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#disabling-the-postgresql-jit-to-improve-enum-datatype-handling
     if settings.POSTGRES_CLIENT_NAME:
         assert isinstance(settings.POSTGRES_CLIENT_NAME, str)  # nosec
-        server_settings = {
-            "application_name": settings.POSTGRES_CLIENT_NAME,
-        }
+        server_settings.update(
+            {
+                "application_name": settings.POSTGRES_CLIENT_NAME,
+            }
+        )
 
     engine = create_async_engine(
         settings.dsn_with_async_sqlalchemy,

packages/service-library/tests/aiohttp/with_postgres/test_aiopg_utils.py

@@ -78,7 +78,7 @@ async def test_create_pg_engine(postgres_service_with_fake_data: DataSourceName)
     dsn = postgres_service_with_fake_data
 
     # using raw call and dsn.asdict to fill create_engine arguments!
-    engine1 = await aiopg.sa.create_engine(minsize=1, maxsize=5, **asdict(dsn))
+    engine1 = await aiopg.sa.create_engine(minsize=2, maxsize=5, **asdict(dsn))
 
     # just creating engine
     engine2 = await create_pg_engine(dsn)
@@ -114,7 +114,7 @@ async def test_engine_when_idle_for_some_time():
         database="db",
         application_name="test-app",
     )
-    engine = await create_pg_engine(dsn, minsize=1, maxsize=1)
+    engine = await create_pg_engine(dsn, minsize=2, maxsize=2)
     init_pg_tables(dsn, metadata)
     assert not engine.closed  # does not mean anything!!!
     # pylint: disable=no-value-for-parameter

packages/settings-library/src/settings_library/postgres.py

@@ -31,10 +31,10 @@ class PostgresSettings(BaseCustomSettings):
 
     # pool connection limits
     POSTGRES_MINSIZE: Annotated[
-        int, Field(description="Minimum number of connections in the pool", ge=1)
-    ] = 1
+        int, Field(description="Minimum number of connections in the pool", ge=2)
+    ] = 2  # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
     POSTGRES_MAXSIZE: Annotated[
-        int, Field(description="Maximum number of connections in the pool", ge=1)
+        int, Field(description="Maximum number of connections in the pool", ge=2)
     ] = 50
 
     POSTGRES_CLIENT_NAME: Annotated[
@@ -124,7 +124,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "POSTGRES_USER": "usr",
                         "POSTGRES_PASSWORD": "secret",
                         "POSTGRES_DB": "db",
-                        "POSTGRES_MINSIZE": 1,
+                        "POSTGRES_MINSIZE": 2,
                         "POSTGRES_MAXSIZE": 50,
                         "POSTGRES_CLIENT_NAME": "my_app",  # first-choice
                         "HOST": "should be ignored",

packages/settings-library/tests/data/.env-compact

@@ -3,4 +3,4 @@
 APP_HOST=localhost
 APP_PORT=80
 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
-APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
+APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'

packages/settings-library/tests/data/.env-granular

@@ -13,7 +13,7 @@ POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
 # Minimum number of connections in the pool
-POSTGRES_MINSIZE=1
+POSTGRES_MINSIZE=2
 # Maximum number of connections in the pool
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)

packages/settings-library/tests/data/.env-mixed

@@ -13,7 +13,7 @@ POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
 # Minimum number of connections in the pool
-POSTGRES_MINSIZE=1
+POSTGRES_MINSIZE=2
 # Maximum number of connections in the pool
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)

packages/settings-library/tests/data/.env-sample

@@ -9,7 +9,7 @@ POSTGRES_PASSWORD=secret
 # Database name
 POSTGRES_DB=foodb
 # Maximum number of connections in the pool
-POSTGRES_MINSIZE=1
+POSTGRES_MINSIZE=2
 # Minimum number of connections in the pool
 POSTGRES_MAXSIZE=50
 

packages/settings-library/tests/test_base_w_postgres.py

@@ -55,8 +55,8 @@ class _FakePostgresSettings(BaseCustomSettings):
             POSTGRES_PASSWORD: str
 
             POSTGRES_DB: str
-            POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1
-            POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50
+            POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2
+            POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50
 
             POSTGRES_CLIENT_NAME: Annotated[
                 str | None,
@@ -200,7 +200,7 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MINSIZE": 2,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -215,7 +215,7 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MINSIZE": 2,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -262,7 +262,7 @@ def test_parse_compact_env(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MINSIZE": 2,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }
@@ -372,7 +372,7 @@ def test_parse_from_mixed_envs(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MINSIZE": 2,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }

packages/settings-library/tests/test_utils_cli.py

@@ -74,7 +74,7 @@ def fake_granular_env_file_content() -> str:
         POSTGRES_USER=foo
         POSTGRES_PASSWORD=secret
         POSTGRES_DB=foodb
-        POSTGRES_MINSIZE=1
+        POSTGRES_MINSIZE=2
         POSTGRES_MAXSIZE=50
         POSTGRES_CLIENT_NAME=None
         MODULE_VALUE=10
@@ -188,7 +188,7 @@ def test_cli_default_settings_envs(
                 "POSTGRES_USER": "foo",
                 "POSTGRES_PASSWORD": "secret",
                 "POSTGRES_DB": "foodb",
-                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MINSIZE": 2,
                 "POSTGRES_MAXSIZE": 50,
                 "POSTGRES_CLIENT_NAME": None,
             },
@@ -219,7 +219,7 @@ def test_cli_compact_settings_envs(
                 "POSTGRES_USER": "foo",
                 "POSTGRES_PASSWORD": "secret",
                 "POSTGRES_DB": "foodb",
-                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MINSIZE": 2,
                 "POSTGRES_MAXSIZE": 50,
                 "POSTGRES_CLIENT_NAME": None,
             },
@@ -244,7 +244,7 @@ def test_cli_compact_settings_envs(
             "APP_HOST": "localhost",
             "APP_PORT": "80",
             "APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}',
-            "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}',
+            "APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}',
         }
 
         settings_2 = fake_settings_class()
@@ -261,7 +261,7 @@ def test_compact_format(
         APP_HOST=localhost
         APP_PORT=80
         APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
-        APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
+        APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
         """,
     )
 
@@ -293,7 +293,7 @@ def test_granular_format(
     # Database name
     POSTGRES_DB=foodb
     # Minimum number of connections in the pool
-    POSTGRES_MINSIZE=1
+    POSTGRES_MINSIZE=2
     # Maximum number of connections in the pool
     POSTGRES_MAXSIZE=50
     # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
@@ -313,7 +313,7 @@ def test_granular_format(
             "POSTGRES_USER": "foo",
             "POSTGRES_PASSWORD": "secret",
             "POSTGRES_DB": "foodb",
-            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MINSIZE": 2,
             "POSTGRES_MAXSIZE": 50,
             "POSTGRES_CLIENT_NAME": None,
         },

services/api-server/tests/unit/conftest.py

@@ -184,7 +184,7 @@ def auth(
     # mock engine if db was not init
     if app.state.settings.API_SERVER_POSTGRES is None:
         engine = mocker.MagicMock()
-        engine.minsize = 1
+        engine.minsize = 2
         engine.size = 10
         engine.freesize = 3
         engine.maxsize = 10

services/docker-compose.yml

@@ -726,6 +726,8 @@ services:
       POSTGRES_HOST: ${POSTGRES_HOST}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_PORT: ${POSTGRES_PORT}
+      POSTGRES_MINSIZE: ${POSTGRES_MINSIZE}
+      POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE}
       POSTGRES_USER: ${POSTGRES_USER}
 
       # WEBSERVER_DIAGNOSTICS

services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py

@@ -164,8 +164,10 @@ async def check_running_services(app: FastAPI) -> None:
     base_start_timestamp = datetime.now(tz=UTC)
 
     # Get all current running services (across all products)
-    total_count: PositiveInt = await service_runs_db.total_service_runs_with_running_status_across_all_products(
-        _db_engine
+    total_count: PositiveInt = (
+        await service_runs_db.total_service_runs_with_running_status_across_all_products(
+            _db_engine
+        )
     )
 
     for offset in range(0, total_count, _BATCH_SIZE):

services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py

@@ -1,3 +1,4 @@
+import asyncio
 from collections.abc import Callable, Iterator
 from datetime import UTC, datetime, timedelta
 
@@ -120,6 +121,9 @@ def resource_tracker_setup_db(
         con.execute(resource_tracker_service_runs.delete())
 
 
+_PROD_RUN_INTERVAL_SEC = 1  # in reality in production this is 5 mins
+
+
 async def test_process_event_functions(
     create_rabbitmq_client: Callable[[str], RabbitMQClient],
     mocked_redis_server: None,
@@ -132,6 +136,7 @@ async def test_process_event_functions(
 
     for _ in range(app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL):
         await check_running_services(initialized_app)
+        await asyncio.sleep(_PROD_RUN_INTERVAL_SEC)
         # NOTE: As we are doing check that the modified field needs to be older then some
         # threshold, we need to make this field artificaly older in this test
         with postgres_db.connect() as con:

services/web/server/tests/data/default_app_config-unit.yaml

@@ -29,7 +29,7 @@ db:
     host: 127.0.0.1
     port: 5432
     maxsize: 5
-    minsize: 1
+    minsize: 2
     endpoint: 127.0.0.1:5432
 diagnostics:
   enabled: false