Skip to content

Commit ffd604e

Browse files
sandereggsanderegg
authored
🎨Maintenance & Performance: Properly setup asyncpg DB engine (#8322)
1 parent a58201f commit ffd604e

File tree

21 files changed

+148
-168
lines changed

21 files changed

+148
-168
lines changed

‎.env-devel‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,13 +194,14 @@ PAYMENTS_USERNAME=admin
194194
PAYMENTS_TRACING={}
195195

196196
POSTGRES_DB=simcoredb
197-
POSTGRES_ENDPOINT=postgres:5432
198197
POSTGRES_HOST=postgres
199198
POSTGRES_PASSWORD=adminadmin
200199
POSTGRES_PORT=5432
201200
POSTGRES_USER=scu
202-
POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
201+
POSTGRES_MINSIZE=1
203202
POSTGRES_MAXSIZE=50
203+
POSTGRES_MAX_POOLSIZE=10
204+
POSTGRES_MAX_OVERFLOW=20
204205
POSTGRES_READONLY_PASSWORD=readonly
205206
POSTGRES_READONLY_USER=postgres_readonly
206207

‎packages/pytest-simcore/src/pytest_simcore/postgres_service.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,6 @@ def postgres_env_vars_dict(postgres_dsn: PostgresTestConfig) -> EnvVarsDict:
250250
"POSTGRES_DB": postgres_dsn["database"],
251251
"POSTGRES_HOST": postgres_dsn["host"],
252252
"POSTGRES_PORT": f"{postgres_dsn['port']}",
253-
"POSTGRES_ENDPOINT": f"{postgres_dsn['host']}:{postgres_dsn['port']}",
254253
}
255254

256255

‎packages/service-library/src/servicelib/db_asyncpg_utils.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ async def create_async_engine_and_database_ready(
3939

4040
engine = create_async_engine(
4141
settings.dsn_with_async_sqlalchemy,
42-
pool_size=settings.POSTGRES_MINSIZE,
43-
max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
42+
pool_size=settings.POSTGRES_MAX_POOLSIZE,
43+
max_overflow=settings.POSTGRES_MAX_OVERFLOW,
4444
connect_args={"server_settings": server_settings},
4545
pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
4646
future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released
@@ -90,8 +90,8 @@ async def with_async_pg_engine(
9090

9191
engine = create_async_engine(
9292
settings.dsn_with_async_sqlalchemy,
93-
pool_size=settings.POSTGRES_MINSIZE,
94-
max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
93+
pool_size=settings.POSTGRES_MAX_POOLSIZE,
94+
max_overflow=settings.POSTGRES_MAX_OVERFLOW,
9595
connect_args={"server_settings": server_settings},
9696
pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
9797
future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released

‎packages/settings-library/src/settings_library/postgres.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from pydantic import (
66
AliasChoices,
77
Field,
8+
NonNegativeInt,
89
PostgresDsn,
910
SecretStr,
1011
model_validator,
@@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings):
3031

3132
# pool connection limits
3233
POSTGRES_MINSIZE: Annotated[
33-
int, Field(description="Minimum number of connections in the pool", ge=2)
34-
] = 2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
34+
int,
35+
Field(
36+
description="Minimum number of connections in the pool that are always created and kept",
37+
ge=1,
38+
),
39+
] = 1
3540
POSTGRES_MAXSIZE: Annotated[
36-
int, Field(description="Maximum number of connections in the pool", ge=2)
41+
int,
42+
Field(
43+
description="Maximum number of connections in the pool that are kept",
44+
ge=1,
45+
),
3746
] = 50
47+
POSTGRES_MAX_POOLSIZE: Annotated[
48+
int,
49+
Field(
50+
description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand"
51+
),
52+
] = 10
53+
POSTGRES_MAX_OVERFLOW: Annotated[
54+
NonNegativeInt, Field(description="Maximal overflow connections")
55+
] = 20
3856

3957
POSTGRES_CLIENT_NAME: Annotated[
4058
str | None,
@@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
125143
"POSTGRES_USER": "usr",
126144
"POSTGRES_PASSWORD": "secret",
127145
"POSTGRES_DB": "db",
128-
"POSTGRES_MINSIZE": 2,
146+
"POSTGRES_MINSIZE": 1,
129147
"POSTGRES_MAXSIZE": 50,
148+
"POSTGRES_MAX_POOLSIZE": 10,
149+
"POSTGRES_MAX_OVERFLOW": 20,
130150
"POSTGRES_CLIENT_NAME": "my_app", # first-choice
131151
"HOST": "should be ignored",
132152
"HOST_NAME": "should be ignored",
@@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
136156
)
137157

138158
model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)
139-
model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)

‎packages/settings-library/tests/data/.env-compact‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
APP_HOST=localhost
44
APP_PORT=80
55
APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
6-
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
6+
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'

‎packages/settings-library/tests/data/.env-granular‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ POSTGRES_USER=foo
1212
POSTGRES_PASSWORD=**********
1313
# Database name
1414
POSTGRES_DB=foodb
15-
# Minimum number of connections in the pool
16-
POSTGRES_MINSIZE=2
17-
# Maximum number of connections in the pool
15+
POSTGRES_MINSIZE=1
16+
POSTGRES_MAX_POOLSIZE=10
17+
POSTGRES_MAX_OVERFLOW=20
1818
POSTGRES_MAXSIZE=50
1919
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
2020
POSTGRES_CLIENT_NAME=None

‎packages/settings-library/tests/data/.env-mixed‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ POSTGRES_USER=foo
1212
POSTGRES_PASSWORD=**********
1313
# Database name
1414
POSTGRES_DB=foodb
15-
# Minimum number of connections in the pool
16-
POSTGRES_MINSIZE=2
17-
# Maximum number of connections in the pool
15+
POSTGRES_MINSIZE=1
16+
POSTGRES_MAX_POOLSIZE=10
17+
POSTGRES_MAX_OVERFLOW=20
1818
POSTGRES_MAXSIZE=50
1919
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
2020
POSTGRES_CLIENT_NAME=None

‎packages/settings-library/tests/data/.env-sample‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ POSTGRES_USER=foo
88
POSTGRES_PASSWORD=secret
99
# Database name
1010
POSTGRES_DB=foodb
11-
# Maximum number of connections in the pool
12-
POSTGRES_MINSIZE=2
13-
# Minimum number of connections in the pool
11+
POSTGRES_MINSIZE=1
12+
POSTGRES_MAX_POOLSIZE=10
13+
POSTGRES_MAX_OVERFLOW=20
1414
POSTGRES_MAXSIZE=50
1515

1616
# --- APP_MODULE_FIELD ---

‎packages/settings-library/tests/test_base_w_postgres.py‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,10 @@ class _FakePostgresSettings(BaseCustomSettings):
5555
POSTGRES_PASSWORD: str
5656

5757
POSTGRES_DB: str
58-
POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2
59-
POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50
58+
POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1
59+
POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50
60+
POSTGRES_MAX_POOLSIZE: int = 10
61+
POSTGRES_MAX_OVERFLOW: Annotated[int, Field(ge=0)] = 20
6062

6163
POSTGRES_CLIENT_NAME: Annotated[
6264
str | None,
@@ -126,7 +128,6 @@ class S5(BaseCustomSettings):
126128
def test_parse_from_empty_envs(
127129
postgres_envvars_unset: None, model_classes_factory: Callable
128130
):
129-
130131
S1, S2, S3, S4, S5 = model_classes_factory()
131132

132133
with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info:
@@ -160,7 +161,6 @@ def test_parse_from_individual_envs(
160161
monkeypatch: pytest.MonkeyPatch,
161162
model_classes_factory: Callable,
162163
):
163-
164164
S1, S2, S3, S4, S5 = model_classes_factory()
165165

166166
# environment
@@ -200,7 +200,9 @@ def test_parse_from_individual_envs(
200200
"POSTGRES_PASSWORD": "shh",
201201
"POSTGRES_DB": "db",
202202
"POSTGRES_MAXSIZE": 50,
203-
"POSTGRES_MINSIZE": 2,
203+
"POSTGRES_MINSIZE": 1,
204+
"POSTGRES_MAX_POOLSIZE": 10,
205+
"POSTGRES_MAX_OVERFLOW": 20,
204206
"POSTGRES_CLIENT_NAME": None,
205207
}
206208
}
@@ -215,7 +217,9 @@ def test_parse_from_individual_envs(
215217
"POSTGRES_PASSWORD": "shh",
216218
"POSTGRES_DB": "db",
217219
"POSTGRES_MAXSIZE": 50,
218-
"POSTGRES_MINSIZE": 2,
220+
"POSTGRES_MINSIZE": 1,
221+
"POSTGRES_MAX_POOLSIZE": 10,
222+
"POSTGRES_MAX_OVERFLOW": 20,
219223
"POSTGRES_CLIENT_NAME": None,
220224
}
221225
}
@@ -228,7 +232,6 @@ def test_parse_from_individual_envs(
228232
def test_parse_compact_env(
229233
postgres_envvars_unset: None, monkeypatch, model_classes_factory
230234
):
231-
232235
S1, S2, S3, S4, S5 = model_classes_factory()
233236

234237
# environment
@@ -262,7 +265,9 @@ def test_parse_compact_env(
262265
"POSTGRES_PASSWORD": "shh2",
263266
"POSTGRES_DB": "db2",
264267
"POSTGRES_MAXSIZE": 50,
265-
"POSTGRES_MINSIZE": 2,
268+
"POSTGRES_MINSIZE": 1,
269+
"POSTGRES_MAX_POOLSIZE": 10,
270+
"POSTGRES_MAX_OVERFLOW": 20,
266271
"POSTGRES_CLIENT_NAME": None,
267272
}
268273
}
@@ -342,7 +347,6 @@ def test_parse_compact_env(
342347
def test_parse_from_mixed_envs(
343348
postgres_envvars_unset: None, monkeypatch, model_classes_factory
344349
):
345-
346350
S1, S2, S3, S4, S5 = model_classes_factory()
347351

348352
# environment
@@ -372,7 +376,9 @@ def test_parse_from_mixed_envs(
372376
"POSTGRES_PASSWORD": "shh2",
373377
"POSTGRES_DB": "db2",
374378
"POSTGRES_MAXSIZE": 50,
375-
"POSTGRES_MINSIZE": 2,
379+
"POSTGRES_MINSIZE": 1,
380+
"POSTGRES_MAX_POOLSIZE": 10,
381+
"POSTGRES_MAX_OVERFLOW": 20,
376382
"POSTGRES_CLIENT_NAME": None,
377383
}
378384
}
@@ -472,7 +478,6 @@ def test_parse_from_mixed_envs(
472478
def test_toggle_plugin_1(
473479
postgres_envvars_unset: None, monkeypatch, model_classes_factory
474480
):
475-
476481
*_, S4, S5 = model_classes_factory()
477482

478483
# empty environ
@@ -535,7 +540,6 @@ def test_toggle_plugin_3(
535540
def test_toggle_plugin_4(
536541
postgres_envvars_unset: None, monkeypatch, model_classes_factory
537542
):
538-
539543
*_, S4, S5 = model_classes_factory()
540544
JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}'
541545

@@ -565,7 +569,6 @@ def test_toggle_plugin_4(
565569
)
566570

567571
with monkeypatch.context() as patch:
568-
569572
# Enables both but remove individuals
570573
setenvs_from_envfile(
571574
patch,

‎packages/settings-library/tests/test_utils_cli.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,10 @@ def fake_granular_env_file_content() -> str:
7474
POSTGRES_USER=foo
7575
POSTGRES_PASSWORD=secret
7676
POSTGRES_DB=foodb
77-
POSTGRES_MINSIZE=2
77+
POSTGRES_MINSIZE=1
7878
POSTGRES_MAXSIZE=50
79+
POSTGRES_MAX_POOLSIZE=10
80+
POSTGRES_MAX_OVERFLOW=20
7981
POSTGRES_CLIENT_NAME=None
8082
MODULE_VALUE=10
8183
"""
@@ -188,8 +190,10 @@ def test_cli_default_settings_envs(
188190
"POSTGRES_USER": "foo",
189191
"POSTGRES_PASSWORD": "secret",
190192
"POSTGRES_DB": "foodb",
191-
"POSTGRES_MINSIZE": 2,
193+
"POSTGRES_MINSIZE": 1,
192194
"POSTGRES_MAXSIZE": 50,
195+
"POSTGRES_MAX_POOLSIZE": 10,
196+
"POSTGRES_MAX_OVERFLOW": 20,
193197
"POSTGRES_CLIENT_NAME": None,
194198
},
195199
}
@@ -219,8 +223,10 @@ def test_cli_compact_settings_envs(
219223
"POSTGRES_USER": "foo",
220224
"POSTGRES_PASSWORD": "secret",
221225
"POSTGRES_DB": "foodb",
222-
"POSTGRES_MINSIZE": 2,
226+
"POSTGRES_MINSIZE": 1,
223227
"POSTGRES_MAXSIZE": 50,
228+
"POSTGRES_MAX_POOLSIZE": 10,
229+
"POSTGRES_MAX_OVERFLOW": 20,
224230
"POSTGRES_CLIENT_NAME": None,
225231
},
226232
}
@@ -244,7 +250,7 @@ def test_cli_compact_settings_envs(
244250
"APP_HOST": "localhost",
245251
"APP_PORT": "80",
246252
"APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}',
247-
"APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}',
253+
"APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50,"POSTGRES_MAX_POOLSIZE":10,"POSTGRES_MAX_OVERFLOW":20,"POSTGRES_CLIENT_NAME":null}',
248254
}
249255

250256
settings_2 = fake_settings_class()
@@ -261,7 +267,7 @@ def test_compact_format(
261267
APP_HOST=localhost
262268
APP_PORT=80
263269
APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
264-
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
270+
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "None"}'
265271
""",
266272
)
267273

@@ -292,10 +298,10 @@ def test_granular_format(
292298
POSTGRES_PASSWORD=secret
293299
# Database name
294300
POSTGRES_DB=foodb
295-
# Minimum number of connections in the pool
296-
POSTGRES_MINSIZE=2
297-
# Maximum number of connections in the pool
301+
POSTGRES_MINSIZE=1
298302
POSTGRES_MAXSIZE=50
303+
POSTGRES_MAX_POOLSIZE=10
304+
POSTGRES_MAX_OVERFLOW=20
299305
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
300306
POSTGRES_CLIENT_NAME=None
301307
""",
@@ -313,8 +319,10 @@ def test_granular_format(
313319
"POSTGRES_USER": "foo",
314320
"POSTGRES_PASSWORD": "secret",
315321
"POSTGRES_DB": "foodb",
316-
"POSTGRES_MINSIZE": 2,
322+
"POSTGRES_MINSIZE": 1,
317323
"POSTGRES_MAXSIZE": 50,
324+
"POSTGRES_MAX_POOLSIZE": 10,
325+
"POSTGRES_MAX_OVERFLOW": 20,
318326
"POSTGRES_CLIENT_NAME": None,
319327
},
320328
)

0 commit comments

Comments
 (0)