Skip to content

Commit 2050b04

Browse files
sandereggsanderegg
committed
added new envs
1 parent 33608bc commit 2050b04

File tree

12 files changed

+88
-34
lines changed

12 files changed

+88
-34
lines changed

‎.env-devel‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,8 +198,10 @@ POSTGRES_HOST=postgres
198198
POSTGRES_PASSWORD=adminadmin
199199
POSTGRES_PORT=5432
200200
POSTGRES_USER=scu
201-
POSTGRES_MINSIZE=10 # NOTE: this is currently asyncpg pool size and aiopg min size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660
202-
POSTGRES_MAXSIZE=30 # NOTE: this is currently asyncpg 20 overflow size and aiopg max size see https://github.com/ITISFoundation/osparc-simcore/issues/7829#issuecomment-3258204660
201+
POSTGRES_MINSIZE=1
202+
POSTGRES_MAXSIZE=50
203+
POSTGRES_MAX_POOLSIZE=10
204+
POSTGRES_MAX_OVERFLOW=20
203205
POSTGRES_READONLY_PASSWORD=readonly
204206
POSTGRES_READONLY_USER=postgres_readonly
205207

‎packages/service-library/src/servicelib/db_asyncpg_utils.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from collections.abc import AsyncIterator
55
from datetime import timedelta
66

7+
import orjson
78
from models_library.healthchecks import IsNonResponsive, IsResponsive, LivenessResult
89
from settings_library.postgres import PostgresSettings
910
from sqlalchemy.exc import SQLAlchemyError
@@ -39,11 +40,13 @@ async def create_async_engine_and_database_ready(
3940

4041
engine = create_async_engine(
4142
settings.dsn_with_async_sqlalchemy,
42-
pool_size=settings.POSTGRES_MINSIZE,
43-
max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
43+
pool_size=settings.POSTGRES_MAX_POOLSIZE,
44+
max_overflow=settings.POSTGRES_MAX_OVERFLOW,
4445
connect_args={"server_settings": server_settings},
4546
pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
4647
future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released
48+
json_serializer=orjson.dumps,
49+
json_deserializer=orjson.loads,
4750
)
4851

4952
try:
@@ -90,11 +93,13 @@ async def with_async_pg_engine(
9093

9194
engine = create_async_engine(
9295
settings.dsn_with_async_sqlalchemy,
93-
pool_size=settings.POSTGRES_MINSIZE,
94-
max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
96+
pool_size=settings.POSTGRES_MAX_POOLSIZE,
97+
max_overflow=settings.POSTGRES_MAX_OVERFLOW,
9598
connect_args={"server_settings": server_settings},
9699
pool_pre_ping=True, # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
97100
future=True, # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released
101+
json_serializer=orjson.dumps,
102+
json_deserializer=orjson.loads,
98103
)
99104
yield engine
100105
finally:

‎packages/settings-library/src/settings_library/postgres.py‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from pydantic import (
66
AliasChoices,
77
Field,
8+
NonNegativeInt,
89
PostgresDsn,
910
SecretStr,
1011
model_validator,
@@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings):
3031

3132
# pool connection limits
3233
POSTGRES_MINSIZE: Annotated[
33-
int, Field(description="Minimum number of connections in the pool", ge=2)
34-
] = 2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
34+
int,
35+
Field(
36+
description="Minimum number of connections in the pool that are always created and kept",
37+
ge=1,
38+
),
39+
] = 1
3540
POSTGRES_MAXSIZE: Annotated[
36-
int, Field(description="Maximum number of connections in the pool", ge=2)
41+
int,
42+
Field(
43+
description="Maximum number of connections in the pool that are kept",
44+
ge=1,
45+
),
3746
] = 50
47+
POSTGRES_MAX_POOLSIZE: Annotated[
48+
int,
49+
Field(
50+
description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand"
51+
),
52+
] = 10
53+
POSTGRES_MAX_OVERFLOW: Annotated[
54+
NonNegativeInt, Field(description="Maximal overflow connections")
55+
] = 20
3856

3957
POSTGRES_CLIENT_NAME: Annotated[
4058
str | None,
@@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
125143
"POSTGRES_USER": "usr",
126144
"POSTGRES_PASSWORD": "secret",
127145
"POSTGRES_DB": "db",
128-
"POSTGRES_MINSIZE": 2,
146+
"POSTGRES_MINSIZE": 1,
129147
"POSTGRES_MAXSIZE": 50,
148+
"POSTGRES_MAX_POOLSIZE": 10,
149+
"POSTGRES_MAX_OVERFLOW": 20,
130150
"POSTGRES_CLIENT_NAME": "my_app", # first-choice
131151
"HOST": "should be ignored",
132152
"HOST_NAME": "should be ignored",
@@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
136156
)
137157

138158
model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)
139-
model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)

‎packages/settings-library/tests/data/.env-granular‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ POSTGRES_USER=foo
1212
POSTGRES_PASSWORD=**********
1313
# Database name
1414
POSTGRES_DB=foodb
15-
# Minimum number of connections in the pool
16-
POSTGRES_MINSIZE=2
17-
# Maximum number of connections in the pool
15+
POSTGRES_MINSIZE=1
1816
POSTGRES_MAXSIZE=50
17+
POSTGRES_MAX_POOLSIZE=10
18+
POSTGRES_MAX_OVERFLOW=20
1919
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
2020
POSTGRES_CLIENT_NAME=None

‎packages/settings-library/tests/data/.env-mixed‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ POSTGRES_USER=foo
1212
POSTGRES_PASSWORD=**********
1313
# Database name
1414
POSTGRES_DB=foodb
15-
# Minimum number of connections in the pool
16-
POSTGRES_MINSIZE=2
17-
# Maximum number of connections in the pool
15+
POSTGRES_MINSIZE=1
1816
POSTGRES_MAXSIZE=50
17+
POSTGRES_MAX_POOLSIZE=10
18+
POSTGRES_MAX_OVERFLOW=20
1919
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
2020
POSTGRES_CLIENT_NAME=None

‎packages/settings-library/tests/data/.env-sample‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ POSTGRES_USER=foo
88
POSTGRES_PASSWORD=secret
99
# Database name
1010
POSTGRES_DB=foodb
11-
# Maximum number of connections in the pool
12-
POSTGRES_MINSIZE=2
13-
# Minimum number of connections in the pool
11+
POSTGRES_MINSIZE=1
1412
POSTGRES_MAXSIZE=50
13+
POSTGRES_MAX_POOLSIZE=10
14+
POSTGRES_MAX_OVERFLOW=20
1515

1616
# --- APP_MODULE_FIELD ---
1717
# Some value for module 1

‎packages/settings-library/tests/test_base_w_postgres.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,9 @@ def test_parse_from_individual_envs(
200200
"POSTGRES_PASSWORD": "shh",
201201
"POSTGRES_DB": "db",
202202
"POSTGRES_MAXSIZE": 50,
203-
"POSTGRES_MINSIZE": 2,
203+
"POSTGRES_MINSIZE": 1,
204+
"POSTGRES_MAX_POOLSIZE": 10,
205+
"POSTGRES_MAX_OVERFLOW": 20,
204206
"POSTGRES_CLIENT_NAME": None,
205207
}
206208
}
@@ -215,7 +217,9 @@ def test_parse_from_individual_envs(
215217
"POSTGRES_PASSWORD": "shh",
216218
"POSTGRES_DB": "db",
217219
"POSTGRES_MAXSIZE": 50,
218-
"POSTGRES_MINSIZE": 2,
220+
"POSTGRES_MINSIZE": 1,
221+
"POSTGRES_MAX_POOLSIZE": 10,
222+
"POSTGRES_MAX_OVERFLOW": 20,
219223
"POSTGRES_CLIENT_NAME": None,
220224
}
221225
}
@@ -262,7 +266,9 @@ def test_parse_compact_env(
262266
"POSTGRES_PASSWORD": "shh2",
263267
"POSTGRES_DB": "db2",
264268
"POSTGRES_MAXSIZE": 50,
265-
"POSTGRES_MINSIZE": 2,
269+
"POSTGRES_MINSIZE": 1,
270+
"POSTGRES_MAX_POOLSIZE": 10,
271+
"POSTGRES_MAX_OVERFLOW": 20,
266272
"POSTGRES_CLIENT_NAME": None,
267273
}
268274
}
@@ -372,7 +378,9 @@ def test_parse_from_mixed_envs(
372378
"POSTGRES_PASSWORD": "shh2",
373379
"POSTGRES_DB": "db2",
374380
"POSTGRES_MAXSIZE": 50,
375-
"POSTGRES_MINSIZE": 2,
381+
"POSTGRES_MINSIZE": 1,
382+
"POSTGRES_MAX_POOLSIZE": 10,
383+
"POSTGRES_MAX_OVERFLOW": 20,
376384
"POSTGRES_CLIENT_NAME": None,
377385
}
378386
}

‎packages/settings-library/tests/test_utils_cli.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,10 @@ def fake_granular_env_file_content() -> str:
7474
POSTGRES_USER=foo
7575
POSTGRES_PASSWORD=secret
7676
POSTGRES_DB=foodb
77-
POSTGRES_MINSIZE=2
77+
POSTGRES_MINSIZE=1
7878
POSTGRES_MAXSIZE=50
79+
POSTGRES_MAX_POOLSIZE=10
80+
POSTGRES_MAX_OVERFLOW=20
7981
POSTGRES_CLIENT_NAME=None
8082
MODULE_VALUE=10
8183
"""
@@ -188,8 +190,10 @@ def test_cli_default_settings_envs(
188190
"POSTGRES_USER": "foo",
189191
"POSTGRES_PASSWORD": "secret",
190192
"POSTGRES_DB": "foodb",
191-
"POSTGRES_MINSIZE": 2,
193+
"POSTGRES_MINSIZE": 1,
192194
"POSTGRES_MAXSIZE": 50,
195+
"POSTGRES_MAX_POOLSIZE": 10,
196+
"POSTGRES_MAX_OVERFLOW": 20,
193197
"POSTGRES_CLIENT_NAME": None,
194198
},
195199
}
@@ -219,8 +223,10 @@ def test_cli_compact_settings_envs(
219223
"POSTGRES_USER": "foo",
220224
"POSTGRES_PASSWORD": "secret",
221225
"POSTGRES_DB": "foodb",
222-
"POSTGRES_MINSIZE": 2,
226+
"POSTGRES_MINSIZE": 1,
223227
"POSTGRES_MAXSIZE": 50,
228+
"POSTGRES_MAX_POOLSIZE": 10,
229+
"POSTGRES_MAX_OVERFLOW": 20,
224230
"POSTGRES_CLIENT_NAME": None,
225231
},
226232
}
@@ -244,7 +250,7 @@ def test_cli_compact_settings_envs(
244250
"APP_HOST": "localhost",
245251
"APP_PORT": "80",
246252
"APP_OPTIONAL_ADDON": '{"MODULE_VALUE":10,"MODULE_VALUE_DEFAULT":42}',
247-
"APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":2,"POSTGRES_MAXSIZE":50,"POSTGRES_CLIENT_NAME":null}',
253+
"APP_REQUIRED_PLUGIN": '{"POSTGRES_HOST":"localhost","POSTGRES_PORT":5432,"POSTGRES_USER":"foo","POSTGRES_PASSWORD":"secret","POSTGRES_DB":"foodb","POSTGRES_MINSIZE":1,"POSTGRES_MAXSIZE":50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME":null}',
248254
}
249255

250256
settings_2 = fake_settings_class()
@@ -261,7 +267,7 @@ def test_compact_format(
261267
APP_HOST=localhost
262268
APP_PORT=80
263269
APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
264-
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
270+
APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "secret", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAXSIZE": 50, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_CLIENT_NAME": "None"}'
265271
""",
266272
)
267273

@@ -292,10 +298,10 @@ def test_granular_format(
292298
POSTGRES_PASSWORD=secret
293299
# Database name
294300
POSTGRES_DB=foodb
295-
# Minimum number of connections in the pool
296-
POSTGRES_MINSIZE=2
297-
# Maximum number of connections in the pool
301+
POSTGRES_MINSIZE=1
298302
POSTGRES_MAXSIZE=50
303+
POSTGRES_MAX_POOLSIZE=10
304+
POSTGRES_MAX_OVERFLOW=20
299305
# Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
300306
POSTGRES_CLIENT_NAME=None
301307
""",
@@ -313,8 +319,10 @@ def test_granular_format(
313319
"POSTGRES_USER": "foo",
314320
"POSTGRES_PASSWORD": "secret",
315321
"POSTGRES_DB": "foodb",
316-
"POSTGRES_MINSIZE": 2,
322+
"POSTGRES_MINSIZE": 1,
317323
"POSTGRES_MAXSIZE": 50,
324+
"POSTGRES_MAX_POOLSIZE": 10,
325+
"POSTGRES_MAX_OVERFLOW": 20,
318326
"POSTGRES_CLIENT_NAME": None,
319327
},
320328
)

‎services/api-server/tests/conftest.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,10 @@ def default_app_env_vars(
7272
env_vars["API_SERVER_DEV_FEATURES_ENABLED"] = "1"
7373
env_vars["API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED"] = "1"
7474
env_vars["API_SERVER_PROMETHEUS_INSTRUMENTATION_ENABLED"] = "0"
75-
env_vars["POSTGRES_MINSIZE"] = "2"
75+
env_vars["POSTGRES_MINSIZE"] = "1"
7676
env_vars["POSTGRES_MAXSIZE"] = "10"
77+
env_vars["POSTGRES_MAX_POOLSIZE"] = "10"
78+
env_vars["POSTGRES_MAX_OVERFLOW"] = "20"
7779
env_vars["API_SERVER_CELERY"] = "null"
7880
env_vars["API_SERVER_RABBITMQ"] = "null"
7981

‎services/docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ x-postgres-settings: &postgres_settings
2424
POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE}
2525
POSTGRES_MINSIZE: ${POSTGRES_MINSIZE}
2626
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
27+
POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE},
28+
POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW},
2729
POSTGRES_PORT: ${POSTGRES_PORT}
2830
POSTGRES_USER: ${POSTGRES_USER}
2931

0 commit comments

Comments
 (0)