Merge branch 'master' into 8292-create-custom-GenerateJsonSchema-for-resolving-references

Author: bisgaard-itis

.env-devel

@@ -194,13 +194,14 @@ PAYMENTS_USERNAME=admin
 PAYMENTS_TRACING={}
 
 POSTGRES_DB=simcoredb
-POSTGRES_ENDPOINT=postgres:5432
 POSTGRES_HOST=postgres
 POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
-POSTGRES_MINSIZE=2 # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+POSTGRES_MINSIZE=1
 POSTGRES_MAXSIZE=50
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_READONLY_PASSWORD=readonly
 POSTGRES_READONLY_USER=postgres_readonly
 

packages/pytest-simcore/src/pytest_simcore/postgres_service.py

@@ -250,7 +250,6 @@ def postgres_env_vars_dict(postgres_dsn: PostgresTestConfig) -> EnvVarsDict:
         "POSTGRES_DB": postgres_dsn["database"],
         "POSTGRES_HOST": postgres_dsn["host"],
         "POSTGRES_PORT": f"{postgres_dsn['port']}",
-        "POSTGRES_ENDPOINT": f"{postgres_dsn['host']}:{postgres_dsn['port']}",
     }
 
 

packages/service-library/src/servicelib/db_asyncpg_utils.py

@@ -39,8 +39,8 @@ async def create_async_engine_and_database_ready(
 
     engine = create_async_engine(
         settings.dsn_with_async_sqlalchemy,
-        pool_size=settings.POSTGRES_MINSIZE,
-        max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
+        pool_size=settings.POSTGRES_MAX_POOLSIZE,
+        max_overflow=settings.POSTGRES_MAX_OVERFLOW,
         connect_args={"server_settings": server_settings},
         pool_pre_ping=True,  # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
         future=True,  # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released
@@ -90,8 +90,8 @@ async def with_async_pg_engine(
 
             engine = create_async_engine(
                 settings.dsn_with_async_sqlalchemy,
-                pool_size=settings.POSTGRES_MINSIZE,
-                max_overflow=settings.POSTGRES_MAXSIZE - settings.POSTGRES_MINSIZE,
+                pool_size=settings.POSTGRES_MAX_POOLSIZE,
+                max_overflow=settings.POSTGRES_MAX_OVERFLOW,
                 connect_args={"server_settings": server_settings},
                 pool_pre_ping=True,  # https://docs.sqlalchemy.org/en/14/core/pooling.html#dealing-with-disconnects
                 future=True,  # this uses sqlalchemy 2.0 API, shall be removed when sqlalchemy 2.0 is released

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/functions/functions_rpc_interface.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Literal
 
 from models_library.api_schemas_webserver import WEBSERVER_RPC_NAMESPACE
 from models_library.api_schemas_webserver.functions import (
@@ -18,6 +19,7 @@
 )
 from models_library.functions import (
     FunctionClass,
+    FunctionGroupAccessRights,
     FunctionJobStatus,
     FunctionOutputs,
     FunctionUserAccessRights,
@@ -418,6 +420,7 @@ async def update_function_job_status(
     product_name: ProductName,
     function_job_id: FunctionJobID,
     job_status: FunctionJobStatus,
+    check_write_permissions: bool = True,
 ) -> FunctionJobStatus:
     result = await rabbitmq_rpc_client.request(
         WEBSERVER_RPC_NAMESPACE,
@@ -426,6 +429,7 @@ async def update_function_job_status(
         job_status=job_status,
         user_id=user_id,
         product_name=product_name,
+        check_write_permissions=check_write_permissions,
     )
     return TypeAdapter(FunctionJobStatus).validate_python(result)
 
@@ -438,6 +442,7 @@ async def update_function_job_outputs(
     product_name: ProductName,
     function_job_id: FunctionJobID,
     outputs: FunctionOutputs,
+    check_write_permissions: bool = True,
 ) -> FunctionOutputs:
     result = await rabbitmq_rpc_client.request(
         WEBSERVER_RPC_NAMESPACE,
@@ -446,6 +451,7 @@ async def update_function_job_outputs(
         outputs=outputs,
         user_id=user_id,
         product_name=product_name,
+        check_write_permissions=check_write_permissions,
     )
     return TypeAdapter(FunctionOutputs).validate_python(result)
 
@@ -578,3 +584,37 @@ async def get_functions_user_api_access_rights(
         product_name=product_name,
     )
     return TypeAdapter(FunctionUserApiAccessRights).validate_python(result)
+
+
+@log_decorator(_logger, level=logging.DEBUG)
+async def set_group_permissions(
+    rabbitmq_rpc_client: RabbitMQRPCClient,
+    *,
+    user_id: UserID,
+    product_name: ProductName,
+    object_type: Literal["function", "function_job", "function_job_collection"],
+    object_ids: list[FunctionID | FunctionJobID | FunctionJobCollectionID],
+    permission_group_id: int,
+    read: bool | None = None,
+    write: bool | None = None,
+    execute: bool | None = None,
+) -> list[
+    tuple[
+        FunctionID | FunctionJobID | FunctionJobCollectionID, FunctionGroupAccessRights
+    ]
+]:
+    result = await rabbitmq_rpc_client.request(
+        WEBSERVER_RPC_NAMESPACE,
+        TypeAdapter(RPCMethodName).validate_python("set_group_permissions"),
+        user_id=user_id,
+        product_name=product_name,
+        object_type=object_type,
+        object_ids=object_ids,
+        permission_group_id=permission_group_id,
+        read=read,
+        write=write,
+        execute=execute,
+    )
+    return TypeAdapter(
+        list[tuple[FunctionID | FunctionJobID, FunctionGroupAccessRights]]
+    ).validate_python(result)

packages/settings-library/src/settings_library/postgres.py

@@ -5,6 +5,7 @@
 from pydantic import (
     AliasChoices,
     Field,
+    NonNegativeInt,
     PostgresDsn,
     SecretStr,
     model_validator,
@@ -30,11 +31,28 @@ class PostgresSettings(BaseCustomSettings):
 
     # pool connection limits
     POSTGRES_MINSIZE: Annotated[
-        int, Field(description="Minimum number of connections in the pool", ge=2)
-    ] = 2  # see https://github.com/ITISFoundation/osparc-simcore/pull/8199
+        int,
+        Field(
+            description="Minimum number of connections in the pool that are always created and kept",
+            ge=1,
+        ),
+    ] = 1
     POSTGRES_MAXSIZE: Annotated[
-        int, Field(description="Maximum number of connections in the pool", ge=2)
+        int,
+        Field(
+            description="Maximum number of connections in the pool that are kept",
+            ge=1,
+        ),
     ] = 50
+    POSTGRES_MAX_POOLSIZE: Annotated[
+        int,
+        Field(
+            description="Maximal number of connection in asyncpg pool (without overflow), lazily created on demand"
+        ),
+    ] = 10
+    POSTGRES_MAX_OVERFLOW: Annotated[
+        NonNegativeInt, Field(description="Maximal overflow connections")
+    ] = 20
 
     POSTGRES_CLIENT_NAME: Annotated[
         str | None,
@@ -125,8 +143,10 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
                         "POSTGRES_USER": "usr",
                         "POSTGRES_PASSWORD": "secret",
                         "POSTGRES_DB": "db",
-                        "POSTGRES_MINSIZE": 2,
+                        "POSTGRES_MINSIZE": 1,
                         "POSTGRES_MAXSIZE": 50,
+                        "POSTGRES_MAX_POOLSIZE": 10,
+                        "POSTGRES_MAX_OVERFLOW": 20,
                         "POSTGRES_CLIENT_NAME": "my_app",  # first-choice
                         "HOST": "should be ignored",
                         "HOST_NAME": "should be ignored",
@@ -136,4 +156,3 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
         )
 
     model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)
-    model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)

packages/settings-library/tests/data/.env-compact

@@ -3,4 +3,4 @@
 APP_HOST=localhost
 APP_PORT=80
 APP_OPTIONAL_ADDON='{"MODULE_VALUE": 10, "MODULE_VALUE_DEFAULT": 42}'
-APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 2, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'
+APP_REQUIRED_PLUGIN='{"POSTGRES_HOST": "localhost", "POSTGRES_PORT": 5432, "POSTGRES_USER": "foo", "POSTGRES_PASSWORD": "**********", "POSTGRES_DB": "foodb", "POSTGRES_MINSIZE": 1, "POSTGRES_MAX_POOLSIZE": 10, "POSTGRES_MAX_OVERFLOW": 20, "POSTGRES_MAXSIZE": 50, "POSTGRES_CLIENT_NAME": "None"}'

packages/settings-library/tests/data/.env-granular

@@ -12,9 +12,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
-# Minimum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Maximum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
 POSTGRES_CLIENT_NAME=None

packages/settings-library/tests/data/.env-mixed

@@ -12,9 +12,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=**********
 # Database name
 POSTGRES_DB=foodb
-# Minimum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Maximum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 # Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)
 POSTGRES_CLIENT_NAME=None

packages/settings-library/tests/data/.env-sample

@@ -8,9 +8,9 @@ POSTGRES_USER=foo
 POSTGRES_PASSWORD=secret
 # Database name
 POSTGRES_DB=foodb
-# Maximum number of connections in the pool
-POSTGRES_MINSIZE=2
-# Minimum number of connections in the pool
+POSTGRES_MINSIZE=1
+POSTGRES_MAX_POOLSIZE=10
+POSTGRES_MAX_OVERFLOW=20
 POSTGRES_MAXSIZE=50
 
 # --- APP_MODULE_FIELD ---

packages/settings-library/tests/test_base_w_postgres.py

@@ -55,8 +55,10 @@ class _FakePostgresSettings(BaseCustomSettings):
             POSTGRES_PASSWORD: str
 
             POSTGRES_DB: str
-            POSTGRES_MINSIZE: Annotated[int, Field(ge=2)] = 2
-            POSTGRES_MAXSIZE: Annotated[int, Field(ge=2)] = 50
+            POSTGRES_MINSIZE: Annotated[int, Field(ge=1)] = 1
+            POSTGRES_MAXSIZE: Annotated[int, Field(ge=1)] = 50
+            POSTGRES_MAX_POOLSIZE: int = 10
+            POSTGRES_MAX_OVERFLOW: Annotated[int, Field(ge=0)] = 20
 
             POSTGRES_CLIENT_NAME: Annotated[
                 str | None,
@@ -126,7 +128,6 @@ class S5(BaseCustomSettings):
 def test_parse_from_empty_envs(
     postgres_envvars_unset: None, model_classes_factory: Callable
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     with pytest.raises(ValidationError, match="WEBSERVER_POSTGRES") as exc_info:
@@ -160,7 +161,6 @@ def test_parse_from_individual_envs(
     monkeypatch: pytest.MonkeyPatch,
     model_classes_factory: Callable,
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -200,7 +200,9 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 2,
+            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MAX_POOLSIZE": 10,
+            "POSTGRES_MAX_OVERFLOW": 20,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -215,7 +217,9 @@ def test_parse_from_individual_envs(
             "POSTGRES_PASSWORD": "shh",
             "POSTGRES_DB": "db",
             "POSTGRES_MAXSIZE": 50,
-            "POSTGRES_MINSIZE": 2,
+            "POSTGRES_MINSIZE": 1,
+            "POSTGRES_MAX_POOLSIZE": 10,
+            "POSTGRES_MAX_OVERFLOW": 20,
             "POSTGRES_CLIENT_NAME": None,
         }
     }
@@ -228,7 +232,6 @@ def test_parse_from_individual_envs(
 def test_parse_compact_env(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -262,7 +265,9 @@ def test_parse_compact_env(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }
@@ -342,7 +347,6 @@ def test_parse_compact_env(
 def test_parse_from_mixed_envs(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     S1, S2, S3, S4, S5 = model_classes_factory()
 
     # environment
@@ -372,7 +376,9 @@ def test_parse_from_mixed_envs(
                 "POSTGRES_PASSWORD": "shh2",
                 "POSTGRES_DB": "db2",
                 "POSTGRES_MAXSIZE": 50,
-                "POSTGRES_MINSIZE": 2,
+                "POSTGRES_MINSIZE": 1,
+                "POSTGRES_MAX_POOLSIZE": 10,
+                "POSTGRES_MAX_OVERFLOW": 20,
                 "POSTGRES_CLIENT_NAME": None,
             }
         }
@@ -472,7 +478,6 @@ def test_parse_from_mixed_envs(
 def test_toggle_plugin_1(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     *_, S4, S5 = model_classes_factory()
 
     # empty environ
@@ -535,7 +540,6 @@ def test_toggle_plugin_3(
 def test_toggle_plugin_4(
     postgres_envvars_unset: None, monkeypatch, model_classes_factory
 ):
-
     *_, S4, S5 = model_classes_factory()
     JSON_VALUE = '{"POSTGRES_HOST":"pg2", "POSTGRES_USER":"test2", "POSTGRES_PASSWORD":"shh2", "POSTGRES_DB":"db2"}'
 
@@ -565,7 +569,6 @@ def test_toggle_plugin_4(
         )
 
     with monkeypatch.context() as patch:
-
         # Enables both but remove individuals
         setenvs_from_envfile(
             patch,