diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 069fe6f42454..dacab64e334b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -21,7 +21,7 @@ This document provides guidelines and best practices for using GitHub Copilot in - ensure we use `sqlalchemy` >2 compatible code. - ensure we use `pydantic` >2 compatible code. - ensure we use `fastapi` >0.100 compatible code - +- use f-string formatting ## Node.js-Specific Instructions diff --git a/packages/models-library/src/models_library/clusters.py b/packages/models-library/src/models_library/clusters.py index e18f3681b4db..14f3b77b3903 100644 --- a/packages/models-library/src/models_library/clusters.py +++ b/packages/models-library/src/models_library/clusters.py @@ -3,6 +3,7 @@ from typing import Literal, TypeAlias from pydantic import AnyUrl, BaseModel, ConfigDict, Field, HttpUrl, field_validator +from pydantic.config import JsonDict from pydantic.types import NonNegativeInt from .groups import GroupID @@ -36,18 +37,22 @@ class TLSAuthentication(_AuthenticationBase): tls_client_cert: Path tls_client_key: Path - model_config = ConfigDict( - json_schema_extra={ - "examples": [ - { - "type": "tls", - "tls_ca_file": "/path/to/ca_file", - "tls_client_cert": "/path/to/cert_file", - "tls_client_key": "/path/to/key_file", - }, - ] - } - ) + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "type": "tls", + "tls_ca_file": "/path/to/ca_file", + "tls_client_cert": "/path/to/cert_file", + "tls_client_key": "/path/to/key_file", + }, + ] + } + ) + + model_config = ConfigDict(json_schema_extra=_update_json_schema_extra) ClusterAuthentication: TypeAlias = NoAuthentication | TLSAuthentication @@ -71,36 +76,41 @@ class BaseCluster(BaseModel): create_enums_pre_validator(ClusterTypeInModel) ) - model_config = ConfigDict( - use_enum_values=True, - json_schema_extra={ - "examples": [ - { - "name": "My awesome cluster", - "type": ClusterTypeInModel.ON_PREMISE, - "owner": 12, - "endpoint": "https://registry.osparc-development.fake.dev", - "authentication": { - "type": "tls", - "tls_ca_file": "/path/to/ca_file", - "tls_client_cert": "/path/to/cert_file", - "tls_client_key": "/path/to/key_file", + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "name": "My awesome cluster", + "type": ClusterTypeInModel.ON_PREMISE, + "owner": 12, + "endpoint": "https://registry.osparc-development.fake.dev", + "authentication": { + "type": "tls", + "tls_ca_file": "/path/to/ca_file", + "tls_client_cert": "/path/to/cert_file", + "tls_client_key": "/path/to/key_file", + }, }, - }, - { - "name": "My AWS cluster", - "type": ClusterTypeInModel.AWS, - "owner": 154, - "endpoint": "https://registry.osparc-development.fake.dev", - "authentication": { - "type": "tls", - "tls_ca_file": "/path/to/ca_file", - "tls_client_cert": "/path/to/cert_file", - "tls_client_key": "/path/to/key_file", + { + "name": "My AWS cluster", + "type": ClusterTypeInModel.AWS, + "owner": 154, + "endpoint": "https://registry.osparc-development.fake.dev", + "authentication": { + "type": "tls", + "tls_ca_file": "/path/to/ca_file", + "tls_client_cert": "/path/to/cert_file", + "tls_client_key": "/path/to/key_file", + }, }, - }, - ] - }, + ] + } + ) + + model_config = ConfigDict( + use_enum_values=True, json_schema_extra=_update_json_schema_extra ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/monkeypatch_envs.py b/packages/pytest-simcore/src/pytest_simcore/helpers/monkeypatch_envs.py index 25eff962f73e..d81356144304 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/monkeypatch_envs.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/monkeypatch_envs.py @@ -30,6 +30,9 @@ def setenvs_from_dict( if isinstance(value, bool): v = "true" if value else "false" + if isinstance(value, int | float): + v = f"{value}" + assert isinstance(v, str), ( "caller MUST explicitly stringify values since some cannot be done automatically" f"e.g. json-like values. Check {key=},{value=}" diff --git a/packages/service-library/tests/fastapi/test_rabbitmq_lifespan.py b/packages/service-library/tests/fastapi/test_rabbitmq_lifespan.py index 550aaeb5c815..f143070da9e7 100644 --- a/packages/service-library/tests/fastapi/test_rabbitmq_lifespan.py +++ b/packages/service-library/tests/fastapi/test_rabbitmq_lifespan.py @@ -39,7 +39,7 @@ def mock_rabbitmq_connection(mocker: MockerFixture) -> MockType: def mock_rabbitmq_rpc_client_class(mocker: MockerFixture) -> MockType: mock_rpc_client_instance = mocker.AsyncMock() mocker.patch.object( - servicelib.rabbitmq._client_rpc.RabbitMQRPCClient, + servicelib.rabbitmq._client_rpc.RabbitMQRPCClient, # noqa: SLF001 "create", return_value=mock_rpc_client_instance, ) @@ -87,7 +87,6 @@ async def my_app_rpc_server(app: FastAPI, state: State) -> AsyncIterator[State]: # setup rpc-client using rabbitmq_rpc_client_context async def my_app_rpc_client(app: FastAPI, state: State) -> AsyncIterator[State]: - assert "RABBIT_CONNECTIVITY_LIFESPAN_NAME" in state async with rabbitmq_rpc_client_context( @@ -122,7 +121,6 @@ async def test_lifespan_rabbitmq_in_an_app( startup_timeout=None if is_pdb_enabled else 10, shutdown_timeout=None if is_pdb_enabled else 10, ): - # Verify that RabbitMQ responsiveness was checked mock_rabbitmq_connection.assert_called_once_with( app.state.settings.RABBITMQ.dsn diff --git a/packages/settings-library/src/settings_library/rabbit.py b/packages/settings-library/src/settings_library/rabbit.py index 1e95bdbec2ea..5e59010c3ee8 100644 --- a/packages/settings-library/src/settings_library/rabbit.py +++ b/packages/settings-library/src/settings_library/rabbit.py @@ -1,5 +1,7 @@ from functools import cached_property +from typing import ClassVar +from pydantic.config import JsonDict from pydantic.networks import AnyUrl from pydantic.types import SecretStr from pydantic_settings import SettingsConfigDict @@ -9,7 +11,7 @@ class RabbitDsn(AnyUrl): - allowed_schemes = {"amqp", "amqps"} + allowed_schemes: ClassVar[set[str]] = {"amqp", "amqps"} class RabbitSettings(BaseCustomSettings): @@ -35,16 +37,31 @@ def dsn(self) -> str: ) return rabbit_dsn + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "RABBIT_HOST": "rabbitmq.example.com", + "RABBIT_USER": "guest", + "RABBIT_PASSWORD": "guest-password", + "RABBIT_SECURE": False, + "RABBIT_PORT": 5672, + }, + { + "RABBIT_HOST": "secure.rabbitmq.example.com", + "RABBIT_USER": "guest", + "RABBIT_PASSWORD": "guest-password", + "RABBIT_SECURE": True, + "RABBIT_PORT": 15672, + }, + ] + } + ) + model_config = SettingsConfigDict( - json_schema_extra={ - "examples": [ - # minimal required - { - "RABBIT_SECURE": "1", - "RABBIT_HOST": "localhost", - "RABBIT_USER": "user", - "RABBIT_PASSWORD": "foobar", # NOSONAR - } - ], - } + extra="ignore", + populate_by_name=True, + json_schema_extra=_update_json_schema_extra, ) diff --git a/services/autoscaling/tests/unit/test_modules_dask.py b/services/autoscaling/tests/unit/test_modules_dask.py index 36c45a707526..9c53865cfa30 100644 --- a/services/autoscaling/tests/unit/test_modules_dask.py +++ b/services/autoscaling/tests/unit/test_modules_dask.py @@ -43,7 +43,7 @@ _authentication_types = [ NoAuthentication(), TLSAuthentication.model_construct( - **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + **TLSAuthentication.model_json_schema()["examples"][0] ), ] @@ -267,7 +267,9 @@ def _add_fct(x: int, y: int) -> int: ) assert isinstance(exc, RuntimeError) else: - result = await future_queued_task.result(timeout=_DASK_SCHEDULER_REACTION_TIME_S) # type: ignore + result = await future_queued_task.result( + timeout=_DASK_SCHEDULER_REACTION_TIME_S + ) # type: ignore assert result == 7 await _wait_for_dask_scheduler_to_change_state() diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index af120c9d4cc8..525148fa257e 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -252,6 +252,14 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings): ), ] = "172.20.0.0/14" # nosec + PRIMARY_EC2_INSTANCES_RABBIT: Annotated[ + RabbitSettings | None, + Field( + description="defines the Rabbit settings for the primary instance (may be disabled)", + json_schema_extra={"auto_default_from_env": True}, + ), + ] + @field_validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def _check_valid_instance_names( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml index d0e829c151f5..87f4ef94560f 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml @@ -91,6 +91,7 @@ services: AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} AUTOSCALING_NODES_MONITORING: null AUTOSCALING_POLL_INTERVAL: 00:00:10 + AUTOSCALING_RABBITMQ: ${AUTOSCALING_RABBITMQ} DASK_MONITORING_URL: tls://dask-scheduler:8786 DASK_SCHEDULER_AUTH: '{"type":"tls","tls_ca_file":"${DASK_TLS_CA_FILE}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}' EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} @@ -188,6 +189,7 @@ services: networks: cluster: + configs: prometheus-config: file: ./prometheus.yml @@ -200,6 +202,7 @@ volumes: redis-data: prometheus-data: + secrets: dask_tls_ca: file: ${DASK_TLS_CA_FILE} diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 5a9402ba0930..83aa56c425a4 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -9,6 +9,8 @@ import yaml from aws_library.ec2 import EC2InstanceBootSpecific, EC2InstanceData, EC2Tags from aws_library.ec2._models import CommandStr +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from fastapi.encoders import jsonable_encoder from models_library.api_schemas_clusters_keeper.clusters import ( ClusterState, @@ -81,6 +83,8 @@ def _convert_to_env_list(entries: list[Any]) -> str: def _convert_to_env_dict(entries: dict[str, Any]) -> str: return f"'{json.dumps(jsonable_encoder(entries))}'" + assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES # nosec + return [ f"CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ACCESS_KEY_ID}", f"CLUSTERS_KEEPER_EC2_ENDPOINT={app_settings.CLUSTERS_KEEPER_EC2_ACCESS.EC2_ENDPOINT or 'null'}", @@ -102,6 +106,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str: f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING}", f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}", + f"AUTOSCALING_RABBITMQ={json_dumps(model_dump_with_secrets(app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_RABBIT, show_secrets=True)) if app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_RABBIT else 'null'}", ] diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index 77fbff6bd65a..a80776951ded 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -224,6 +224,7 @@ def disable_clusters_management_background_task( @pytest.fixture def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("CLUSTERS_KEEPER_RABBITMQ", "null") + monkeypatch.delenv("RABBIT_HOST", raising=False) @pytest.fixture @@ -331,7 +332,7 @@ async def _do(num: int) -> list[str]: "Tags": [ { "Key": "Name", - "Value": f"{get_cluster_name(app_settings,user_id=user_id,wallet_id=wallet_id,is_manager=False)}_blahblah", + "Value": f"{get_cluster_name(app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False)}_blahblah", } ], } diff --git a/services/clusters-keeper/tests/unit/test_modules_dask.py b/services/clusters-keeper/tests/unit/test_modules_dask.py index 97c831ea789b..bc8d8739f54b 100644 --- a/services/clusters-keeper/tests/unit/test_modules_dask.py +++ b/services/clusters-keeper/tests/unit/test_modules_dask.py @@ -25,7 +25,7 @@ _authentication_types = [ NoAuthentication(), TLSAuthentication.model_construct( - **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + **TLSAuthentication.model_json_schema()["examples"][0] ), ] diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index 96983dd34d5e..a2d9397e9524 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -2,6 +2,7 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +import random import re import subprocess from collections.abc import Callable @@ -25,6 +26,7 @@ ) from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from settings_library.rabbit import RabbitSettings from simcore_service_clusters_keeper.core.settings import ApplicationSettings from simcore_service_clusters_keeper.utils.clusters import ( _prepare_environment_variables, @@ -34,6 +36,12 @@ ) from types_aiobotocore_ec2.literals import InstanceStateNameType +pytest_simcore_core_services_selection = [ + "rabbit", +] + +pytest_simcore_ops_services_selection = [] + @pytest.fixture def cluster_machines_name_prefix(faker: Faker) -> str: @@ -69,9 +77,9 @@ def app_environment( monkeypatch, { "CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": json_dumps( - TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + TLSAuthentication.model_json_schema()["examples"][0] if isinstance(backend_cluster_auth, TLSAuthentication) - else NoAuthentication.model_config["json_schema_extra"]["examples"][0] + else NoAuthentication.model_json_schema()["examples"][0] ) }, ) @@ -105,7 +113,9 @@ def test_create_deploy_cluster_stack_script( clusters_keeper_docker_compose: dict[str, Any], ): additional_custom_tags = { - AWSTagKey("pytest-tag-key"): AWSTagValue("pytest-tag-value") + TypeAdapter(AWSTagKey) + .validate_python("pytest-tag-key"): TypeAdapter(AWSTagValue) + .validate_python("pytest-tag-value") } deploy_script = create_deploy_cluster_stack_script( app_settings, @@ -175,6 +185,9 @@ def test_create_deploy_cluster_stack_script( for i in dict_settings ) + # check that the RabbitMQ settings are null since rabbit is disabled + assert re.search(r"AUTOSCALING_RABBITMQ=null", deploy_script) + # check the additional tags are in assert all( f'"{key}": "{value}"' in deploy_script @@ -182,6 +195,65 @@ def test_create_deploy_cluster_stack_script( ) +@pytest.fixture( + params=["default", "custom"], ids=["defaultRabbitMQ", "specialClusterRabbitMQ"] +) +def rabbitmq_settings_fixture( + app_environment: EnvVarsDict, + enabled_rabbitmq: RabbitSettings, + request: pytest.FixtureRequest, + monkeypatch: pytest.MonkeyPatch, + faker: Faker, +) -> RabbitSettings | None: + if request.param == "custom": + # Create random RabbitMQ settings using faker + custom_rabbit_settings = random.choice( # noqa: S311 + RabbitSettings.model_json_schema()["examples"] + ) + monkeypatch.setenv( + "PRIMARY_EC2_INSTANCES_RABBIT", json_dumps(custom_rabbit_settings) + ) + return RabbitSettings.model_validate(custom_rabbit_settings) + assert request.param == "default" + return enabled_rabbitmq + + +def test_rabbitmq_settings_are_passed_with_pasword_clear( + docker_swarm: None, + rabbitmq_settings_fixture: RabbitSettings | None, + mocked_ec2_server_envs: EnvVarsDict, + mocked_ssm_server_envs: EnvVarsDict, + mocked_redis_server: None, + app_settings: ApplicationSettings, + cluster_machines_name_prefix: str, + clusters_keeper_docker_compose: dict[str, Any], +): + assert app_settings.CLUSTERS_KEEPER_RABBITMQ + assert app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES + assert ( + rabbitmq_settings_fixture + == app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_RABBIT + ) + + additional_custom_tags = { + TypeAdapter(AWSTagKey) + .validate_python("pytest-tag-key"): TypeAdapter(AWSTagValue) + .validate_python("pytest-tag-value") + } + deploy_script = create_deploy_cluster_stack_script( + app_settings, + cluster_machines_name_prefix=cluster_machines_name_prefix, + additional_custom_tags=additional_custom_tags, + ) + assert isinstance(deploy_script, str) + + match = re.search(r"AUTOSCALING_RABBITMQ=({.*?})", deploy_script) + assert match, "AUTOSCALING_RABBITMQ is not present in the deploy script!" + autoscaling_rabbitmq = match.group(1) + passed_settings = RabbitSettings.model_validate_json(autoscaling_rabbitmq) + assert passed_settings == rabbitmq_settings_fixture + + def test_create_deploy_cluster_stack_script_below_64kb( disabled_rabbitmq: None, mocked_ec2_server_envs: EnvVarsDict, @@ -192,7 +264,9 @@ def test_create_deploy_cluster_stack_script_below_64kb( clusters_keeper_docker_compose: dict[str, Any], ): additional_custom_tags = { - AWSTagKey("pytest-tag-key"): AWSTagValue("pytest-tag-value") + TypeAdapter(AWSTagKey) + .validate_python("pytest-tag-key"): TypeAdapter(AWSTagValue) + .validate_python("pytest-tag-value") } deploy_script = create_deploy_cluster_stack_script( app_settings, @@ -239,7 +313,9 @@ def test__prepare_environment_variables_defines_all_envs_for_docker_compose( clusters_keeper_docker_compose_file: Path, ): additional_custom_tags = { - AWSTagKey("pytest-tag-key"): AWSTagValue("pytest-tag-value") + TypeAdapter(AWSTagKey) + .validate_python("pytest-tag-key"): TypeAdapter(AWSTagValue) + .validate_python("pytest-tag-value") } environment_variables = _prepare_environment_variables( app_settings, @@ -285,9 +361,7 @@ def test__prepare_environment_variables_defines_all_envs_for_docker_compose( "authentication", [ NoAuthentication(), - TLSAuthentication( - **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] - ), + TLSAuthentication(**TLSAuthentication.model_json_schema()["examples"][0]), ], ) def test_create_cluster_from_ec2_instance( diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 3378df3272ed..45eca7fd7ffe 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -220,6 +220,7 @@ services: PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD: ${PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD} PRIMARY_EC2_INSTANCES_MAX_START_TIME: ${PRIMARY_EC2_INSTANCES_MAX_START_TIME} PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL: ${PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL} + PRIMARY_EC2_INSTANCES_RABBIT_SETTINGS: ${PRIMARY_EC2_INSTANCES_RABBIT_SETTINGS} RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT}