[DOP-19992] - split settings to different classes

maxim-lixakov · maxim-lixakov · commit 47f09481d338 · 2024-11-19T17:36:05.000+03:00
diff --git a/.env.docker b/.env.docker
@@ -7,14 +7,25 @@ SYNCMASTER__SERVER__DEBUG=true
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
-SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
-
-# Session
-SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
 # Encrypt / Decrypt credentials data
 SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
+# Worker settings
+SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+SYNCMASTER__WORKER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+# Scheduler settings
+SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200
+
+# Session
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
 # Postgres
 SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
diff --git a/.env.local b/.env.local
@@ -7,7 +7,18 @@ export SYNCMASTER__SERVER__DEBUG=true
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
-export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+
+# Worker settings
+export SYNCMASTER__WORKER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+export SYNCMASTER__WORKER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+export SYNCMASTER__WORKER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+export SYNCMASTER__WORKER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+export SYNCMASTER__SCHEDULER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+export SYNCMASTER__SCHEDULER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200
 
 # Session
 export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__WORKER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SCHEDULER__DATABASE__URL=postgresql+psycopg://fake:fake@127.0.0.1:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__WORKER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__SCHEDULER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__WORKER__CRYPTO_KEY=crypto_key SYNCMASTER__SCHEDULER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py
diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker
@@ -39,7 +39,7 @@ COPY ./syncmaster/ /app/syncmaster/
 
 FROM base as test
 
-ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
+ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
 
 # CI runs tests in the worker container, so we need backend dependencies too
 RUN poetry install --no-root --all-extras --with test --without docs,dev
diff --git a/syncmaster/scheduler/__main__.py b/syncmaster/scheduler/__main__.py
@@ -3,7 +3,7 @@
 import asyncio
 import logging
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.scheduler.settings import SchedulerSettings as Settings
 from syncmaster.scheduler.transfer_fetcher import TransferFetcher
 from syncmaster.scheduler.transfer_job_manager import TransferJobManager
 
@@ -30,7 +30,7 @@ async def main():
             transfer_fetcher.last_updated_at = max(t.updated_at for t in transfers)
             logger.info("Scheduler state has been updated. Last updated at: %s", transfer_fetcher.last_updated_at)
 
-        await asyncio.sleep(settings.SCHEDULER_TRANSFER_FETCHING_TIMEOUT)
+        await asyncio.sleep(settings.TRANSFER_FETCHING_TIMEOUT)
 
 
 if __name__ == "__main__":
diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py
@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+
+from syncmaster.settings import SyncmasterSettings
+
+
+class SchedulerSettings(SyncmasterSettings):
+    """Celery scheduler settings.
+
+    Examples
+    --------
+
+    .. code-block:: bash
+
+        SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT=200
+    """
+
+    TRANSFER_FETCHING_TIMEOUT: int = 180  # seconds
+    MISFIRE_GRACE_TIME: int = 300  # seconds
+
+    class Config:
+        env_prefix = "SYNCMASTER__SCHEDULER__"
+        env_nested_delimiter = "__"
diff --git a/syncmaster/scheduler/transfer_fetcher.py b/syncmaster/scheduler/transfer_fetcher.py
@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 from sqlalchemy import select
 
-from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import Transfer
+from syncmaster.scheduler.settings import SchedulerSettings as Settings
 from syncmaster.scheduler.utils import get_async_session
 
 
diff --git a/syncmaster/scheduler/transfer_job_manager.py b/syncmaster/scheduler/transfer_job_manager.py
@@ -7,9 +7,9 @@
 from kombu.exceptions import KombuError
 
 from syncmaster.backend.services.unit_of_work import UnitOfWork
-from syncmaster.backend.settings import BackendSettings as Settings
 from syncmaster.db.models import RunType, Status, Transfer
 from syncmaster.exceptions.run import CannotConnectToTaskQueueError
+from syncmaster.scheduler.settings import SchedulerSettings as Settings
 from syncmaster.scheduler.utils import get_async_session
 from syncmaster.schemas.v1.connections.connection import ReadAuthDataSchema
 from syncmaster.worker.config import celery
@@ -35,15 +35,15 @@ def update_jobs(self, transfers: list[Transfer]) -> None:
                 self.scheduler.modify_job(
                     job_id=job_id,
                     trigger=CronTrigger.from_crontab(transfer.schedule),
-                    misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME,
+                    misfire_grace_time=self.settings.MISFIRE_GRACE_TIME,
                     args=(transfer.id,),
                 )
             else:
                 self.scheduler.add_job(
                     func=TransferJobManager.send_job_to_celery,
                     id=job_id,
                     trigger=CronTrigger.from_crontab(transfer.schedule),
-                    misfire_grace_time=self.settings.SCHEDULER_MISFIRE_GRACE_TIME,
+                    misfire_grace_time=self.settings.MISFIRE_GRACE_TIME,
                     args=(transfer.id,),
                 )
 
diff --git a/syncmaster/scheduler/utils.py b/syncmaster/scheduler/utils.py
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.scheduler.settings import SchedulerSettings as Settings
 
 
 def get_async_session(settings: Settings) -> AsyncSession:
diff --git a/syncmaster/settings/__init__.py b/syncmaster/settings/__init__.py
@@ -41,12 +41,8 @@ class SyncmasterSettings(BaseSettings):
         SYNCMASTER__SERVER__DEBUG=True
     """
 
-    crypto_key: str
-
-    # TODO: move settings to corresponding classes (scheduler also)
     TZ: str = "UTC"
-    SCHEDULER_TRANSFER_FETCHING_TIMEOUT: int = 180  # seconds
-    SCHEDULER_MISFIRE_GRACE_TIME: int = 300  # seconds
+    crypto_key: str = Field(description="Key for encrypt/decrypt credentials data")
 
     database: DatabaseSettings = Field(description=":ref:`Database settings <backend-configuration-database>`")
     broker: RabbitMQSettings = Field(description=":ref:`Broker settings <backend-configuration-broker>`")
diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py
@@ -16,7 +16,11 @@ class WorkerSettings(SyncmasterSettings):
         SYNCMASTER__WORKER__LOGGING__PRESET=colored
     """
 
-    CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID"
     LOG_URL_TEMPLATE: str = ""
+    CORRELATION_CELERY_HEADER_ID: str = "CORRELATION_CELERY_HEADER_ID"
 
     CREATE_SPARK_SESSION_FUNCTION: ImportString = "syncmaster.worker.spark.get_worker_spark_session"
+
+    class Config:
+        env_prefix = "SYNCMASTER__WORKER__"
+        env_nested_delimiter = "__"
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -21,6 +21,7 @@
 from syncmaster.backend.settings.auth.jwt import JWTSettings
 from syncmaster.backend.utils.jwt import sign_jwt
 from syncmaster.db.models import Base
+from syncmaster.scheduler.settings import SchedulerSettings
 from tests.mocks import UserTestRoles
 from tests.settings import TestSettings
 from tests.utils import prepare_new_database, run_async_migrations
@@ -70,6 +71,11 @@ def settings(request: pytest.FixtureRequest) -> Settings:
     return Settings.parse_obj(request.param)
 
 
+@pytest.fixture(scope="session", params=[{}])
+def scheduler_settings(request: pytest.FixtureRequest) -> SchedulerSettings:
+    return SchedulerSettings.parse_obj(request.param)
+
+
 @pytest.fixture(scope="session")
 def test_settings():
     return TestSettings()
diff --git a/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py b/tests/test_unit/test_scheduler/scheduler_fixtures/transfer_job_manager_fixture.py
@@ -4,13 +4,16 @@
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from syncmaster.backend.settings import BackendSettings as Settings
+from syncmaster.scheduler.settings import SchedulerSettings as Settings
 from syncmaster.scheduler.transfer_job_manager import TransferJobManager
 
 
 @pytest_asyncio.fixture
-async def transfer_job_manager(session: AsyncSession, settings: Settings) -> AsyncGenerator[TransferJobManager, None]:
-    transfer_job_manager = TransferJobManager(settings)
+async def transfer_job_manager(
+    session: AsyncSession,
+    scheduler_settings: Settings,
+) -> AsyncGenerator[TransferJobManager, None]:
+    transfer_job_manager = TransferJobManager(scheduler_settings)
     transfer_job_manager.scheduler.start()
 
     yield transfer_job_manager
