[DOP-19992] - split settings to different classes (#136)

* [DOP-19992] - split settings to different classes

* [DOP-19992] - remove importing class WorkerSettings

* [DOP-19992] - split settings into ServerAppSettings & WorkerAppSettings & SchedulerSettings & add corresponding configuration documentation

* [DOP-19992] - move log_url_template to server_settings

Author: maxim-lixakov

.env.docker

@@ -1,31 +1,35 @@
 TZ=UTC
 ENV=LOCAL
 
-# Debug
+# Server Settngs
 SYNCMASTER__SERVER__DEBUG=true
+SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
 
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
-SYNCMASTER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+# Encrypt / Decrypt credentials data
+SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+
+# Scheduler settings
+SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Session
 SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
-# Encrypt / Decrypt credentials data
-SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
-
 # Postgres
 SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
 # TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
 # KEYCLOAK Auth
-SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080
-SYNCMASTER__AUTH__REALM_NAME=manually_created
-SYNCMASTER__AUTH__CLIENT_ID=manually_created
-SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
-SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/v1/auth/callback
-SYNCMASTER__AUTH__SCOPE=email
+SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
+SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
+SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
+SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
 SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth

.env.local

@@ -1,30 +1,34 @@
 export TZ=UTC
 export ENV=LOCAL
 
-# Debug
+# Server Settngs
 export SYNCMASTER__SERVER__DEBUG=true
+export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
 
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
-export SYNCMASTER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Session
 export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
 
 # Encrypt / Decrypt credentials data
-export SYNCMASTER__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
 # Postgres
 export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
 # Keycloack Auth
-export SYNCMASTER__AUTH__SERVER_URL=http://keycloak:8080
-export SYNCMASTER__AUTH__REALM_NAME=manually_created
-export SYNCMASTER__AUTH__CLIENT_ID=manually_created
-export SYNCMASTER__AUTH__CLIENT_SECRET=generated_by_keycloak
-export SYNCMASTER__AUTH__REDIRECT_URI=http://localhost:8000/auth/callback
-export SYNCMASTER__AUTH__SCOPE=email
+export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
+export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
+export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:[email protected]:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:[email protected]:5432/fake SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=crypto_key SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=fakepython python -m syncmaster.backend.scripts.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py

Makefile

@@ -113,7 +113,7 @@ dev-server: db-start ##@Application Run development server (without docker)
 	${POETRY} run python -m syncmaster.backend $(ARGS)
 
 dev-worker: db-start broker-start ##@Application Run development broker (without docker)
-	${POETRY} run python -m celery -A syncmaster.worker.config.celery worker --max-tasks-per-child=1 $(ARGS)
+	${POETRY} run python -m celery -A syncmaster.worker.celery worker --max-tasks-per-child=1 $(ARGS)
 
 
 

docker/Dockerfile.worker

@@ -39,7 +39,7 @@ COPY ./syncmaster/ /app/syncmaster/
 
 FROM base as test
 
-ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
+ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
 
 # CI runs tests in the worker container, so we need backend dependencies too
 RUN poetry install --no-root --all-extras --with test --without docs,dev

docker/entrypoint_worker.sh

@@ -5,5 +5,5 @@ set -e
 # Required to start each Celery task in separated process, avoiding issues with global Spark session object
 
 # exec is required to forward all signals to the main process
-exec python -m celery -A syncmaster.worker.config.celery worker --max-tasks-per-child=1 "$@"
+exec python -m celery -A syncmaster.worker.celery worker --max-tasks-per-child=1 "$@"
 

docs/backend/configuration/index.rst

@@ -18,5 +18,5 @@ Configuration
     static_files
     openapi
 
-.. autopydantic_settings:: syncmaster.settings.Settings
-.. autopydantic_settings:: syncmaster.settings.server.ServerSettings
+.. autopydantic_settings:: syncmaster.backend.settings.ServerAppSettings
+.. autopydantic_settings:: syncmaster.backend.settings.server.ServerSettings

docs/index.rst

@@ -20,6 +20,22 @@
     backend/openapi
     backend/configuration/index
 
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Worker
+    :hidden:
+
+    worker/configuration/index
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Scheduler
+    :hidden:
+
+    scheduler/configuration/index
+
 .. toctree::
     :maxdepth: 2
     :caption: Development

docs/scheduler/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/scheduler/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings