[DOP-19907] - change settings configuration (#112)

* [DOP-19796] - add log_url before sedning run task to celery worker

* [DOP-19907] - refactor repository

* [DOP-19907] - add broker configuration

* [DOP-19907] - remove markers

* [DOP-19907] - update psycopg2 to psycopg3

* [DOP-19907] - configure middlewares

* [DOP-19907] - bring back asyncpg in backend

* [DOP-19907] - add flake8 to pyproject.toml and remove setup.cfg

* [DOP-19907] - minor fixes

* [DOP-19907] - update ENV in Dockerfile.worker

Author: maxim-lixakov

.env.docker

@@ -1,18 +1,18 @@
 TZ=UTC
 ENV=LOCAL
 
+# Debug
+SYNCMASTER__SERVER__DEBUG=true
+
+# Logging
+SYNCMASTER__SERVER__LOGGING__SETUP=True
+SYNCMASTER__SERVER__LOGGING__PRESET=colored
+
 # Postgres
-POSTGRES_HOST=db
-POSTGRES_PORT=5432
-POSTGRES_DB=syncmaster
-POSTGRES_USER=syncmaster
-POSTGRES_PASSWORD=changeme
+SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
 # RabbitMQ
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_PORT=5672
-RABBITMQ_USER=guest
-RABBITMQ_PASSWORD=guest
+SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
 
 # S3
 MINIO_ACCESS_KEY=syncmaster
@@ -28,6 +28,13 @@ TEST_S3_ACCESS_KEY=syncmaster
 TEST_S3_SECRET_KEY=changeme
 TEST_S3_BUCKET=syncmaster
 
+# needed for test-postgres container
+POSTGRES_HOST=test-postgres
+POSTGRES_PORT=5432
+POSTGRES_DB=syncmaster
+POSTGRES_USER=syncmaster
+POSTGRES_PASSWORD=changeme
+
 TEST_POSTGRES_HOST_FOR_CONFTEST=test-postgres
 TEST_POSTGRES_PORT_FOR_CONFTEST=5432
 TEST_POSTGRES_HOST_FOR_WORKER=test-postgres

.env.local

@@ -1,18 +1,18 @@
 export TZ=UTC
 export ENV=LOCAL
 
+# Debug
+export SYNCMASTER__SERVER__DEBUG=true
+
+# Logging
+export SYNCMASTER__SERVER__LOGGING__SETUP=True
+export SYNCMASTER__SERVER__LOGGING__PRESET=colored
+
 # Postgres
-export POSTGRES_HOST=localhost
-export POSTGRES_PORT=5432
-export POSTGRES_DB=syncmaster
-export POSTGRES_USER=syncmaster
-export POSTGRES_PASSWORD=changeme
+export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
 # RabbitMQ
-export RABBITMQ_HOST=localhost
-export RABBITMQ_PORT=5672
-export RABBITMQ_USER=guest
-export RABBITMQ_PASSWORD=guest
+export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
 
 export TEST_S3_HOST_FOR_CONFTEST=localhost
 export TEST_S3_PORT_FOR_CONFTEST=9010

.pre-commit-config.yaml

@@ -49,7 +49,7 @@ repos:
         files: ^(.*__main__\.py)$
       - id: insert-license
         files: .*\.py$
-        exclude: ^(tests/.*\.py)$
+        exclude: ^(syncmaster/backend/dependencies/stub.py|docs/.*\.py|tests/.*\.py)$
         args:
           - --license-filepath
           - .spdx-license-header.txt
@@ -100,6 +100,14 @@ repos:
 
   - repo: local
     hooks:
+      - id: flake8
+        name: flake8
+        entry: flake8
+        language: python
+        types: [python]
+        files: ^syncmaster/.*$
+        pass_filenames: true
+
       - id: mypy
         name: mypy
         entry: mypy ./syncmaster/backend --config-file ./pyproject.toml
@@ -136,6 +144,7 @@ repos:
 
 ci:
   skip:
+    - flake8 # checked with Github Actions
     - mypy  # checked with Github Actions
     - chmod  # failing in pre-commit.ci
     - docker-compose-check  # cannot run on pre-commit.ci

.readthedocs.yaml

@@ -17,7 +17,7 @@ build:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry install --no-root --all-extras --with docs --without dev,test
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m poetry show -v
       - python -m pip list -v
-      - POSTGRES_HOST=db POSTGRES_PORT=5432 POSTGRES_DB=syncmaster POSTGRES_USER=user POSTGRES_PASSWORD=password RABBITMQ_HOST=rabbitmq RABBITMQ_PORT=5672 RABBITMQ_USER=user RABBITMQ_PASSWORD=password python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
+      - SYNCMASTER__DATABASE__URL=postgresql+psycopg://fake:[email protected]:5432/fake SYNCMASTER__BROKER__URL=amqp://fake:faket@fake:5672/ python -m syncmaster.backend.export_openapi_schema docs/_static/openapi.json
 
 sphinx:
   configuration: docs/conf.py

docker/Dockerfile.worker

@@ -39,7 +39,7 @@ COPY ./syncmaster/ /app/syncmaster/
 
 FROM base as test
 
-ENV CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
+ENV SYNCMASTER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
 
 # CI runs tests in the worker container, so we need backend dependencies too
 RUN poetry install --no-root --all-extras --with test --without docs,dev

docs/backend/configuration/broker.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-broker:
+
+Broker settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings

docs/backend/configuration/cors.rst

@@ -0,0 +1,8 @@
+.. _backend-configuration-cors:
+
+CORS settings
+=============
+
+These settings used to control `CORS <https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS>`_ options.
+
+.. autopydantic_model:: syncmaster.settings.server.cors.CORSSettings

docs/backend/configuration/database.rst

@@ -0,0 +1,6 @@
+.. _backend-configuration-database:
+
+Database settings
+=================
+
+.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings

docs/backend/configuration/index.rst

@@ -0,0 +1,18 @@
+.. _backend-configuration:
+
+Configuration
+=============
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Configuration
+    :hidden:
+
+    database
+    broker
+    logging
+    cors
+    debug
+
+.. autopydantic_settings:: syncmaster.settings.Settings
+.. autopydantic_settings:: syncmaster.settings.server.ServerSettings

docs/backend/configuration/logging.rst

@@ -0,0 +1,7 @@
+.. _backend-configuration-logging:
+
+Logging settings
+================
+
+
+.. autopydantic_model:: syncmaster.settings.server.log.LoggingSettings