[DOP-29496] Load settings from config.yml

Author: dolfinus

.env.docker

@@ -54,7 +54,6 @@ jobs:
 
       - name: Generate OpenAPI Schema
         run: |
-          source .env.local
           poetry run python -m syncmaster.server.scripts.export_openapi_schema docs/_static/openapi.json
 
       - name: Fix logo in Readme

.env.local

@@ -65,7 +65,6 @@ jobs:
 
       - name: Run Unit Tests
         run: |
-          source .env.local
           source .env.local.test
           poetry run coverage run -m pytest -vvv -s -k "test_unit or test_database"
 

.github/workflows/release.yaml

@@ -101,7 +101,7 @@ Then start development server:
 
 And open http://localhost:8000/docs
 
-Settings are stored in ``.env.local`` file.
+Settings are stored in ``config.yml`` file.
 
 To start development worker, open a new terminal window/tab, and run:
 
@@ -181,6 +181,7 @@ To run specific integration tests:
     make test-integration-hdfs
 
 This starts database, broker & worker containers, and also HDFS container. Then it runs only HDFS-related integration tests.
+DB/filesystem addresses and credentials are stored in ``.env.local`` file.
 
 To run full test suite:
 
@@ -219,7 +220,7 @@ And then start all necessary services:
 
 Then open http://localhost:8000/docs
 
-Settings are stored in ``.env.docker`` file.
+Settings are stored in ``config.yml`` file.
 
 Build documentation
 ~~~~~~~~~~~~~~~~~~~

.github/workflows/unit-test.yml

@@ -1,6 +1,5 @@
 #!make
 
-include .env.local
 include .env.local.test
 
 VERSION = develop

CONTRIBUTING.rst

@@ -0,0 +1,68 @@
+database:
+  url: postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+
+broker:
+  url: amqp://guest:guest@broker:5672
+
+encryption:
+  # Encrypt / Decrypt credentials data using this Fernet key.
+  # !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
+  secret_key: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+
+
+auth:
+  # Dummy Auth
+  provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
+  access_token:
+    secret_key: generate_another_random_string
+
+  # Keycloak Auth
+  # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
+  # server_url: http://keycloak:8080
+  # realm_name: manually_created
+  # client_id: manually_created
+  # client_secret: generated_by_keycloak
+  # redirect_uri: http://localhost:3000/auth/callback
+  # scope: email
+  # verify_ssl: False
+
+
+ui:
+  api_browser_url: http://localhost:8000
+  auth_provider: dummyAuth
+  # auth_provider: keycloakAuth
+
+
+server:
+  debug: true # !!! NEVER USE ON PRODUCTION !!!
+
+  session:
+    secret_key: generate_some_random_string
+    max_age: 86400
+
+  cors:
+    enabled: true
+
+  allow_origins: [http://localhost:3000]
+  allow_credentials: true
+  allow_methods: ['*']
+  allow_headers: ['*']
+  expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials]
+
+
+
+scheduler:
+  transfer_fetching_timeout_seconds: 200
+
+
+worker:
+  log_url_template: https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+
+hwm_store:
+  enabled: true
+  type: horizon
+  url: http://horizon:8020
+  namespace: syncmaster_namespace
+  user: admin
+  password: 123UsedForTestOnly@!

Makefile

@@ -0,0 +1,71 @@
+database:
+  url: postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+
+broker:
+  url: amqp://guest:guest@localhost:5672
+
+encryption:
+  # Encrypt / Decrypt credentials data using this Fernet key.
+  # !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
+  secret_key: UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
+
+
+auth:
+  # Dummy Auth
+  provider: syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
+  access_token:
+    secret_key: generate_another_random_string
+
+  # Keycloak Auth
+  # provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
+  # server_url: http://localhost:8080
+  # realm_name: manually_created
+  # client_id: manually_created
+  # client_secret: generated_by_keycloak
+  # redirect_uri: http://localhost:3000/auth/callback
+  # scope: email
+  # verify_ssl: False
+
+
+ui:
+  api_browser_url: http://localhost:8000
+  auth_provider: dummyAuth
+  # auth_provider: keycloakAuth
+
+
+server:
+  debug: true # !!! NEVER USE ON PRODUCTION !!!
+
+  session:
+    secret_key: generate_some_random_string
+    max_age: 86400
+
+  cors:
+    enabled: true
+    allow_origins: [http://localhost:3000]
+    allow_credentials: true
+    allow_methods: ['*']
+    allow_headers: ['*']
+    expose_headers: [X-Request-ID, Location, Access-Control-Allow-Credentials]
+
+
+
+scheduler:
+  transfer_fetching_timeout_seconds: 200
+
+
+worker:
+  log_url_template: https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+
+hwm_store:
+  enabled: true
+  type: horizon
+  url: http://localhost:8020
+  namespace: syncmaster_namespace
+  user: admin
+  password: 123UsedForTestOnly@!
+
+
+superusers:
+  - admin

config.docker.yml

@@ -42,10 +42,10 @@ services:
       context: .
       target: test
     volumes:
+      - ./config.docker.yml:/app/config.yml
       - ./syncmaster:/app/syncmaster
       - ./tests:/app/tests
     entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
-    env_file: .env.docker
     depends_on:
       db:
         condition: service_healthy
@@ -57,10 +57,10 @@ services:
       dockerfile: docker/Dockerfile.server
       context: .
       target: test
-    env_file: .env.docker
     ports:
       - 8000:8000
     volumes:
+      - ./config.docker.yml:/app/config.yml
       - ./syncmaster:/app/syncmaster
       - ./docs/_static:/app/docs/_static
       - ./reports:/app/reports
@@ -82,8 +82,8 @@ services:
       dockerfile: docker/Dockerfile.scheduler
       context: .
       target: test
-    env_file: .env.docker
     volumes:
+      - ./config.docker.yml:/app/config.yml
       - ./syncmaster:/app/syncmaster
       - ./tests:/app/tests
       - ./reports:/app/reports
@@ -106,13 +106,12 @@ services:
       target: test
     command: --loglevel=info -Q 123-test_queue
     entrypoint: [coverage, run, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1]
-    env_file:
-      - .env.docker
-      - .env.docker.test
+    env_file: .env.docker.test
     environment:
       # CI runs tests in the worker container, so we need to turn off interaction with static files for it
       - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=false
     volumes:
+      - ./config.docker.yml:/app/config.yml
       - ./syncmaster:/app/syncmaster
       - ./reports:/app/reports
       - ./tests:/app/tests