[DOP-23077] Improve documentation

dolfinus · dolfinus · commit 162b0a0794f8 · 2024-12-23T14:51:31.000+03:00
diff --git a/.env.docker b/.env.docker
@@ -1,43 +1,44 @@
 TZ=UTC
 ENV=LOCAL
 
-# Server Settngs
-SYNCMASTER__SERVER__DEBUG=true
-SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
-
-# Session
-SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+# Logging options
+SYNCMASTER__LOGGING__SETUP=True
+SYNCMASTER__LOGGING__PRESET=colored
 
-# Postgres
+# Common DB options
 SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
-# RabbitMQ
-SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+# Encrypt / Decrypt credentials data using this Fernet key.
+# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
+SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
-# Logging
-SYNCMASTER__LOGGING__SETUP=True
-SYNCMASTER__LOGGING__PRESET=colored
+# Common RabbitMQ options
+SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672
 
-# Encrypt / Decrypt credentials data
-SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
-
-# Scheduler settings
-SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
+# Server options
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string
+SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+# !!! NEVER USE ON PRODUCTION !!!
+SYNCMASTER__SERVER__DEBUG=true
 
-# KEYCLOAK Auth
+# Keycloak Auth
+#SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
 SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
 SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
 SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
 SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
 SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
 SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
-SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth
 SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
-SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
+SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string
+
+# Scheduler options
+SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
+# Tests-only
 # S3
 MINIO_ACCESS_KEY=syncmaster
 MINIO_SECRET_KEY=changeme
diff --git a/.env.local b/.env.local
@@ -1,43 +1,44 @@
 export TZ=UTC
 export ENV=LOCAL
 
-# Server Settngs
-export SYNCMASTER__SERVER__DEBUG=true
-export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
-
-# Session
-export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+# Logging options
+export SYNCMASTER__LOGGING__SETUP=True
+export SYNCMASTER__LOGGING__PRESET=colored
 
-# Postgres
+# Common DB options
 export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
 
-# RabbitMQ
-export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
+# Encrypt / Decrypt credentials data using this Fernet key.
+# !!! GENERATE YOUR OWN COPY FOR PRODUCTION USAGE !!!
+export SYNCMASTER__ENCRYPTION__SECRET_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
-# Logging
-export SYNCMASTER__LOGGING__SETUP=True
-export SYNCMASTER__LOGGING__PRESET=colored
+# Common RabbitMQ options
+export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672
 
-# Encrypt / Decrypt credentials data
-export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
-
-# Scheduler settings
-export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
+# Server options
+export SYNCMASTER__SERVER__SESSION__SECRET_KEY=generate_some_random_string
+export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://logs.location.example.com/syncmaster-worker?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+# !!! NEVER USE ON PRODUCTION !!!
+export SYNCMASTER__SERVER__DEBUG=true
 
-# Keycloack Auth
-export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+# Keycloak Auth
+#export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
+export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://localhost:8080
 export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
 export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=manually_created
 export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=generated_by_keycloak
 export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
 export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
 export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
-export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider
 
 # Dummy Auth
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.server.providers.auth.dummy_provider.DummyAuthProvider
-export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
+export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=generate_another_random_string
+
+# Scheduler options
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
+# Tests-only
 export TEST_S3_HOST_FOR_CONFTEST=localhost
 export TEST_S3_PORT_FOR_CONFTEST=9010
 export TEST_S3_HOST_FOR_WORKER=test-s3
diff --git a/docs/index.rst b/docs/index.rst
@@ -26,7 +26,7 @@
     :caption: Worker
     :hidden:
 
-    worker/start_worker
+    worker/install
     worker/log_url
     worker/configuration/index
 
@@ -36,7 +36,7 @@
     :caption: Scheduler
     :hidden:
 
-    scheduler/start_scheduler
+    scheduler/install
     scheduler/configuration/index
 
 .. toctree::
diff --git a/docs/scheduler/configuration/broker.rst b/docs/scheduler/configuration/broker.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-broker:
+.. _scheduler-configuration-broker:
 
 Broker settings
 =================
diff --git a/docs/scheduler/configuration/credentials.rst b/docs/scheduler/configuration/credentials.rst
@@ -0,0 +1,6 @@
+.. _scheduler-configuration-credentials:
+
+Credentials settings
+====================
+
+.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings
diff --git a/docs/scheduler/configuration/database.rst b/docs/scheduler/configuration/database.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-database:
+.. _scheduler-configuration-database:
 
 Database settings
 =================
diff --git a/docs/scheduler/configuration/index.rst b/docs/scheduler/configuration/index.rst
@@ -10,6 +10,7 @@ Configuration
 
     database
     broker
+    credentials
     logging
 
 .. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings
diff --git a/docs/scheduler/configuration/logging.rst b/docs/scheduler/configuration/logging.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-logging:
+.. _scheduler-configuration-logging:
 
 Logging settings
 ================
diff --git a/docs/scheduler/install.rst b/docs/scheduler/install.rst
@@ -1,5 +1,7 @@
-Starting the Scheduler
-======================
+.. _server-install:
+
+Install & run scheduler
+=======================
 
 With docker
 -----------
@@ -13,12 +15,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp
 .. dropdown:: ``docker-compose.yml``
 
     .. literalinclude:: ../../docker-compose.yml
-        :emphasize-lines: 90-103
+        :emphasize-lines: 93-107
 
 .. dropdown:: ``.env.docker``
 
     .. literalinclude:: ../../.env.docker
-        :emphasize-lines: 11-25
+        :emphasize-lines: 4-16,38-39
 
 To start the worker container you need to run the command:
 
diff --git a/docs/server/configuration/broker.rst b/docs/server/configuration/broker.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-broker:
+.. _server-configuration-broker:
 
 Broker settings
 =================
diff --git a/docs/server/configuration/credentials.rst b/docs/server/configuration/credentials.rst
@@ -0,0 +1,6 @@
+.. _server-configuration-credentials:
+
+Credentials settings
+====================
+
+.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings
diff --git a/docs/server/configuration/database.rst b/docs/server/configuration/database.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-database:
+.. _server-configuration-database:
 
 Database settings
 =================
diff --git a/docs/server/configuration/debug.rst b/docs/server/configuration/debug.rst
@@ -0,0 +1,71 @@
+.. _server-configuration-debug:
+
+Enabling debug
+===============
+
+Return debug info in REST API responses
+---------------------------------------
+
+By default, server does not add error details to response bodies,
+to avoid exposing instance-specific information to end users.
+
+You can change this by setting:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__SERVER__DEBUG=False
+    $ # start REST API server
+    $ curl -XPOST http://localhost:8000/failing/endpoint ...
+    {
+        "error": {
+            "code": "unknown",
+            "message": "Got unhandled exception. Please contact support",
+            "details": null,
+        },
+    }
+
+.. code-block:: console
+
+    $ export SYNCMASTER__SERVER__DEBUG=True
+    $ # start REST API server
+    $ curl -XPOST http://localhost:8000/failing/endpoint ...
+    Traceback (most recent call last):
+    File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi
+        result = await app(  # type: ignore[func-returns-value]
+                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    File ".../site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__
+        return await self.app(scope, receive, send)
+
+.. DANGER::
+
+    This is only for development environment only. Do **NOT** use on production!
+
+Print debug logs on backend
+---------------------------
+
+See :ref:`server-configuration-logging`, but replace log level ``INFO`` with ``DEBUG``.
+
+Fill up ``X-Request-ID`` header on backend
+------------------------------------------
+
+Server can add ``X-Request-ID`` header to responses, which allows to match request on client with backend response.
+
+This is done by ``request_id`` middleware, which is enabled by default and can configured as described below:
+
+.. autopydantic_model:: syncmaster.server.settings.server.request_id.RequestIDSettings
+
+Print request ID  to backend logs
+---------------------------------
+
+This is done by adding a specific filter to logging handler:
+
+.. dropdown:: ``logging.yml``
+
+    .. literalinclude:: ../../../syncmaster/settings/log/plain.yml
+        :emphasize-lines: 6-12,17-18,25
+
+Resulting logs look like:
+
+.. code-block:: text
+
+    2023-12-18 17:14:11.711 uvicorn.access:498 [INFO] 018c15e97a068ae09484f8c25e2799dd 127.0.0.1:34884 - "GET /monitoring/ping HTTP/1.1" 200
diff --git a/docs/server/configuration/index.rst b/docs/server/configuration/index.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration:
+.. _server-configuration:
 
 Configuration
 =============
@@ -10,13 +10,15 @@ Configuration
 
     database
     broker
+    credentials
     logging
     session
     cors
     debug
     monitoring
     static_files
     openapi
+    debug
 
 .. autopydantic_settings:: syncmaster.server.settings.ServerAppSettings
 .. autopydantic_settings:: syncmaster.server.settings.server.ServerSettings
diff --git a/docs/server/configuration/logging.rst b/docs/server/configuration/logging.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-logging:
+.. _server-configuration-logging:
 
 Logging settings
 ================
diff --git a/docs/server/install.rst b/docs/server/install.rst
@@ -21,10 +21,18 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp
 .. dropdown:: ``docker-compose.yml``
 
     .. literalinclude:: ../../docker-compose.yml
+        :emphasize-lines: 44-75
 
 .. dropdown:: ``.env.docker``
 
     .. literalinclude:: ../../.env.docker
+        :emphasize-lines: 4-36
+
+To start the worker container you need to run the command:
+
+.. code-block:: bash
+
+    docker compose up server -d --wait --wait-timeout 200
 
 After container is started and ready, open http://localhost:8000/docs.
 
diff --git a/docs/worker/configuration/broker.rst b/docs/worker/configuration/broker.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-broker:
+.. _worker-configuration-broker:
 
 Broker settings
 =================
diff --git a/docs/worker/configuration/credentials.rst b/docs/worker/configuration/credentials.rst
@@ -0,0 +1,6 @@
+.. _worker-configuration-credentials:
+
+Credentials settings
+====================
+
+.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings
diff --git a/docs/worker/configuration/database.rst b/docs/worker/configuration/database.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-database:
+.. _server-configuration-database:
 
 Database settings
 =================
diff --git a/docs/worker/configuration/index.rst b/docs/worker/configuration/index.rst
@@ -10,6 +10,7 @@ Configuration
 
     database
     broker
+    credentials
     logging
 
 .. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings
diff --git a/docs/worker/configuration/logging.rst b/docs/worker/configuration/logging.rst
@@ -1,4 +1,4 @@
-.. _backend-configuration-logging:
+.. _worker-configuration-logging:
 
 Logging settings
 ================
diff --git a/docs/worker/install.rst b/docs/worker/install.rst
diff --git a/docs/worker/log_url.rst b/docs/worker/log_url.rst
diff --git a/syncmaster/db/repositories/utils.py b/syncmaster/db/repositories/utils.py
diff --git a/syncmaster/scheduler/settings/__init__.py b/syncmaster/scheduler/settings/__init__.py
diff --git a/syncmaster/server/settings/__init__.py b/syncmaster/server/settings/__init__.py
diff --git a/syncmaster/server/settings/server/__init__.py b/syncmaster/server/settings/server/__init__.py
diff --git a/syncmaster/settings/credentials.py b/syncmaster/settings/credentials.py
diff --git a/syncmaster/worker/settings/__init__.py b/syncmaster/worker/settings/__init__.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-.. _backend-configuration-broker:`
	`1`	`+.. _scheduler-configuration-broker:`
`2`	`2`
`3`	`3`	`Broker settings`
`4`	`4`	`=================`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-.. _backend-configuration-database:`
	`1`	`+.. _scheduler-configuration-database:`
`2`	`2`
`3`	`3`	`Database settings`
`4`	`4`	`=================`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-.. _backend-configuration-logging:`
	`1`	`+.. _scheduler-configuration-logging:`
`2`	`2`
`3`	`3`	`Logging settings`
`4`	`4`	`================`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-.. _backend-configuration-broker:`
	`1`	`+.. _server-configuration-broker:`
`2`	`2`
`3`	`3`	`Broker settings`
`4`	`4`	`=================`