Merge branch 'develop' into feature/DOP-21350

Author: IlyasDevelopment

.pre-commit-config.yaml

@@ -40,13 +40,13 @@ repos:
       - id: chmod
         args: ['644']
         exclude_types: [shell]
-        exclude: ^(.*__main__\.py)$
+        exclude: ^(.*__main__\.py|syncmaster/backend/export_openapi_schema\.py)$
       - id: chmod
         args: ['755']
         types: [shell]
       - id: chmod
         args: ['755']
-        files: ^(.*__main__\.py)$
+        files: ^(.*__main__\.py|syncmaster/backend/export_openapi_schema\.py)$
       - id: insert-license
         files: .*\.py$
         exclude: ^(syncmaster/backend/dependencies/stub.py|docs/.*\.py|tests/.*\.py)$

Makefile

@@ -29,9 +29,11 @@ venv: venv-cleanup  venv-install##@Env Init venv and install poetry dependencies
 
 venv-cleanup: ##@Env Cleanup venv
 	@rm -rf .venv || true
-	python3.12 -m venv .venv
+	python -m venv .venv
 	${PIP} install -U setuptools wheel pip
 	${PIP} install poetry
+	${PIP} install -U flake8-commas
+	${PIP} install --no-deps sphinx-plantuml
 
 venv-install: ##@Env Install requirements to venv
 	${POETRY} config virtualenvs.create false

README.rst

@@ -33,8 +33,11 @@ Data.SyncMaster is as low-code ETL tool for transfering data between databases a
 List of currently supported connections:
 
 * Apache Hive
+* Clickhouse
 * Postgres
 * Oracle
+* MSSQL
+* MySQL
 * HDFS
 * S3
 

docker-compose.test.yml

@@ -80,6 +80,9 @@ services:
     command: --loglevel=info -Q test_queue
     entrypoint: [python, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1]
     env_file: .env.docker
+    environment:
+      # CI runs tests in the worker container, so we need to turn off interaction with static files for it
+      - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=false
     volumes:
       - ./syncmaster:/app/syncmaster
       - ./cached_jars:/root/.ivy2

docker-compose.yml

@@ -36,6 +36,13 @@ services:
       context: .
     ports:
       - 8000:8000
+    # PROMETHEUS_MULTIPROC_DIR is required for multiple workers, see:
+    # https://prometheus.github.io/client_python/multiprocess/
+    environment:
+      PROMETHEUS_MULTIPROC_DIR: /tmp/prometheus-metrics
+    # tmpfs dir is cleaned up each container restart
+    tmpfs:
+      - /tmp/prometheus-metrics
     env_file: .env.docker
     depends_on:
       db:

docker/Dockerfile.backend

@@ -19,6 +19,20 @@ ENTRYPOINT ["/app/entrypoint.sh"]
 FROM base AS prod
 
 COPY ./syncmaster/ /app/syncmaster/
+# add this when logo will be ready
+# COPY ./docs/_static/*.svg ./syncmaster/backend/static/
+
+# Swagger UI
+ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui-bundle.js /app/syncmaster/backend/static/swagger/swagger-ui-bundle.js
+ADD https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui.css /app/syncmaster/backend/static/swagger/swagger-ui.css
+
+# Redoc
+ADD https://cdn.jsdelivr.net/npm/redoc@latest/bundles/redoc.standalone.js /app/syncmaster/backend/static/redoc/redoc.standalone.js
+
+ENV SYNCMASTER__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bundle.js \
+    SYNCMASTER__SERVER__OPENAPI__SWAGGER__CSS_URL=/static/swagger/swagger-ui.css \
+    SYNCMASTER__SERVER__OPENAPI__REDOC__JS_URL=/static/redoc/redoc.standalone.js \
+    SYNCMASTER__SERVER__STATIC_FILES__DIRECTORY=/app/syncmaster/backend/static
 
 
 FROM base as test

docs/backend/configuration/cors.rst

@@ -1,4 +1,4 @@
-.. _backend-configuration-cors:
+.. _backend-configuration-server-cors:
 
 CORS settings
 =============

docs/backend/configuration/index.rst

@@ -13,6 +13,9 @@ Configuration
     logging
     cors
     debug
+    monitoring
+    static_files
+    openapi
 
 .. autopydantic_settings:: syncmaster.settings.Settings
 .. autopydantic_settings:: syncmaster.settings.server.ServerSettings

docs/backend/configuration/logging.rst

@@ -1,4 +1,4 @@
-.. _configuration-logging:
+.. _backend-configuration-logging:
 
 Logging settings
 ================

docs/backend/configuration/monitoring.rst

@@ -0,0 +1,16 @@
+.. _backend-configuration-server-monitoring:
+
+Server monitoring
+=================
+
+REST API server provides the following endpoints with Prometheus compatible metrics:
+
+* ``GET /monitoring/metrics`` - server metrics, like number of requests per path and response status, CPU and RAM usage, and so on.
+
+.. dropdown:: Example
+
+    .. literalinclude:: ../../_static/metrics.prom
+
+These endpoints are enabled and configured using settings below:
+
+.. autopydantic_model:: syncmaster.settings.server.monitoring.MonitoringSettings