[DOP-19933] - update worker, scheduler, keycloak documentation

Author: maxim-lixakov

.env.docker

@@ -5,6 +5,15 @@ ENV=LOCAL
 SYNCMASTER__SERVER__DEBUG=true
 SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
 
+# Session
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+
+# RabbitMQ
+SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
@@ -15,13 +24,6 @@ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 # Scheduler settings
 SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
-# Session
-SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
-# Postgres
-SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
-
-# TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
 # KEYCLOAK Auth
 SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
@@ -36,9 +38,6 @@ SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.K
 SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
 SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
 
-# RabbitMQ
-SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
-
 # S3
 MINIO_ACCESS_KEY=syncmaster
 MINIO_SECRET_KEY=changeme

.env.local

@@ -3,23 +3,26 @@ export ENV=LOCAL
 
 # Server Settngs
 export SYNCMASTER__SERVER__DEBUG=true
-export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
+export SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
+
+# Session
+export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+
+# RabbitMQ
+export SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
 
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
 
-# Scheduler settings
-export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
-
-# Session
-export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
 # Encrypt / Decrypt credentials data
 export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
-# Postgres
-export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Keycloack Auth
 export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080

docker-compose.test.yml

@@ -53,15 +53,17 @@ services:
     profiles: [backend, all]
 
   scheduler:
-    image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test}
+    image: mtsrus/syncmaster-scheduler:${SCHEDULER_IMAGE_TAG:-test}
     restart: unless-stopped
     build:
-      dockerfile: docker/Dockerfile.backend
+      dockerfile: docker/Dockerfile.scheduler
       context: .
       target: test
     env_file: .env.docker
     volumes:
       - ./syncmaster:/app/syncmaster
+      - ./tests:/app/tests
+      - ./reports:/app/reports
       - ./pyproject.toml:/app/pyproject.toml
     depends_on:
       db:

docker-compose.yml

@@ -65,6 +65,23 @@ services:
       rabbitmq:
         condition: service_healthy
 
+  scheduler:
+    image: mtsrus/syncmaster-scheduler:${TAG:-develop}
+    restart: unless-stopped
+    build:
+      dockerfile: docker/Dockerfile.scheduler
+      context: .
+    env_file: .env.docker
+    volumes:
+      - ./syncmaster:/app/syncmaster
+      - ./pyproject.toml:/app/pyproject.toml
+    depends_on:
+      db:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    entrypoint: [python, -m, syncmaster.scheduler]
+
 volumes:
   postgres_data:
   rabbitmq_data:

docker/Dockerfile.backend

@@ -1,7 +1,11 @@
 ARG BASE_IMAGE=python:3.12-slim
 FROM $BASE_IMAGE AS base
 
-RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry \
+RUN apt-get update && apt-get install -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+    && ln -s /root/.local/bin/poetry /usr/local/bin/poetry \
     && poetry config virtualenvs.create false
 
 WORKDIR /app

docker/Dockerfile.scheduler

@@ -0,0 +1,24 @@
+FROM python:3.12-slim AS base
+
+RUN apt-get update && apt-get install -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+    && ln -s /root/.local/bin/poetry /usr/local/bin/poetry \
+    && poetry config virtualenvs.create false
+
+WORKDIR /app
+ENV PYTHONPATH=/app
+
+COPY ./pyproject.toml ./poetry.lock* /app/
+
+RUN pip install --upgrade pip setuptools wheel packaging
+RUN poetry install --no-root --extras "scheduler" --without test,docs,dev
+
+FROM base AS prod
+
+COPY ./syncmaster/ /app/syncmaster/
+
+FROM base as test
+
+RUN poetry install --no-root --all-extras --with test --without docs,dev

docker/Dockerfile.worker

@@ -17,7 +17,12 @@ RUN apt-get update && apt-get install -y \
         libffi-dev \
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --no-cache-dir --timeout 3 --retries 3 poetry && poetry config virtualenvs.create false
+RUN apt-get update && apt-get install -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+    && ln -s /root/.local/bin/poetry /usr/local/bin/poetry \
+    && poetry config virtualenvs.create false
 
 WORKDIR /app
 ENV PYTHONPATH=/app

docs/_static/architecture.png

@@ -12,7 +12,7 @@ Syncmaster supports different auth provider implementations. You can change impl
     :caption: Auth providers
 
     dummy
-    keycloak
+    keycloak/index
 
 .. toctree::
     :maxdepth: 2