[DOP-22054] Move migrations to separated script

Author: dolfinus

docker-compose.test.yml

@@ -28,6 +28,22 @@ services:
       timeout: 5s
       retries: 3
 
+  migrations:
+    image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test}
+    restart: unless-stopped
+    build:
+      dockerfile: docker/Dockerfile.backend
+      context: .
+      target: test
+    volumes:
+      - ./syncmaster:/app/syncmaster
+    entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
+    env_file: .env.docker
+    depends_on:
+      db:
+        condition: service_healthy
+    profiles: [backend, worker, scheduler, all]
+
   backend:
     image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test}
     restart: unless-stopped
@@ -41,15 +57,22 @@ services:
     volumes:
       - ./syncmaster:/app/syncmaster
       - ./docs/_static:/app/docs/_static
-      - ./cached_jars:/root/.ivy2
       - ./reports:/app/reports
       - ./tests:/app/tests
       - ./pyproject.toml:/app/pyproject.toml
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
+    healthcheck:
+      test: [CMD-SHELL, curl -f http://localhost:8000/monitoring/ping]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 5s
     profiles: [backend, all]
 
   scheduler:
@@ -68,6 +91,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
     profiles: [scheduler, all]
@@ -94,6 +119,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
     profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, all]

docker-compose.yml

@@ -28,6 +28,18 @@ services:
       timeout: 5s
       retries: 3
 
+  migrations:
+    image: mtsrus/syncmaster-backend:${TAG:-develop}
+    restart: unless-stopped
+    build:
+      dockerfile: docker/Dockerfile.backend
+      context: .
+    entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
+    env_file: .env.docker
+    depends_on:
+      db:
+        condition: service_healthy
+
   backend:
     image: mtsrus/syncmaster-backend:${TAG:-develop}
     restart: unless-stopped
@@ -49,6 +61,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
     healthcheck:
@@ -68,6 +82,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
 
@@ -81,6 +97,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
 

docker/entrypoint_backend.sh

@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-python -m syncmaster.db.migrations upgrade head
-
 if [[ "x${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" != "x" ]]; then
   superusers=$(echo "${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" | tr "," " ")
   python -m syncmaster.backend.scripts.manage_superusers add ${superusers}

docs/changelog/next_release/163.breaking.rst

@@ -0,0 +1 @@
+Now migrations are executed in a dedicated one-off container, instead of being run as a part of ``backend`` container.

docs/scheduler/start_scheduler.rst

@@ -1,7 +1,6 @@
 Starting the Scheduler
 ======================
 
-
 With docker
 -----------
 
@@ -14,12 +13,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp
 .. dropdown:: ``docker-compose.yml``
 
     .. literalinclude:: ../../docker-compose.yml
-       :emphasize-lines: 68-83
+        :emphasize-lines: 90-103
 
 .. dropdown:: ``.env.docker``
 
     .. literalinclude:: ../../.env.docker
-       :emphasize-lines: 11-25
+        :emphasize-lines: 11-25
 
 To start the worker container you need to run the command:
 
@@ -28,7 +27,6 @@ To start the worker container you need to run the command:
     docker compose up scheduler -d --wait --wait-timeout 200
 
 
-
 Without docker
 --------------
 

docs/worker/start_worker.rst

@@ -6,25 +6,24 @@ Starting the Celery Worker
     Before starting the worker you need to create a queue.
     The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details).
 
-
 With docker
 -----------
 
 Installation process
 ~~~~~~~~~~~~~~~~~~~~
 
-Docker will download worker image of syncmaster worker & broker, and run them.
+Docker will download worker image, and then start worker container with dependencies.
 Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml``
 
 .. dropdown:: ``docker-compose.yml``
 
     .. literalinclude:: ../../docker-compose.yml
-       :emphasize-lines: 55-66
+        :emphasize-lines: 75-88
 
 .. dropdown:: ``.env.docker``
 
     .. literalinclude:: ../../.env.docker
-       :emphasize-lines: 11-22
+        :emphasize-lines: 11-22
 
 To start the worker container you need to run the command:
 
@@ -33,21 +32,20 @@ To start the worker container you need to run the command:
     docker compose up worker -d --wait --wait-timeout 200
 
 
-
 Without docker
 --------------
 
 To start the worker you need to run the command
 
 .. code-block:: bash
 
-    python -m celery -A syncmaster.worker.celery worker
+    python -m celery -A syncmaster.worker.celery worker --max-tasks-per-child=1
 
 You can specify options like concurrency and queues by adding additional flags:
 
 .. code-block:: bash
 
-    celery -A -A syncmaster.worker.celery  worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info
+    python -m celery -A syncmaster.worker.celery worker --concurrency=4 --max-tasks-per-child=1 --loglevel=info
 
 
 Refer to the `Celery <https://docs.celeryq.dev/en/stable/>`_ documentation for more advanced start options.