Skip to content

Commit 85dd321

Browse files
dolfinusdolfinus
committed
[DOP-22054] Move migrations to separated script
1 parent 41a481f commit 85dd321

File tree

14 files changed

+74
-76
lines changed

14 files changed

+74
-76
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ repos:
88
- id: check-added-large-files
99
- id: check-ast
1010
- id: check-merge-conflict
11-
- id: check-yaml
1211
- id: debug-statements
1312
- id: detect-private-key
1413
- id: end-of-file-fixer

‎Makefile‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,13 @@ dev-worker: db-start broker-start ##@Application Run development broker (without
130130

131131

132132
prod-build-server: ##@Application Build docker image for server
133-
docker build --progress=plain -t mtsrus/syncmaster-backend:develop -f ./docker/Dockerfile.backend $(ARGS) .
133+
docker build --progress=plain -t mtsrus/syncmaster-backend:develop -f ./docker/Dockerfile.backend --target=prod $(ARGS) .
134+
135+
prod-build-scheduler: ##@Application Build docker image for scheduler
136+
docker build --progress=plain -t mtsrus/syncmaster-scheduler:develop -f ./docker/Dockerfile.scheduler --target=prod $(ARGS) .
134137

135138
prod-build-worker: ##@Application Build docker image for worker
136-
docker build --progress=plain -t mtsrus/syncmaster-worker:develop -f ./docker/Dockerfile.worker $(ARGS) .
139+
docker build --progress=plain -t mtsrus/syncmaster-worker:develop -f ./docker/Dockerfile.worker --target=prod $(ARGS) .
137140

138141
prod-build: prod-build-server prod-build-worker ##@Application Build docker images
139142

‎docker-compose.test.yml‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,24 @@ services:
2828
timeout: 5s
2929
retries: 3
3030

31+
# Using worker image here to skip building backend image build in CI.
32+
# Worker test image already has all neccessary dependencies
33+
migrations:
34+
image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
35+
restart: none
36+
build:
37+
dockerfile: docker/Dockerfile.worker
38+
context: .
39+
target: test
40+
volumes:
41+
- ./syncmaster:/app/syncmaster
42+
- ./tests:/app/tests
43+
entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
44+
env_file: .env.docker
45+
depends_on:
46+
db:
47+
condition: service_healthy
48+
3149
backend:
3250
image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test}
3351
restart: unless-stopped
@@ -41,15 +59,22 @@ services:
4159
volumes:
4260
- ./syncmaster:/app/syncmaster
4361
- ./docs/_static:/app/docs/_static
44-
- ./cached_jars:/root/.ivy2
4562
- ./reports:/app/reports
4663
- ./tests:/app/tests
4764
- ./pyproject.toml:/app/pyproject.toml
4865
depends_on:
4966
db:
5067
condition: service_healthy
68+
migrations:
69+
condition: service_completed_successfully
5170
rabbitmq:
5271
condition: service_healthy
72+
healthcheck:
73+
test: [CMD-SHELL, curl -f http://localhost:8000/monitoring/ping]
74+
interval: 30s
75+
timeout: 5s
76+
retries: 3
77+
start_period: 5s
5378
profiles: [backend, all]
5479

5580
scheduler:
@@ -68,6 +93,8 @@ services:
6893
depends_on:
6994
db:
7095
condition: service_healthy
96+
migrations:
97+
condition: service_completed_successfully
7198
rabbitmq:
7299
condition: service_healthy
73100
profiles: [scheduler, all]
@@ -94,6 +121,8 @@ services:
94121
depends_on:
95122
db:
96123
condition: service_healthy
124+
migrations:
125+
condition: service_completed_successfully
97126
rabbitmq:
98127
condition: service_healthy
99128
profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, all]

‎docker-compose.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,26 @@ services:
2828
timeout: 5s
2929
retries: 3
3030

31+
migrations:
32+
image: mtsrus/syncmaster-backend:${TAG:-develop}
33+
restart: none
34+
build:
35+
dockerfile: docker/Dockerfile.backend
36+
context: .
37+
target: prod
38+
entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
39+
env_file: .env.docker
40+
depends_on:
41+
db:
42+
condition: service_healthy
43+
3144
backend:
3245
image: mtsrus/syncmaster-backend:${TAG:-develop}
3346
restart: unless-stopped
3447
build:
3548
dockerfile: docker/Dockerfile.backend
3649
context: .
50+
target: prod
3751
ports:
3852
- 8000:8000
3953
environment:
@@ -49,6 +63,8 @@ services:
4963
depends_on:
5064
db:
5165
condition: service_healthy
66+
migrations:
67+
condition: service_completed_successfully
5268
rabbitmq:
5369
condition: service_healthy
5470

@@ -58,10 +74,13 @@ services:
5874
build:
5975
dockerfile: docker/Dockerfile.worker
6076
context: .
77+
target: prod
6178
env_file: .env.docker
6279
depends_on:
6380
db:
6481
condition: service_healthy
82+
migrations:
83+
condition: service_completed_successfully
6584
rabbitmq:
6685
condition: service_healthy
6786

@@ -71,10 +90,13 @@ services:
7190
build:
7291
dockerfile: docker/Dockerfile.scheduler
7392
context: .
93+
target: prod
7494
env_file: .env.docker
7595
depends_on:
7696
db:
7797
condition: service_healthy
98+
migrations:
99+
condition: service_completed_successfully
78100
rabbitmq:
79101
condition: service_healthy
80102

‎docker/Dockerfile.backend‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ ENV SYNCMASTER__SERVER__OPENAPI__SWAGGER__JS_URL=/static/swagger/swagger-ui-bund
3939
SYNCMASTER__SERVER__STATIC_FILES__DIRECTORY=/app/syncmaster/backend/static
4040

4141

42-
FROM base as test
42+
FROM base AS test
4343

4444
RUN poetry install --no-root --extras "backend" --with test --without docs,dev
4545
RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh

‎docker/Dockerfile.scheduler‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ FROM base AS prod
2222

2323
COPY ./syncmaster/ /app/syncmaster/
2424

25-
FROM base as test
25+
26+
FROM base AS test
2627

2728
RUN poetry install --no-root --all-extras --with test --without docs,dev
28-
RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh
29+
RUN sed -i 's/python -m/coverage run -m/g' /app/entrypoint.sh

‎docker/Dockerfile.worker‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ FROM base as prod
4343
COPY ./syncmaster/ /app/syncmaster/
4444

4545

46-
FROM base as test
46+
FROM base AS test
4747

4848
ENV SYNCMASTER__WORKER__CREATE_SPARK_SESSION_FUNCTION=tests.spark.get_worker_spark_session
4949

‎docker/entrypoint_backend.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
#!/usr/bin/env bash
22
set -e
33

4-
python -m syncmaster.db.migrations upgrade head
5-
64
if [[ "x${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" != "x" ]]; then
75
superusers=$(echo "${SYNCMASTER__ENTRYPOINT__SUPERUSERS}" | tr "," " ")
86
python -m syncmaster.backend.scripts.manage_superusers add ${superusers}

‎docs/changelog/next_release/163.breaking.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Now migrations are executed in a dedicated one-off container, instead of being run as a part of ``backend`` container.

‎docs/scheduler/start_scheduler.rst‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
Starting the Scheduler
22
======================
33

4-
54
With docker
65
-----------
76

@@ -14,12 +13,12 @@ Options can be set via ``.env`` file or ``environment`` section in ``docker-comp
1413
.. dropdown:: ``docker-compose.yml``
1514

1615
.. literalinclude:: ../../docker-compose.yml
17-
:emphasize-lines: 68-83
16+
:emphasize-lines: 90-103
1817

1918
.. dropdown:: ``.env.docker``
2019

2120
.. literalinclude:: ../../.env.docker
22-
:emphasize-lines: 11-25
21+
:emphasize-lines: 11-25
2322

2423
To start the worker container you need to run the command:
2524

@@ -28,7 +27,6 @@ To start the worker container you need to run the command:
2827
docker compose up scheduler -d --wait --wait-timeout 200
2928
3029
31-
3230
Without docker
3331
--------------
3432

0 commit comments

Comments
 (0)