[DOP-25282] Use docker-compose profiles

dolfinus · dolfinus · commit a1a031377388 · 2025-04-03T15:06:00.000+03:00
diff --git a/.github/labels.yml b/.github/labels.yml
@@ -19,8 +19,8 @@
     color: '5319e7'
     from_name: component:backend
 
-  - name: component:client
-    description: Client-related changes
+  - name: component:consumer
+    description: Consumer-related changes
     color: 75f526
 
   - name: kind:bug
diff --git a/Makefile b/Makefile
@@ -2,8 +2,7 @@
 
 include .env.local
 
-BAKEND_IMAGE = develop
-FRONTEND_IMAGE = develop
+VERSION = develop
 VENV = .venv
 PIP = ${VENV}/bin/pip
 POETRY = ${VENV}/bin/poetry
@@ -108,10 +107,10 @@ prod-build: ##@Application Build docker image
 	docker build --progress=plain --network=host -t mtsrus/data-rentgen:develop -f ./docker/Dockerfile $(ARGS) .
 
 prod: ##@Application Run production containers
-	docker compose -f docker-compose.yml up -d
+	docker compose -f docker-compose.yml --profile all up -d $(ARGS)
 
-prod-cleanup: ##@Application Stop production containers
-	docker compose -f docker-compose.yml down --remove-orphans $(ARGS)
+prod-stop: ##@Application Stop production containers
+	docker compose -f docker-compose.yml --profile all down --remove-orphans $(ARGS)
 
 
 .PHONY: docs
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -14,8 +14,26 @@ services:
       timeout: 5s
       retries: 3
 
+  db-migration:
+    image: mtsrus/data-rentgen:${VERSION:-latest}
+    command: |
+      python -m data_rentgen.db.migrations upgrade head && python -m data_rentgen.db.scripts.create_partitions
+    env_file: .env.docker
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db-views:
+    image: mtsrus/data-rentgen:${VERSION:-latest}
+    command: |
+      python -m data_rentgen.db.scripts.refresh_analytic_views
+    env_file: .env.docker
+    depends_on:
+      db-migration:
+        condition: service_completed_successfully
+
   server:
-    image: mtsrus/data-rentgen:${BACKEND_IMAGE:-latest}
+    image: mtsrus/data-rentgen:${VERSION:-latest}
     command: python -m data_rentgen.server --host 0.0.0.0 --port 8000
     restart: unless-stopped
     env_file: .env.docker
@@ -39,6 +57,10 @@ services:
         condition: service_healthy
       db-migration:
         condition: service_completed_successfully
+    profiles:
+      - server
+      - frontend
+      - all
 
   broker:
     image: bitnami/kafka:3.9
@@ -53,9 +75,13 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+    profiles:
+      - consumer
+      - broker
+      - all
 
   consumer:
-    image: mtsrus/data-rentgen:${BACKEND_IMAGE:-latest}
+    image: mtsrus/data-rentgen:${VERSION:-latest}
     command: python -m data_rentgen.consumer --host 0.0.0.0 --port 8000
     restart: unless-stopped
     env_file: .env.docker
@@ -70,27 +96,12 @@ services:
         condition: service_healthy
       db-migration:
         condition: service_completed_successfully
-
-  db-migration:
-    image: mtsrus/data-rentgen:${BACKEND_IMAGE:-latest}
-    command: |
-      python -m data_rentgen.db.migrations upgrade head && python -m data_rentgen.db.scripts.create_partitions
-    env_file: .env.docker
-    depends_on:
-      db:
-        condition: service_healthy
-
-  db-views:
-    image: mtsrus/data-rentgen:${BACKEND_IMAGE:-latest}
-    command: |
-      python -m data_rentgen.db.scripts.refresh_analytic_views
-    env_file: .env.docker
-    depends_on:
-      db-migration:
-        condition: service_completed_successfully
+    profiles:
+      - consumer
+      - all
 
   frontend:
-    image: mtsrus/data-rentgen-ui:${FRONTEND_IMAGE:-latest}
+    image: mtsrus/data-rentgen-ui:${VERSION:-latest}
     restart: unless-stopped
     env_file: .env.docker
     ports:
@@ -104,6 +115,9 @@ services:
     depends_on:
       server:
         condition: service_healthy
+    profiles:
+      - frontend
+      - all
 
 volumes:
   postgres_data:
diff --git a/docs/quickstart/install.rst b/docs/quickstart/install.rst
@@ -27,7 +27,7 @@ Then start containers using ``docker-compose``:
 
 .. code:: console
 
-    $ docker compose up -d --wait
+    $ docker compose --profile all up -d --wait
 
 ``docker-compose`` will download required images, create containers and start them in a proper order.
 Options can be set via ``.env.docker`` file or ``environment`` section in ``docker-compose.yml``.
diff --git a/docs/reference/broker/index.rst b/docs/reference/broker/index.rst
@@ -22,7 +22,7 @@ With Docker
 
   .. code:: console
 
-      $ docker compose up -d broker
+      $ docker compose --profile broker up -d
 
   ``docker-compose`` will download Apache Kafka image, create container and volume, and then start container.
 
@@ -32,7 +32,7 @@ With Docker
   .. dropdown:: ``docker-compose.yml``
 
       .. literalinclude:: ../../../docker-compose.yml
-          :emphasize-lines: 43-55,95
+          :emphasize-lines: 65-81,124
 
   .. dropdown:: ``.env.docker``
 
diff --git a/docs/reference/consumer/index.rst b/docs/reference/consumer/index.rst
@@ -18,7 +18,7 @@ With docker
 
   .. code:: console
 
-      $ docker compose up -d db consumer
+      $ docker compose --profile consumer up -d
 
   ``docker-compose`` will download all necessary images, create containers, and then start consumer process.
 
@@ -27,7 +27,7 @@ With docker
   .. dropdown:: ``docker-compose.yml``
 
       .. literalinclude:: ../../../docker-compose.yml
-          :emphasize-lines: 57-72
+          :emphasize-lines: 83-101
 
   .. dropdown:: ``.env.docker``
 
diff --git a/docs/reference/database/index.rst b/docs/reference/database/index.rst
@@ -9,6 +9,10 @@ After a database is started, it is required to run migration script.
 For empty database, it creates all the required tables and indexes.
 For non-empty database, it will perform database structure upgrade, using `Alembic <https://alembic.sqlalchemy.org/>`_.
 
+.. warning::
+
+    Other containers (consumer, server) should be stopped while running migrations, to prevent interference.
+
 After migrations are performed, it is required to run script which creates partitions for some tables in the database.
 By default, it creates monthly partitions, for current and next month. This can be changed by overriding command args.
 This script should run on schedule, for example by adding a dedicated entry to `crontab <https://help.ubuntu.com/community/CronHowto>`_.
@@ -17,18 +21,16 @@ Along with migrations analytics views are created. By default these materialized
 In order to fill these tables with data you need to run refresh script. The command for this shown below.
 Views based on data in ``output`` and ``input`` tables and has such structure:
 
-  .. code:: text
-
-    dataset_name - Name of dataset.
-    dataset_location - Name of dataset location (e.g. clusster name).
-    dataset_location_type - Type of dataset location (e.g. hive, hdfs, postgres).
-    user_id - Internal user id.
-    user_name - Internal user name (e.g. name of user which run spark job).
-    last_interaction_dt - Time when user lat time interact with dataset. Read or write depens on base table.
-    num_of_interactions - Number of interactions in given interval.
-    sum_bytes - Sum of bytes in given interval. ``num_bytes`` - column.
-    sum_rows - Sum of rows in given interval. ``num_rows`` - column.
-    sum_files - Sum of files in given interval. ``num_files`` - column.
+* ``dataset_name`` - Name of dataset.
+* ``dataset_location`` - Name of dataset location (e.g. clusster name).
+* ``dataset_location_type`` - Type of dataset location (e.g. hive, hdfs, postgres).
+* ``user_id`` - Internal user id.
+* ``user_name`` - Internal user name (e.g. name of user which run spark job).
+* ``last_interaction_dt`` - Time when user lat time interact with dataset. Read or write depens on base table.
+* ``num_of_interactions`` - Number of interactions in given interval.
+* ``sum_bytes`` - Sum of bytes in given interval.
+* ``sum_rows`` - Sum of rows in given interval.
+* ``sum_files`` - Sum of files in given interval.
 
 We provide three types of views: ``day``, ``week`` and ``month``, based on the time period in which the aggregation occur.
 Views are created for all intervals and by default, script refresh all views.
@@ -64,7 +66,7 @@ With Docker
   .. dropdown:: ``docker-compose.yml``
 
       .. literalinclude:: ../../../docker-compose.yml
-          :emphasize-lines: 1-15,108-110
+          :emphasize-lines: 1-33,123
 
   .. dropdown:: ``.env.docker``
 
diff --git a/docs/reference/frontend/index.rst b/docs/reference/frontend/index.rst
@@ -19,7 +19,7 @@ With Docker
 
   .. code:: console
 
-    $ docker compose up -d db frontend
+    $ docker compose --profile frontend up -d
 
   ``docker-compose`` will download Data.Rentgen UI image, create containers, and then start them.
 
@@ -28,7 +28,7 @@ With Docker
   .. dropdown:: ``docker-compose.yml``
 
       .. literalinclude:: ../../../docker-compose.yml
-          :emphasize-lines: 92-106
+          :emphasize-lines: 103-120
 
   .. dropdown:: ``.env.docker``
 
diff --git a/docs/reference/server/index.rst b/docs/reference/server/index.rst
@@ -19,7 +19,7 @@ With docker
 
   .. code:: console
 
-    $ docker compose up -d db server
+    $ docker compose --profile server up -d
 
   ``docker-compose`` will download all necessary images, create containers, and then start the server.
 
@@ -28,7 +28,7 @@ With docker
   .. dropdown:: ``docker-compose.yml``
 
       .. literalinclude:: ../../../docker-compose.yml
-          :emphasize-lines: 17-41
+          :emphasize-lines: 35-63
 
   .. dropdown:: ``.env.docker``
 
