Skip to content

Commit caa2f9d

Browse files
sitaowang1998junhaoliao
sitaowang1998
and
junhaoliao
authored
refactor(deployment): Refactor Package Docker Compose files using service-level compositions. (#1522)
Co-authored-by: Junhao Liao <[email protected]>
1 parent f2da1db commit caa2f9d

File tree

5 files changed

+273
-165
lines changed

5 files changed

+273
-165
lines changed

‎components/clp-package-utils/clp_package_utils/controller.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -743,8 +743,7 @@ def start(self) -> None:
743743
logger.info(f"Starting CLP using Docker Compose ({deployment_type} deployment)...")
744744

745745
cmd = ["docker", "compose", "--project-name", self._project_name]
746-
if deployment_type == DeploymentType.BASE:
747-
cmd += ["--file", "docker-compose.base.yaml"]
746+
cmd += ["--file", self._get_docker_file_name()]
748747
if self._clp_config.mcp_server is not None:
749748
cmd += ["--profile", "mcp"]
750749
cmd += ["up", "--detach", "--wait"]
@@ -795,6 +794,15 @@ def _get_num_workers() -> int:
795794
# This will change when we move from single to multi-container workers. See y-scope/clp#1424
796795
return multiprocessing.cpu_count() // 2
797796

797+
def _get_docker_file_name(self) -> str:
798+
"""
799+
:return: The Docker Compose file name to use based on the config.
800+
"""
801+
deployment_type = self._clp_config.get_deployment_type()
802+
if deployment_type == DeploymentType.BASE:
803+
return "docker-compose-base.yaml"
804+
return "docker-compose.yaml"
805+
798806

799807
def get_or_create_instance_id(clp_config: CLPConfig) -> str:
800808
"""

‎tools/deployment/package/docker-compose.base.yaml‎ renamed to ‎tools/deployment/package/docker-compose-all.yaml‎

Lines changed: 137 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: "clp-package-base"
1+
name: "clp-package-all"
22

33
# Common service defaults.
44
x-service-defaults: &service_defaults
@@ -40,14 +40,6 @@ x-volume-definitions:
4040
source: "${CLP_LOGS_DIR_HOST:-./var/log}"
4141
target: "/var/log"
4242

43-
volumes:
44-
# Dummy volume to use when a bind mount is not desired.
45-
empty:
46-
driver_opts:
47-
device: "tmpfs"
48-
type: "tmpfs"
49-
size: 0
50-
5143
services:
5244
database:
5345
<<: *service_defaults
@@ -358,3 +350,139 @@ services:
358350
"-m", "job_orchestration.garbage_collector.garbage_collector",
359351
"--config", "/etc/clp-config.yml",
360352
]
353+
354+
query-scheduler:
355+
<<: *service_defaults
356+
hostname: "query_scheduler"
357+
stop_grace_period: "10s"
358+
environment:
359+
BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\
360+
:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672"
361+
CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
362+
CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}"
363+
CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}"
364+
CLP_LOGS_DIR: "/var/log/query_scheduler"
365+
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
366+
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\
367+
/${CLP_REDIS_BACKEND_DB_QUERY:-0}"
368+
volumes:
369+
- *volume_clp_config_readonly
370+
- *volume_clp_logs
371+
depends_on:
372+
db-table-creator:
373+
condition: "service_completed_successfully"
374+
queue:
375+
condition: "service_healthy"
376+
redis:
377+
condition: "service_healthy"
378+
command: [
379+
"python3",
380+
"-u",
381+
"-m", "job_orchestration.scheduler.query.query_scheduler",
382+
"--config", "/etc/clp-config.yml"
383+
]
384+
healthcheck:
385+
<<: *healthcheck_defaults
386+
test: [
387+
"CMD",
388+
"bash",
389+
"-c",
390+
"< /dev/tcp/query_scheduler/7000"
391+
]
392+
393+
query-worker:
394+
<<: *service_defaults
395+
hostname: "query_worker"
396+
environment:
397+
BROKER_URL: "amqp://${CLP_QUEUE_USER:?Please set a value.}\
398+
:${CLP_QUEUE_PASS:?Please set a value.}@queue:5672"
399+
CLP_CONFIG_PATH: "/etc/clp-config.yml"
400+
CLP_HOME: "/opt/clp"
401+
CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}"
402+
CLP_LOGS_DIR: "/var/log/query_worker"
403+
CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log"
404+
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
405+
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\
406+
/${CLP_REDIS_BACKEND_DB_QUERY:-0}"
407+
volumes:
408+
- *volume_clp_config_readonly
409+
- *volume_clp_logs
410+
- "${CLP_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/archives"
411+
- "${CLP_AWS_CONFIG_DIR_HOST:-empty}:/opt/clp/.aws:ro"
412+
- "${CLP_STAGED_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/staged-streams"
413+
- "${CLP_STREAM_OUTPUT_DIR_HOST:-empty}:/var/data/streams"
414+
command: [
415+
"python3",
416+
"-u",
417+
"/opt/clp/lib/python3/site-packages/bin/celery",
418+
"-A", "job_orchestration.executor.query",
419+
"worker",
420+
"--concurrency", "${CLP_QUERY_WORKER_CONCURRENCY:-1}",
421+
"--loglevel", "WARNING",
422+
"-f", "/var/log/query_worker/worker.log",
423+
"-Q", "query",
424+
"-n", "query-worker"
425+
]
426+
427+
reducer:
428+
<<: *service_defaults
429+
hostname: "reducer"
430+
stop_grace_period: "10s"
431+
environment:
432+
CLP_HOME: "/opt/clp"
433+
CLP_LOGGING_LEVEL: "${CLP_REDUCER_LOGGING_LEVEL:-INFO}"
434+
CLP_LOGS_DIR: "/var/log/reducer"
435+
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
436+
volumes:
437+
- *volume_clp_config_readonly
438+
- *volume_clp_logs
439+
depends_on:
440+
query-scheduler:
441+
condition: "service_healthy"
442+
results-cache-indices-creator:
443+
condition: "service_completed_successfully"
444+
command: [
445+
"python3", "-u",
446+
"-m", "job_orchestration.reducer.reducer",
447+
"--config", "/etc/clp-config.yml",
448+
"--concurrency", "${CLP_REDUCER_CONCURRENCY:-1}",
449+
"--upsert-interval", "${CLP_REDUCER_UPSERT_INTERVAL:-100}"
450+
]
451+
452+
mcp-server:
453+
<<: *service_defaults
454+
hostname: "mcp_server"
455+
profiles: ["mcp"]
456+
environment:
457+
CLP_LOGGING_LEVEL: "${CLP_MCP_LOGGING_LEVEL:-INFO}"
458+
CLP_LOGS_DIR: "/var/log/mcp_server"
459+
CLP_DB_USER: "${CLP_DB_USER}"
460+
CLP_DB_PASS: "${CLP_DB_PASS}"
461+
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
462+
ports:
463+
- host_ip: "${CLP_MCP_HOST:-127.0.0.1}"
464+
published: "${CLP_MCP_PORT:-8000}"
465+
target: 8000
466+
volumes:
467+
- *volume_clp_config_readonly
468+
- *volume_clp_logs
469+
depends_on:
470+
db-table-creator:
471+
condition: "service_completed_successfully"
472+
results-cache-indices-creator:
473+
condition: "service_completed_successfully"
474+
command: [
475+
"python3", "-u",
476+
"-m", "clp_mcp_server.clp_mcp_server",
477+
"--host", "mcp_server",
478+
"--port", "8000",
479+
"--config-path", "/etc/clp-config.yml",
480+
]
481+
healthcheck:
482+
<<: *healthcheck_defaults
483+
test: [
484+
"CMD",
485+
"curl",
486+
"-f",
487+
"http://mcp_server:8000/health"
488+
]

‎tools/deployment/package/docker-compose-base.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
name: "clp-package-base"
2+
3+
include:
4+
- "docker-compose.utils.yaml"
5+
6+
services:
7+
database:
8+
extends:
9+
file: "docker-compose-all.yaml"
10+
service: "database"
11+
12+
db-table-creator:
13+
extends:
14+
file: "docker-compose-all.yaml"
15+
service: "db-table-creator"
16+
17+
queue:
18+
extends:
19+
file: "docker-compose-all.yaml"
20+
service: "queue"
21+
22+
redis:
23+
extends:
24+
file: "docker-compose-all.yaml"
25+
service: "redis"
26+
27+
results-cache:
28+
extends:
29+
file: "docker-compose-all.yaml"
30+
service: "results-cache"
31+
32+
results-cache-indices-creator:
33+
extends:
34+
file: "docker-compose-all.yaml"
35+
service: "results-cache-indices-creator"
36+
37+
compression-scheduler:
38+
extends:
39+
file: "docker-compose-all.yaml"
40+
service: "compression-scheduler"
41+
42+
compression-worker:
43+
extends:
44+
file: "docker-compose-all.yaml"
45+
service: "compression-worker"
46+
47+
webui:
48+
extends:
49+
file: "docker-compose-all.yaml"
50+
service: "webui"
51+
52+
garbage-collector:
53+
extends:
54+
file: "docker-compose-all.yaml"
55+
service: "garbage-collector"

‎tools/deployment/package/docker-compose.utils.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
volumes:
2+
# Dummy volume to use when a bind mount is not desired.
3+
empty:
4+
driver_opts:
5+
device: "tmpfs"
6+
type: "tmpfs"
7+
size: 0

0 commit comments

Comments
 (0)