Skip to content

Commit dbc1799

Browse files
junhaoliaojunhaoliao
authored
fix(clp-package): Restore CLP_*_PORT env vars for bundled services (fixes #2065); Separate published ports from connection ports. (#2066)
1 parent 35b5ef1 commit dbc1799

File tree

2 files changed

+33
-21
lines changed

2 files changed

+33
-21
lines changed

‎components/clp-package-utils/clp_package_utils/controller.py‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -196,14 +196,17 @@ def _set_up_env_for_database(self) -> EnvVarsDict:
196196

197197
if BundledService.DATABASE not in self._clp_config.bundled:
198198
env_vars |= {
199-
"CLP_DB_PORT": str(self._clp_config.database.port),
199+
"CLP_DB_CONNECT_PORT": str(self._clp_config.database.port),
200200
"CLP_EXTRA_HOST_DATABASE_NAME": DB_COMPONENT_NAME,
201201
"CLP_EXTRA_HOST_DATABASE_ADDR": _resolve_external_host(
202202
self._clp_config.database.host
203203
),
204204
}
205205
else:
206-
env_vars["CLP_DB_HOST"] = _get_ip_from_hostname(self._clp_config.database.host)
206+
env_vars |= {
207+
"CLP_DB_HOST": _get_ip_from_hostname(self._clp_config.database.host),
208+
"CLP_DB_PORT": str(self._clp_config.database.port),
209+
}
207210

208211
# Credentials
209212
credentials = self._clp_config.database.credentials
@@ -273,12 +276,15 @@ def _set_up_env_for_queue(self) -> EnvVarsDict:
273276
# Connection config
274277
if BundledService.QUEUE not in self._clp_config.bundled:
275278
env_vars |= {
276-
"CLP_QUEUE_PORT": str(self._clp_config.queue.port),
279+
"CLP_QUEUE_CONNECT_PORT": str(self._clp_config.queue.port),
277280
"CLP_EXTRA_HOST_QUEUE_NAME": QUEUE_COMPONENT_NAME,
278281
"CLP_EXTRA_HOST_QUEUE_ADDR": _resolve_external_host(self._clp_config.queue.host),
279282
}
280283
else:
281-
env_vars["CLP_QUEUE_HOST"] = _get_ip_from_hostname(self._clp_config.queue.host)
284+
env_vars |= {
285+
"CLP_QUEUE_HOST": _get_ip_from_hostname(self._clp_config.queue.host),
286+
"CLP_QUEUE_PORT": str(self._clp_config.queue.port),
287+
}
282288

283289
# Credentials
284290
env_vars |= {
@@ -359,12 +365,15 @@ def _set_up_env_for_redis(self) -> EnvVarsDict:
359365
# Connection config
360366
if BundledService.REDIS not in self._clp_config.bundled:
361367
env_vars |= {
362-
"CLP_REDIS_PORT": str(self._clp_config.redis.port),
368+
"CLP_REDIS_CONNECT_PORT": str(self._clp_config.redis.port),
363369
"CLP_EXTRA_HOST_REDIS_NAME": REDIS_COMPONENT_NAME,
364370
"CLP_EXTRA_HOST_REDIS_ADDR": _resolve_external_host(self._clp_config.redis.host),
365371
}
366372
else:
367-
env_vars["CLP_REDIS_HOST"] = _get_ip_from_hostname(self._clp_config.redis.host)
373+
env_vars |= {
374+
"CLP_REDIS_HOST": _get_ip_from_hostname(self._clp_config.redis.host),
375+
"CLP_REDIS_PORT": str(self._clp_config.redis.port),
376+
}
368377

369378
# Credentials
370379
env_vars |= {
@@ -463,16 +472,19 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict:
463472
}
464473
if BundledService.RESULTS_CACHE not in self._clp_config.bundled:
465474
env_vars |= {
466-
"CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port),
475+
"CLP_RESULTS_CACHE_CONNECT_PORT": str(self._clp_config.results_cache.port),
467476
"CLP_EXTRA_HOST_RESULTS_CACHE_NAME": RESULTS_CACHE_COMPONENT_NAME,
468477
"CLP_EXTRA_HOST_RESULTS_CACHE_ADDR": _resolve_external_host(
469478
self._clp_config.results_cache.host
470479
),
471480
}
472481
else:
473-
env_vars["CLP_RESULTS_CACHE_HOST"] = _get_ip_from_hostname(
474-
self._clp_config.results_cache.host
475-
)
482+
env_vars |= {
483+
"CLP_RESULTS_CACHE_HOST": _get_ip_from_hostname(
484+
self._clp_config.results_cache.host
485+
),
486+
"CLP_RESULTS_CACHE_PORT": str(self._clp_config.results_cache.port),
487+
}
476488

477489
return env_vars
478490

‎tools/deployment/package/docker-compose-all.yaml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ services:
233233
"-u",
234234
"-m", "clp_py_utils.initialize-results-cache",
235235
"--uri", "mongodb://${CLP_EXTRA_HOST_RESULTS_CACHE_ADDR:-results_cache}\
236-
:${CLP_RESULTS_CACHE_PORT:-27017}/${CLP_RESULTS_CACHE_DB_NAME:-clp-query-results}",
236+
:${CLP_RESULTS_CACHE_CONNECT_PORT:-27017}/${CLP_RESULTS_CACHE_DB_NAME:-clp-query-results}",
237237
"--stream-collection", "${CLP_RESULTS_CACHE_STREAM_COLLECTION_NAME:-stream-files}",
238238
]
239239

@@ -253,7 +253,7 @@ services:
253253
"/opt/clp/bin/spider_scheduler",
254254
"--host", "spider_scheduler",
255255
"--port", "6000",
256-
"--storage_url", "jdbc:mariadb://database:${CLP_DB_PORT:-3306}/\
256+
"--storage_url", "jdbc:mariadb://database:${CLP_DB_CONNECT_PORT:-3306}/\
257257
${SPIDER_DB_NAME:-spider-db}?\
258258
user=${SPIDER_DB_USER:-spider-user}\
259259
&password=${SPIDER_DB_PASS:?Please set a value.}",
@@ -265,14 +265,14 @@ services:
265265
stop_grace_period: "300s"
266266
environment:
267267
BROKER_URL: "amqp://${CLP_QUEUE_USER:-clp-user}:${CLP_QUEUE_PASS:?Please set a value.}\
268-
@queue:${CLP_QUEUE_PORT:-5672}"
268+
@queue:${CLP_QUEUE_CONNECT_PORT:-5672}"
269269
CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
270270
CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
271271
CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}"
272272
CLP_LOGS_DIR: "/var/log/compression_scheduler"
273273
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
274274
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}\
275-
@redis:${CLP_REDIS_PORT:-6379}\
275+
@redis:${CLP_REDIS_CONNECT_PORT:-6379}\
276276
/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}"
277277
volumes:
278278
- *volume_clp_config_readonly
@@ -298,15 +298,15 @@ services:
298298
hostname: "compression_worker"
299299
environment:
300300
BROKER_URL: "amqp://${CLP_QUEUE_USER:-clp-user}:${CLP_QUEUE_PASS:?Please set a value.}\
301-
@queue:${CLP_QUEUE_PORT:-5672}"
301+
@queue:${CLP_QUEUE_CONNECT_PORT:-5672}"
302302
CLP_CONFIG_PATH: "/etc/clp-config.yaml"
303303
CLP_HOME: "/opt/clp"
304304
CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_WORKER_LOGGING_LEVEL:-INFO}"
305305
CLP_LOGS_DIR: "/var/log/compression_worker"
306306
CLP_WORKER_LOG_PATH: "/var/log/compression_worker/worker.log"
307307
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
308308
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}\
309-
@redis:${CLP_REDIS_PORT:-6379}\
309+
@redis:${CLP_REDIS_CONNECT_PORT:-6379}\
310310
/${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}"
311311
volumes:
312312
- *volume_clp_config_readonly
@@ -356,7 +356,7 @@ services:
356356
"-m", "job_orchestration.executor.start-spider-worker",
357357
"--host", "compression_worker",
358358
"--num-workers", "${CLP_COMPRESSION_WORKER_CONCURRENCY:-1}",
359-
"--storage-url", "jdbc:mariadb://database:${CLP_DB_PORT:-3306}/\
359+
"--storage-url", "jdbc:mariadb://database:${CLP_DB_CONNECT_PORT:-3306}/\
360360
${SPIDER_DB_NAME:-spider-db}?\
361361
user=${SPIDER_DB_USER:-spider-user}\
362362
&password=${SPIDER_DB_PASS:?Please set a value.}",
@@ -446,14 +446,14 @@ services:
446446
stop_grace_period: "10s"
447447
environment:
448448
BROKER_URL: "amqp://${CLP_QUEUE_USER:-clp-user}:${CLP_QUEUE_PASS:?Please set a value.}\
449-
@queue:${CLP_QUEUE_PORT:-5672}"
449+
@queue:${CLP_QUEUE_CONNECT_PORT:-5672}"
450450
CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
451451
CLP_DB_USER: "${CLP_DB_USER:-clp-user}"
452452
CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}"
453453
CLP_LOGS_DIR: "/var/log/query_scheduler"
454454
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
455455
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}\
456-
@redis:${CLP_REDIS_PORT:-6379}\
456+
@redis:${CLP_REDIS_CONNECT_PORT:-6379}\
457457
/${CLP_REDIS_BACKEND_DB_QUERY:-0}"
458458
volumes:
459459
- *volume_clp_config_readonly
@@ -485,15 +485,15 @@ services:
485485
hostname: "query_worker"
486486
environment:
487487
BROKER_URL: "amqp://${CLP_QUEUE_USER:-clp-user}:${CLP_QUEUE_PASS:?Please set a value.}\
488-
@queue:${CLP_QUEUE_PORT:-5672}"
488+
@queue:${CLP_QUEUE_CONNECT_PORT:-5672}"
489489
CLP_CONFIG_PATH: "/etc/clp-config.yaml"
490490
CLP_HOME: "/opt/clp"
491491
CLP_LOGGING_LEVEL: "${CLP_QUERY_WORKER_LOGGING_LEVEL:-INFO}"
492492
CLP_LOGS_DIR: "/var/log/query_worker"
493493
CLP_WORKER_LOG_PATH: "/var/log/query_worker/worker.log"
494494
PYTHONPATH: "/opt/clp/lib/python3/site-packages"
495495
RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}\
496-
@redis:${CLP_REDIS_PORT:-6379}\
496+
@redis:${CLP_REDIS_CONNECT_PORT:-6379}\
497497
/${CLP_REDIS_BACKEND_DB_QUERY:-0}"
498498
volumes:
499499
- *volume_clp_config_readonly

0 commit comments

Comments
 (0)