fix(clp-package): Use component-specific log directories for compression-scheduler and query-scheduler (fixes #1483). (#1484)

Author: junhaoliao

components/clp-package-utils/clp_package_utils/controller.py

@@ -288,6 +288,9 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict:
         component_name = COMPRESSION_SCHEDULER_COMPONENT_NAME
         logger.info(f"Setting up environment for {component_name}...")
 
+        logs_dir = self._clp_config.logs_directory / component_name
+        logs_dir.mkdir(parents=True, exist_ok=True)
+
         env_vars = EnvVarsDict()
 
         # Logging config
@@ -308,6 +311,9 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict:
         component_name = QUERY_SCHEDULER_COMPONENT_NAME
         logger.info(f"Setting up environment for {component_name}...")
 
+        logs_dir = self._clp_config.logs_directory / component_name
+        logs_dir.mkdir(parents=True, exist_ok=True)
+
         env_vars = EnvVarsDict()
 
         # Logging config

tools/deployment/package/docker-compose.base.yaml

@@ -222,7 +222,7 @@ services:
       CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
       CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}"
       CLP_LOGGING_LEVEL: "${CLP_COMPRESSION_SCHEDULER_LOGGING_LEVEL:-INFO}"
-      CLP_LOGS_DIR: "/var/log"
+      CLP_LOGS_DIR: "/var/log/compression_scheduler"
       PYTHONPATH: "/opt/clp/lib/python3/site-packages"
       RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\
         /${CLP_REDIS_BACKEND_DB_COMPRESSION:-1}"

tools/deployment/package/docker-compose.yaml

@@ -38,7 +38,7 @@ services:
       CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}"
       CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}"
       CLP_LOGGING_LEVEL: "${CLP_QUERY_SCHEDULER_LOGGING_LEVEL:-INFO}"
-      CLP_LOGS_DIR: "/var/log"
+      CLP_LOGS_DIR: "/var/log/query_scheduler"
       PYTHONPATH: "/opt/clp/lib/python3/site-packages"
       RESULT_BACKEND: "redis://default:${CLP_REDIS_PASS:?Please set a value.}@redis:6379\
         /${CLP_REDIS_BACKEND_DB_QUERY:-0}"