From d5ffb333917b241ca7443e46d5de33d1d60c7e39 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:24:56 -0400 Subject: [PATCH 1/7] Add garbage collector to a new components list --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index ea42413b27..56d4042536 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -48,7 +48,10 @@ RESULTS_CACHE_COMPONENT_NAME, WEBUI_COMPONENT_NAME, } -ALL_COMPONENTS = COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS +RETENTION_CONTROL_COMPONENTS = {GARBAGE_COLLECTOR_NAME} +ALL_COMPONENTS = ( + COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | RETENTION_CONTROL_COMPONENTS +) # Target names ALL_TARGET_NAME = "" From 0f556b5ee3be3e725498500c0c56458dbc27a749 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:08:56 -0400 Subject: [PATCH 2/7] Rename and add to presto --- components/clp-py-utils/clp_py_utils/clp_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 56d4042536..6f3d319185 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -48,9 +48,9 @@ RESULTS_CACHE_COMPONENT_NAME, WEBUI_COMPONENT_NAME, } -RETENTION_CONTROL_COMPONENTS = {GARBAGE_COLLECTOR_NAME} +STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_NAME} ALL_COMPONENTS = ( - COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | RETENTION_CONTROL_COMPONENTS + COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS ) # Target names @@ -874,7 +874,7 @@ def load_redis_credentials_from_file(self): def get_runnable_components(self) -> Set[str]: if QueryEngine.PRESTO == self.package.query_engine: - return COMPRESSION_COMPONENTS | UI_COMPONENTS + return COMPRESSION_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS else: return ALL_COMPONENTS From d9afb696c6de530947dd9571f10e09e07a11dd2a Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:13:10 -0400 Subject: [PATCH 3/7] Disable garbage collection for Presto and add garbage collection to the controller targets --- components/clp-py-utils/clp_py_utils/clp_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 6f3d319185..597a028b92 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -63,7 +63,8 @@ | { COMPRESSION_SCHEDULER_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, - }, + } + | STORAGE_MANAGEMENT_COMPONENTS, } QUERY_JOBS_TABLE_NAME = "query_jobs" @@ -874,7 +875,7 @@ def load_redis_credentials_from_file(self): def get_runnable_components(self) -> Set[str]: if QueryEngine.PRESTO == self.package.query_engine: - return COMPRESSION_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS + return COMPRESSION_COMPONENTS | UI_COMPONENTS else: return ALL_COMPONENTS From 7197b0945d07b4eecd2f7b8fbaec2102918d8b5a Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 16:21:01 -0400 Subject: [PATCH 4/7] Fix --- .../clp-package-utils/clp_package_utils/scripts/start_clp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index de3ee1a41a..d152d4a7dd 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1330,7 +1330,8 @@ def main(argv): if WEBUI_COMPONENT_NAME in components_to_start: start_webui(instance_id, clp_config, container_clp_config, mounts) - if target in (ALL_TARGET_NAME, GARBAGE_COLLECTOR_NAME): + + if GARBAGE_COLLECTOR_NAME in components_to_start: start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) except Exception as ex: From ad8d23aed4be4b077d672f36066418e7645862ce Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 17:36:33 -0400 Subject: [PATCH 5/7] Add warning message --- .../clp_package_utils/general.py | 9 ++ .../clp_package_utils/scripts/start_clp.py | 97 ++++++++++--------- 2 files changed, 61 insertions(+), 45 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index dc75bca377..4fa4816e63 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -16,6 +16,7 @@ CLP_DEFAULT_CREDENTIALS_FILE_PATH, CLPConfig, DB_COMPONENT_NAME, + QueryEngine, QUEUE_COMPONENT_NAME, REDIS_COMPONENT_NAME, REDUCER_COMPONENT_NAME, @@ -582,6 +583,14 @@ def validate_dataset_name(clp_table_prefix: str, dataset_name: str) -> None: ) +def validate_retention_config(clp_config: CLPConfig) -> None: + clp_query_engine = clp_config.package.query_engine + if is_retention_period_configured(clp_config) and clp_query_engine == QueryEngine.PRESTO: + raise ValueError( + f"Retention control is not supported with query_engine `{clp_query_engine}`" + ) + + def is_retention_period_configured(clp_config: CLPConfig) -> bool: if clp_config.archive_output.retention_period is not None: return True diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index d152d4a7dd..6d72448195 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -69,6 +69,7 @@ validate_redis_config, validate_reducer_config, validate_results_cache_config, + validate_retention_config, validate_webui_config, ) @@ -1251,6 +1252,12 @@ def main(argv): GARBAGE_COLLECTOR_NAME, ): validate_output_storage_config(clp_config) + if target in ( + ALL_TARGET_NAME, + CONTROLLER_TARGET_NAME, + GARBAGE_COLLECTOR_NAME, + ): + validate_retention_config(clp_config) clp_config.validate_data_dir() clp_config.validate_logs_dir() @@ -1288,51 +1295,51 @@ def main(argv): conf_dir = clp_home / "etc" - # Start components - if DB_COMPONENT_NAME in components_to_start: - start_db(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components - ) or DB_COMPONENT_NAME in components_to_start: - create_db_tables(instance_id, clp_config, container_clp_config, mounts) - - if QUEUE_COMPONENT_NAME in components_to_start: - start_queue(instance_id, clp_config) - - if REDIS_COMPONENT_NAME in components_to_start: - start_redis(instance_id, clp_config, conf_dir) - - if RESULTS_CACHE_COMPONENT_NAME in components_to_start: - start_results_cache(instance_id, clp_config, conf_dir) - - if ( - target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components - ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: - create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: - start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: - start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) - - if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: - start_compression_worker( - instance_id, clp_config, container_clp_config, num_workers, mounts - ) - - if QUERY_WORKER_COMPONENT_NAME in components_to_start: - start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if REDUCER_COMPONENT_NAME in components_to_start: - start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) - - if WEBUI_COMPONENT_NAME in components_to_start: - start_webui(instance_id, clp_config, container_clp_config, mounts) - - if GARBAGE_COLLECTOR_NAME in components_to_start: - start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) + # # Start components + # if DB_COMPONENT_NAME in components_to_start: + # start_db(instance_id, clp_config, conf_dir) + # + # if ( + # target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components + # ) or DB_COMPONENT_NAME in components_to_start: + # create_db_tables(instance_id, clp_config, container_clp_config, mounts) + # + # if QUEUE_COMPONENT_NAME in components_to_start: + # start_queue(instance_id, clp_config) + # + # if REDIS_COMPONENT_NAME in components_to_start: + # start_redis(instance_id, clp_config, conf_dir) + # + # if RESULTS_CACHE_COMPONENT_NAME in components_to_start: + # start_results_cache(instance_id, clp_config, conf_dir) + # + # if ( + # target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components + # ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: + # create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) + # + # if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: + # start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) + # + # if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: + # start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) + # + # if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: + # start_compression_worker( + # instance_id, clp_config, container_clp_config, num_workers, mounts + # ) + # + # if QUERY_WORKER_COMPONENT_NAME in components_to_start: + # start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) + # + # if REDUCER_COMPONENT_NAME in components_to_start: + # start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) + # + # if WEBUI_COMPONENT_NAME in components_to_start: + # start_webui(instance_id, clp_config, container_clp_config, mounts) + # + # if GARBAGE_COLLECTOR_NAME in components_to_start: + # start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) except Exception as ex: if type(ex) == ValueError: From bf73952c01249cb4757ef11213621a09f5e74cc7 Mon Sep 17 00:00:00 2001 From: haiqi96 <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 17:40:21 -0400 Subject: [PATCH 6/7] Revert unintended changes --- .../clp_package_utils/scripts/start_clp.py | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index 6d72448195..c7fcc8d2e4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -1295,51 +1295,51 @@ def main(argv): conf_dir = clp_home / "etc" - # # Start components - # if DB_COMPONENT_NAME in components_to_start: - # start_db(instance_id, clp_config, conf_dir) - # - # if ( - # target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components - # ) or DB_COMPONENT_NAME in components_to_start: - # create_db_tables(instance_id, clp_config, container_clp_config, mounts) - # - # if QUEUE_COMPONENT_NAME in components_to_start: - # start_queue(instance_id, clp_config) - # - # if REDIS_COMPONENT_NAME in components_to_start: - # start_redis(instance_id, clp_config, conf_dir) - # - # if RESULTS_CACHE_COMPONENT_NAME in components_to_start: - # start_results_cache(instance_id, clp_config, conf_dir) - # - # if ( - # target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components - # ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: - # create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) - # - # if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: - # start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) - # - # if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: - # start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) - # - # if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: - # start_compression_worker( - # instance_id, clp_config, container_clp_config, num_workers, mounts - # ) - # - # if QUERY_WORKER_COMPONENT_NAME in components_to_start: - # start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) - # - # if REDUCER_COMPONENT_NAME in components_to_start: - # start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) - # - # if WEBUI_COMPONENT_NAME in components_to_start: - # start_webui(instance_id, clp_config, container_clp_config, mounts) - # - # if GARBAGE_COLLECTOR_NAME in components_to_start: - # start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) + # Start components + if DB_COMPONENT_NAME in components_to_start: + start_db(instance_id, clp_config, conf_dir) + + if ( + target == CONTROLLER_TARGET_NAME and DB_COMPONENT_NAME in runnable_components + ) or DB_COMPONENT_NAME in components_to_start: + create_db_tables(instance_id, clp_config, container_clp_config, mounts) + + if QUEUE_COMPONENT_NAME in components_to_start: + start_queue(instance_id, clp_config) + + if REDIS_COMPONENT_NAME in components_to_start: + start_redis(instance_id, clp_config, conf_dir) + + if RESULTS_CACHE_COMPONENT_NAME in components_to_start: + start_results_cache(instance_id, clp_config, conf_dir) + + if ( + target == CONTROLLER_TARGET_NAME and RESULTS_CACHE_COMPONENT_NAME in runnable_components + ) or RESULTS_CACHE_COMPONENT_NAME in components_to_start: + create_results_cache_indices(instance_id, clp_config, container_clp_config, mounts) + + if COMPRESSION_SCHEDULER_COMPONENT_NAME in components_to_start: + start_compression_scheduler(instance_id, clp_config, container_clp_config, mounts) + + if QUERY_SCHEDULER_COMPONENT_NAME in components_to_start: + start_query_scheduler(instance_id, clp_config, container_clp_config, mounts) + + if COMPRESSION_WORKER_COMPONENT_NAME in components_to_start: + start_compression_worker( + instance_id, clp_config, container_clp_config, num_workers, mounts + ) + + if QUERY_WORKER_COMPONENT_NAME in components_to_start: + start_query_worker(instance_id, clp_config, container_clp_config, num_workers, mounts) + + if REDUCER_COMPONENT_NAME in components_to_start: + start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts) + + if WEBUI_COMPONENT_NAME in components_to_start: + start_webui(instance_id, clp_config, container_clp_config, mounts) + + if GARBAGE_COLLECTOR_NAME in components_to_start: + start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) except Exception as ex: if type(ex) == ValueError: From 2bd23f0d8e73538707b7c654b6cd4ea61f59a820 Mon Sep 17 00:00:00 2001 From: Haiqi Xu <14502009+haiqi96@users.noreply.github.com> Date: Thu, 14 Aug 2025 21:54:00 -0400 Subject: [PATCH 7/7] fix naming --- .../clp_package_utils/scripts/start_clp.py | 14 +++++++------- .../clp_package_utils/scripts/stop_clp.py | 8 ++++---- components/clp-py-utils/clp_py_utils/clp_config.py | 4 ++-- components/clp-py-utils/clp_py_utils/s3_utils.py | 4 ++-- .../garbage_collector/garbage_collector.py | 10 ++++++---- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index c7fcc8d2e4..faa087a26b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -22,7 +22,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, get_components_for_target, QUERY_JOBS_TABLE_NAME, QUERY_SCHEDULER_COMPONENT_NAME, @@ -1068,7 +1068,7 @@ def start_garbage_collector( container_clp_config: CLPConfig, mounts: CLPDockerMounts, ): - component_name = GARBAGE_COLLECTOR_NAME + component_name = GARBAGE_COLLECTOR_COMPONENT_NAME if not is_retention_period_configured(clp_config): logger.info(f"Retention period is not configured, skipping {component_name} creation...") @@ -1180,7 +1180,7 @@ def main(argv): reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME) add_num_workers_argument(reducer_server_parser) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_NAME) + component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -1214,7 +1214,7 @@ def main(argv): ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, WEBUI_COMPONENT_NAME, @@ -1249,13 +1249,13 @@ def main(argv): ALL_TARGET_NAME, COMPRESSION_WORKER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, ): validate_output_storage_config(clp_config) if target in ( ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, ): validate_retention_config(clp_config) @@ -1338,7 +1338,7 @@ def main(argv): if WEBUI_COMPONENT_NAME in components_to_start: start_webui(instance_id, clp_config, container_clp_config, mounts) - if GARBAGE_COLLECTOR_NAME in components_to_start: + if GARBAGE_COLLECTOR_COMPONENT_NAME in components_to_start: start_garbage_collector(instance_id, clp_config, container_clp_config, mounts) except Exception as ex: diff --git a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py index f70a45d45f..899b7a96a3 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/stop_clp.py @@ -11,7 +11,7 @@ COMPRESSION_WORKER_COMPONENT_NAME, CONTROLLER_TARGET_NAME, DB_COMPONENT_NAME, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, QUEUE_COMPONENT_NAME, @@ -85,7 +85,7 @@ def main(argv): component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME) component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME) component_args_parser.add_parser(WEBUI_COMPONENT_NAME) - component_args_parser.add_parser(GARBAGE_COLLECTOR_NAME) + component_args_parser.add_parser(GARBAGE_COLLECTOR_COMPONENT_NAME) parsed_args = args_parser.parse_args(argv[1:]) @@ -132,8 +132,8 @@ def main(argv): already_exited_containers = [] force = parsed_args.force - if target in (ALL_TARGET_NAME, GARBAGE_COLLECTOR_NAME): - container_name = f"clp-{GARBAGE_COLLECTOR_NAME}-{instance_id}" + if target in (ALL_TARGET_NAME, GARBAGE_COLLECTOR_COMPONENT_NAME): + container_name = f"clp-{GARBAGE_COLLECTOR_COMPONENT_NAME}-{instance_id}" stop_running_container(container_name, already_exited_containers, force) if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME): container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}" diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 597a028b92..7e4fcb23dd 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -26,7 +26,7 @@ COMPRESSION_WORKER_COMPONENT_NAME = "compression_worker" QUERY_WORKER_COMPONENT_NAME = "query_worker" WEBUI_COMPONENT_NAME = "webui" -GARBAGE_COLLECTOR_NAME = "garbage_collector" +GARBAGE_COLLECTOR_COMPONENT_NAME = "garbage_collector" # Component groups GENERAL_SCHEDULING_COMPONENTS = { @@ -48,7 +48,7 @@ RESULTS_CACHE_COMPONENT_NAME, WEBUI_COMPONENT_NAME, } -STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_NAME} +STORAGE_MANAGEMENT_COMPONENTS = {GARBAGE_COLLECTOR_COMPONENT_NAME} ALL_COMPONENTS = ( COMPRESSION_COMPONENTS | QUERY_COMPONENTS | UI_COMPONENTS | STORAGE_MANAGEMENT_COMPONENTS ) diff --git a/components/clp-py-utils/clp_py_utils/s3_utils.py b/components/clp-py-utils/clp_py_utils/s3_utils.py index c0cf29745b..9e376a4e68 100644 --- a/components/clp-py-utils/clp_py_utils/s3_utils.py +++ b/components/clp-py-utils/clp_py_utils/s3_utils.py @@ -14,7 +14,7 @@ COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, FsStorage, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, S3Config, @@ -119,7 +119,7 @@ def generate_container_auth_options( elif component_type in (WEBUI_COMPONENT_NAME,): output_storages_by_component_type = [clp_config.stream_output.storage] elif component_type in ( - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, QUERY_SCHEDULER_COMPONENT_NAME, QUERY_WORKER_COMPONENT_NAME, ): diff --git a/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py b/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py index 3c16e32b9d..108e7c244e 100644 --- a/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py +++ b/components/job-orchestration/job_orchestration/garbage_collector/garbage_collector.py @@ -9,7 +9,7 @@ from clp_py_utils.clp_config import ( CLPConfig, - GARBAGE_COLLECTOR_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, ) from clp_py_utils.clp_logging import get_logger from clp_py_utils.core import read_yaml_config_file @@ -24,18 +24,20 @@ from job_orchestration.garbage_collector.utils import configure_logger from pydantic import ValidationError -logger = get_logger(GARBAGE_COLLECTOR_NAME) +logger = get_logger(GARBAGE_COLLECTOR_COMPONENT_NAME) async def main(argv: List[str]) -> int: - args_parser = argparse.ArgumentParser(description=f"Spin up the {GARBAGE_COLLECTOR_NAME}.") + args_parser = argparse.ArgumentParser( + description=f"Spin up the {GARBAGE_COLLECTOR_COMPONENT_NAME}." + ) args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.") parsed_args = args_parser.parse_args(argv[1:]) # Setup logging to file logs_directory = Path(os.getenv("CLP_LOGS_DIR")) logging_level = os.getenv("CLP_LOGGING_LEVEL") - configure_logger(logger, logging_level, logs_directory, GARBAGE_COLLECTOR_NAME) + configure_logger(logger, logging_level, logs_directory, GARBAGE_COLLECTOR_COMPONENT_NAME) # Load configuration config_path = Path(parsed_args.config)