feat(clp-package)!: Containerize the scripts in sbin via Docker Compose; Remove redundant assets from the package (resolves #1358).

components/clp-package-utils/clp_package_utils/controller.py

@@ -37,6 +37,7 @@
     get_datasets_table_name,
     get_files_table_name,
 )
+from clp_py_utils.core import resolve_host_path
 
 from clp_package_utils.general import (
     check_docker_dependencies,
@@ -119,9 +120,12 @@ def _set_up_env_for_database(self) -> EnvVarsDict:
         logs_dir = self._clp_config.logs_directory / component_name
         validate_db_config(self._clp_config, conf_logging_file, data_dir, logs_dir)
 
-        data_dir.mkdir(exist_ok=True, parents=True)
-        logs_dir.mkdir(exist_ok=True, parents=True)
-        _chown_paths_if_root(data_dir, logs_dir)
+        resolved_data_dir = resolve_host_path(data_dir)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+
+        resolved_data_dir.mkdir(exist_ok=True, parents=True)
+        resolved_logs_dir.mkdir(exist_ok=True, parents=True)
+        _chown_paths_if_root(resolved_data_dir, resolved_logs_dir)
 
         env_vars = EnvVarsDict()
 
@@ -166,8 +170,10 @@ def _set_up_env_for_queue(self) -> EnvVarsDict:
         logs_dir = self._clp_config.logs_directory / component_name
         validate_queue_config(self._clp_config, logs_dir)
 
-        logs_dir.mkdir(exist_ok=True, parents=True)
-        _chown_paths_if_root(logs_dir)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+
+        resolved_logs_dir.mkdir(exist_ok=True, parents=True)
+        _chown_paths_if_root(resolved_logs_dir)
 
         env_vars = EnvVarsDict()
 
@@ -204,9 +210,12 @@ def _set_up_env_for_redis(self) -> EnvVarsDict:
         logs_dir = self._clp_config.logs_directory / component_name
         validate_redis_config(self._clp_config, conf_file, data_dir, logs_dir)
 
-        data_dir.mkdir(exist_ok=True, parents=True)
-        logs_dir.mkdir(exist_ok=True, parents=True)
-        _chown_paths_if_root(data_dir, logs_dir)
+        resolved_data_dir = resolve_host_path(data_dir)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+
+        resolved_data_dir.mkdir(exist_ok=True, parents=True)
+        resolved_logs_dir.mkdir(exist_ok=True, parents=True)
+        _chown_paths_if_root(resolved_data_dir, resolved_logs_dir)
 
         env_vars = EnvVarsDict()
 
@@ -252,9 +261,12 @@ def _set_up_env_for_results_cache(self) -> EnvVarsDict:
         logs_dir = self._clp_config.logs_directory / component_name
         validate_results_cache_config(self._clp_config, conf_file, data_dir, logs_dir)
 
-        data_dir.mkdir(exist_ok=True, parents=True)
-        logs_dir.mkdir(exist_ok=True, parents=True)
-        _chown_paths_if_root(data_dir, logs_dir)
+        resolved_data_dir = resolve_host_path(data_dir)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+
+        resolved_data_dir.mkdir(exist_ok=True, parents=True)
+        resolved_logs_dir.mkdir(exist_ok=True, parents=True)
+        _chown_paths_if_root(resolved_data_dir, resolved_logs_dir)
 
         env_vars = EnvVarsDict()
 
@@ -291,7 +303,8 @@ def _set_up_env_for_compression_scheduler(self) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -314,7 +327,8 @@ def _set_up_env_for_query_scheduler(self) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -336,7 +350,8 @@ def _set_up_env_for_compression_worker(self, num_workers: int) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -365,7 +380,8 @@ def _set_up_env_for_query_worker(self, num_workers: int) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -392,7 +408,8 @@ def _set_up_env_for_reducer(self, num_workers: int) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -426,8 +443,12 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict:
         server_settings_json_path = (
             self._clp_home / "var" / "www" / "webui" / "server" / "dist" / "settings.json"
         )
+        resolved_client_settings_json_path = resolve_host_path(client_settings_json_path)
+        resolved_server_settings_json_path = resolve_host_path(server_settings_json_path)
         validate_webui_config(
-            self._clp_config, client_settings_json_path, server_settings_json_path
+            self._clp_config,
+            client_settings_json_path,
+            server_settings_json_path,
         )
 
         # Read, update, and write back client's and server's settings.json
@@ -455,9 +476,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict:
             "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME,
         }
         client_settings_json = self._read_and_update_settings_json(
-            client_settings_json_path, client_settings_json_updates
+            resolved_client_settings_json_path, client_settings_json_updates
         )
-        with open(client_settings_json_path, "w") as client_settings_json_file:
+        with open(resolved_client_settings_json_path, "w") as client_settings_json_file:
             client_settings_json_file.write(json.dumps(client_settings_json))
 
         server_settings_json_updates = {
@@ -510,9 +531,9 @@ def _set_up_env_for_webui(self, container_clp_config: CLPConfig) -> EnvVarsDict:
             server_settings_json_updates["PrestoPort"] = None
 
         server_settings_json = self._read_and_update_settings_json(
-            server_settings_json_path, server_settings_json_updates
+            resolved_server_settings_json_path, server_settings_json_updates
         )
-        with open(server_settings_json_path, "w") as settings_json_file:
+        with open(resolved_server_settings_json_path, "w") as settings_json_file:
             settings_json_file.write(json.dumps(server_settings_json))
 
         env_vars = EnvVarsDict()
@@ -544,7 +565,8 @@ def _set_up_env_for_mcp_server(self) -> EnvVarsDict:
 
         logs_dir = self._clp_config.logs_directory / component_name
         validate_mcp_server_config(self._clp_config, logs_dir)
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -581,7 +603,8 @@ def _set_up_env_for_garbage_collector(self) -> EnvVarsDict:
         logger.info(f"Setting up environment for {component_name}...")
 
         logs_dir = self._clp_config.logs_directory / component_name
-        logs_dir.mkdir(parents=True, exist_ok=True)
+        resolved_logs_dir = resolve_host_path(logs_dir)
+        resolved_logs_dir.mkdir(parents=True, exist_ok=True)
 
         env_vars = EnvVarsDict()
 
@@ -696,8 +719,10 @@ def set_up_env(self) -> None:
             env_vars["CLP_LOGS_INPUT_DIR_HOST"] = str(self._clp_config.logs_input.directory)
 
         # Output config
-        archive_output_dir_str = str(self._clp_config.archive_output.get_directory())
-        stream_output_dir_str = str(self._clp_config.stream_output.get_directory())
+        archive_output_dir = self._clp_config.archive_output.get_directory()
+        stream_output_dir = self._clp_config.stream_output.get_directory()
+        archive_output_dir_str = str(archive_output_dir)
+        stream_output_dir_str = str(stream_output_dir)
         if self._clp_config.archive_output.storage.type == StorageType.FS:
             env_vars["CLP_ARCHIVE_OUTPUT_DIR_HOST"] = archive_output_dir_str
         if self._clp_config.archive_output.storage.type == StorageType.S3:
@@ -804,13 +829,14 @@ def get_or_create_instance_id(clp_config: CLPConfig) -> str:
     :return: The instance ID.
     """
     instance_id_file_path = clp_config.logs_directory / "instance-id"
+    resolved_instance_id_file_path = resolve_host_path(instance_id_file_path)
 
-    if instance_id_file_path.exists():
-        with open(instance_id_file_path, "r") as f:
+    if resolved_instance_id_file_path.exists():
+        with open(resolved_instance_id_file_path, "r") as f:
             instance_id = f.readline()
     else:
         instance_id = str(uuid.uuid4())[-4:]
-        with open(instance_id_file_path, "w") as f:
+        with open(resolved_instance_id_file_path, "w") as f:
             f.write(instance_id)
 
     return instance_id

components/clp-package-utils/clp_package_utils/general.py

@@ -38,6 +38,7 @@
     get_config_value,
     make_config_path_absolute,
     read_yaml_config_file,
+    resolve_host_path,
     validate_path_could_be_dir,
 )
 from strenum import KebabCaseStrEnum
@@ -355,7 +356,8 @@ def dump_container_config(
     """
     config_file_path_on_host = clp_config.logs_directory / config_filename
     config_file_path_on_container = container_clp_config.logs_directory / config_filename
-    with open(config_file_path_on_host, "w") as f:
+    resolved_config_file_path_on_host = resolve_host_path(config_file_path_on_host)
+    with open(resolved_config_file_path_on_host, "w") as f:
         yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
 
     return config_file_path_on_container, config_file_path_on_host
@@ -427,8 +429,9 @@ def validate_config_key_existence(config, key):
 def load_config_file(
     config_file_path: pathlib.Path, default_config_file_path: pathlib.Path, clp_home: pathlib.Path
 ):
-    if config_file_path.exists():
-        raw_clp_config = read_yaml_config_file(config_file_path)
+    resolved_config_file_path = resolve_host_path(config_file_path)
+    if resolved_config_file_path.exists():
+        raw_clp_config = read_yaml_config_file(resolved_config_file_path)
         if raw_clp_config is None:
             clp_config = CLPConfig()
         else:
@@ -464,16 +467,17 @@ def validate_credentials_file_path(
     clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool
 ):
     credentials_file_path = clp_config.credentials_file_path
-    if not credentials_file_path.exists():
+    resolved_credentials_file_path = resolve_host_path(credentials_file_path)
+    if not resolved_credentials_file_path.exists():
         if (
             make_config_path_absolute(clp_home, CLP_DEFAULT_CREDENTIALS_FILE_PATH)
             == credentials_file_path
             and generate_default_file
         ):
-            generate_credentials_file(credentials_file_path)
+            generate_credentials_file(resolved_credentials_file_path)
         else:
             raise ValueError(f"Credentials file path '{credentials_file_path}' does not exist.")
-    elif not credentials_file_path.is_file():
+    elif not resolved_credentials_file_path.is_file():
         raise ValueError(f"Credentials file path '{credentials_file_path}' is not a file.")
 
 
@@ -504,7 +508,8 @@ def validate_db_config(
     data_dir: pathlib.Path,
     logs_dir: pathlib.Path,
 ):
-    if not component_config.exists():
+    resolved_component_config = resolve_host_path(component_config)
+    if not resolved_component_config.exists():
         raise ValueError(f"{DB_COMPONENT_NAME} configuration file missing: '{component_config}'.")
     _validate_data_directory(data_dir, DB_COMPONENT_NAME)
     _validate_log_directory(logs_dir, DB_COMPONENT_NAME)
@@ -524,7 +529,8 @@ def validate_redis_config(
     data_dir: pathlib.Path,
     logs_dir: pathlib.Path,
 ):
-    if not component_config.exists():
+    resolved_component_config = resolve_host_path(component_config)
+    if not resolved_component_config.exists():
         raise ValueError(
             f"{REDIS_COMPONENT_NAME} configuration file missing: '{component_config}'."
         )
@@ -551,7 +557,8 @@ def validate_results_cache_config(
     data_dir: pathlib.Path,
     logs_dir: pathlib.Path,
 ):
-    if not component_config.exists():
+    resolved_component_config = resolve_host_path(component_config)
+    if not resolved_component_config.exists():
         raise ValueError(
             f"{RESULTS_CACHE_COMPONENT_NAME} configuration file missing: '{component_config}'."
         )
@@ -583,7 +590,8 @@ def validate_webui_config(
     server_settings_json_path: pathlib.Path,
 ):
     for path in [client_settings_json_path, server_settings_json_path]:
-        if not path.exists():
+        resolved_path = resolve_host_path(path)
+        if not resolved_path.exists():
             raise ValueError(f"{WEBUI_COMPONENT_NAME} {path} is not a valid path to settings.json")
 
     validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port)
@@ -765,7 +773,7 @@ def _is_docker_compose_project_running(project_name: str) -> bool:
 
 def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None:
     try:
-        validate_path_could_be_dir(data_dir)
+        validate_path_could_be_dir(data_dir, True)
     except ValueError as ex:
         raise ValueError(f"{component_name} data directory is invalid: {ex}")
 
@@ -779,6 +787,6 @@ def _validate_log_directory(logs_dir: pathlib.Path, component_name: str):
     :raise ValueError: If the path is invalid or can't be a directory.
     """
     try:
-        validate_path_could_be_dir(logs_dir)
+        validate_path_could_be_dir(logs_dir, True)
     except ValueError as ex:
         raise ValueError(f"{component_name} logs directory is invalid: {ex}")

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -14,6 +14,7 @@
     StorageEngine,
     StorageType,
 )
+from clp_py_utils.core import resolve_host_path
 
 from clp_package_utils.general import (
     CLPConfig,
@@ -220,7 +221,6 @@ def main(argv: List[str]) -> int:
     )
 
     necessary_mounts: List[Optional[DockerMount]] = [
-        mounts.clp_home,
         mounts.logs_dir,
         mounts.archives_output_dir,
     ]
@@ -281,7 +281,8 @@ def main(argv: List[str]) -> int:
         logger.debug(f"Docker command failed: {shlex.join(cmd)}")
 
     # Remove generated files
-    generated_config_path_on_host.unlink()
+    resolved_generated_config_path_on_host = resolve_host_path(generated_config_path_on_host)
+    resolved_generated_config_path_on_host.unlink()
 
     return ret_code