refactor(package): Standardize how configuration and credentials are passed to containers (resolves #1149). (#1152)

Author: junhaoliao

components/clp-package-utils/clp_package_utils/general.py

@@ -9,11 +9,12 @@
 import typing
 import uuid
 from enum import auto
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import yaml
 from clp_py_utils.clp_config import (
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
+    CLP_SHARED_CONFIG_FILENAME,
     CLPConfig,
     DB_COMPONENT_NAME,
     QueryEngine,
@@ -95,6 +96,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
         self.archives_output_dir: typing.Optional[DockerMount] = None
         self.stream_output_dir: typing.Optional[DockerMount] = None
         self.aws_config_dir: typing.Optional[DockerMount] = None
+        self.generated_config_file: typing.Optional[DockerMount] = None
 
 
 def _validate_data_directory(data_dir: pathlib.Path, component_name: str) -> None:
@@ -285,6 +287,18 @@ def generate_container_config(
             container_clp_config.stream_output.get_directory(),
         )
 
+    if not is_path_already_mounted(
+        clp_home,
+        CONTAINER_CLP_HOME,
+        clp_config.get_shared_config_file_path(),
+        container_clp_config.get_shared_config_file_path(),
+    ):
+        docker_mounts.generated_config_file = DockerMount(
+            DockerMountType.BIND,
+            clp_config.get_shared_config_file_path(),
+            container_clp_config.get_shared_config_file_path(),
+        )
+
     # Only create the mount if the directory exists
     if clp_config.aws_config_directory is not None:
         container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY
@@ -308,33 +322,57 @@ def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig:
     return worker_config
 
 
+def get_container_config_filename(container_name: str) -> str:
+    return f".{container_name}-config.yml"
+
+
 def dump_container_config(
-    container_clp_config: CLPConfig, clp_config: CLPConfig, container_name: str
-) -> Tuple[pathlib.Path, pathlib.Path]:
+    container_clp_config: CLPConfig, clp_config: CLPConfig, config_filename: str
+):
     """
-    Writes the given config to the logs directory so that it's accessible in the container.
+    Writes the given container config to the logs directory, so that it's accessible in the
+    container.
+
     :param container_clp_config: The config to write.
     :param clp_config: The corresponding config on the host (used to determine the logs directory).
-    :param container_name:
+    :param config_filename:
     :return: The path to the config file in the container and on the host.
     """
-    container_config_filename = f".{container_name}-config.yml"
-    config_file_path_on_host = clp_config.logs_directory / container_config_filename
-    config_file_path_on_container = container_clp_config.logs_directory / container_config_filename
+    config_file_path_on_host = clp_config.logs_directory / config_filename
+    config_file_path_on_container = container_clp_config.logs_directory / config_filename
     with open(config_file_path_on_host, "w") as f:
         yaml.safe_dump(container_clp_config.dump_to_primitive_dict(), f)
 
     return config_file_path_on_container, config_file_path_on_host
 
 
+def dump_shared_container_config(
+    container_clp_config: CLPConfig, clp_config: CLPConfig
+) -> Tuple[pathlib.Path, pathlib.Path]:
+    """
+    Dumps the given container config to `CLP_SHARED_CONFIG_FILENAME` in the logs directory, so that
+    it's accessible in the container.
+
+    :param container_clp_config:
+    :param clp_config:
+    """
+    return dump_container_config(container_clp_config, clp_config, CLP_SHARED_CONFIG_FILENAME)
+
+
 def generate_container_start_cmd(
-    container_name: str, container_mounts: List[Optional[DockerMount]], container_image: str
+    container_name: str,
+    container_mounts: List[Optional[DockerMount]],
+    container_image: str,
+    extra_env_vars: Optional[Dict[str, str]] = None,
 ) -> List[str]:
     """
-    Generates the command to start a container with the given mounts and name.
+    Generates the command to start a container with the given mounts, environment variables, and
+    name.
+
     :param container_name:
     :param container_mounts:
     :param container_image:
+    :param extra_env_vars: Environment variables to set on top of the predefined ones.
     :return: The command.
     """
     clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
@@ -350,6 +388,12 @@ def generate_container_start_cmd(
         "--name", container_name,
         "--log-driver", "local"
     ]
+    env_vars = {
+        "PYTHONPATH": clp_site_packages_dir,
+        **(extra_env_vars if extra_env_vars is not None else {}),
+    }
+    for key, value in env_vars.items():
+        container_start_cmd.extend(["-e", f"{key}={value}"])
     for mount in container_mounts:
         if mount:
             container_start_cmd.append("--mount")
@@ -428,21 +472,21 @@ def validate_and_load_db_credentials_file(
     clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool
 ):
     validate_credentials_file_path(clp_config, clp_home, generate_default_file)
-    clp_config.load_database_credentials_from_file()
+    clp_config.database.load_credentials_from_file(clp_config.credentials_file_path)
 
 
 def validate_and_load_queue_credentials_file(
     clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool
 ):
     validate_credentials_file_path(clp_config, clp_home, generate_default_file)
-    clp_config.load_queue_credentials_from_file()
+    clp_config.queue.load_credentials_from_file(clp_config.credentials_file_path)
 
 
 def validate_and_load_redis_credentials_file(
     clp_config: CLPConfig, clp_home: pathlib.Path, generate_default_file: bool
 ):
     validate_credentials_file_path(clp_config, clp_home, generate_default_file)
-    clp_config.load_redis_credentials_from_file()
+    clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path)
 
 
 def validate_db_config(clp_config: CLPConfig, data_dir: pathlib.Path, logs_dir: pathlib.Path):
@@ -599,3 +643,68 @@ def is_retention_period_configured(clp_config: CLPConfig) -> bool:
         return True
 
     return False
+
+
+def get_common_env_vars_list(
+    include_clp_home_env_var=True,
+) -> List[str]:
+    """
+    :param include_clp_home_env_var:
+    :return: A list of common environment variables for Docker containers, in the format
+    "KEY=VALUE".
+    """
+    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
+    env_vars = [f"PYTHONPATH={clp_site_packages_dir}"]
+
+    if include_clp_home_env_var:
+        env_vars.append(f"CLP_HOME={CONTAINER_CLP_HOME}")
+
+    return env_vars
+
+
+def get_credential_env_vars_list(
+    container_clp_config: CLPConfig,
+    include_db_credentials=False,
+    include_queue_credentials=False,
+    include_redis_credentials=False,
+) -> List[str]:
+    """
+    :param container_clp_config:
+    :param include_db_credentials:
+    :param include_queue_credentials:
+    :param include_redis_credentials:
+    :return: A list of credential environment variables for Docker containers, in the format
+    "KEY=VALUE".
+    """
+    env_vars = []
+
+    if include_db_credentials:
+        env_vars.append(f"CLP_DB_USER={container_clp_config.database.username}")
+        env_vars.append(f"CLP_DB_PASS={container_clp_config.database.password}")
+
+    if include_queue_credentials:
+        env_vars.append(f"CLP_QUEUE_USER={container_clp_config.queue.username}")
+        env_vars.append(f"CLP_QUEUE_PASS={container_clp_config.queue.password}")
+
+    if include_redis_credentials:
+        env_vars.append(f"CLP_REDIS_PASS={container_clp_config.redis.password}")
+
+    return env_vars
+
+
+def get_celery_connection_env_vars_list(container_clp_config: CLPConfig) -> List[str]:
+    """
+    :param container_clp_config:
+    :return: A list of Celery connection environment variables for Docker containers, in the format
+    "KEY=VALUE".
+    """
+    env_vars = [
+        f"BROKER_URL=amqp://"
+        f"{container_clp_config.queue.username}:{container_clp_config.queue.password}@"
+        f"{container_clp_config.queue.host}:{container_clp_config.queue.port}",
+        f"RESULT_BACKEND=redis://default:{container_clp_config.redis.password}@"
+        f"{container_clp_config.redis.host}:{container_clp_config.redis.port}/"
+        f"{container_clp_config.redis.query_backend_database}",
+    ]
+
+    return env_vars

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -6,6 +6,8 @@
 from typing import Final, List, Optional
 
 from clp_py_utils.clp_config import (
+    CLP_DB_PASS_ENV_VAR_NAME,
+    CLP_DB_USER_ENV_VAR_NAME,
     CLP_DEFAULT_DATASET_NAME,
     StorageEngine,
     StorageType,
@@ -20,6 +22,7 @@
     generate_container_name,
     generate_container_start_cmd,
     get_clp_home,
+    get_container_config_filename,
     load_config_file,
     validate_and_load_db_credentials_file,
     validate_dataset_name,
@@ -212,16 +215,20 @@ def main(argv: List[str]) -> int:
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        container_clp_config, clp_config, container_name
+        container_clp_config, clp_config, get_container_config_filename(container_name)
     )
 
     necessary_mounts: List[Optional[DockerMount]] = [
         mounts.clp_home,
         mounts.logs_dir,
         mounts.archives_output_dir,
     ]
+    extra_env_vars = {
+        CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
+        CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
+    }
     container_start_cmd: List[str] = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/compress.py

@@ -7,6 +7,8 @@
 from typing import List, Optional
 
 from clp_py_utils.clp_config import (
+    CLP_DB_PASS_ENV_VAR_NAME,
+    CLP_DB_USER_ENV_VAR_NAME,
     CLP_DEFAULT_DATASET_NAME,
     StorageEngine,
 )
@@ -20,6 +22,7 @@
     generate_container_name,
     generate_container_start_cmd,
     get_clp_home,
+    get_container_config_filename,
     JobType,
     load_config_file,
     validate_and_load_db_credentials_file,
@@ -202,7 +205,7 @@ def main(argv):
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        container_clp_config, clp_config, container_name
+        container_clp_config, clp_config, get_container_config_filename(container_name)
     )
 
     necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir]
@@ -222,8 +225,12 @@ def main(argv):
 
     _generate_logs_list(clp_config.logs_input.type, container_logs_list_path, parsed_args)
 
+    extra_env_vars = {
+        CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
+        CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
     compress_cmd = _generate_compress_cmd(
         parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container

components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py

@@ -19,6 +19,7 @@
     generate_container_name,
     generate_container_start_cmd,
     get_clp_home,
+    get_container_config_filename,
     load_config_file,
     validate_and_load_db_credentials_file,
     validate_dataset_name,
@@ -124,7 +125,7 @@ def main(argv: List[str]) -> int:
 
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        container_clp_config, clp_config, container_name
+        container_clp_config, clp_config, get_container_config_filename(container_name)
     )
 
     necessary_mounts = [

components/clp-package-utils/clp_package_utils/scripts/decompress.py

@@ -6,6 +6,8 @@
 from typing import Optional
 
 from clp_py_utils.clp_config import (
+    CLP_DB_PASS_ENV_VAR_NAME,
+    CLP_DB_USER_ENV_VAR_NAME,
     CLP_DEFAULT_DATASET_NAME,
     CLPConfig,
     StorageEngine,
@@ -24,6 +26,7 @@
     generate_container_name,
     generate_container_start_cmd,
     get_clp_home,
+    get_container_config_filename,
     JobType,
     load_config_file,
     validate_and_load_db_credentials_file,
@@ -99,7 +102,7 @@ def handle_extract_file_cmd(
     container_name = generate_container_name(str(JobType.FILE_EXTRACTION))
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        container_clp_config, clp_config, container_name
+        container_clp_config, clp_config, get_container_config_filename(container_name)
     )
 
     # Set up mounts
@@ -122,8 +125,13 @@ def handle_extract_file_cmd(
                 container_paths_to_extract_file_path,
             )
         )
+
+    extra_env_vars = {
+        CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
+        CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off
@@ -191,11 +199,15 @@ def handle_extract_stream_cmd(
     container_name = generate_container_name(str(JobType.IR_EXTRACTION))
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
-        container_clp_config, clp_config, container_name
+        container_clp_config, clp_config, get_container_config_filename(container_name)
     )
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]
+    extra_env_vars = {
+        CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
+        CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py

@@ -194,6 +194,7 @@ def main(argv: List[str]) -> int:
             config_file_path, default_config_file_path, clp_home
         )
         clp_config.validate_logs_dir()
+        clp_config.database.load_credentials_from_env()
     except:
         logger.exception("Failed to load config.")
         return -1

components/clp-package-utils/clp_package_utils/scripts/native/compress.py

@@ -226,6 +226,7 @@ def main(argv):
         clp_config = load_config_file(config_file_path, default_config_file_path, clp_home)
         clp_config.validate_logs_input_config()
         clp_config.validate_logs_dir()
+        clp_config.database.load_credentials_from_env()
     except:
         logger.exception("Failed to load config.")
         return -1