refactor(package): Standardize how configuration and credentials are passed to containers (resolves #1149).

components/clp-package-utils/clp_package_utils/general.py

@@ -1,5 +1,6 @@
 import enum
 import errno
+import logging
 import os
 import pathlib
 import re
@@ -36,6 +37,8 @@
 )
 from strenum import KebabCaseStrEnum
 
+logger = logging.getLogger(__file__)
+
 # CONSTANTS
 EXTRACT_FILE_CMD = "x"
 EXTRACT_IR_CMD = "i"
@@ -94,6 +97,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
         self.archives_output_dir: typing.Optional[DockerMount] = None
         self.stream_output_dir: typing.Optional[DockerMount] = None
         self.aws_config_dir: typing.Optional[DockerMount] = None
+        self.generated_config_dir: typing.Optional[DockerMount] = None
 
 
 def get_clp_home():
@@ -278,6 +282,19 @@ def generate_container_config(
             clp_config.aws_config_directory,
             container_clp_config.aws_config_directory,
         )
+
+    if not is_path_already_mounted(
+        clp_home,
+        CONTAINER_CLP_HOME,
+        clp_config.generated_config_file_path,
+        container_clp_config.generated_config_file_path,
+    ):
+        docker_mounts.generated_config_dir = DockerMount(
+            DockerMountType.BIND,
+            clp_config.generated_config_file_path,
+            container_clp_config.generated_config_file_path,
+        )
+
     return container_clp_config, docker_mounts
 
 
@@ -313,11 +330,15 @@ def dump_container_config(
 
 
 def generate_container_start_cmd(
-    container_name: str, container_mounts: List[Optional[DockerMount]], container_image: str
+    container_name: str,
+    extra_env_vars: dict[str, str],
+    container_mounts: List[Optional[DockerMount]],
+    container_image: str,
 ) -> List[str]:
     """
     Generates the command to start a container with the given mounts and name.
     :param container_name:
+    :param extra_env_vars:
     :param container_mounts:
     :param container_image:
     :return: The command.
@@ -335,6 +356,8 @@ def generate_container_start_cmd(
         "--name", container_name,
         "--log-driver", "local"
     ]
+    for key, value in extra_env_vars.items():
+        container_start_cmd.extend(["-e", f"{key}={value}"])
     for mount in container_mounts:
         if mount:
             container_start_cmd.append("--mount")
@@ -377,6 +400,11 @@ def load_config_file(
     hostname = socket.gethostname()
     clp_config.data_directory /= hostname
     clp_config.logs_directory /= hostname
+    clp_config.generated_config_file_path = (
+        clp_config.generated_config_file_path.parent
+        / hostname
+        / clp_config.generated_config_file_path.name
+    )
 
     return clp_config
 
@@ -590,3 +618,59 @@ def validate_dataset_name(clp_table_prefix: str, dataset_name: str) -> None:
             f"Invalid dataset name: `{dataset_name}`. Names can only be a maximum of"
             f" {dataset_name_max_len} characters long."
         )
+
+
+def generate_common_environment_variables(
+    container_clp_config,
+    include_clp_home=True,
+    include_db_credentials=False,
+    include_queue_credentials=False,
+    include_redis_credentials=False,
+    include_celery_connection_params=False,
+    extra_vars=None,
+) -> List[str]:
+    """
+    Generate a list of common environment variables for Docker containers.
+
+    :param container_clp_config: The CLPConfig object for the container.
+    :param include_clp_home:
+    :param include_db_credentials:
+    :param include_queue_credentials:
+    :param include_redis_credentials:
+    :param include_celery_connection_params:
+    :param extra_vars: Additional environment variables to include in the format "KEY=VALUE".
+    :return: List of environment variable strings in the format "KEY=VALUE".
+    """
+    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
+    env_vars = [f"PYTHONPATH={clp_site_packages_dir}"]
+
+    if include_clp_home:
+        env_vars.append(f"CLP_HOME={CONTAINER_CLP_HOME}")
+
+    if include_db_credentials:
+        env_vars.append(f"CLP_DB_USER={container_clp_config.database.username}")
+        env_vars.append(f"CLP_DB_PASS={container_clp_config.database.password}")
+
+    if include_queue_credentials:
+        env_vars.append(f"CLP_QUEUE_USER={container_clp_config.queue.username}")
+        env_vars.append(f"CLP_QUEUE_PASS={container_clp_config.queue.password}")
+
+    if include_redis_credentials:
+        env_vars.append(f"CLP_REDIS_PASS={container_clp_config.redis.password}")
+
+    if include_celery_connection_params:
+        env_vars.append(
+            f"BROKER_URL=amqp://"
+            f"{container_clp_config.queue.username}:{container_clp_config.queue.password}@"
+            f"{container_clp_config.queue.host}:{container_clp_config.queue.port}"
+        )
+        env_vars.append(
+            f"RESULT_BACKEND=redis://default:{container_clp_config.redis.password}@"
+            f"{container_clp_config.redis.host}:{container_clp_config.redis.port}/"
+            f"{container_clp_config.redis.query_backend_database}"
+        )
+
+    if extra_vars:
+        env_vars.extend(extra_vars)
+
+    return env_vars

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -206,14 +206,17 @@ def main(argv: typing.List[str]) -> int:
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
-
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     necessary_mounts: typing.List[CLPDockerMounts] = [
         mounts.clp_home,
         mounts.logs_dir,
         mounts.archives_output_dir,
     ]
     container_start_cmd: typing.List[str] = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, extra_env_vars, necessary_mounts, clp_config.execution_container
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/compress.py

@@ -205,6 +205,11 @@ def main(argv):
         container_clp_config, clp_config, container_name
     )
 
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
+
     necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir]
     if InputType.FS == input_type:
         necessary_mounts.append(mounts.input_logs_dir)
@@ -223,7 +228,7 @@ def main(argv):
     _generate_logs_list(clp_config.logs_input.type, container_logs_list_path, parsed_args)
 
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, extra_env_vars, necessary_mounts, clp_config.execution_container
     )
     compress_cmd = _generate_compress_cmd(
         parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container

components/clp-package-utils/clp_package_utils/scripts/decompress.py

@@ -102,6 +102,11 @@ def handle_extract_file_cmd(
         container_clp_config, clp_config, container_name
     )
 
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
+
     # Set up mounts
     extraction_dir.mkdir(exist_ok=True)
     container_extraction_dir = pathlib.Path("/") / "mnt" / "extraction-dir"
@@ -123,7 +128,7 @@ def handle_extract_file_cmd(
             )
         )
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, extra_env_vars, necessary_mounts, clp_config.execution_container
     )
 
     # fmt: off
@@ -203,9 +208,13 @@ def handle_extract_stream_cmd(
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, extra_env_vars, necessary_mounts, clp_config.execution_container
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import os
 import shutil
 import sys
 import typing
@@ -188,6 +189,12 @@ def main(argv: typing.List[str]) -> int:
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     database_config: Database = clp_config.database
     dataset = parsed_args.dataset

components/clp-package-utils/clp_package_utils/scripts/native/compress.py

@@ -1,6 +1,7 @@
 import argparse
 import datetime
 import logging
+import os
 import pathlib
 import sys
 import time
@@ -229,6 +230,12 @@ def main(argv):
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     comp_jobs_dir = clp_config.logs_directory / "comp-jobs"
     comp_jobs_dir.mkdir(parents=True, exist_ok=True)

components/clp-package-utils/clp_package_utils/scripts/native/decompress.py

@@ -1,6 +1,7 @@
 import argparse
 import asyncio
 import logging
+import os
 import pathlib
 import subprocess
 import sys
@@ -112,11 +113,18 @@ def handle_extract_stream_cmd(
     :return: 0 on success, -1 otherwise.
     """
     # Validate and load config file
-    clp_config = validate_and_load_config_file(
+    clp_config = load_and_validate_config_file(
         clp_home, pathlib.Path(parsed_args.config), default_config_file_path
     )
     if clp_config is None:
         return -1
+    # Override credentials with environment variables
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     command = parsed_args.command
 
@@ -222,6 +230,13 @@ def handle_extract_file_cmd(
     )
     if clp_config is None:
         return -1
+    # Override credentials with environment variables
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     paths = parsed_args.paths
     list_path = parsed_args.files_from

components/clp-package-utils/clp_package_utils/scripts/native/search.py

@@ -4,6 +4,7 @@
 import asyncio
 import ipaddress
 import logging
+import os
 import pathlib
 import socket
 import sys
@@ -297,6 +298,12 @@ def main(argv):
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     database_config: Database = clp_config.database
     dataset = parsed_args.dataset

components/clp-package-utils/clp_package_utils/scripts/search.py

@@ -117,10 +117,13 @@ def main(argv):
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
-
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, extra_env_vars, necessary_mounts, clp_config.execution_container
     )
 
     # fmt: off