refactor(package): Standardize how configuration and credentials are passed to containers (resolves #1149).

components/clp-package-utils/clp_package_utils/general.py

@@ -1,5 +1,6 @@
 import enum
 import errno
+import logging
 import os
 import pathlib
 import re
@@ -9,7 +10,7 @@
 import typing
 import uuid
 from enum import auto
-from typing import List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import yaml
 from clp_py_utils.clp_config import (
@@ -94,6 +95,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
         self.archives_output_dir: typing.Optional[DockerMount] = None
         self.stream_output_dir: typing.Optional[DockerMount] = None
         self.aws_config_dir: typing.Optional[DockerMount] = None
+        self.generated_config_dir: typing.Optional[DockerMount] = None
 
 
 def get_clp_home():
@@ -270,6 +272,18 @@ def generate_container_config(
             container_clp_config.stream_output.get_directory(),
         )
 
+    if not is_path_already_mounted(
+        clp_home,
+        CONTAINER_CLP_HOME,
+        clp_config.generated_config_file_path,
+        container_clp_config.generated_config_file_path,
+    ):
+        docker_mounts.generated_config_dir = DockerMount(
+            DockerMountType.BIND,
+            clp_config.generated_config_file_path,
+            container_clp_config.generated_config_file_path,
+        )
+
     # Only create the mount if the directory exists
     if clp_config.aws_config_directory is not None:
         container_clp_config.aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY
@@ -278,6 +292,7 @@ def generate_container_config(
             clp_config.aws_config_directory,
             container_clp_config.aws_config_directory,
         )
+
     return container_clp_config, docker_mounts
 
 
@@ -313,13 +328,17 @@ def dump_container_config(
 
 
 def generate_container_start_cmd(
-    container_name: str, container_mounts: List[Optional[DockerMount]], container_image: str
+    container_name: str,
+    container_mounts: List[Optional[DockerMount]],
+    container_image: str,
+    extra_env_vars: Optional[Dict[str, str]] = None
 ) -> List[str]:
     """
     Generates the command to start a container with the given mounts and name.
     :param container_name:
     :param container_mounts:
     :param container_image:
+    :param extra_env_vars:
     :return: The command.
     """
     clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
@@ -335,6 +354,9 @@ def generate_container_start_cmd(
         "--name", container_name,
         "--log-driver", "local"
     ]
+    if extra_env_vars:
+        for key, value in extra_env_vars.items():
+            container_start_cmd.extend(["-e", f"{key}={value}"])
     for mount in container_mounts:
         if mount:
             container_start_cmd.append("--mount")
@@ -377,6 +399,11 @@ def load_config_file(
     hostname = socket.gethostname()
     clp_config.data_directory /= hostname
     clp_config.logs_directory /= hostname
+    clp_config.generated_config_file_path = (
+        clp_config.generated_config_file_path.parent
+        / hostname
+        / clp_config.generated_config_file_path.name
+    )
 
     return clp_config
 
@@ -590,3 +617,71 @@ def validate_dataset_name(clp_table_prefix: str, dataset_name: str) -> None:
             f"Invalid dataset name: `{dataset_name}`. Names can only be a maximum of"
             f" {dataset_name_max_len} characters long."
         )
+
+
+def generate_common_environment_variables(
+    include_clp_home=True,
+) -> List[str]:
+    """
+    Generate a list of common environment variables for Docker containers.
+
+    :param include_clp_home:
+    :return: List of environment variable strings in the format "KEY=VALUE".
+    """
+    clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
+    env_vars = [f"PYTHONPATH={clp_site_packages_dir}"]
+
+    if include_clp_home:
+        env_vars.append(f"CLP_HOME={CONTAINER_CLP_HOME}")
+
+    return env_vars
+
+
+def generate_credential_environment_variables(
+    container_clp_config: CLPConfig,
+    include_db_credentials=False,
+    include_queue_credentials=False,
+    include_redis_credentials=False,
+) -> List[str]:
+    """
+    Generate a list of credential environment variables for Docker containers.
+
+    :param container_clp_config:
+    :param include_db_credentials:
+    :param include_queue_credentials:
+    :param include_redis_credentials:
+    :return: List of environment variable strings in the format "KEY=VALUE".
+    """
+    env_vars = []
+
+    if include_db_credentials:
+        env_vars.append(f"CLP_DB_USER={container_clp_config.database.username}")
+        env_vars.append(f"CLP_DB_PASS={container_clp_config.database.password}")
+
+    if include_queue_credentials:
+        env_vars.append(f"CLP_QUEUE_USER={container_clp_config.queue.username}")
+        env_vars.append(f"CLP_QUEUE_PASS={container_clp_config.queue.password}")
+
+    if include_redis_credentials:
+        env_vars.append(f"CLP_REDIS_PASS={container_clp_config.redis.password}")
+
+    return env_vars
+
+
+def generate_celery_connection_environment_variables(container_clp_config: CLPConfig) -> List[str]:
+    """
+    Generate a list of Celery connection environment variables for Docker containers.
+
+    :param container_clp_config:
+    :return: List of environment variable strings in the format "KEY=VALUE".
+    """
+    env_vars = [
+        f"BROKER_URL=amqp://"
+        f"{container_clp_config.queue.username}:{container_clp_config.queue.password}@"
+        f"{container_clp_config.queue.host}:{container_clp_config.queue.port}",
+        f"RESULT_BACKEND=redis://default:{container_clp_config.redis.password}@"
+        f"{container_clp_config.redis.host}:{container_clp_config.redis.port}/"
+        f"{container_clp_config.redis.query_backend_database}",
+    ]
+
+    return env_vars

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -206,14 +206,17 @@ def main(argv: typing.List[str]) -> int:
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
-
     necessary_mounts: typing.List[CLPDockerMounts] = [
         mounts.clp_home,
         mounts.logs_dir,
         mounts.archives_output_dir,
     ]
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     container_start_cmd: typing.List[str] = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/compress.py

@@ -222,8 +222,12 @@ def main(argv):
 
     _generate_logs_list(clp_config.logs_input.type, container_logs_list_path, parsed_args)
 
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
     compress_cmd = _generate_compress_cmd(
         parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container

components/clp-package-utils/clp_package_utils/scripts/decompress.py

@@ -122,8 +122,13 @@ def handle_extract_file_cmd(
                 container_paths_to_extract_file_path,
             )
         )
+
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off
@@ -203,9 +208,13 @@ def handle_extract_stream_cmd(
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off

components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import os
 import shutil
 import sys
 import typing
@@ -188,6 +189,12 @@ def main(argv: typing.List[str]) -> int:
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     database_config: Database = clp_config.database
     dataset = parsed_args.dataset

components/clp-package-utils/clp_package_utils/scripts/native/compress.py

@@ -1,6 +1,7 @@
 import argparse
 import datetime
 import logging
+import os
 import pathlib
 import sys
 import time
@@ -229,6 +230,12 @@ def main(argv):
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     comp_jobs_dir = clp_config.logs_directory / "comp-jobs"
     comp_jobs_dir.mkdir(parents=True, exist_ok=True)

components/clp-package-utils/clp_package_utils/scripts/native/decompress.py

@@ -1,6 +1,7 @@
 import argparse
 import asyncio
 import logging
+import os
 import pathlib
 import subprocess
 import sys
@@ -117,6 +118,13 @@ def handle_extract_stream_cmd(
     )
     if clp_config is None:
         return -1
+    # Override credentials with environment variables
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     command = parsed_args.command
 
@@ -222,6 +230,13 @@ def handle_extract_file_cmd(
     )
     if clp_config is None:
         return -1
+    # Override credentials with environment variables
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     paths = parsed_args.paths
     list_path = parsed_args.files_from

components/clp-package-utils/clp_package_utils/scripts/native/search.py

@@ -4,6 +4,7 @@
 import asyncio
 import ipaddress
 import logging
+import os
 import pathlib
 import socket
 import sys
@@ -297,6 +298,12 @@ def main(argv):
     except:
         logger.exception("Failed to load config.")
         return -1
+    try:
+        clp_config.database.username = os.environ["CLP_DB_USER"]
+        clp_config.database.password = os.environ["CLP_DB_PASS"]
+    except KeyError as e:
+        logger.error(f"Missing environment variable: {e}")
+        return -1
 
     database_config: Database = clp_config.database
     dataset = parsed_args.dataset

components/clp-package-utils/clp_package_utils/scripts/search.py

@@ -117,10 +117,13 @@ def main(argv):
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
         container_clp_config, clp_config, container_name
     )
-
     necessary_mounts = [mounts.clp_home, mounts.logs_dir]
+    extra_env_vars = {
+        "CLP_DB_USER": clp_config.database.username,
+        "CLP_DB_PASS": clp_config.database.password,
+    }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container
+        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
     )
 
     # fmt: off