99import typing
1010import uuid
1111from enum import auto
12- from typing import List , Optional , Tuple
12+ from typing import Dict , List , Optional , Tuple
1313
1414import yaml
1515from clp_py_utils .clp_config import (
1616 CLP_DEFAULT_CREDENTIALS_FILE_PATH ,
17+ CLP_SHARED_CONFIG_FILENAME ,
1718 CLPConfig ,
1819 DB_COMPONENT_NAME ,
1920 QueryEngine ,
@@ -95,6 +96,7 @@ def __init__(self, clp_home: pathlib.Path, docker_clp_home: pathlib.Path):
9596 self .archives_output_dir : typing .Optional [DockerMount ] = None
9697 self .stream_output_dir : typing .Optional [DockerMount ] = None
9798 self .aws_config_dir : typing .Optional [DockerMount ] = None
99+ self .generated_config_file : typing .Optional [DockerMount ] = None
98100
99101
100102def _validate_data_directory (data_dir : pathlib .Path , component_name : str ) -> None :
@@ -285,6 +287,18 @@ def generate_container_config(
285287 container_clp_config .stream_output .get_directory (),
286288 )
287289
290+ if not is_path_already_mounted (
291+ clp_home ,
292+ CONTAINER_CLP_HOME ,
293+ clp_config .get_shared_config_file_path (),
294+ container_clp_config .get_shared_config_file_path (),
295+ ):
296+ docker_mounts .generated_config_file = DockerMount (
297+ DockerMountType .BIND ,
298+ clp_config .get_shared_config_file_path (),
299+ container_clp_config .get_shared_config_file_path (),
300+ )
301+
288302 # Only create the mount if the directory exists
289303 if clp_config .aws_config_directory is not None :
290304 container_clp_config .aws_config_directory = CONTAINER_AWS_CONFIG_DIRECTORY
@@ -308,33 +322,57 @@ def generate_worker_config(clp_config: CLPConfig) -> WorkerConfig:
308322 return worker_config
309323
310324
325+ def get_container_config_filename (container_name : str ) -> str :
326+ return f".{ container_name }
327+
328+
311329def dump_container_config (
312- container_clp_config : CLPConfig , clp_config : CLPConfig , container_name : str
313- ) -> Tuple [ pathlib . Path , pathlib . Path ] :
330+ container_clp_config : CLPConfig , clp_config : CLPConfig , config_filename : str
331+ ):
314332 """
315- Writes the given config to the logs directory so that it's accessible in the container.
333+ Writes the given container config to the logs directory, so that it's accessible in the
334+ container.
335+
316336 :param container_clp_config: The config to write.
317337 :param clp_config: The corresponding config on the host (used to determine the logs directory).
318- :param container_name :
338+ :param config_filename :
319339 :return: The path to the config file in the container and on the host.
320340 """
321- container_config_filename = f".{ container_name }
322- config_file_path_on_host = clp_config .logs_directory / container_config_filename
323- config_file_path_on_container = container_clp_config .logs_directory / container_config_filename
341+ config_file_path_on_host = clp_config .logs_directory / config_filename
342+ config_file_path_on_container = container_clp_config .logs_directory / config_filename
324343 with open (config_file_path_on_host , "w" ) as f :
325344 yaml .safe_dump (container_clp_config .dump_to_primitive_dict (), f )
326345
327346 return config_file_path_on_container , config_file_path_on_host
328347
329348
349+ def dump_shared_container_config (
350+ container_clp_config : CLPConfig , clp_config : CLPConfig
351+ ) -> Tuple [pathlib .Path , pathlib .Path ]:
352+ """
353+ Dumps the given container config to `CLP_SHARED_CONFIG_FILENAME` in the logs directory, so that
354+ it's accessible in the container.
355+
356+ :param container_clp_config:
357+ :param clp_config:
358+ """
359+ return dump_container_config (container_clp_config , clp_config , CLP_SHARED_CONFIG_FILENAME )
360+
361+
330362def generate_container_start_cmd (
331- container_name : str , container_mounts : List [Optional [DockerMount ]], container_image : str
363+ container_name : str ,
364+ container_mounts : List [Optional [DockerMount ]],
365+ container_image : str ,
366+ extra_env_vars : Optional [Dict [str , str ]] = None ,
332367) -> List [str ]:
333368 """
334- Generates the command to start a container with the given mounts and name.
369+ Generates the command to start a container with the given mounts, environment variables, and
370+ name.
371+
335372 :param container_name:
336373 :param container_mounts:
337374 :param container_image:
375+ :param extra_env_vars: Environment variables to set on top of the predefined ones.
338376 :return: The command.
339377 """
340378 clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
@@ -350,6 +388,12 @@ def generate_container_start_cmd(
350388 "--name" , container_name ,
351389 "--log-driver" , "local"
352390 ]
391+ env_vars = {
392+ "PYTHONPATH" : clp_site_packages_dir ,
393+ ** (extra_env_vars if extra_env_vars is not None else {}),
394+ }
395+ for key , value in env_vars .items ():
396+ container_start_cmd .extend (["-e" , f"{ key } { value } ])
353397 for mount in container_mounts :
354398 if mount :
355399 container_start_cmd .append ("--mount" )
@@ -428,21 +472,21 @@ def validate_and_load_db_credentials_file(
428472 clp_config : CLPConfig , clp_home : pathlib .Path , generate_default_file : bool
429473):
430474 validate_credentials_file_path (clp_config , clp_home , generate_default_file )
431- clp_config .load_database_credentials_from_file ( )
475+ clp_config .database . load_credentials_from_file ( clp_config . credentials_file_path )
432476
433477
434478def validate_and_load_queue_credentials_file (
435479 clp_config : CLPConfig , clp_home : pathlib .Path , generate_default_file : bool
436480):
437481 validate_credentials_file_path (clp_config , clp_home , generate_default_file )
438- clp_config .load_queue_credentials_from_file ( )
482+ clp_config .queue . load_credentials_from_file ( clp_config . credentials_file_path )
439483
440484
441485def validate_and_load_redis_credentials_file (
442486 clp_config : CLPConfig , clp_home : pathlib .Path , generate_default_file : bool
443487):
444488 validate_credentials_file_path (clp_config , clp_home , generate_default_file )
445- clp_config .load_redis_credentials_from_file ( )
489+ clp_config .redis . load_credentials_from_file ( clp_config . credentials_file_path )
446490
447491
448492def validate_db_config (clp_config : CLPConfig , data_dir : pathlib .Path , logs_dir : pathlib .Path ):
@@ -599,3 +643,68 @@ def is_retention_period_configured(clp_config: CLPConfig) -> bool:
599643 return True
600644
601645 return False
646+
647+
648+ def get_common_env_vars_list (
649+ include_clp_home_env_var = True ,
650+ ) -> List [str ]:
651+ """
652+ :param include_clp_home_env_var:
653+ :return: A list of common environment variables for Docker containers, in the format
654+ "KEY=VALUE".
655+ """
656+ clp_site_packages_dir = CONTAINER_CLP_HOME / "lib" / "python3" / "site-packages"
657+ env_vars = [f"PYTHONPATH={ clp_site_packages_dir } ]
658+
659+ if include_clp_home_env_var :
660+ env_vars .append (f"CLP_HOME={ CONTAINER_CLP_HOME } )
661+
662+ return env_vars
663+
664+
665+ def get_credential_env_vars_list (
666+ container_clp_config : CLPConfig ,
667+ include_db_credentials = False ,
668+ include_queue_credentials = False ,
669+ include_redis_credentials = False ,
670+ ) -> List [str ]:
671+ """
672+ :param container_clp_config:
673+ :param include_db_credentials:
674+ :param include_queue_credentials:
675+ :param include_redis_credentials:
676+ :return: A list of credential environment variables for Docker containers, in the format
677+ "KEY=VALUE".
678+ """
679+ env_vars = []
680+
681+ if include_db_credentials :
682+ env_vars .append (f"CLP_DB_USER={ container_clp_config .database .username } )
683+ env_vars .append (f"CLP_DB_PASS={ container_clp_config .database .password } )
684+
685+ if include_queue_credentials :
686+ env_vars .append (f"CLP_QUEUE_USER={ container_clp_config .queue .username } )
687+ env_vars .append (f"CLP_QUEUE_PASS={ container_clp_config .queue .password } )
688+
689+ if include_redis_credentials :
690+ env_vars .append (f"CLP_REDIS_PASS={ container_clp_config .redis .password } )
691+
692+ return env_vars
693+
694+
695+ def get_celery_connection_env_vars_list (container_clp_config : CLPConfig ) -> List [str ]:
696+ """
697+ :param container_clp_config:
698+ :return: A list of Celery connection environment variables for Docker containers, in the format
699+ "KEY=VALUE".
700+ """
701+ env_vars = [
702+ f"BROKER_URL=amqp://"
703+ f"{ container_clp_config .queue .username } { container_clp_config .queue .password }
704+ f"{ container_clp_config .queue .host } { container_clp_config .queue .port } ,
705+ f"RESULT_BACKEND=redis://default:{ container_clp_config .redis .password }
706+ f"{ container_clp_config .redis .host } { container_clp_config .redis .port }
707+ f"{ container_clp_config .redis .query_backend_database } ,
708+ ]
709+
710+ return env_vars
0 commit comments