From 52f59187ea53da8d0e01c5d0747dc9a87095e53a Mon Sep 17 00:00:00 2001 From: Donny Wong Date: Mon, 24 Feb 2025 00:20:56 -0500 Subject: [PATCH 1/3] capturing supervisor's child process logs --- server/.gitignore | 3 ++- server/start_stop.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/.gitignore b/server/.gitignore index 1b54bf31..b0c85321 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -6,4 +6,5 @@ autotest_server/testers/java/lib/ .idea __pycache__ .DS_Store -.hypothesis/ \ No newline at end of file +.hypothesis/ +supervisor_child_log/ \ No newline at end of file diff --git a/server/start_stop.py b/server/start_stop.py index 218dabb2..089a098c 100644 --- a/server/start_stop.py +++ b/server/start_stop.py @@ -40,6 +40,12 @@ autorestart=true stopasgroup=true killasgroup=true +stdout_logfile={stdout_logfile} +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=10 +stderr_logfile={stderr_logfile} +stderr_logfile_maxbytes=1MB +stderr_logfile_backups=10 """ @@ -57,6 +63,8 @@ def create_enqueuer_wrapper(rq): queues=" ".join(worker_data["queues"]), numprocs=1, directory=os.path.dirname(os.path.realpath(__file__)), + stdout_logfile=os.path.join(_THIS_DIR, f'supervisor_child_log/{worker_data["user"]}_stdout.log'), + stderr_logfile=os.path.join(_THIS_DIR, f'supervisor_child_log/{worker_data["user"]}_stderr.log'), ) f.write(c) From e3feef508029a45b1e691efefeaf8bf1361ce10c Mon Sep 17 00:00:00 2001 From: Donny Wong Date: Mon, 24 Feb 2025 12:26:20 -0500 Subject: [PATCH 2/3] making supervisor child log folder path a configuration --- compose.yaml | 1 + server/.gitignore | 3 +-- server/autotest_server/settings.yml | 1 + server/install.py | 6 ++++++ server/start_stop.py | 8 ++++++-- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/compose.yaml b/compose.yaml index 9b8571ad..7f198558 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,7 @@ services: - SUPERVISOR_URL=127.0.0.1:9001 - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml - STACK_ROOT=/home/docker/.autotesting/.stack + - SUPERVISOR_CHILD_LOG=/home/docker/.autotesting/supervisor_child_log depends_on: - postgres - redis diff --git a/server/.gitignore b/server/.gitignore index b0c85321..1b54bf31 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -6,5 +6,4 @@ autotest_server/testers/java/lib/ .idea __pycache__ .DS_Store -.hypothesis/ -supervisor_child_log/ \ No newline at end of file +.hypothesis/ \ No newline at end of file diff --git a/server/autotest_server/settings.yml b/server/autotest_server/settings.yml index 0325289b..2dc1cfe9 100644 --- a/server/autotest_server/settings.yml +++ b/server/autotest_server/settings.yml @@ -1,6 +1,7 @@ workspace: !ENV ${WORKSPACE} redis_url: !ENV ${REDIS_URL} supervisor_url: !ENV ${SUPERVISOR_URL} +supervisor_child_log: !ENV ${SUPERVISOR_CHILD_LOG} workers: - user: !ENV ${USER} queues: diff --git a/server/install.py b/server/install.py index 878763fc..9ff8e08c 100644 --- a/server/install.py +++ b/server/install.py @@ -62,6 +62,11 @@ def create_workspace(): os.makedirs(config["workspace"], exist_ok=True) +def create_supervisor_child_log(): + _print(f'creating supervisor log folder for its children at {config["supervisor_child_log"]}') + os.makedirs(config["supervisor_child_log"], exist_ok=True) + + def install_all_testers(): settings = install_testers() skeleton_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "autotest_server", "schema_skeleton.json") @@ -76,6 +81,7 @@ def install(): check_dependencies() check_users_exist() create_workspace() + create_supervisor_child_log() install_all_testers() diff --git a/server/start_stop.py b/server/start_stop.py index 089a098c..9496e046 100644 --- a/server/start_stop.py +++ b/server/start_stop.py @@ -63,8 +63,12 @@ def create_enqueuer_wrapper(rq): queues=" ".join(worker_data["queues"]), numprocs=1, directory=os.path.dirname(os.path.realpath(__file__)), - stdout_logfile=os.path.join(_THIS_DIR, f'supervisor_child_log/{worker_data["user"]}_stdout.log'), - stderr_logfile=os.path.join(_THIS_DIR, f'supervisor_child_log/{worker_data["user"]}_stderr.log'), + stdout_logfile=os.path.join( + _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stdout.log' + ), + stderr_logfile=os.path.join( + _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stderr.log' + ), ) f.write(c) From 2a2bb06917a430ecd7223ddce1c7d0cc63447005 Mon Sep 17 00:00:00 2001 From: Donny Wong Date: Fri, 28 Feb 2025 15:20:48 -0500 Subject: [PATCH 3/3] changed directory name to worker_log_dir to reflect worker logs --- README.md | 2 ++ compose.yaml | 2 +- server/autotest_server/settings.yml | 2 +- server/install.py | 8 ++++---- server/start_stop.py | 8 ++------ 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f159fe9d..2f3a6e21 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,8 @@ redis_url: # url of the redis database. default is: redis://127.0.0.1:6379/0 supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:9001' # This can also be set with the SUPERVISOR_URL environment variable. +worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs. + rlimit_settings: # RLIMIT settings (see details below) nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300 - 300 diff --git a/compose.yaml b/compose.yaml index 7f198558..03995839 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,7 +17,7 @@ services: - SUPERVISOR_URL=127.0.0.1:9001 - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml - STACK_ROOT=/home/docker/.autotesting/.stack - - SUPERVISOR_CHILD_LOG=/home/docker/.autotesting/supervisor_child_log + - WORKER_LOG_DIR=/home/docker/.autotesting/worker_log_dir depends_on: - postgres - redis diff --git a/server/autotest_server/settings.yml b/server/autotest_server/settings.yml index 2dc1cfe9..93502031 100644 --- a/server/autotest_server/settings.yml +++ b/server/autotest_server/settings.yml @@ -1,7 +1,7 @@ workspace: !ENV ${WORKSPACE} redis_url: !ENV ${REDIS_URL} supervisor_url: !ENV ${SUPERVISOR_URL} -supervisor_child_log: !ENV ${SUPERVISOR_CHILD_LOG} +worker_log_dir: !ENV ${WORKER_LOG_DIR} workers: - user: !ENV ${USER} queues: diff --git a/server/install.py b/server/install.py index 9ff8e08c..469c3dc7 100644 --- a/server/install.py +++ b/server/install.py @@ -62,9 +62,9 @@ def create_workspace(): os.makedirs(config["workspace"], exist_ok=True) -def create_supervisor_child_log(): - _print(f'creating supervisor log folder for its children at {config["supervisor_child_log"]}') - os.makedirs(config["supervisor_child_log"], exist_ok=True) +def create_worker_log_dir(): + _print(f'creating worker log directory at {config["worker_log_dir"]}') + os.makedirs(config["worker_log_dir"], exist_ok=True) def install_all_testers(): @@ -81,7 +81,7 @@ def install(): check_dependencies() check_users_exist() create_workspace() - create_supervisor_child_log() + create_worker_log_dir() install_all_testers() diff --git a/server/start_stop.py b/server/start_stop.py index 9496e046..e791d4c1 100644 --- a/server/start_stop.py +++ b/server/start_stop.py @@ -63,12 +63,8 @@ def create_enqueuer_wrapper(rq): queues=" ".join(worker_data["queues"]), numprocs=1, directory=os.path.dirname(os.path.realpath(__file__)), - stdout_logfile=os.path.join( - _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stdout.log' - ), - stderr_logfile=os.path.join( - _THIS_DIR, f'{config["supervisor_child_log"]}/{worker_data["user"]}_stderr.log' - ), + stdout_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stdout.log'), + stderr_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stderr.log'), ) f.write(c)