diff --git a/README.md b/README.md index f159fe9d..2f3a6e21 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,8 @@ redis_url: # url of the redis database. default is: redis://127.0.0.1:6379/0 supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:9001' # This can also be set with the SUPERVISOR_URL environment variable. +worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs. + rlimit_settings: # RLIMIT settings (see details below) nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300 - 300 diff --git a/compose.yaml b/compose.yaml index 9b8571ad..03995839 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,7 @@ services: - SUPERVISOR_URL=127.0.0.1:9001 - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml - STACK_ROOT=/home/docker/.autotesting/.stack + - WORKER_LOG_DIR=/home/docker/.autotesting/worker_log_dir depends_on: - postgres - redis diff --git a/server/autotest_server/settings.yml b/server/autotest_server/settings.yml index 0325289b..93502031 100644 --- a/server/autotest_server/settings.yml +++ b/server/autotest_server/settings.yml @@ -1,6 +1,7 @@ workspace: !ENV ${WORKSPACE} redis_url: !ENV ${REDIS_URL} supervisor_url: !ENV ${SUPERVISOR_URL} +worker_log_dir: !ENV ${WORKER_LOG_DIR} workers: - user: !ENV ${USER} queues: diff --git a/server/install.py b/server/install.py index 878763fc..469c3dc7 100644 --- a/server/install.py +++ b/server/install.py @@ -62,6 +62,11 @@ def create_workspace(): os.makedirs(config["workspace"], exist_ok=True) +def create_worker_log_dir(): + _print(f'creating worker log directory at {config["worker_log_dir"]}') + os.makedirs(config["worker_log_dir"], exist_ok=True) + + def install_all_testers(): settings = install_testers() skeleton_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "autotest_server", "schema_skeleton.json") @@ -76,6 +81,7 @@ def install(): check_dependencies() check_users_exist() create_workspace() + create_worker_log_dir() install_all_testers() diff --git a/server/start_stop.py b/server/start_stop.py index 218dabb2..e791d4c1 100644 --- a/server/start_stop.py +++ b/server/start_stop.py @@ -40,6 +40,12 @@ autorestart=true stopasgroup=true killasgroup=true +stdout_logfile={stdout_logfile} +stdout_logfile_maxbytes=1MB +stdout_logfile_backups=10 +stderr_logfile={stderr_logfile} +stderr_logfile_maxbytes=1MB +stderr_logfile_backups=10 """ @@ -57,6 +63,8 @@ def create_enqueuer_wrapper(rq): queues=" ".join(worker_data["queues"]), numprocs=1, directory=os.path.dirname(os.path.realpath(__file__)), + stdout_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stdout.log'), + stderr_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stderr.log'), ) f.write(c)