Logging supervisor's child process logs

README.md

@@ -196,6 +196,8 @@ redis_url: # url of the redis database. default is: redis://127.0.0.1:6379/0
 supervisor_url: # url used by the supervisor process. default is: '127.0.0.1:9001'
                 # This can also be set with the SUPERVISOR_URL environment variable.
 
+worker_log_dir: # an absolute path to a directory containing the worker's stdout and stderr logs.
+
 rlimit_settings: # RLIMIT settings (see details below)
   nproc: # for example, this setting sets the hard and soft limits for the number of processes available to 300
     - 300

compose.yaml

@@ -17,6 +17,7 @@ services:
       - SUPERVISOR_URL=127.0.0.1:9001
       - AUTOTESTER_CONFIG=/app/.dockerfiles/docker-config.yml
       - STACK_ROOT=/home/docker/.autotesting/.stack
+      - WORKER_LOG_DIR=/home/docker/.autotesting/worker_log_dir
     depends_on:
       - postgres
       - redis

server/autotest_server/settings.yml

@@ -1,6 +1,7 @@
 workspace: !ENV ${WORKSPACE}
 redis_url: !ENV ${REDIS_URL}
 supervisor_url: !ENV ${SUPERVISOR_URL}
+worker_log_dir: !ENV ${WORKER_LOG_DIR}
 workers:
   - user: !ENV ${USER}
     queues:

server/install.py

@@ -62,6 +62,11 @@ def create_workspace():
     os.makedirs(config["workspace"], exist_ok=True)
 
 
+def create_worker_log_dir():
+    _print(f'creating worker log directory at {config["worker_log_dir"]}')
+    os.makedirs(config["worker_log_dir"], exist_ok=True)
+
+
 def install_all_testers():
     settings = install_testers()
     skeleton_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "autotest_server", "schema_skeleton.json")
@@ -76,6 +81,7 @@ def install():
     check_dependencies()
     check_users_exist()
     create_workspace()
+    create_worker_log_dir()
     install_all_testers()
 
 

server/start_stop.py

@@ -40,6 +40,12 @@
 autorestart=true
 stopasgroup=true
 killasgroup=true
+stdout_logfile={stdout_logfile}
+stdout_logfile_maxbytes=1MB
+stdout_logfile_backups=10
+stderr_logfile={stderr_logfile}
+stderr_logfile_maxbytes=1MB
+stderr_logfile_backups=10
 
 """
 
@@ -57,6 +63,8 @@ def create_enqueuer_wrapper(rq):
                 queues=" ".join(worker_data["queues"]),
                 numprocs=1,
                 directory=os.path.dirname(os.path.realpath(__file__)),
+                stdout_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stdout.log'),
+                stderr_logfile=os.path.join(_THIS_DIR, f'{config["worker_log_dir"]}/{worker_data["user"]}_stderr.log'),
             )
             f.write(c)