Separate stdout and stderr logs for event handler execution

This to avoid to have unexpected interleaved stream of stdout and stderr

Signed-off-by: Luca Carrogu <[email protected]>

Author: lukeseawalker

attributes/default.rb

@@ -132,7 +132,8 @@
 # BYOS event handler
 default['cluster']['scheduler_plugin']['home'] = '/home/pcluster-scheduler-plugin'
 default['cluster']['scheduler_plugin']['handler_dir'] = '/home/pcluster-scheduler-plugin/.parallelcluster'
-default['cluster']['scheduler_plugin']['handler_log'] = '/var/log/parallelcluster/scheduler-plugin.log'
+default['cluster']['scheduler_plugin']['handler_log_out'] = '/var/log/parallelcluster/scheduler-plugin.out.log'
+default['cluster']['scheduler_plugin']['handler_log_err'] = '/var/log/parallelcluster/scheduler-plugin.err.log'
 default['cluster']['scheduler_plugin']['shared_dir'] = "#{node['cluster']['shared_dir']}/scheduler-plugin"
 default['cluster']['scheduler_plugin']['local_dir'] = "#{node['cluster']['base_dir']}/scheduler-plugin"
 default['cluster']['scheduler_plugin']['scheduler_plugin_substack_outputs_path'] = "#{node['cluster']['shared_dir']}/scheduler-plugin-substack-outputs.json"

cookbooks/aws-parallelcluster-config/files/default/cloudwatch_agent/cloudwatch_log_files.json

@@ -411,8 +411,26 @@
     },
     {
       "timestamp_format_key": "default",
-      "file_path": "/var/log/parallelcluster/scheduler-plugin.log",
-      "log_stream_name": "scheduler-plugin",
+      "file_path": "/var/log/parallelcluster/scheduler-plugin.out.log",
+      "log_stream_name": "scheduler-plugin-out",
+      "schedulers": [
+        "plugin"
+      ],
+      "platforms": [
+        "centos",
+        "ubuntu",
+        "amazon"
+      ],
+      "node_roles": [
+        "ComputeFleet",
+        "HeadNode"
+      ],
+      "feature_conditions": []
+    },
+    {
+      "timestamp_format_key": "default",
+      "file_path": "/var/log/parallelcluster/scheduler-plugin.err.log",
+      "log_stream_name": "scheduler-plugin-err",
       "schedulers": [
         "plugin"
       ],

cookbooks/aws-parallelcluster-scheduler-plugin/resources/execute_event_handler.rb

@@ -17,24 +17,25 @@
     return
   end
 
-  event_log = node['cluster']['scheduler_plugin']['handler_log']
+  event_log_out = node['cluster']['scheduler_plugin']['handler_log_out']
+  event_log_err = node['cluster']['scheduler_plugin']['handler_log_err']
   event_cwd = node['cluster']['scheduler_plugin']['home']
   event_user = node['cluster']['scheduler_plugin']['user']
   event_timeout = 3600
   event_env = build_env
   event_log_prefix_error = "%Y-%m-%d %H:%M:%S,000 - [#{new_resource.event_name}] - ERROR:"
   event_log_prefix_info = "%Y-%m-%d %H:%M:%S,000 - [#{new_resource.event_name}] - INFO:"
-  Chef::Log.info("Executing Event #{new_resource.event_name}, with user (#{event_user}), cwd (#{event_cwd}), command (#{new_resource.event_command}), log (#{event_log})")
+  Chef::Log.info("Executing Event #{new_resource.event_name}, with user (#{event_user}), cwd (#{event_cwd}), command (#{new_resource.event_command}), log out (#{event_log_out}), log err (#{event_log_err})")
   # shellout https://github.com/chef/mixlib-shellout
   # switch stderr/stdout with (2>&1 1>&3-), process error (now on stdout), switch back stdout/stderr with (3>&1 1>&2) and then process output
-  event_command = Shellwords.escape("set -o pipefail; { (#{new_resource.event_command}) 2>&1 1>&3- | ts '#{event_log_prefix_error}' | tee -a #{event_log}; } " \
-    "3>&1 1>&2 | ts '#{event_log_prefix_info}' | tee -a #{event_log}")
+  event_command = Shellwords.escape("set -o pipefail; { (#{new_resource.event_command}) 2>&1 1>&3- | ts '#{event_log_prefix_error}' | tee -a #{event_log_err}; } " \
+    "3>&1 1>&2 | ts '#{event_log_prefix_info}' | tee -a #{event_log_out}")
   cmd = Mixlib::ShellOut.new("/bin/bash -c #{event_command}", user: event_user, group: event_user, login: true, env: event_env, cwd: event_cwd, timeout: event_timeout)
   cmd.run_command
 
   if cmd.error?
     raise "Expected Event #{new_resource.event_name} to exit with #{cmd.valid_exit_codes.inspect}," \
-      " but received '#{cmd.exitstatus}', complete log in #{event_log}\n #{format_stderr(cmd)}"
+      " but received '#{cmd.exitstatus}', complete log info in #{event_log_out} and error in #{event_log_err}\n #{format_stderr(cmd)}"
   end
 end