diff --git a/files/oom b/files/oom new file mode 100644 index 0000000..299f917 --- /dev/null +++ b/files/oom @@ -0,0 +1 @@ +OOM_MSG oom-kill:%{GREEDYDATA:KEY_EQ_VALUEDATA_COMMA} diff --git a/tests/data/nfs b/tests/data/nfs index 18d5fb1..07b86f5 100644 --- a/tests/data/nfs +++ b/tests/data/nfs @@ -21,5 +21,14 @@ data = [ 'syslog_version': '1', } }, - -] \ No newline at end of file +{ + "raw": "<5>1 2025-07-11T16:15:23.882825+02:00 login1 kernel: - kernel: nfs: server icts-n-hpc-01.icts.leuven.vsc not responding, timed out", + "expected": { + "appname": "kernel", + "program": "kernel", + "@source_host": "login1", + "nfsreason": "not responding, timed out", + "nfsserver": "icts-n-hpc-01.icts.leuven.vsc", + } +}, +] diff --git a/tests/data/oom b/tests/data/oom new file mode 100644 index 0000000..a6ac7af --- /dev/null +++ b/tests/data/oom @@ -0,0 +1,32 @@ +data = [ + { + "raw": "<6>1 2025-05-13T08:30:04.107564+02:00 node300 kernel: - kernel: oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=step_0,mems_allowed=0-3,oom_memcg=/slurm/uid_2510668/job_10755228,task_memcg=/slurm/uid_2510668/job_10755228/step_0/task_0,task=vasp_std,pid=44266,uid=2510668", + "expected": { + "@source_host": "node300", + "program": "kernel", + } + }, + { + "raw": "<6>1 2025-04-28T11:29:49.162661+02:00 node618 kernel: - kernel: oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=step_batch,mems_allowed=0-15,oom_memcg=/slurm/uid_2511201/job_10666405,task_memcg=/slurm/uid_2511201/job_10666405/step_batch/task_0,task=ase,pid=857594,uid=2511201", + "expected": { + "@source_host": "node618", + "program": "kernel", + } + }, + { + "raw": "<6>1 2025-09-03T11:27:36.368324+02:00 node706 kernel: - kernel: oom-kill:constraint=CONSTRAINT_MEMORY_POLICY,nodemask=3,cpuset=gpfs.service,mems_allowed=0-31,global_oom,task_memcg=/system.slice/slurmstepd.scope/job_11048783/step_batch/user/task_0,task=python,pid=549012,uid=2510053", + "expected": { + "@source_host": "node706", + "program": "kernel", + "constraint": "CONSTRAINT_MEMORY_POLICY", + "nodemask": "3", + "cpuset": "gpfs.service", + "mems_allowed": "0-31", + "task_memcg": "/system.slice/slurmstepd.scope/job_11048783/step_batch/user/task_0", + "task": "python", + "pid": 549012, + "uid": 2510053, + "global_oom": "true", + } + }, +] diff --git a/tests/logstash_7.6.2.conf b/tests/logstash_7.6.2.conf index 6c7b7d0..8f9c0c3 100644 --- a/tests/logstash_7.6.2.conf +++ b/tests/logstash_7.6.2.conf @@ -39,6 +39,7 @@ filter "%{RSYSLOGPREFIX}%{QUATTOR_MSG}", "%{RSYSLOGPREFIX}%{SNOOPY_MSG}", "%{RSYSLOGPREFIX}%{APACHE_MSG}", + "%{RSYSLOGPREFIX}%{OOM_MSG}", # Last resort, this should be one to last "%{RSYSLOGPREFIX}%{KEYVALUE_MSG}", # RSYSLOGCUSTOM always last (and no PREFIX)! @@ -52,6 +53,19 @@ filter source => "KEY_EQ_VALUEDATA" } + if ([KEY_EQ_VALUEDATA_COMMA]) { + mutate { + gsub => [ + "KEY_EQ_VALUEDATA_COMMA", ",global_oom(,|$)", ",global_oom=true\1" + ] + } + } + + kv { + source => "KEY_EQ_VALUEDATA_COMMA" + field_split => "," + } + date { match => [ "syslog_timestamp", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZZ", "yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd HH:mm:ss.SSSSSS", "MMM d HH:mm:ss", "MMM d HH:mm:ss" ] } @@ -74,7 +88,7 @@ filter if ("_grokparsefailure" not in [tags]) { mutate { - remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "int" ] + remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp", "KEY_EQ_VALUEDATA", "KEY_EQ_VALUEDATA_COMMA", "int" ] convert => { "success" => "boolean" } # we need MB converted to MiB for bytes2human