From 9faf64396da56b6f2b2d5fb517d24bb44a2c1195 Mon Sep 17 00:00:00 2001 From: Valters Jansons Date: Tue, 17 Sep 2019 18:10:01 +0300 Subject: [PATCH 1/2] Handle PID as part of default log format Field 1 on the default HAProxy log format is set as `process_name '[' pid ']:'` which is also displayed in the example message in README. The existing rules do not allow for the PID brackets. This commit properly resolves them while also maintaining backwards compatibility for a PID-less log format. --- content_pack.json | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/content_pack.json b/content_pack.json index 812c52c..df04dd3 100644 --- a/content_pack.json +++ b/content_pack.json @@ -17,7 +17,7 @@ "title" : "Remote Address", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:\\s+(\\S+):" + "regex_value" : "haproxy(?:\\[\\d+\\])?:\\s+(\\S+):" }, "converters" : [ ], "order" : 1, @@ -25,12 +25,12 @@ "target_field" : "remote_addr", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Timestamp", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[(.+?)\\]" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[(.+?)\\]" }, "converters" : [ { "type" : "DATE", @@ -43,12 +43,12 @@ "target_field" : "timestamp", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Frontend", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\]\\s(\\S+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\]\\s(\\S+)" }, "converters" : [ ], "order" : 4, @@ -56,12 +56,12 @@ "target_field" : "frontend", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Server", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?/(\\S+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\].+?/(\\S+)" }, "converters" : [ ], "order" : 6, @@ -69,12 +69,12 @@ "target_field" : "server", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Backend", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?(\\S+)/" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\].+?(\\S+)/" }, "converters" : [ ], "order" : 5, @@ -82,12 +82,12 @@ "target_field" : "backend", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Response Status", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s(\\d+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s(\\d+)" }, "converters" : [ { "type" : "NUMERIC", @@ -98,12 +98,12 @@ "target_field" : "response_status", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Response Bytes", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s\\d+\\s(\\d+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\].+?\\s\\S+\\s\\S+\\s\\d+\\s(\\d+)" }, "converters" : [ { "type" : "NUMERIC", @@ -114,12 +114,12 @@ "target_field" : "response_bytes", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Verb", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\"(\\w+)\\s" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\"(\\w+)\\s" }, "converters" : [ ], "order" : 9, @@ -127,12 +127,12 @@ "target_field" : "request_verb", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Path", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\"\\w+\\s(\\S+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\"\\w+\\s(\\S+)" }, "converters" : [ ], "order" : 10, @@ -140,12 +140,12 @@ "target_field" : "request_path", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "HTTP Version", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+HTTP/(\\S+)\"" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+HTTP/(\\S+)\"" }, "converters" : [ ], "order" : 2, @@ -153,12 +153,12 @@ "target_field" : "http_version", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Message", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+\"(.+)\"" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+\"(.+)\"" }, "converters" : [ ], "order" : 11, @@ -166,12 +166,12 @@ "target_field" : "message", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy:" + "condition_value" : "^haproxy(?:\\[\\d+\\])?:" }, { "title" : "Timings", "type" : "REGEX", "configuration" : { - "regex_value" : "haproxy:.+?\\[.+?\\].+?\\S+/\\S+\\s(\\S+)" + "regex_value" : "haproxy(?:\\[\\d+\\])?:.+?\\[.+?\\].+?\\S+/\\S+\\s(\\S+)" }, "converters" : [ ], "order" : 0, @@ -179,7 +179,7 @@ "target_field" : "timings", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "haproxy:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Milliseconds spent waiting for the client to send a full HTTP request", "type" : "SPLIT_AND_INDEX", @@ -196,7 +196,7 @@ "target_field" : "tq", "source_field" : "timings", "condition_type" : "NONE", - "condition_value" : "haproxy:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Milliseconds spent waiting in queues", "type" : "SPLIT_AND_INDEX", From 7087a5a166f0fbb807bcca1961f547b48ac76a45 Mon Sep 17 00:00:00 2001 From: Valters Jansons Date: Tue, 17 Sep 2019 18:14:44 +0300 Subject: [PATCH 2/2] Match `condition_value` inside a log line In case there is a prefix applied to the message, the `^haproxy` condition match would fail. There are already some of the extractors which have dropped the line-start requirement from their regular expressions. This commit removes the requirement from all the extractors for consistency and input flexibility. --- content_pack.json | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/content_pack.json b/content_pack.json index df04dd3..08f451e 100644 --- a/content_pack.json +++ b/content_pack.json @@ -25,7 +25,7 @@ "target_field" : "remote_addr", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Timestamp", "type" : "REGEX", @@ -43,7 +43,7 @@ "target_field" : "timestamp", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Frontend", "type" : "REGEX", @@ -56,7 +56,7 @@ "target_field" : "frontend", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Server", "type" : "REGEX", @@ -69,7 +69,7 @@ "target_field" : "server", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Backend", "type" : "REGEX", @@ -82,7 +82,7 @@ "target_field" : "backend", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Response Status", "type" : "REGEX", @@ -98,7 +98,7 @@ "target_field" : "response_status", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Response Bytes", "type" : "REGEX", @@ -114,7 +114,7 @@ "target_field" : "response_bytes", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Verb", "type" : "REGEX", @@ -127,7 +127,7 @@ "target_field" : "request_verb", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Request Path", "type" : "REGEX", @@ -140,7 +140,7 @@ "target_field" : "request_path", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "HTTP Version", "type" : "REGEX", @@ -153,7 +153,7 @@ "target_field" : "http_version", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Message", "type" : "REGEX", @@ -166,7 +166,7 @@ "target_field" : "message", "source_field" : "message", "condition_type" : "REGEX", - "condition_value" : "^haproxy(?:\\[\\d+\\])?:" + "condition_value" : "haproxy(?:\\[\\d+\\])?:" }, { "title" : "Timings", "type" : "REGEX",