Handle ECS v1 fields as dot notated fileds

Signed-off-by: Hiroshi Hatake <[email protected]>

Author: cosmo0920

lib/fluent/plugin/grok.rb

@@ -134,8 +134,15 @@ def expand_pattern(pattern)
         curr_pattern = @pattern_map[m["pattern"]]
         raise GrokPatternNotFoundError, "grok pattern not found: #{pattern}" unless curr_pattern
         if m["subname"]
-          replacement_pattern = "(?<#{m["subname"]}>#{curr_pattern})"
-          type_map[m["subname"]] = m["type"] || "string"
+          ecs = /(?<ecs-key>(^\[.*\]$))/.match(m["subname"])
+          subname = if ecs
+                      # remove starting "[" and trailing "]" on matched data
+                      ecs["ecs-key"][1..-2].split("][").join('.')
+                    else
+                      m["subname"]
+                    end
+          replacement_pattern = "(?<#{subname}>#{curr_pattern})"
+          type_map[subname] = m["type"] || "string"
         else
           replacement_pattern = "(?:#{curr_pattern})"
         end

test/test_grok_parser.rb

@@ -79,14 +79,14 @@ class GrokParserTest < ::Test::Unit::TestCase
       internal_test_grok_pattern("%{HTTPD_COMBINEDLOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
                                   str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
                                   {
-                                    "[apache][access][user][identity]" => "192.168.0.1",
-                                    "[http][request][method]"          => "GET",
-                                    "[http][response][body][bytes]"    => 777,
-                                    "[http][response][status_code]"    => 200,
-                                    "[http][version]"                  => "1.1",
-                                    "[source][address]"                => "127.0.0.1",
-                                    "[url][original]"                  => "/",
-                                    "[user_agent][original]"           => "Opera/12.0",
+                                    "apache.access.user.identity" => "192.168.0.1",
+                                    "http.request.method"         => "GET",
+                                    "http.response.body.bytes"    => 777,
+                                    "http.response.status_code"   => 200,
+                                    "http.version"                => "1.1",
+                                    "source.address"              => "127.0.0.1",
+                                    "url.original"                => "/",
+                                    "user_agent.original"         => "Opera/12.0",
                                   },
                                   "time_key" => "timestamp",
                                   "time_format" => "%d/%b/%Y:%H:%M:%S %z",
@@ -394,14 +394,14 @@ class GrokParserTest < ::Test::Unit::TestCase
           </grok>
         ])
         expected_record = {
-          "[apache][access][user][identity]" => "192.168.0.1",
-          "[http][request][method]"          => "GET",
-          "[http][response][body][bytes]"    => 777,
-          "[http][response][status_code]"    => 200,
-          "[http][version]"                  => "1.1",
-          "[source][address]"                => "127.0.0.1",
-          "[url][original]"                  => "/",
-          "[user_agent][original]"           => "Opera/12.0"
+          "apache.access.user.identity" => "192.168.0.1",
+          "http.request.method"         => "GET",
+          "http.response.body.bytes"    => 777,
+          "http.response.status_code"   => 200,
+          "http.version"                => "1.1",
+          "source.address"              => "127.0.0.1",
+          "url.original"                => "/",
+          "user_agent.original"         => "Opera/12.0"
         }
         d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
           assert_equal(expected_record, record)