support grokker

Author: mhoff

logprep/util/grok/grok.py

@@ -25,7 +25,6 @@
 
 import re
 import string
-import sys
 from hashlib import md5
 from importlib import resources
 from itertools import chain
@@ -38,7 +37,7 @@
 from logprep.util.decorators import timeout
 
 DEFAULT_PATTERNS_DIRS = [str(resources.files(__package__) / "patterns/ecs-v1")]
-LOGSTASH_NOTATION = r"(([^\[\]\{\}\.:]*)?(\[[^\[\]\{\}\.:]*\])*)"
+LOGSTASH_NOTATION = r"(([^\[\]\{\}\.:]*)?(\[[^\[\]\{\}:]*\])*)"
 GROK = r"%\{" + rf"([A-Z0-9_]*)(:({LOGSTASH_NOTATION}))?(:(int|float))?" + r"\}"
 ONIGURUMA = r"\(\?<([^()]*)>\(?(([^()]*|\(([^()]*|\([^()]*\))*\))*)\)?\)"
 NON_RESOLVED_ONIGURUMA = r"\(\?<[^md5].*>"
@@ -53,7 +52,15 @@ class Grok:
     grok_pattern = re.compile(GROK)
     oniguruma = re.compile(ONIGURUMA)
 
-    pattern: str = field(validator=validators.instance_of((str, list)))
+    pattern: str | list[str] = field(
+        validator=validators.or_(
+            validators.instance_of(str),
+            validators.deep_iterable(
+                iterable_validator=validators.instance_of(list),
+                member_validator=validators.instance_of(str),
+            ),
+        )
+    )
     custom_patterns_dir: str = field(default="")
     custom_patterns: dict = field(factory=dict)
     fullmatch: bool = field(default=True)
@@ -128,7 +135,7 @@ def _to_dundered_field(fields: str) -> str:
     def _to_dotted_field(fields: str) -> str:
         if not "__" in fields:
             return fields
-        return fields.replace("__", ".")
+        return fields.replace(".", "\\.").replace("__", ".")
 
     def _resolve_grok(self, match: re.Match) -> str:
         name = match.group(1)

tests/unit/ng/processor/grokker/test_grokker.py

@@ -197,6 +197,33 @@
             "parent": {"some_ip": "123.123.123.123", "port": 1234},
         },
     ),
+    (
+        "normalization from escaped & nested grok",
+        {
+            "filter": "win\\.log.event\\._id: 123456789",
+            "grokker": {
+                "mapping": {
+                    "win\\.log.event_data.normalize me!": r"%{IP:[par\\ent][...]} \w+ %{NUMBER:[par\\ent][\\port\\]:int} %[ts]+ %{NUMBER:te\\.st\\:int}"
+                },
+            },
+        },
+        {
+            "win.log": {
+                "api": "wineventlog",
+                "event._id": 123456789,
+                "event_data": {"normalize me!": "123.123.123.123 555 1234 %ttss 11"},
+            }
+        },
+        {
+            "win.log": {
+                "api": "wineventlog",
+                "event._id": 123456789,
+                "event_data": {"normalize me!": "123.123.123.123 555 1234 %ttss 11"},
+            },
+            "te.st\\": 11,
+            "par\\ent": {"...": "123.123.123.123", "\\port\\": 1234},
+        },
+    ),
     (
         "example log message",
         {

tests/unit/processor/grokker/test_grokker.py

@@ -192,6 +192,33 @@
             "parent": {"some_ip": "123.123.123.123", "port": 1234},
         },
     ),
+    (
+        "normalization from escaped & nested grok",
+        {
+            "filter": "win\\.log.event\\._id: 123456789",
+            "grokker": {
+                "mapping": {
+                    "win\\.log.event_data.normalize me!": r"%{IP:[par\\ent][...]} \w+ %{NUMBER:[par\\ent][\\port\\]:int} %[ts]+ %{NUMBER:te\\.st\\:int}"
+                },
+            },
+        },
+        {
+            "win.log": {
+                "api": "wineventlog",
+                "event._id": 123456789,
+                "event_data": {"normalize me!": "123.123.123.123 555 1234 %ttss 11"},
+            }
+        },
+        {
+            "win.log": {
+                "api": "wineventlog",
+                "event._id": 123456789,
+                "event_data": {"normalize me!": "123.123.123.123 555 1234 %ttss 11"},
+            },
+            "te.st\\": 11,
+            "par\\ent": {"...": "123.123.123.123", "\\port\\": 1234},
+        },
+    ),
     (
         "example log message",
         {