fix spliting elements with one item (#924)

ekneg54 · Pablu23 · kaya-david · web-flow · commit 36ab0e7d52fd · 2026-02-18T09:17:04.000+01:00
* fix spliting elements with one item * put changes to ng * add multiple empty use case * Add remove_whitespace flag, fix tests, use pytest.param * Rename, fix documentation test_cases maybe, add check for non Empty instead of only space * Try and make template account for pytest.param * Change from item.id is defined, to item.values * Add pydoc, remove lsp comments, fix pydoc * release: 18.0.1 (#925) * Update changelog.md * Update pre commit and black dependency * Reformat with black * refactor: switch to uv with lockfile, decouple PyPI and optimize Docker builds (#914) Documentation * doc: update README.md * doc: update installation docs * doc: update CHANGELOG.md Build & Dependencies * build: replace pip with uv pip * build: add uv.lock * chore: cap major versions for production dependencies * fix: CVE-2025-69223 Docker * feat: adapt Dockerfile for uv-based installation * feat: optimize Docker build using uv caching * feat: simplify Dockerfile * refactor: small Dockerfile improvements * fix: remove LOGPREP_VERSION build argument CI / CD * feat: adapt CI/CD workflow files * feat: update actions/checkout to v6 * build: add CI job to verify uv.lock consistency (#927) * build: add CI job to verify uv.lock consistency * feat: make mapping of Generic Resolver yaml compliant (#928) * Add list[tuple[str]] as valid resolve_list type, change test to account for tuple, COMMITED WITH NO-VERIFY * Add converter func * Apparently resolve_list is a mapping from str to dict * Add test to test converter * update changelog * Add typevar and rename func * Rename INPUT_TYPE and add KEY_TYPE, extract to own utils file, with own utils test, add wrapper function and keep merge_dicts relativly clean * Remove assert isinstance for now, rework converters, update tests * Add FieldValue as Dict value * pull up path * Update tests and add doc * Update logprep/processor/generic_resolver/rule.py Co-authored-by: Michael Hoff <mail@michael-hoff.net> * fix review comments * Convert from file load, fix tests, fix Key and Value vartypes * Upgrade uv.lock for protobuf cve fix * Update logprep/processor/generic_resolver/rule.py Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Rename test id, add pydoc to converters * Split long comment line into multiple * Change test to expect ValueError not InvalidConfigurationError --------- Co-authored-by: Michael Hoff <mail@michael-hoff.net> Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * feat: add service account to chart (#931) * Add service account to chart, no-verify because yamlfmt doesnt like go templating? * update Changelog * Make serviceAccount disable able * Remove spec: as it is not spec * Update tests, enable service accounts by default * Remove enabled as it is redundant * remove unused chart.yaml.j2 * Update charts/logprep/templates/deployment.yaml Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Update charts/logprep/templates/service-account.yaml Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Update CHANGELOG.md Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Update charts/logprep/values.yaml Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Update default values and tests * Remove if enabled * Remove chart.lock --------- Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Update CHANGELOG * Add pytest.param improvement, add more rules, enhance pydoc * Update logprep/processor/string_splitter/rule.py Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com> * Fix closing qoutes --------- Co-authored-by: Pablu23 <me@pablu.de> Co-authored-by: Pablu23 <43807157+Pablu23@users.noreply.github.com> Co-authored-by: kaya-david <david.kaya@bwi.de> Co-authored-by: kaya-david <205917832+kaya-david@users.noreply.github.com> Co-authored-by: Michael Hoff <mail@michael-hoff.net> Co-authored-by: Michael Hoff <9436725+mhoff@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,10 +4,12 @@
 ### Features
 * add uv as dependency management, including uv.lock
 * allow configuration (and auto-creation) of service accounts in helm chart
+* add new drop_empty flag to allow the `string_splitter` to drop resulting fields that would be empty (e.g. whitespace)
 * generic_resolver now handles all FieldValue types (including None)
 
 ### Improvements
 * simplify Dockerfile and remove docker build support for `LOGPREP_VERSION`
+* pytest.param now works with test_cases document generation
 
 ### Bugfix
 * generic_resolver now follows yaml standard and accepts a list instead of relying on the ordering of a dict
diff --git a/doc/source/_templates/testcase-renderer.tmpl b/doc/source/_templates/testcase-renderer.tmpl
@@ -1,10 +1,21 @@
 .. -*- mode: rst -*-
 {% for item in data.test_cases %}
 
+{% if item.values %}
+{# handles pytest.param #}
+*{{ item.id }}:*
+
+- rule: :code:`{{ item.values[0] }}`
+- message: :code:`{{ item.values[1] }}`
+- processed: :code:`{{ item.values[2] }}`
+
+{% else %}
+{# handles legacy tuple style #}
 *{{ item[0] }}:*
 
 - rule: :code:`{{ item[1] }}`
 - message: :code:`{{ item[2] }}`
 - processed: :code:`{{ item[3] }}`
+{% endif %}
 
 {% endfor %}
diff --git a/logprep/ng/processor/string_splitter/processor.py b/logprep/ng/processor/string_splitter/processor.py
@@ -24,8 +24,13 @@
 .. automodule:: logprep.processor.string_splitter.rule
 """
 
+import typing
+
+from typing_extensions import override
+
 from logprep.ng.processor.field_manager.processor import FieldManager
 from logprep.processor.base.exceptions import ProcessingWarning
+from logprep.processor.base.rule import Rule
 from logprep.processor.string_splitter.rule import StringSplitterRule
 from logprep.util.helper import get_dotted_field_value
 
@@ -35,11 +40,17 @@ class StringSplitter(FieldManager):
 
     rule_class = StringSplitterRule
 
-    def _apply_rules(self, event: dict, rule: StringSplitterRule) -> None:
-        source_field = rule.source_fields[0]
+    @override
+    def _apply_rules(self, event: dict, rule: Rule) -> None:
+        _rule = typing.cast(StringSplitterRule, rule)
+
+        source_field = _rule.source_fields[0]
         source_field_content = get_dotted_field_value(event, source_field)
-        self._handle_missing_fields(event, rule, rule.source_fields, [source_field_content])
+        self._handle_missing_fields(event, _rule, _rule.source_fields, [source_field_content])
         if not isinstance(source_field_content, str):
             raise ProcessingWarning(f"source_field '{source_field}' is not a string", rule, event)
-        result = source_field_content.split(rule.delimiter)
+        result = source_field_content.split(_rule.delimiter)
+
+        if _rule.drop_empty:
+            result = [item for item in result if item != "" and not item.isspace()]
         self._write_target_field(event, rule, result)
diff --git a/logprep/processor/string_splitter/processor.py b/logprep/processor/string_splitter/processor.py
@@ -42,4 +42,7 @@ def _apply_rules(self, event: dict, rule: StringSplitterRule):
         if not isinstance(source_field_content, str):
             raise ProcessingWarning(f"source_field '{source_field}' is not a string", rule, event)
         result = source_field_content.split(rule.delimiter)
+
+        if rule.drop_empty:
+            result = [item for item in result if item != "" and not item.isspace()]
         self._write_target_field(event, rule, result)
diff --git a/logprep/processor/string_splitter/rule.py b/logprep/processor/string_splitter/rule.py
@@ -41,6 +41,8 @@
 
 """
 
+import typing
+
 from attrs import define, field, validators
 
 from logprep.processor.field_manager.rule import FieldManagerRule
@@ -60,13 +62,28 @@ class Config(FieldManagerRule.Config):
                 validators.min_len(1),
                 validators.max_len(1),
             ],
+            default=[],
         )
         delimiter: str = field(validator=validators.instance_of(str), default=" ")
         """The delimiter for splitting. Defaults to whitespace"""
-        mapping: dict = field(default="", init=False, repr=False, eq=False)
+        mapping: dict = field(default={}, init=False, repr=False, eq=False)
         ignore_missing_fields: bool = field(default=False, init=False, repr=False, eq=False)
+        drop_empty: bool = field(default=False)
+        """If empty list values (as a result of the splitting operation) should be dropped or kept.
+        By this definition, the empty string (no characters) and strings containing only `whitespace <https://docs.python.org/3/library/stdtypes.html#str.isspace>`_ count as 'empty'.
+        The default setting is to keep empty list values."""
+
+    @property
+    def config(self) -> Config:
+        """returns the config as typed StringSplitterRule.Config"""
+        return typing.cast(StringSplitterRule.Config, self._config)
 
     @property
-    def delimiter(self):
+    def delimiter(self) -> str:
         """returns the configured delimiter"""
-        return self._config.delimiter
+        return self.config.delimiter
+
+    @property
+    def drop_empty(self) -> bool:
+        """returns the configured drop_empty flag"""
+        return self.config.drop_empty
diff --git a/tests/unit/ng/processor/string_splitter/test_string_splitter.py b/tests/unit/ng/processor/string_splitter/test_string_splitter.py
@@ -13,52 +13,146 @@
 from tests.unit.ng.processor.base import BaseProcessorTestCase
 
 test_cases = [
-    (
-        "splits without delimeter on whitespace",
+    pytest.param(
         {
             "filter": "message",
-            "string_splitter": {"source_fields": ["message"], "target_field": "result"},
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "drop_empty": True,
+            },
         },
         {"message": "this is the message"},
-        {"message": "this is the message", "result": ["this", "is", "the", "message"]},
+        ["this", "is", "the", "message"],
+        id="splits_without_explicit_set_delimiter_on_whitespace",
     ),
-    (
-        "splits with delimeter",
+    pytest.param(
         {
             "filter": "message",
             "string_splitter": {
                 "source_fields": ["message"],
                 "target_field": "result",
                 "delimiter": ", ",
+                "drop_empty": True,
             },
         },
         {"message": "this, is, the, message"},
-        {"message": "this, is, the, message", "result": ["this", "is", "the", "message"]},
+        ["this", "is", "the", "message"],
+        id="splits_with_delimiter",
     ),
-]  # testcase, rule, event, expected
-
-failure_test_cases = [
-    (
-        "splits without delimeter on whitespace",
+    pytest.param(
         {
             "filter": "message",
-            "string_splitter": {"source_fields": ["message"], "target_field": "result"},
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
         },
-        {"message": ["this", "is", "the", "message"]},
-        {"message": ["this", "is", "the", "message"], "tags": ["_string_splitter_failure"]},
-        ".*ProcessingWarning.*",
+        {"message": "this,"},
+        ["this"],
+        id="splits_one_item_with_delimiter",
     ),
-    (
-        "splits without delimeter on whitespace",
+    pytest.param(
         {
             "filter": "message",
-            "string_splitter": {"source_fields": ["message"], "target_field": "message"},
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
         },
-        {"message": "this is the message"},
-        {"message": "this is the message", "tags": ["_string_splitter_failure"]},
-        ".*FieldExistsWarning.*",
+        {"message": ",,this,,"},
+        ["this"],
+        id="splits_one_item_with_multiple_delimiter_and_drop_empty",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": False,
+            },
+        },
+        {"message": ",,this,,"},
+        ["", "", "this", "", ""],
+        id="splits_one_item_with_multiple_delimiter_and_no_drop_empty",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
+        },
+        {"message": " , ,this, ,"},
+        ["this"],
+        id="splits_one_item_with_multiple_delimiter_and_empty_fields",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
+        },
+        {"message": ",, this , , "},
+        [" this "],
+        id="splits_one_item_with_multiple_delimiter_and_whitespace",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
+        },
+        {"message": "\n,,this,\t, "},
+        ["this"],
+        id="splits_one_item_with_multiple_delimiter_and_newline",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
+        },
+        {"message": ",, this, , "},
+        [" this"],
+        id="splits_one_item_with_multiple_delimiter_and_whitespace_only_in_front",
+    ),
+    pytest.param(
+        {
+            "filter": "message",
+            "string_splitter": {
+                "source_fields": ["message"],
+                "target_field": "result",
+                "delimiter": ",",
+                "drop_empty": True,
+            },
+        },
+        {"message": "hello , world,this, is a very complex,\n , and even multiline, text,,, "},
+        ["hello ", " world", "this", " is a very complex", " and even multiline", " text"],
+        id="splits_one_item_with_multiple_delimiter_and_whitespace_only_in_front",
     ),
-]  # testcase, rule, event, expected, error_message
+]
 
 
 class TestStringSplitter(BaseProcessorTestCase):
@@ -67,18 +161,42 @@ class TestStringSplitter(BaseProcessorTestCase):
         "rules": ["tests/testdata/unit/string_splitter/rules"],
     }
 
-    @pytest.mark.parametrize("testcase, rule, event, expected", test_cases)
-    def test_testcases(self, testcase, rule, event, expected):  # pylint: disable=unused-argument
+    @pytest.mark.parametrize(["rule", "event", "expected"], test_cases)
+    def test_testcases(self, rule, event, expected):
         self._load_rule(rule)
         event = LogEvent(event, original=b"")
         self.object.process(event)
-        assert event.data == expected, testcase
+        assert event.data["result"] == expected
 
-    @pytest.mark.parametrize("testcase, rule, event, expected, error_message", failure_test_cases)
-    def test_testcases_failure_handling(self, testcase, rule, event, expected, error_message):
+    @pytest.mark.parametrize(
+        ["rule", "event", "expected", "error_message"],
+        [
+            pytest.param(
+                {
+                    "filter": "message",
+                    "string_splitter": {"source_fields": ["message"], "target_field": "result"},
+                },
+                {"message": ["this", "is", "the", "message"]},
+                {"message": ["this", "is", "the", "message"], "tags": ["_string_splitter_failure"]},
+                ".*ProcessingWarning.*",
+                id="splits_without_delimiter_on_whitespace_with_no_string",
+            ),
+            pytest.param(
+                {
+                    "filter": "message",
+                    "string_splitter": {"source_fields": ["message"], "target_field": "message"},
+                },
+                {"message": "this is the message"},
+                {"message": "this is the message", "tags": ["_string_splitter_failure"]},
+                ".*FieldExistsWarning.*",
+                id="splits_without_delimiter_on_whitespace_with_existing_field",
+            ),
+        ],
+    )
+    def test_testcases_failure_handling(self, rule, event, expected, error_message):
         self._load_rule(rule)
         event = LogEvent(event, original=b"")
         result = self.object.process(event)
         assert len(result.warnings) == 1
         assert re.match(error_message, str(result.warnings[0]))
-        assert event.data == expected, testcase
+        assert event.data == expected
diff --git a/tests/unit/processor/string_splitter/test_string_splitter.py b/tests/unit/processor/string_splitter/test_string_splitter.py
