fix(mappers): Configuration schema now reflects that casting to bool(...) in stream map __filter__ expressions is not required (#3350)

fix(mapper): Configuration schema now reflects that casting to `bool(...)` in stream map `__filter__` expressions is not required

Author: edgarrmondragon

singer_sdk/helpers/capabilities.py

@@ -66,7 +66,7 @@
                     ),
                     Property(
                         "__filter__",
-                        StringType(pattern=r"^bool\((.*)\)$"),
+                        StringType(),
                         title="Filter",
                         description=(
                             "Filter out records from a stream. A string expression "
@@ -79,7 +79,7 @@
                     ),
                     Property(
                         "__key_properties__",
-                        ArrayType(StringType),
+                        ArrayType(StringType()),
                         title="Key Properties",
                         description="Primary key properties for the stream.",
                         nullable=False,

singer_sdk/mapper.py

@@ -443,10 +443,9 @@ def _init_functions_and_schema(  # noqa: PLR0912, PLR0915, C901
         """
         stream_map = copy.copy(stream_map)
 
-        filter_rule: str | None = None
+        filter_rule: str | None = stream_map.pop(MAPPER_FILTER_OPTION, None)
         include_by_default = True
-        if stream_map and MAPPER_FILTER_OPTION in stream_map:
-            filter_rule = stream_map.pop(MAPPER_FILTER_OPTION)
+        if filter_rule is not None:
             try:
                 filter_rule_parsed: ast.Expr = ast.parse(filter_rule).body[0]  # type: ignore[assignment]
             except (SyntaxError, IndexError) as ex:

tests/core/test_mapper.py

@@ -977,11 +977,17 @@ def discover_streams(self):
             id="json_dumps",
         ),
         pytest.param(
-            {"mystream": {"__filter__": "bool(count < 20)"}},
+            {"mystream": {"__filter__": "count < 20"}},
             {"flattening_enabled": False, "flattening_max_depth": 0},
             "filter_records.jsonl",
             id="filter_records",
         ),
+        pytest.param(
+            {"mystream": {"__filter__": "bool(count < 20)"}},
+            {"flattening_enabled": False, "flattening_max_depth": 0},
+            "filter_records_cast_to_bool.jsonl",
+            id="filter_records_cast_to_bool",
+        ),
         pytest.param(
             {"mystream": "__NULL__"},
             {"flattening_enabled": False, "flattening_max_depth": 0},

tests/snapshots/mapped_stream/filter_records_cast_to_bool.jsonl

@@ -0,0 +1,4 @@
+{"type":"SCHEMA","stream":"mystream","schema":{"properties":{"email":{"type":["string"]},"count":{"type":["integer","null"]},"user":{"properties":{"id":{"type":["integer","null"]},"sub":{"properties":{"num":{"type":["integer","null"]},"custom_obj":{"type":["string","null"]}},"type":["object","null"],"additionalProperties":true},"some_numbers":{"items":{"type":["number"]},"type":["array","null"]}},"type":["object","null"],"additionalProperties":true},"joined_at":{"format":"date-time","type":["string","null"]}},"type":"object","required":["email"],"$schema":"https://json-schema.org/draft/2020-12/schema"},"key_properties":[]}
+{"type":"RECORD","stream":"mystream","record":{"email":"bob@example.com","count":13,"user":{"id":2,"sub":{"num":2,"custom_obj":"obj-world"},"some_numbers":[10.32,1.618]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"RECORD","stream":"mystream","record":{"email":"charlie@example.com","count":19,"user":{"id":3,"sub":{"num":3,"custom_obj":"obj-hello"},"some_numbers":[1.414,1.732]},"joined_at":"2022-01-01T00:00:00Z"},"time_extracted":"2022-01-01T00:00:00+00:00"}
+{"type":"STATE","value":{"bookmarks":{"mystream":{}}}}