Refactor IqrOutlierCheck to consistently exclude date filters from WHERE clauses and data requirements

Author: Benjamin Gutzmann

CHANGELOG.md

@@ -20,6 +20,7 @@ Types of changes:
 
 - Refactor executor to bulk load data into DuckDB memory and run checks from there
 - Update ValuesInSetCheck config to require value_set as a set of str, bool, or int
+- Refactor IqrOutlierCheck to consistently exclude date filters from WHERE clauses and data requirements
 
 ### Fixed
 

src/koality/checks.py

@@ -474,7 +474,11 @@ def assemble_query(self) -> str:
         """Assemble the complete SQL query for this check."""
         main_query = self.query_boilerplate(self.transformation_statement())
 
-        if where_statement := self.assemble_where_statement(self.filters):
+        filters = self.filters.copy()
+        if isinstance(self, IqrOutlierCheck):
+            filters = {name: cfg for name, cfg in filters.items() if cfg.get("type") != "date"}
+
+        if where_statement := self.assemble_where_statement(filters):
             return main_query + "\n" + where_statement
 
         return main_query
@@ -1579,9 +1583,6 @@ def __init__(
             monitor_only=monitor_only,
         )
 
-        # Remove date filter from WHERE clause (it's used in the interval SQL, not WHERE)
-        self.filters = {name: cfg for name, cfg in self.filters.items() if cfg.get("type") != "date"}
-
     def transformation_statement(self) -> str:
         """Return the SQL statement for IQR-based outlier detection."""
         # TODO: currently we only raise an error if there is no data for the date
@@ -1592,10 +1593,12 @@ def transformation_statement(self) -> str:
         date_col = self.date_filter["column"]
         date_val = self.date_filter["value"]
 
-        if self.filters:
-            filter_columns = ",\n".join([v["column"] for v in self.filters.values()])
+        filters = {k: v for k, v in self.filters.items() if v["type"] != "date"}
+
+        if filters:
+            filter_columns = ",\n".join([v["column"] for v in filters.values()])
             filter_columns = ",\n" + filter_columns
-            where_statement = self.assemble_where_statement(self.filters)
+            where_statement = self.assemble_where_statement(filters)
             where_statement = "\nAND\n" + where_statement.removeprefix("WHERE\n")
         return f"""
         WITH
@@ -1670,7 +1673,9 @@ def assemble_data_exists_query(self) -> str:
         date_col = self.date_filter["column"]
         date_val = self.date_filter["value"]
 
-        where_statement = self.assemble_where_statement(self.filters)
+        filters = {k: v for k, v in self.filters.items() if v["type"] != "date"}
+
+        where_statement = self.assemble_where_statement(filters)
         if where_statement:
             where_statement = f"{where_statement} AND CAST({date_col} AS DATE) = DATE '{date_val}'"
         else:

src/koality/executor.py

@@ -202,7 +202,7 @@ def _get_dataset_cache_key(check_instance: DataQualityCheck) -> tuple:
 
         return table, database_accessor, date_value, filters_key
 
-    def get_data_requirements(self) -> defaultdict[str, defaultdict[str, set]]:
+    def get_data_requirements(self) -> defaultdict[str, defaultdict[str, set]]:  # noqa: C901
         """Aggregate data requirements from all checks.
 
         This method collects all required tables, columns, and filter configurations
@@ -216,10 +216,11 @@ def get_data_requirements(self) -> defaultdict[str, defaultdict[str, set]]:
         data_requirements = defaultdict(lambda: defaultdict(set))
         for check in self.checks:
             table_name = check.table
+            check_filters = check.filters
             # Add check-specific columns and filter columns to the requirements
             if check.check_column and check.check_column != "*":
                 data_requirements[table_name]["columns"].add(check.check_column)
-            for _filter in check.filters.values():
+            for _filter in check_filters.values():
                 if "column" in _filter:
                     data_requirements[table_name]["columns"].add(_filter["column"])
 
@@ -234,8 +235,11 @@ def get_data_requirements(self) -> defaultdict[str, defaultdict[str, set]]:
                     if "column" in _filter:
                         data_requirements[check.right_table]["columns"].add(_filter["column"])
 
+            if isinstance(check, IqrOutlierCheck):
+                check_filters = {k: v for k, v in check.filters.items() if v.get("type") != "date"}
+
             # Store unique filter configurations for each table
-            filter_key = frozenset((name, frozenset(config.items())) for name, config in check.filters.items())
+            filter_key = frozenset((name, frozenset(config.items())) for name, config in check_filters.items())
             data_requirements[table_name]["filters"].add(filter_key)
         return data_requirements
 

tests/integration/test_column_checks.py

@@ -646,6 +646,7 @@ def test_iqr_outlier_check_two_shops_success(duckdb_client_iqr_two_shops: duckdb
         },
     )
     result = check(duckdb_client_iqr_two_shops)
+    assert result["METRIC_NAME"] == "VALUE_outlier_iqr_both_1_5"
     assert result["VALUE"] == 101.0
     assert result["RESULT"] == "SUCCESS"
     assert result["IDENTIFIER"] == "SHOP_ID=abcd"