Skip to content

fix: BI-0 raise an error on unknown fields in BEFORE FILTER BY or IGNORE DIMENSIONS #1439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MCPN wants to merge 4 commits into
base: main
Choose a base branch
from
Open

fix: BI-0 raise an error on unknown fields in BEFORE FILTER BY or IGNORE DIMENSIONS #1439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c37bfe7
fix: BI-0 raise an error on unknown fields in BEFORE FILTER BY or IGN…
MCPN Jan 4, 2026
303dc9f
lint
MCPN Jan 4, 2026
4207ad1
copilot
MCPN Jan 4, 2026
60b2b18
lint
MCPN Jan 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_lookup_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import re
from typing import Optional

import pytest

from dl_api_client.dsmaker.primitives import WhereClause
from dl_api_client.dsmaker.shortcuts.dataset import (
add_formulas_to_dataset,
Expand Down Expand Up @@ -602,7 +600,6 @@ def test_ago_with_different_measures(self, control_api, data_api, saved_dataset)
check_ago_data(data_rows=data_rows, date_idx=0, value_idx=1, ago_idx=3, day_offset=1)
check_ago_data(data_rows=data_rows, date_idx=0, value_idx=2, ago_idx=4, day_offset=1)

@pytest.mark.xfail(reason="https://github.com/datalens-tech/datalens-backend/issues/531") # FIXME
def test_id_with_unknown_field(self, control_api, data_api, saved_dataset):
ds = add_formulas_to_dataset(
api_v1=control_api,
Expand Down
1 change: 0 additions & 1 deletion lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_window_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -815,7 +815,6 @@ def get_data(measures: tuple[str, ...]) -> list[list[str]]:
assert row_no_sum[0] == row_with_sum[0] # The dimension
assert row_no_sum[1] == row_with_sum[1] # The measure

@pytest.mark.xfail(reason="https://github.com/datalens-tech/datalens-backend/issues/531") # FIXME
def test_bfb_with_unknown_field(self, control_api, data_api, saved_dataset):
ds = add_formulas_to_dataset(
api_v1=control_api,
Expand Down
4 changes: 4 additions & 0 deletions lib/dl_formula/dl_formula/core/exc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,10 @@ class ValidationError(FormulaError):
default_code = FormulaError.default_code + ("VALIDATION",)


class UnknownBFBFieldError(ValidationError):
default_code = ValidationError.default_code + ("BFB_UNKNOWN_FIELD",)


# Aggregation errors


Expand Down
2 changes: 2 additions & 0 deletions lib/dl_formula/dl_formula/mutation/lookup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,8 @@ def make_replacement(
err_node = _get_arg_error_node(old)
if err_node is not None:
return err_node
if ignore_dim_errors := old.ignore_dimensions.list_node_type(aux_nodes.ErrorNode):
return ignore_dim_errors[0]

mutator = LOOKUP_MUTATOR_REGISTRY[func_name]

Expand Down
1 change: 0 additions & 1 deletion lib/dl_formula/dl_formula/validation/aggregation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import dl_formula.core.nodes as nodes
from dl_formula.inspect.env import InspectionEnvironment
import dl_formula.inspect.expression
import dl_formula.inspect.function
import dl_formula.inspect.node
from dl_formula.validation.env import ValidationEnvironment
from dl_formula.validation.validator import (
Expand Down
44 changes: 44 additions & 0 deletions lib/dl_formula/dl_formula/validation/bfb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
from typing import Collection

import attr

import dl_formula.core.exc as exc
import dl_formula.core.nodes as nodes
import dl_formula.inspect.expression
import dl_formula.inspect.node
from dl_formula.validation.validator import (
Checker,
ValidatorProxy,
)


@attr.s
class BFBChecker(Checker):
_field_ids: Collection[str] = attr.ib(factory=frozenset, kw_only=True)

def perform_node_check(
self,
validator: ValidatorProxy,
node: nodes.FormulaItem,
parent_stack: tuple[nodes.FormulaItem, ...],
) -> None:
children_w_stacks = dl_formula.inspect.expression.autonomous_children_w_stack(
node,
parent_stack=parent_stack,
)
for child, stack in children_w_stacks:
with validator.handle_error(node=node):
self.check_node(validator=validator, node=child, parent_stack=stack)

if not isinstance(node, nodes.FuncCall):
return

bfb = node.before_filter_by
for bfb_name in bfb.field_names:
if bfb_name not in self._field_ids:
with validator.handle_error(node=node):
raise exc.UnknownBFBFieldError(
f"Function's BEFORE FILTER BY clause refers to unknown field {bfb_name}",
token=dl_formula.inspect.node.get_token(bfb),
position=bfb.position,
)
1 change: 0 additions & 1 deletion lib/dl_formula/dl_formula/validation/window.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import dl_formula.core.nodes as nodes
from dl_formula.inspect.env import InspectionEnvironment
import dl_formula.inspect.expression
import dl_formula.inspect.function
import dl_formula.inspect.node
from dl_formula.validation.validator import (
Checker,
Expand Down
181 changes: 181 additions & 0 deletions lib/dl_formula/dl_formula_tests/unit/validation/test_bfb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
from __future__ import annotations

import pytest

from dl_formula.core import (
exc,
nodes,
)
from dl_formula.validation.bfb import BFBChecker
from dl_formula.validation.env import ValidationEnvironment
from dl_formula.validation.validator import validate


def validate_bfb(
node: nodes.FormulaItem,
env: ValidationEnvironment,
field_ids: frozenset[str],
collect_errors: bool = False,
) -> None:
validate(
node,
env=env,
checkers=[BFBChecker(field_ids=field_ids)],
collect_errors=collect_errors,
)


def test_no_bfb():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

# simple expression
validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="+",
args=[
nodes.Field.make("Dim Field"),
nodes.LiteralInteger.make(10),
],
),
),
env=env,
field_ids=field_ids,
)

# nested function calls
validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="sum",
args=[
nodes.FuncCall.make(
name="+",
args=[
nodes.Field.make("Dim Field"),
nodes.Field.make("Some Field"),
],
),
],
),
),
env=env,
field_ids=field_ids,
)


def test_correct_bfb_top_level():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="sum",
args=[nodes.Field.make("Some Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Dim Field"}),
),
),
env=env,
field_ids=field_ids,
)


def test_correct_bfb_nested():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="+",
args=[
nodes.LiteralInteger.make(100),
nodes.FuncCall.make(
name="sum",
args=[nodes.Field.make("Some Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Dim Field"}),
),
],
),
),
env=env,
field_ids=field_ids,
)


def test_unknown_bfb_top_level():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

with pytest.raises(exc.ValidationError) as exc_info:
validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="sum",
args=[nodes.Field.make("Some Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Unknown Field"}),
),
),
env=env,
field_ids=field_ids,
)
assert exc_info.value.errors[0].code == exc.UnknownBFBFieldError.default_code


def test_unknown_bfb_nested():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

with pytest.raises(exc.ValidationError) as exc_info:
validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="+",
args=[
nodes.LiteralInteger.make(8),
nodes.FuncCall.make(
name="avg",
args=[nodes.Field.make("Some Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Barley Field"}),
),
],
),
),
env=env,
field_ids=field_ids,
)
assert exc_info.value.errors[0].code == exc.UnknownBFBFieldError.default_code


def test_multiple_unknown_bfbs():
env = ValidationEnvironment()
field_ids = frozenset({"Dim Field", "Some Field"})

with pytest.raises(exc.ValidationError) as exc_info:
validate_bfb(
node=nodes.Formula(
nodes.FuncCall.make(
name="+",
args=[
nodes.FuncCall.make(
name="sum",
args=[nodes.Field.make("Dim Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Rye Field"}),
),
nodes.FuncCall.make(
name="avg",
args=[nodes.Field.make("Some Field")],
before_filter_by=nodes.BeforeFilterBy.make(field_names={"Barley Field"}),
),
],
),
),
env=env,
field_ids=field_ids,
collect_errors=True,
)
assert len(exc_info.value.errors) == 2
for error in exc_info.value.errors:
assert error.code == exc.UnknownBFBFieldError.default_code
4 changes: 4 additions & 0 deletions lib/dl_query_processing/dl_query_processing/compilation/formula_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
)
from dl_formula.parser.base import FormulaParser
from dl_formula.validation.aggregation import AggregationChecker
from dl_formula.validation.bfb import BFBChecker
from dl_formula.validation.env import ValidationEnvironment
from dl_formula.validation.validator import (
Checker,
Expand Down Expand Up @@ -885,6 +886,9 @@ def _validate_field_formula(
unselected_dimension_ids=unselected_dimension_ids,
)
)
checkers.append(
BFBChecker(field_ids=self._fields.ids),
)
validate(node=formula_obj, env=self._valid_env, collect_errors=collect_errors, checkers=checkers)

@implements_stage(ProcessingStage.base)
Expand Down
Loading