diff --git a/docs/changelog/118324.yaml b/docs/changelog/118324.yaml new file mode 100644 index 0000000000000..729ff56f6a253 --- /dev/null +++ b/docs/changelog/118324.yaml @@ -0,0 +1,6 @@ +pr: 118324 +summary: Allow the data type of `null` in filters +area: ES|QL +type: bug +issues: + - 116351 diff --git a/docs/changelog/125832.yaml b/docs/changelog/125832.yaml new file mode 100644 index 0000000000000..4877a02e9e6d0 --- /dev/null +++ b/docs/changelog/125832.yaml @@ -0,0 +1,6 @@ +pr: 125832 +summary: "ESQL: Fix `NULL` handling in `IN` clause" +area: ES|QL +type: bug +issues: + - 119950 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec index 3ed435c80aa1e..ccf8dde563f37 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/null.csv-spec @@ -191,3 +191,28 @@ emp_no:integer | languages:integer | height:double | x:double | y:double | z:dou 10020 | null | 1.41 | 1.41 | 1.41 | 40031.0 | 40031 10021 | null | 1.47 | 1.47 | 1.47 | 60408.0 | 60408 ; + +whereNull +FROM employees +| WHERE NULL and emp_no <= 10021 +| SORT first_name, last_name +| EVAL fullname = CONCAT(first_name, " ", last_name) +| KEEP fullname, job_positions, salary, salary_change +| limit 5 +; + +fullname:keyword | job_positions:keyword | salary:integer | salary_change:double +; + +inConvertedNull +required_capability: filter_in_converted_null +FROM employees +| WHERE emp_no in (10021, 10022, null::int) +| KEEP emp_no, first_name, last_name +| SORT emp_no +; + +emp_no:integer | first_name:keyword | last_name:keyword +10021 | Ramzi | Erde +10022 | Shahaf | Famili +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 73cb3e599fef6..197d5429e7542 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -806,6 +806,12 @@ public enum Cap { */ AGGREGATE_METRIC_DOUBLE_SORTING(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG), + /** + * Support for filter in converted null. + * See ESQL: Fix `NULL` handling in `IN` clause #125832 + */ + FILTER_IN_CONVERTED_NULL, + /** * When creating constant null blocks in {@link org.elasticsearch.compute.lucene.ValuesSourceReaderOperator}, we also handed off * the ownership of that block - but didn't account for the fact that the caller might close it, leading to double releases diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index bcbcbda33f9ad..126652efd94a1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -478,7 +478,7 @@ private Query translate(TranslatorHandler handler) { List queries = new ArrayList<>(); for (Expression rhs : list()) { - if (DataType.isNull(rhs.dataType()) == false) { + if (Expressions.isGuaranteedNull(rhs) == false) { if (needsTypeSpecificValueHandling(attribute.dataType())) { // delegates to BinaryComparisons translator to ensure consistent handling of date and time values // TODO: diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index ddb8e1870ab26..5065a24fa4789 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -962,6 +962,33 @@ public void testPeriodAndDurationInEval() { public void testFilterNonBoolField() { assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where emp_no")); + + assertEquals( + "1:19: Condition expression needs to be boolean, found [KEYWORD]", + error("from test | where concat(first_name, \"foobar\")") + ); + } + + public void testFilterNullField() { + // `where null` should return empty result set + query("from test | where null"); + + // Value null of type `BOOLEAN` + query("from test | where null::boolean"); + + // Provide `NULL` type in `EVAL` + query("from t | EVAL x = null | where x"); + + // `to_string(null)` is of `KEYWORD` type null, resulting in `to_string(null) == "abc"` being of `BOOLEAN` + query("from t | where to_string(null) == \"abc\""); + + // Other DataTypes can contain null values + assertEquals("1:19: Condition expression needs to be boolean, found [KEYWORD]", error("from test | where null::string")); + assertEquals("1:19: Condition expression needs to be boolean, found [INTEGER]", error("from test | where null::integer")); + assertEquals( + "1:45: Condition expression needs to be boolean, found [DATETIME]", + error("from test | EVAL x = null::datetime | where x") + ); } public void testFilterDateConstant() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index 03a4b063d6294..3ecfe7a7e42be 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -13,19 +13,25 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLikeTests; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import org.junit.AfterClass; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; @@ -79,6 +85,16 @@ private static Literal L(Object value) { return of(EMPTY, value); } + public void testConvertedNull() { + In in = new In( + EMPTY, + new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true)), + Arrays.asList(ONE, new Literal(Source.EMPTY, null, randomFrom(DataType.types())), THREE) + ); + var query = in.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + assertEquals(new TermsQuery(EMPTY, "field", Set.of(1, 3)), query); + } + @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>();