diff --git a/server/src/main/resources/transport/definitions/referable/esql_lookup_join_general_expression.csv b/server/src/main/resources/transport/definitions/referable/esql_lookup_join_general_expression.csv new file mode 100644 index 0000000000000..eab2d4287a8fa --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_lookup_join_general_expression.csv @@ -0,0 +1 @@ +9229000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 3f6ae8e0a806d..7ce0fe03eacf6 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_timestamps_info,9228000 +esql_lookup_join_general_expression,9229000 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index bd42ee08ed384..99125e94e172e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -447,7 +447,7 @@ public static void loadDataSetIntoEs( ); } - private static void loadDataSetIntoEs( + public static void loadDataSetIntoEs( RestClient client, boolean supportsIndexModeLookup, boolean supportsSourceFieldMapping, @@ -698,7 +698,7 @@ record ColumnHeader(String name, String type) {} * - multi-values are comma separated * - commas inside multivalue fields can be escaped with \ (backslash) character */ - private static void loadCsvData(RestClient client, String indexName, URL resource, boolean allowSubFields, Logger logger) + public static void loadCsvData(RestClient client, String indexName, URL resource, boolean allowSubFields, Logger logger) throws IOException { ArrayList failures = new ArrayList<>(); @@ -1031,7 +1031,7 @@ private Settings readSettingsFile() throws IOException { public record EnrichConfig(String policyName, String policyFileName) {} - private interface IndexCreator { + public interface IndexCreator { void createIndex(RestClient client, String indexName, String mapping, Settings indexSettings) throws IOException; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/analysis/MutableAnalyzerContext.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/analysis/MutableAnalyzerContext.java index 69e7b5bdb980f..98b283826bf37 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/analysis/MutableAnalyzerContext.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/analysis/MutableAnalyzerContext.java @@ -52,7 +52,14 @@ public TransportVersion minimumVersion() { public RestoreTransportVersion setTemporaryTransportVersionOnOrAfter(TransportVersion minVersion) { TransportVersion oldVersion = this.currentVersion; // Set to a random version between minVersion and current - this.currentVersion = TransportVersionUtils.randomVersionBetween(ESTestCase.random(), minVersion, TransportVersion.current()); + // have 50+% probability of being the current version, to improve probability of catching a change + // that only affects the version we are trying to check in + boolean useCurrent = ESTestCase.randomBoolean(); + if (useCurrent) { + this.currentVersion = TransportVersion.current(); + } else { + this.currentVersion = TransportVersionUtils.randomVersionBetween(ESTestCase.random(), minVersion, TransportVersion.current()); + } return new RestoreTransportVersion(oldVersion); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec index 24cfaa0018752..fcdd30cd3ad58 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec @@ -1272,3 +1272,667 @@ warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered mu id_left:integer | name_str:keyword | extra1:keyword | other2:integer 1 | Alice | foo | 2000 ; + +lookupJoinGeneralExpressionOnLeftAttributeNonPushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionOnRightAttributeNonPushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_int) > 5 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_int) > 5] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionOnBothSidesNonPushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND ABS(id_int) < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND ABS(id_int) < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionOnLeftAttributePushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND id_left > 5 AND id_left < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND id_left > 5 AND id_left < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionOnRightAttributePushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND id_int > 5 AND id_int < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND id_int > 5 AND id_int < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionMixedPushableNonPushableLeft +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND id_left < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND id_left < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionMixedPushableNonPushableRight +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_int) > 5 AND id_int < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_int) > 5 AND id_int < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionMixedPushableNonPushableBothSides +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND id_int < 15 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(id_left) > 5 AND id_int < 15] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionWithArithmeticOnLeft +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND (id_left + 5) > 10 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND (id_left + 5) > 10] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionWithArithmeticOnRight +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND (id_int + 5) > 10 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND (id_int + 5) > 10] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | null | qux | null | null +5 | Eve | null | quux | null | null +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionWithOther2FieldNonPushable +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2) > 5000 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2) > 5000] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | David | qux | zeta | 6000 +5 | Eve | Eve | quux | eta | 7000 +5 | Eve | Eve | quux | theta | 8000 +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinGeneralExpressionLeftSideComputeLayerWarning +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| EVAL other2_left = CASE(id_int == 1, [5000, 6000], id_int == 2, [7000], id_int == 3, [8000], [9000]) +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2_left) > 5000 +| KEEP id_left, name_left, name_str, extra1, other1, other2_left +| SORT id_left, name_left, name_str, extra1, other1, other2_left +| LIMIT 20 +; + +warning:Line 2:27: evaluation of [id_int == 1] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:27: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 2:54: evaluation of [id_int == 2] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:54: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 2:75: evaluation of [id_int == 3] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:75: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 4:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2_left) > 5000] failed, treating result as null. Only first 20 failures recorded. +warning:Line 4:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value +warning:Line 4:95: evaluation of [ABS(other2_left)] failed, treating result as null. Only first 20 failures recorded. +warning:Line 4:95: java.lang.IllegalArgumentException: single-value function encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2_left:integer +1 | Alice | null | foo | null | [5000, 6000] +[1, 19, 21] | Sophia | null | zyx | null | 9000 +2 | Bob | Bob | bar | gamma | 7000 +3 | Charlie | Charlie | baz | delta | 8000 +3 | Charlie | Charlie | baz | epsilon | 8000 +4 | David | David | qux | zeta | 9000 +5 | Eve | Eve | quux | eta | 9000 +5 | Eve | Eve | quux | theta | 9000 +6 | null | null | corge | null | 9000 +7 | Grace | Grace | grault | kappa | 9000 +8 | Hank | Hank | garply | lambda | 9000 +9 | Ivy | null | waldo | null | 9000 +10 | John | null | fred | null | 9000 +12 | Liam | Liam | xyzzy | nu | 9000 +13 | Mia | Mia | thud | xi | 9000 +14 | Nina | Nina | foo2 | omicron | 9000 +15 | Oscar | null | bar2 | null | 9000 +[17, 18] | Olivia | null | xyz | null | 9000 +null | Kate | null | plugh | null | 9000 + +; + +lookupJoinGeneralExpressionRightSideComputeLayerWarning +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2) > 5000 +| KEEP id_left, name_left, name_str, extra1, other1, other2 +| SORT id_left, name_left, name_str, extra1, other1, other2 +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ABS(other2) > 5000] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword | extra1:keyword | other1:keyword | other2:integer +1 | Alice | null | foo | null | null +[1, 19, 21] | Sophia | null | zyx | null | null +2 | Bob | null | bar | null | null +3 | Charlie | null | baz | null | null +4 | David | David | qux | zeta | 6000 +5 | Eve | Eve | quux | eta | 7000 +5 | Eve | Eve | quux | theta | 8000 +6 | null | null | corge | null | null +7 | Grace | Grace | grault | kappa | 10000 +8 | Hank | Hank | garply | lambda | 11000 +9 | Ivy | null | waldo | null | null +10 | John | null | fred | null | null +12 | Liam | Liam | xyzzy | nu | 13000 +13 | Mia | Mia | thud | xi | 14000 +14 | Nina | Nina | foo2 | omicron | 15000 +15 | Oscar | null | bar2 | null | null +[17, 18] | Olivia | null | xyz | null | null +null | Kate | null | plugh | null | null +; + +lookupJoinComplexExpressionRelatingLeftAndRight +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON (id_left + id_int) == (id_left * 2) +| KEEP id_left, name_left, id_int, name_str +| SORT id_left, id_int +| LIMIT 20 +; + +warning:Line 3:48: evaluation of [id_left + id_int] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:48: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 3:70: evaluation of [id_left * 2] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:70: java.lang.IllegalArgumentException: single-value function encountered multi-value + +id_left:integer | name_left:keyword | id_int:integer | name_str:keyword +1 | Alice | 1 | Alice +1 | Alice | 1 | Alice +[1, 19, 21] | Sophia | null | null +2 | Bob | 2 | Bob +3 | Charlie | 3 | Charlie +3 | Charlie | 3 | Charlie +4 | David | 4 | David +5 | Eve | 5 | Eve +5 | Eve | 5 | Eve +6 | null | 6 | null +7 | Grace | 7 | Grace +8 | Hank | 8 | Hank +9 | Ivy | null | null +10 | John | null | null +12 | Liam | 12 | Liam +13 | Mia | 13 | Mia +14 | Nina | 14 | Nina +15 | Oscar | null | null +[17, 18] | Olivia | null | null +null | Kate | null | null +; + +lookupJoinNoRelationshipBetweenLeftAndRight +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left > 5 AND id_int > 5 +| KEEP id_left, name_left, id_int, name_str +| SORT id_left, name_left, id_int, name_str +| LIMIT 20 +; + +warning:Line 3:47: evaluation of [id_left > 5] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:47: java.lang.IllegalArgumentException: single-value function encountered multi-value + +id_left:integer | name_left:keyword | id_int:integer | name_str:keyword +1 | Alice | null | null +[1, 19, 21] | Sophia | null | null +2 | Bob | null | null +3 | Charlie | null | null +4 | David | null | null +5 | Eve | null | null +6 | null | [1, 19, 20] | Sophia +6 | null | 6 | null +6 | null | 7 | Grace +6 | null | 8 | Hank +6 | null | 12 | Liam +6 | null | 13 | Mia +6 | null | 14 | Nina +6 | null | 16 | Paul +6 | null | [17, 18] | Olivia +7 | Grace | [1, 19, 20] | Sophia +7 | Grace | 6 | null +7 | Grace | 7 | Grace +7 | Grace | 8 | Hank +7 | Grace | 12 | Liam +; + + +lookupJoinFilterOnLeftAndRightComplexAndOr +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ((id_left >= 3 AND id_int >= 3) OR (id_left < 2 AND id_int < 2)) +| KEEP id_left, name_left, name_str +| SORT id_left +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str AND ((id_left >= 3 AND id_int >= 3) OR (id_left < 2 AND id_int < 2))] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword +[1, 19, 21] | Sophia | null +1 | Alice | Alice +1 | Alice | Alice +2 | Bob | null +3 | Charlie | Charlie +3 | Charlie | Charlie +4 | David | David +5 | Eve | Eve +5 | Eve | Eve +6 | null | null +7 | Grace | Grace +8 | Hank | Hank +9 | Ivy | null +10 | John | null +12 | Liam | Liam +13 | Mia | Mia +14 | Nina | Nina +15 | Oscar | null +[17, 18] | Olivia | null +null | Kate | null +; + +lookupJoinFilterOnConditionAndTrue +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND TRUE +| KEEP id_left, name_left, name_str +| SORT id_left +| LIMIT 20 +; + +warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND TRUE] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value + +id_left:integer | name_left:keyword | name_str:keyword +[1, 19, 21] | Sophia | null +1 | Alice | Alice +1 | Alice | Alice +2 | Bob | Bob +3 | Charlie | Charlie +3 | Charlie | Charlie +4 | David | David +5 | Eve | Eve +5 | Eve | Eve +6 | null | null +7 | Grace | Grace +8 | Hank | Hank +9 | Ivy | null +10 | John | null +12 | Liam | Liam +13 | Mia | Mia +14 | Nina | Nina +15 | Oscar | null +[17, 18] | Olivia | null +null | Kate | null +; + +lookupJoinComplexExpressionWithArithmetic +required_capability: join_lookup_v12 +required_capability: lookup_join_with_general_expression + +FROM multi_column_joinable +| RENAME id_int AS id_left, name_str AS name_left +| LOOKUP JOIN multi_column_joinable_lookup ON (id_left * 2) == (id_int + id_int) +| KEEP id_left, name_left, id_int, name_str +| SORT id_left, id_int +| LIMIT 20 +; + +warning:Line 3:48: evaluation of [id_left * 2] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:48: java.lang.IllegalArgumentException: single-value function encountered multi-value +warning:Line 3:65: evaluation of [id_int + id_int] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:65: java.lang.IllegalArgumentException: single-value function encountered multi-value + +id_left:integer | name_left:keyword | id_int:integer | name_str:keyword +1 | Alice | 1 | Alice +1 | Alice | 1 | Alice +[1, 19, 21] | Sophia | null | null +2 | Bob | 2 | Bob +3 | Charlie | 3 | Charlie +3 | Charlie | 3 | Charlie +4 | David | 4 | David +5 | Eve | 5 | Eve +5 | Eve | 5 | Eve +6 | null | 6 | null +7 | Grace | 7 | Grace +8 | Hank | 8 | Hank +9 | Ivy | null | null +10 | John | null | null +12 | Liam | 12 | Liam +13 | Mia | 13 | Mia +14 | Nina | 14 | Nina +15 | Oscar | null | null +[17, 18] | Olivia | null | null +null | Kate | null | null +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java index 2544edda6f53d..7c1ca072546e6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java @@ -56,9 +56,11 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.async.AsyncExecutionId; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -83,6 +85,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -95,6 +98,165 @@ import static org.hamcrest.Matchers.hasSize; public class LookupFromIndexIT extends AbstractEsqlIntegTestCase { + // Precreate all attributes statically to ensure NameId matching + private static final FieldAttribute R_FIELD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "l", + new EsField("l", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY0_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey0", + new EsField("rkey0", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY0_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey0", + new EsField("rkey0", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY1_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey1", + new EsField("rkey1", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY1_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey1", + new EsField("rkey1", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY2_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey2", + new EsField("rkey2", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY2_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey2", + new EsField("rkey2", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY3_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey3", + new EsField("rkey3", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute RKEY3_INTEGER_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "rkey3", + new EsField("rkey3", DataType.INTEGER, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + + // Precreate left-side key attributes (from source index) - up to 4 keys as seen in tests + private static final FieldAttribute KEY0_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key0", + new EsField("key0", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY0_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key0", + new EsField("key0", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY1_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key1", + new EsField("key1", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY1_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key1", + new EsField("key1", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY2_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key2", + new EsField("key2", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY2_LONG_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key2", + new EsField("key2", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY3_KEYWORD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key3", + new EsField("key3", DataType.KEYWORD, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute KEY3_INTEGER_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "key3", + new EsField("key3", DataType.INTEGER, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) + ); + + // Index all attributes by name and type for easy lookup - built dynamically from attribute names + // Key format: "name:type" (e.g., "key0:KEYWORD", "key0:LONG", "rkey0:KEYWORD", "l:LONG") + private static final Map ATTRIBUTES_BY_NAME_AND_TYPE = Map.ofEntries( + // Left-side attributes (from source index) + Map.entry(KEY0_KEYWORD_ATTR.name() + ":" + KEY0_KEYWORD_ATTR.dataType(), KEY0_KEYWORD_ATTR), + Map.entry(KEY0_LONG_ATTR.name() + ":" + KEY0_LONG_ATTR.dataType(), KEY0_LONG_ATTR), + Map.entry(KEY1_KEYWORD_ATTR.name() + ":" + KEY1_KEYWORD_ATTR.dataType(), KEY1_KEYWORD_ATTR), + Map.entry(KEY1_LONG_ATTR.name() + ":" + KEY1_LONG_ATTR.dataType(), KEY1_LONG_ATTR), + Map.entry(KEY2_KEYWORD_ATTR.name() + ":" + KEY2_KEYWORD_ATTR.dataType(), KEY2_KEYWORD_ATTR), + Map.entry(KEY2_LONG_ATTR.name() + ":" + KEY2_LONG_ATTR.dataType(), KEY2_LONG_ATTR), + Map.entry(KEY3_KEYWORD_ATTR.name() + ":" + KEY3_KEYWORD_ATTR.dataType(), KEY3_KEYWORD_ATTR), + Map.entry(KEY3_INTEGER_ATTR.name() + ":" + KEY3_INTEGER_ATTR.dataType(), KEY3_INTEGER_ATTR), + // Right-side attributes (from lookup index) + Map.entry(RKEY0_KEYWORD_ATTR.name() + ":" + RKEY0_KEYWORD_ATTR.dataType(), RKEY0_KEYWORD_ATTR), + Map.entry(RKEY0_LONG_ATTR.name() + ":" + RKEY0_LONG_ATTR.dataType(), RKEY0_LONG_ATTR), + Map.entry(RKEY1_KEYWORD_ATTR.name() + ":" + RKEY1_KEYWORD_ATTR.dataType(), RKEY1_KEYWORD_ATTR), + Map.entry(RKEY1_LONG_ATTR.name() + ":" + RKEY1_LONG_ATTR.dataType(), RKEY1_LONG_ATTR), + Map.entry(RKEY2_KEYWORD_ATTR.name() + ":" + RKEY2_KEYWORD_ATTR.dataType(), RKEY2_KEYWORD_ATTR), + Map.entry(RKEY2_LONG_ATTR.name() + ":" + RKEY2_LONG_ATTR.dataType(), RKEY2_LONG_ATTR), + Map.entry(RKEY3_KEYWORD_ATTR.name() + ":" + RKEY3_KEYWORD_ATTR.dataType(), RKEY3_KEYWORD_ATTR), + Map.entry(RKEY3_INTEGER_ATTR.name() + ":" + RKEY3_INTEGER_ATTR.dataType(), RKEY3_INTEGER_ATTR), + Map.entry(R_FIELD_ATTR.name() + ":" + R_FIELD_ATTR.dataType(), R_FIELD_ATTR) + ); + + /** + * Gets a FieldAttribute by name and type. Throws IllegalArgumentException if not found. + */ + private static FieldAttribute getAttribute(String name, DataType type) { + String key = name + ":" + type; + FieldAttribute attr = ATTRIBUTES_BY_NAME_AND_TYPE.get(key); + if (attr == null) { + throw new IllegalArgumentException("Attribute not found: " + key); + } + return attr; + } + public void testKeywordKey() throws IOException { runLookup(List.of(DataType.KEYWORD), new UsingSingleLookupTable(new Object[][] { new String[] { "aa", "bb", "cc", "dd" } }), null); } @@ -121,8 +283,9 @@ public void testJoinOnThreeKeys() throws IOException { } public void testJoinOnFourKeys() throws IOException { + List keyTypes = List.of(DataType.KEYWORD, DataType.LONG, DataType.KEYWORD, DataType.INTEGER); runLookup( - List.of(DataType.KEYWORD, DataType.LONG, DataType.KEYWORD, DataType.INTEGER), + keyTypes, new UsingSingleLookupTable( new Object[][] { new String[] { "aa", "bb", "cc", "dd" }, @@ -130,15 +293,16 @@ public void testJoinOnFourKeys() throws IOException { new String[] { "one", "two", "three", "four" }, new Integer[] { 1, 2, 3, 4 }, } ), - buildGreaterThanFilter(1L) + buildGreaterThanFilter(1L, keyTypes) ); } public void testLongKey() throws IOException { + List keyTypes = List.of(DataType.LONG); runLookup( - List.of(DataType.LONG), + keyTypes, new UsingSingleLookupTable(new Object[][] { new Long[] { 12L, 33L, 1L } }), - buildGreaterThanFilter(0L) + buildGreaterThanFilter(0L, keyTypes) ); } @@ -146,10 +310,11 @@ public void testLongKey() throws IOException { * LOOKUP multiple results match. */ public void testLookupIndexMultiResults() throws IOException { + List keyTypes = List.of(DataType.KEYWORD); runLookup( - List.of(DataType.KEYWORD), + keyTypes, new UsingSingleLookupTable(new Object[][] { new String[] { "aa", "bb", "bb", "dd" } }), - buildGreaterThanFilter(-1L) + buildGreaterThanFilter(-1L, keyTypes) ); } @@ -239,18 +404,60 @@ public void populate(int docCount, List expected, Predicate fil } } - private PhysicalPlan buildGreaterThanFilter(long value) { - FieldAttribute filterAttribute = new FieldAttribute( - Source.EMPTY, - "l", - new EsField("l", DataType.LONG, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) - ); - Expression greaterThan = new GreaterThan(Source.EMPTY, filterAttribute, new Literal(Source.EMPTY, value, DataType.LONG)); - EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), List.of()); + private PhysicalPlan buildGreaterThanFilter(long value, List keyTypes) { + Expression greaterThan = new GreaterThan(Source.EMPTY, R_FIELD_ATTR, new Literal(Source.EMPTY, value, DataType.LONG)); + List rightSideAttributes = buildRightSideAttributes(keyTypes); + rightSideAttributes.add(R_FIELD_ATTR); + EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), rightSideAttributes); Filter filter = new Filter(Source.EMPTY, esRelation, greaterThan); return new FragmentExec(filter); } + /** + * Builds a rightPreJoinPlan with right-side attributes, optionally including fields referenced in a filter expression. + * This ensures collectRightSideFieldNameIds can always find the right-side fields. + */ + private PhysicalPlan buildRightPreJoinPlan(List keyTypes, Expression filterExpression) { + List rightSideAttributes = buildRightSideAttributes(keyTypes); + // If there's a filter expression, check if it references R_FIELD_ATTR and include it + if (filterExpression != null) { + Set referencedIds = new HashSet<>(); + for (Attribute attr : filterExpression.references()) { + referencedIds.add(attr.id()); + } + // If R_FIELD_ATTR is referenced, add it to the EsRelation + if (referencedIds.contains(R_FIELD_ATTR.id())) { + rightSideAttributes.add(R_FIELD_ATTR); + } + } + EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), rightSideAttributes); + return new FragmentExec(esRelation); + } + + /** + * Gets the left-side attribute for a given index and type. + * This ensures consistent NameId usage across MatchConfig and join conditions. + */ + private FieldAttribute getLeftSideAttribute(int index, DataType keyType) { + return getAttribute("key" + index, keyType); + } + + /** + * Gets the right-side attribute for a given index and type. + * This ensures consistent NameId usage across join conditions and rightPreJoinPlan. + */ + private FieldAttribute getRightSideAttribute(int index, DataType keyType) { + return getAttribute("rkey" + index, keyType); + } + + private List buildRightSideAttributes(List keyTypes) { + List rightSideAttributes = new ArrayList<>(); + for (int i = 0; i < keyTypes.size(); i++) { + rightSideAttributes.add(getRightSideAttribute(i, keyTypes.get(i))); + } + return rightSideAttributes; + } + private void runLookup(List keyTypes, PopulateIndices populateIndices, PhysicalPlan pushedDownFilter) throws IOException { String[] fieldMappers = new String[keyTypes.size() * 2]; for (int i = 0; i < keyTypes.size(); i++) { @@ -377,16 +584,9 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, List joinOnConditions = new ArrayList<>(); if (expressionJoin) { for (int i = 0; i < keyTypes.size(); i++) { - FieldAttribute leftAttr = new FieldAttribute( - Source.EMPTY, - "key" + i, - new EsField("key" + i, keyTypes.get(0), Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) - ); - FieldAttribute rightAttr = new FieldAttribute( - Source.EMPTY, - "rkey" + i, - new EsField("rkey" + i, keyTypes.get(i), Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) - ); + // Use precreated static attributes to ensure NameId consistency + FieldAttribute leftAttr = getLeftSideAttribute(i, keyTypes.get(i)); + FieldAttribute rightAttr = getRightSideAttribute(i, keyTypes.get(i)); joinOnConditions.add(new Equals(Source.EMPTY, leftAttr, rightAttr)); // randomly decide to apply the filter as additional join on filter instead of pushed down filter boolean applyAsJoinOnCondition = EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() @@ -396,14 +596,17 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, && pushedDownFilter instanceof FragmentExec fragmentExec && fragmentExec.fragment() instanceof Filter filter) { joinOnConditions.add(filter.condition()); - pushedDownFilter = null; + pushedDownFilter = new FragmentExec(filter.child()); } } } // the matchFields are shared for both types of join + // Use precreated static attributes to ensure NameId consistency with join conditions for (int i = 0; i < keyTypes.size(); i++) { - matchFields.add(new MatchConfig("key" + i, i + 1, keyTypes.get(i))); + FieldAttribute keyAttr = getLeftSideAttribute(i, keyTypes.get(i)); + matchFields.add(new MatchConfig(keyAttr, i + 1, keyTypes.get(i))); } + PhysicalPlan rightPreJoinPlan = pushedDownFilter != null ? pushedDownFilter : buildRightPreJoinPlan(keyTypes, null); LookupFromIndexOperator.Factory lookup = new LookupFromIndexOperator.Factory( matchFields, "test", @@ -414,7 +617,7 @@ private void runLookup(List keyTypes, PopulateIndices populateIndices, "lookup", List.of(new Alias(Source.EMPTY, "l", new ReferenceAttribute(Source.EMPTY, "l", DataType.LONG))), Source.EMPTY, - pushedDownFilter, + rightPreJoinPlan, Predicates.combineAnd(joinOnConditions) ); DriverContext driverContext = driverContext(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinGeneralExpressionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinGeneralExpressionIT.java new file mode 100644 index 0000000000000..db901ff8cdba5 --- /dev/null +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinGeneralExpressionIT.java @@ -0,0 +1,1047 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.action; + +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.CsvTestsDataLoader; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; + +import java.io.IOException; +import java.net.URL; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.elasticsearch.xpack.esql.action.EsqlQueryRequest.syncEsqlQueryRequest; + +@ClusterScope(scope = SUITE, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) +@TestLogging( + value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", + reason = "debug lookup join expression resolution" +) +public class LookupJoinGeneralExpressionIT extends AbstractEsqlIntegTestCase { + + private static final String MAIN_INDEX = "main_index"; + private static final String LOOKUP_INDEX = "lookup_index"; + private static final String MULTI_COL_MAIN_INDEX = "multi_column_joinable"; + private static final String MULTI_COL_LOOKUP_INDEX = "multi_column_joinable_lookup"; + + @Override + protected Collection> nodePlugins() { + return List.of(EsqlPlugin.class, MapperExtrasPlugin.class); + } + + private void ensureIndicesAndData() { + assumeTrue( + "LOOKUP JOIN ON general expressions requires capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() + ); + if (indexExists(MAIN_INDEX) == false) { + createIndices(); + indexData(); + } + } + + private void ensureMultiColumnIndicesAndData() { + assumeTrue( + "LOOKUP JOIN ON general expressions requires capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() + ); + if (indexExists(MULTI_COL_MAIN_INDEX) == false) { + loadMultiColumnDataFromCsv(); + } + } + + private void createIndices() { + // Create main index + CreateIndexRequestBuilder mainIndexRequest = prepareCreate(MAIN_INDEX).setSettings( + Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0) + ).setMapping(""" + { + "properties": { + "id": {"type": "integer"}, + "name": {"type": "keyword"}, + "value": {"type": "integer"}, + "score": {"type": "double"}, + "description": {"type": "text"}, + "category": {"type": "keyword"}, + "active": {"type": "boolean"} + } + } + """); + + assertAcked(mainIndexRequest); + + // Create lookup index with lookup mode + CreateIndexRequestBuilder lookupIndexRequest = prepareCreate(LOOKUP_INDEX).setSettings( + Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.mode", IndexMode.LOOKUP.getName()) + ).setMapping(""" + { + "properties": { + "lookup_id": {"type": "integer"}, + "lookup_name": {"type": "keyword"}, + "lookup_value": {"type": "integer"}, + "lookup_score": {"type": "double"}, + "lookup_description": {"type": "text"}, + "lookup_category": {"type": "keyword"}, + "lookup_active": {"type": "boolean"}, + "priority": {"type": "integer"} + } + } + """); + + assertAcked(lookupIndexRequest); + } + + private void indexData() { + // Index main index data + client().prepareIndex(MAIN_INDEX).setId("1").setSource(""" + {"id": 1, "name": "Alice", "value": 10, "score": 85.5, "description": "test description", "category": "A", "active": true} + """, XContentType.JSON).get(); + client().prepareIndex(MAIN_INDEX).setId("2").setSource(""" + {"id": 2, "name": "Bob", "value": 20, "score": 90.0, "description": "some text", "category": "B", "active": true} + """, XContentType.JSON).get(); + client().prepareIndex(MAIN_INDEX).setId("3").setSource(""" + {"id": 3, "name": "Charlie", "value": 30, "score": 75.5, "description": "different text", "category": "A", "active": false} + """, XContentType.JSON).get(); + client().prepareIndex(MAIN_INDEX).setId("4").setSource(""" + {"id": 4, "name": "David", "value": 40, "score": 95.0, "description": "description", "category": "C", "active": true} + """, XContentType.JSON).get(); + + // Index lookup index data + client().prepareIndex(LOOKUP_INDEX).setId("1").setSource(""" + {"lookup_id": 1, "lookup_name": "Alice", "lookup_value": 10, "lookup_score": 85.5, + "lookup_description": "test description", "lookup_category": "A", + "lookup_active": true, "priority": 1} + """, XContentType.JSON).get(); + client().prepareIndex(LOOKUP_INDEX).setId("2").setSource(""" + {"lookup_id": 2, "lookup_name": "Bob", "lookup_value": 20, "lookup_score": 90.0, + "lookup_description": "another test", "lookup_category": "B", "lookup_active": true, "priority": 2} + """, XContentType.JSON).get(); + client().prepareIndex(LOOKUP_INDEX).setId("3").setSource(""" + {"lookup_id": 3, "lookup_name": "Eve", "lookup_value": 50, "lookup_score": 80.0, + "lookup_description": "different", "lookup_category": "A", "lookup_active": false, "priority": 3} + """, XContentType.JSON).get(); + client().prepareIndex(LOOKUP_INDEX).setId("4").setSource(""" + {"lookup_id": 4, "lookup_name": "Frank", "lookup_value": 60, "lookup_score": 70.0, + "lookup_description": "test", "lookup_category": "B", "lookup_active": true, "priority": 4} + """, XContentType.JSON).get(); + + refresh(MAIN_INDEX, LOOKUP_INDEX); + } + + @Override + protected boolean addMockHttpTransport() { + return false; // Need real HTTP transport for RestClient + } + + /** + * Loads multi-column indices and data using the existing CsvTestsDataLoader infrastructure. + * Loads only the two datasets needed for multi-column join tests. + */ + private void loadMultiColumnDataFromCsv() { + try { + RestClient restClient = getRestClient(); + org.elasticsearch.logging.Logger logger = org.elasticsearch.logging.LogManager.getLogger(CsvTestsDataLoader.class); + + // Load multi_column_joinable + loadSingleDataset( + restClient, + logger, + MULTI_COL_MAIN_INDEX, + "mapping-multi_column_joinable.json", + "multi_column_joinable.csv", + Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build() + ); + + // Load multi_column_joinable_lookup + Settings lookupSettings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .put("index.mode", IndexMode.LOOKUP.getName()) + .build(); + + loadSingleDataset( + restClient, + logger, + MULTI_COL_LOOKUP_INDEX, + "mapping-multi_column_joinable_lookup.json", + "multi_column_joinable_lookup.csv", + lookupSettings + ); + + // Force refresh to make data available immediately + refresh(MULTI_COL_MAIN_INDEX, MULTI_COL_LOOKUP_INDEX); + } catch (IOException e) { + throw new AssertionError("Failed to load CSV data", e); + } + } + + /** + * Loads a single dataset following the CsvTestsDataLoader.load() pattern. + * Creates the index and loads CSV data using the public loadCsvData() method. + */ + private void loadSingleDataset( + RestClient restClient, + org.elasticsearch.logging.Logger logger, + String indexName, + String mappingFileName, + String csvFileName, + Settings indexSettings + ) throws IOException { + URL mappingResource = CsvTestsDataLoader.class.getResource("/" + mappingFileName); + URL csvResource = CsvTestsDataLoader.class.getResource("/data/" + csvFileName); + if (mappingResource == null || csvResource == null) { + throw new IllegalArgumentException("Cannot find resources for " + indexName); + } + + String mappingContent = CsvTestsDataLoader.readTextFile(mappingResource); + ESRestTestCase.createIndex(restClient, indexName, indexSettings, mappingContent, null); + + // Use the public loadCsvData() method to reuse existing CSV parsing logic + CsvTestsDataLoader.loadCsvData(restClient, indexName, csvResource, false, logger); + } + + private EsqlQueryResponse runQuery(String query) { + return run(syncEsqlQueryRequest(query)); + } + + /** + * Verifies that the query results match the expected data. + * @param actualValues The actual results from the query + * @param expectedRows Expected rows, where each row is an array of expected values in column order + */ + private void verifyResults(List> actualValues, Object[]... expectedRows) { + if (actualValues.size() != expectedRows.length) { + fail( + String.format( + Locale.ROOT, + "Result count mismatch.%nExpected: %d rows%nActual: %d rows%n%nExpected results:%n%s%n%nActual results:%n%s", + expectedRows.length, + actualValues.size(), + formatRows(expectedRows), + formatRows(actualValues) + ) + ); + } + for (int i = 0; i < expectedRows.length; i++) { + List actualRow = actualValues.get(i); + Object[] expectedRow = expectedRows[i]; + if (actualRow.size() != expectedRow.length) { + fail( + String.format( + Locale.ROOT, + "Row %d column count mismatch.%nExpected: %d columns%nActual: %d " + + "columns%n%nExpected row %d: %s%nActual row %d: %s%n%nAll " + + "expected results:%n%s%n%nAll actual results:%n%s", + i, + expectedRow.length, + actualRow.size(), + i, + formatRow(expectedRow), + i, + formatRow(actualRow), + formatRows(expectedRows), + formatRows(actualValues) + ) + ); + } + for (int j = 0; j < expectedRow.length; j++) { + if (Objects.equals(actualRow.get(j), expectedRow[j]) == false) { + fail( + String.format( + Locale.ROOT, + "Row %d, column %d mismatch.%nExpected: %s%nActual: %s%n%n" + + "Expected row %d: %s%nActual row %d: %s%n%nAll expected results:" + + "%n%s%n%nAll actual results:%n%s", + i, + j, + expectedRow[j], + actualRow.get(j), + i, + formatRow(expectedRow), + i, + formatRow(actualRow), + formatRows(expectedRows), + formatRows(actualValues) + ) + ); + } + } + } + } + + private String formatRows(Object[]... rows) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < rows.length; i++) { + sb.append("Row ").append(i).append(": ").append(formatRow(rows[i])).append("\n"); + } + return sb.toString(); + } + + private String formatRows(List> rows) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < rows.size(); i++) { + sb.append("Row ").append(i).append(": ").append(formatRow(rows.get(i))).append("\n"); + } + return sb.toString(); + } + + private String formatRow(Object[] row) { + return java.util.Arrays.toString(row); + } + + private String formatRow(List row) { + return row.toString(); + } + + // Test cases for general expressions in LOOKUP JOIN ON conditions + + public void testFilterOnRightSideLucenePushableMatch() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND lookup_name == "Alice" + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + verifyResults( + values, + new Object[] { 1, "Alice", "Alice", 10 } // id, name, lookup_name, lookup_value + ); + } + } + + public void testFilterOnRightSideNonPushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND ABS(lookup_value) > 15 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_name, lookup_value + | SORT id, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3,4 (ABS(lookup_value) > 15); id=1 excluded (ABS(10) <= 15) + verifyResults( + values, + new Object[] { 2, "Bob", "Bob", 20 }, // id, name, lookup_name, lookup_value + new Object[] { 3, "Charlie", "Eve", 50 }, + new Object[] { 4, "David", "Frank", 60 } + ); + } + } + + public void testFilterOnLeftSideLucenePushableRange() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND name == lookup_name AND value >= 20 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2 (value>=20 and name matches); id=3,4 excluded (name doesn't match) + verifyResults( + values, + new Object[] { 2, "Bob", 20, "Bob", 20 } // id, name, value, lookup_name, lookup_value + ); + } + } + + public void testFilterOnLeftSideNonPushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND name == lookup_name AND ABS(value) > 15 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // The JOIN ON condition evaluates: id == lookup_id AND name == lookup_name AND ABS(value) > 15 + // For each left row, we check if it matches any right row: + // id=1: ABS(10)=10 <= 15, condition fails → excluded + // id=2: ABS(20)=20 > 15, id==lookup_id(2), name=="Bob"==lookup_name → MATCH + // id=3: ABS(30)=30 > 15, id==lookup_id(3), but name=="Charlie" != lookup_name=="Eve" → excluded + // id=4: ABS(40)=40 > 15, id==lookup_id(4), but name=="David" != lookup_name=="Frank" → excluded + // Expected: only id=2 matches + verifyResults( + values, + new Object[] { 2, "Bob", 20, "Bob", 20 } // id, name, value, lookup_name, lookup_value + ); + } + } + + public void testFilterOnBothSidesBothLucenePushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND lookup_category == "A" AND value >= 10 AND value <= 30 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_category + | SORT id, value, lookup_name + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=1,3 (value in range and lookup_category="A"); id=2 excluded (category="B"), id=4 excluded (value>30) + verifyResults( + values, + new Object[] { 1, "Alice", 10, "Alice", "A" }, // id, name, value, lookup_name, lookup_category + new Object[] { 3, "Charlie", 30, "Eve", "A" } + ); + } + } + + public void testFilterOnBothSidesBothNonPushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND ABS(lookup_value) > 15 AND ABS(value) > 15 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3,4 (both ABS conditions > 15); id=1 excluded (ABS(10) <= 15) + verifyResults( + values, + new Object[] { 2, "Bob", 20, "Bob", 20 }, // id, name, value, lookup_name, lookup_value + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testFilterOnBothSidesMixed() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND ABS(lookup_value) > 15 AND value >= 20 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3,4 (value>=20 and ABS(lookup_value)>15) + verifyResults( + values, + new Object[] { 2, "Bob", 20, "Bob", 20 }, // id, name, value, lookup_name, lookup_value + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testGeneralExpressionJoinWithRightSideFilter() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (lookup_name == "Alice" OR lookup_name == "Bob") + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_name, lookup_value + | SORT id, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=1,2 (lookup_name is "Alice" or "Bob") + verifyResults( + values, + new Object[] { 1, "Alice", "Alice", 10 }, // id, name, lookup_name, lookup_value + new Object[] { 2, "Bob", "Bob", 20 } + ); + } + } + + public void testGeneralExpressionJoinWithLeftSideFilter() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND name == lookup_name AND ((value >= 10 AND value <= 20) OR name == "Charlie") + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=1,2 (value in range and name matches); id=3 excluded (name doesn't match) + verifyResults( + values, + new Object[] { 1, "Alice", 10, "Alice", 10 }, // id, name, value, lookup_name, lookup_value + new Object[] { 2, "Bob", 20, "Bob", 20 } + ); + } + } + + public void testGeneralExpressionJoinComplexCondition() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (lookup_name == name OR (lookup_value == value AND lookup_category == category)) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_name, lookup_value, lookup_category + | SORT id, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=1,2 (lookup_name matches name); id=3,4 excluded (neither condition matches) + verifyResults( + values, + new Object[] { 1, "Alice", "Alice", 10, "A" }, // id, name, lookup_name, lookup_value, lookup_category + new Object[] { 2, "Bob", "Bob", 20, "B" } + ); + } + } + + public void testFilterOnRightSideNonPushableNotInOutput() { + ensureIndicesAndData(); + // Test with a non-pushable filter on the right side (ABS(lookup_value) > 15) + // The lookup_value field is used in the filter but NOT in KEEP, ensuring it's not selected to output + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND ABS(lookup_value) > 15 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_name, lookup_category + | SORT id, lookup_name, lookup_category + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + verifyResults( + values, + new Object[] { 2, "Bob", "Bob", "B" }, // id, name, lookup_name, lookup_category + new Object[] { 3, "Charlie", "Eve", "A" }, // id=3 matches lookup_id=3, ABS(50)=50 > 15 + new Object[] { 4, "David", "Frank", "B" } // id=4 matches lookup_id=4, ABS(60)=60 > 15 + ); + } + } + + /* + public void testMatchOnLeftSideOrRightSideFilter() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (MATCH(description, "test") OR lookup_value > 20) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, description, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: 3 rows match + // id=1: MATCH("test description", "test") = true OR lookup_value=10 > 20 = false → true → JOIN succeeds (via MATCH only) + // id=2: MATCH("some text", "test") = false OR lookup_value=20 > 20 = false → false → JOIN fails + // id=3: MATCH("different text", "test") = false OR lookup_value=50 > 20 = true → true → JOIN succeeds (via lookup_value > 20) + // id=4: MATCH("description", "test") = false OR lookup_value=60 > 20 = true → true → JOIN succeeds (via lookup_value > 20) + verifyResults( + values, + new Object[] { 1, "Alice", "test description", "Alice", 10 }, // id, name, description, lookup_name, lookup_value + new Object[] { 3, "Charlie", "different text", "Eve", 50 }, + new Object[] { 4, "David", "description", "Frank", 60 } + ); + } + } + + public void testMatchOnLeftSideOrLeftSidePushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (MATCH(description, "test") OR value >= 25) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, description, value, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: 3 rows match + // id=1: MATCH("test description", "test") = true OR value=10 >= 25 = false → true → JOIN succeeds (via MATCH only) + // id=2: MATCH("some text", "test") = false OR value=20 >= 25 = false → false → JOIN fails + // id=3: MATCH("different text", "test") = false OR value=30 >= 25 = true → true → JOIN succeeds (via value >= 25) + // id=4: MATCH("description", "test") = false OR value=40 >= 25 = true → true → JOIN succeeds (via value >= 25) + verifyResults( + values, + new Object[] { 1, "Alice", "test description", 10, "Alice", 10 }, // id, name, description, value, lookup_name, + // lookup_value + new Object[] { 3, "Charlie", "different text", 30, "Eve", 50 }, + new Object[] { 4, "David", "description", 40, "Frank", 60 } + ); + } + } + + public void testMatchOnLeftSideOrLeftSideNonPushable() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (MATCH(description, "test") OR ABS(value) > 25) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, description, value, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: 3 rows match + // id=1: MATCH("test description", "test") = true OR ABS(10)=10 > 25 = false → true → JOIN succeeds (via MATCH only) + // id=2: MATCH("some text", "test") = false OR ABS(20)=20 > 25 = false → false → JOIN fails + // id=3: MATCH("different text", "test") = false OR ABS(30)=30 > 25 = true → true → JOIN succeeds (via ABS(value) > 25) + // id=4: MATCH("description", "test") = false OR ABS(40)=40 > 25 = true → true → JOIN succeeds (via ABS(value) > 25) + verifyResults( + values, + new Object[] { 1, "Alice", "test description", 10, "Alice", 10 }, // id, name, description, value, lookup_name, + // lookup_value + new Object[] { 3, "Charlie", "different text", 30, "Eve", 50 }, + new Object[] { 4, "David", "description", 40, "Frank", 60 } + ); + } + } */ + + public void testMatchOnRightSideOrLeftSideCondition() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (MATCH(lookup_description, "description") OR value >= 30) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, description, value, lookup_name, lookup_value, lookup_description + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: 3 rows match + // id=1: MATCH("test description", "description") = true OR value=10 >= 30 = false → true → JOIN succeeds (via MATCH) + // id=2: MATCH("another test", "description") = false OR value=20 >= 30 = false → false → JOIN fails + // id=3: MATCH("different", "description") = false OR value=30 >= 30 = true → true → JOIN succeeds (via value >= 30) + // id=4: MATCH("test", "description") = false OR value=40 >= 30 = true → true → JOIN succeeds (via value >= 30) + verifyResults( + values, + new Object[] { 1, "Alice", "test description", 10, "Alice", 10, "test description" }, // id, name, description, value, + // lookup_name, lookup_value, + // lookup_description + new Object[] { 3, "Charlie", "different text", 30, "Eve", 50, "different" }, + new Object[] { 4, "David", "description", 40, "Frank", 60, "test" } + ); + } + } + + public void testArithmeticExpressionInJoinCondition() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id + 1 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, lookup_id, lookup_name, lookup_value + | SORT id, lookup_id, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: 3 rows match + // id=2: matches lookup_id=1 (Alice) + // id=3: matches lookup_id=2 (Bob) + // id=4: matches lookup_id=3 (Eve) + verifyResults( + values, + new Object[] { 2, "Bob", 1, "Alice", 10 }, // id, name, lookup_id, lookup_name, lookup_value + new Object[] { 3, "Charlie", 2, "Bob", 20 }, + new Object[] { 4, "David", 3, "Eve", 50 } + ); + } + } + + public void testMultiColMixedGtEqSwapLeftRight() { + ensureMultiColumnIndicesAndData(); + // Test similar to lookupMultiColMixedGtEqSwapLeftRight from CSV spec + // Uses swapped comparison: id_int < left_id (right field < left field) AND is_active_left == is_active_bool + String query = String.format(Locale.ROOT, """ + FROM %s + | RENAME id_int AS left_id, name_str AS left_name, is_active_bool AS is_active_left + | LOOKUP JOIN %s ON id_int < left_id AND is_active_left == is_active_bool + | KEEP left_id, left_name, id_int, name_str, is_active_left, is_active_bool + | SORT left_id, left_name, id_int, name_str, is_active_left, is_active_bool + | LIMIT 20 + """, MULTI_COL_MAIN_INDEX, MULTI_COL_LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + verifyResults( + values, + new Object[] { 1, "Alice", null, null, true, null }, // left_id=1: no matches (id_int < 1 means no id_int exists) + new Object[] { List.of(1, 19, 21), "Sophia", null, null, true, null }, // left_id=[1,19,21]: no matches + new Object[] { 2, "Bob", null, null, false, null }, // left_id=2: no matches (id_int=1 has is_active_bool=true != + // is_active_left=false) + new Object[] { 3, "Charlie", 1, "Alice", true, true }, // left_id=3: matches id_int=1 (duplicate - first occurrence) + new Object[] { 3, "Charlie", 1, "Alice", true, true }, // left_id=3: matches id_int=1 (duplicate - second occurrence) + new Object[] { 4, "David", 2, "Bob", false, false }, // left_id=4: matches id_int=2 + new Object[] { 4, "David", 3, "Charlie", false, false }, // left_id=4: matches id_int=3 with is_active_bool=false + new Object[] { 5, "Eve", 1, "Alice", true, true }, // left_id=5: matches id_int=1 (duplicate - first occurrence) + new Object[] { 5, "Eve", 1, "Alice", true, true }, // left_id=5: matches id_int=1 (duplicate - second occurrence) + new Object[] { 5, "Eve", 3, "Charlie", true, true }, // left_id=5: matches id_int=3 with is_active_bool=true + new Object[] { 6, null, 1, "Alice", true, true }, // left_id=6: matches id_int=1 (duplicate - first occurrence) + new Object[] { 6, null, 1, "Alice", true, true }, // left_id=6: matches id_int=1 (duplicate - second occurrence) + new Object[] { 6, null, 3, "Charlie", true, true }, // left_id=6: matches id_int=3 with is_active_bool=true + new Object[] { 6, null, 5, "Eve", true, true }, // left_id=6: matches id_int=5 (duplicate - first occurrence) + new Object[] { 6, null, 5, "Eve", true, true }, // left_id=6: matches id_int=5 (duplicate - second occurrence) + new Object[] { 7, "Grace", 2, "Bob", false, false }, // left_id=7: matches id_int=2 + new Object[] { 7, "Grace", 3, "Charlie", false, false }, // left_id=7: matches id_int=3 with is_active_bool=false + new Object[] { 7, "Grace", 4, "David", false, false }, // left_id=7: matches id_int=4 + new Object[] { 8, "Hank", 1, "Alice", true, true }, // left_id=8: matches id_int=1 (duplicate - first occurrence) + new Object[] { 8, "Hank", 1, "Alice", true, true } // left_id=8: matches id_int=1 (duplicate - second occurrence) + // Note: left_id=8 also matches id_int=3 and id_int=5, but the LIMIT 20 cuts off some results + ); + } + } + + public void testComplexExpressionRelatingLeftAndRight() { + ensureIndicesAndData(); + // Test with complex expression relating left and right: (id + lookup_value) == (value + lookup_id) + // This creates a relationship between left and right fields + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON (id + lookup_value) == (value + lookup_id) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_id, lookup_name, lookup_value + | SORT id, value, lookup_id, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Check: (id + lookup_value) == (value + lookup_id) + // id=1: (1 + 10) == (10 + 1) → 11 == 11 → MATCH + // id=2: (2 + 20) == (20 + 2) → 22 == 22 → MATCH + // id=3: (3 + 50) == (30 + 3) → 53 == 33 → NO MATCH + // id=4: (4 + 60) == (40 + 4) → 64 == 44 → NO MATCH + verifyResults(values, new Object[] { 1, "Alice", 10, 1, "Alice", 10 }, new Object[] { 2, "Bob", 20, 2, "Bob", 20 }); + } + } + + public void testNoRelationshipBetweenLeftAndRight() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON value > 15 AND lookup_value > 15 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_id, lookup_name, lookup_value + | SORT id, lookup_id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + verifyResults( + values, + new Object[] { 2, "Bob", 20, 2, "Bob", 20 }, + new Object[] { 2, "Bob", 20, 3, "Eve", 50 }, + new Object[] { 2, "Bob", 20, 4, "Frank", 60 }, + new Object[] { 3, "Charlie", 30, 2, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, 3, "Eve", 50 }, + new Object[] { 3, "Charlie", 30, 4, "Frank", 60 }, + new Object[] { 4, "David", 40, 2, "Bob", 20 }, + new Object[] { 4, "David", 40, 3, "Eve", 50 }, + new Object[] { 4, "David", 40, 4, "Frank", 60 } + ); + } + } + + public void testFilterOnLeftWithLimit() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND value >= 20 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id + | LIMIT 2 + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3 (value >= 20), limited to 2 rows + verifyResults(values, new Object[] { 2, "Bob", 20, "Bob", 20 }, new Object[] { 3, "Charlie", 30, "Eve", 50 }); + } + } + + public void testFilterOnRightWithLimit() { + ensureIndicesAndData(); + // Test filter on right side with LIMIT + // Filter: lookup_value >= 20 + // Limit: 2 + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND lookup_value >= 20 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id + | LIMIT 2 + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3,4 (lookup_value >= 20), limited to 2 rows + verifyResults(values, new Object[] { 2, "Bob", 20, "Bob", 20 }, new Object[] { 3, "Charlie", 30, "Eve", 50 }); + } + } + + public void testFilterOnLeftAndRightWithAnd() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND value >= 20 AND lookup_value >= 20 + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: id=2,3,4 (both value >= 20 AND lookup_value >= 20) + verifyResults( + values, + new Object[] { 2, "Bob", 20, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testFilterOnLeftAndRightWithOr() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND (value >= 30 OR lookup_value >= 50) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: + // id=1: value=10 < 30 AND lookup_value=10 < 50 → NO MATCH + // id=2: value=20 < 30 AND lookup_value=20 < 50 → NO MATCH + // id=3: value=30 >= 30 OR lookup_value=50 >= 50 → MATCH + // id=4: value=40 >= 30 OR lookup_value=60 >= 50 → MATCH + verifyResults(values, new Object[] { 3, "Charlie", 30, "Eve", 50 }, new Object[] { 4, "David", 40, "Frank", 60 }); + } + } + + public void testFilterOnLeftAndRightComplexAndOr() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND ((value >= 20 AND lookup_value >= 20) OR (value < 15 AND lookup_value < 15)) + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: + // id=1: value=10 < 15 AND lookup_value=10 < 15 → MATCH (second OR condition) + // id=2: value=20 >= 20 AND lookup_value=20 >= 20 → MATCH (first OR condition) + // id=3: value=30 >= 20 AND lookup_value=50 >= 20 → MATCH (first OR condition) + // id=4: value=40 >= 20 AND lookup_value=60 >= 20 → MATCH (first OR condition) + verifyResults( + values, + new Object[] { 1, "Alice", 10, "Alice", 10 }, + new Object[] { 2, "Bob", 20, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testFilterOnConditionAndTrue() { + ensureIndicesAndData(); + // Test with just TRUE as filter - should match all rows + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON id == lookup_id AND TRUE + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected: All rows where id == lookup_id (TRUE always evaluates to true) + verifyResults( + values, + new Object[] { 1, "Alice", 10, "Alice", 10 }, + new Object[] { 2, "Bob", 20, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testComplexExpressionWithArithmeticAndComparison() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON (id * 2) + lookup_value == value + lookup_id + | WHERE lookup_id IS NOT NULL + | KEEP id, name, value, lookup_id, lookup_name, lookup_value + | SORT id + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Check: (id * 2) + lookup_value == value + lookup_id + // id=1: (1*2) + 10 == 10 + 1 → 2 + 10 == 11 → 12 == 11 → NO MATCH + // id=2: (2*2) + 20 == 20 + 2 → 4 + 20 == 22 → 24 == 22 → NO MATCH + // id=3: (3*2) + 50 == 30 + 3 → 6 + 50 == 33 → 56 == 33 → NO MATCH + // id=4: (4*2) + 60 == 40 + 4 → 8 + 60 == 44 → 68 == 44 → NO MATCH + // No matches expected + verifyResults(values); + } + } + + public void testComplexExpressionWithMultipleOperations() { + ensureIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | LOOKUP JOIN %s ON ABS(value - lookup_value) < 25 + | KEEP id, name, value, lookup_name, lookup_value + | SORT id, value, lookup_name, lookup_value + """, MAIN_INDEX, LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Results sorted by id, value, lookup_name, lookup_value for consistent ordering + verifyResults( + values, + new Object[] { 1, "Alice", 10, "Alice", 10 }, + new Object[] { 1, "Alice", 10, "Bob", 20 }, + new Object[] { 2, "Bob", 20, "Alice", 10 }, + new Object[] { 2, "Bob", 20, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, "Alice", 10 }, + new Object[] { 3, "Charlie", 30, "Bob", 20 }, + new Object[] { 3, "Charlie", 30, "Eve", 50 }, + new Object[] { 4, "David", 40, "Bob", 20 }, + new Object[] { 4, "David", 40, "Eve", 50 }, + new Object[] { 4, "David", 40, "Frank", 60 } + ); + } + } + + public void testComplexExpressionRelatingLeftAndRightMultiColumn() { + ensureMultiColumnIndicesAndData(); + String query = String.format(Locale.ROOT, """ + FROM %s + | RENAME id_int AS id_left, name_str AS name_left + | LOOKUP JOIN %s ON (id_left + id_int) == (id_left * 2) + | KEEP id_left, name_left, id_int, name_str + | SORT id_left, name_left, id_int, name_str + | LIMIT 20 + """, MULTI_COL_MAIN_INDEX, MULTI_COL_LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + verifyResults( + values, + new Object[] { 1, "Alice", 1, "Alice" }, // id_left=1 matches id_int=1 (first occurrence) + new Object[] { 1, "Alice", 1, "Alice" }, // id_left=1 matches id_int=1 (second occurrence - duplicate) + new Object[] { List.of(1, 19, 21), "Sophia", null, null }, // id_left=[1,19,21] doesn't match + new Object[] { 2, "Bob", 2, "Bob" }, + new Object[] { 3, "Charlie", 3, "Charlie" }, // id_left=3 matches id_int=3 (first occurrence) + new Object[] { 3, "Charlie", 3, "Charlie" }, // id_left=3 matches id_int=3 (second occurrence) + new Object[] { 4, "David", 4, "David" }, + new Object[] { 5, "Eve", 5, "Eve" }, // id_left=5 matches id_int=5 (first occurrence) + new Object[] { 5, "Eve", 5, "Eve" }, // id_left=5 matches id_int=5 (second occurrence - duplicate) + new Object[] { 6, null, 6, null }, // id_left=6 matches id_int=6 (empty name in lookup) + new Object[] { 7, "Grace", 7, "Grace" }, + new Object[] { 8, "Hank", 8, "Hank" }, + new Object[] { 9, "Ivy", null, null }, // id_left=9 doesn't match + new Object[] { 10, "John", null, null }, // id_left=10 doesn't match + new Object[] { 12, "Liam", 12, "Liam" }, // id_left=12 matches id_int=12 + new Object[] { 13, "Mia", 13, "Mia" }, // id_left=13 matches id_int=13 + new Object[] { 14, "Nina", 14, "Nina" }, // id_left=14 matches id_int=14 (CSV has id_int=14, not [14]) + new Object[] { 15, "Oscar", null, null }, // id_left=15 doesn't match + new Object[] { List.of(17, 18), "Olivia", null, null }, // id_left=[17,18] doesn't match + new Object[] { null, "Kate", null, null } // id_left=null doesn't match + ); + } + } + + public void testLookupJoinWithLikeCondition() { + ensureMultiColumnIndicesAndData(); + // Test LOOKUP JOIN with LIKE condition in the ON clause + // Matches rows where other1 ends with "ta" (pattern "*ta") + String query = String.format(Locale.ROOT, """ + FROM %s + | RENAME id_int AS id_left, is_active_bool AS is_active_left + | LOOKUP JOIN %s ON id_int == id_left AND is_active_left == is_active_bool AND other1 LIKE "*ta" + | KEEP id_left, name_str, extra1, other1, other2 + | SORT id_left, name_str, extra1, other1, other2 + """, MULTI_COL_MAIN_INDEX, MULTI_COL_LOOKUP_INDEX); + + try (EsqlQueryResponse response = runQuery(query)) { + List> values = getValuesList(response); + // Expected results based on CSV spec: rows where other1 matches "*ta" pattern + // Note: Some rows may have null values when the LIKE condition doesn't match + verifyResults( + values, + new Object[] { 1, "Alice", "foo", "beta", 2000 }, // other1="beta" ends with "ta" + new Object[] { List.of(1, 19, 21), null, "zyx", null, null }, // no match for multi-value id + new Object[] { 2, null, "bar", null, null }, // no match (other1 doesn't end with "ta") + new Object[] { 3, "Charlie", "baz", "delta", 4000 }, // other1="delta" ends with "ta" + new Object[] { 4, "David", "qux", "zeta", 6000 }, // other1="zeta" ends with "ta" + new Object[] { 5, "Eve", "quux", "eta", 7000 }, // other1="eta" ends with "ta" + new Object[] { 5, "Eve", "quux", "theta", 8000 }, // other1="theta" ends with "ta" + new Object[] { 6, null, "corge", "iota", 9000 }, // other1="iota" ends with "ta" + new Object[] { 7, null, "grault", null, null }, // no match + new Object[] { 8, null, "garply", null, null }, // no match + new Object[] { 9, null, "waldo", null, null }, // no match + new Object[] { 10, null, "fred", null, null }, // no match + new Object[] { 12, null, "xyzzy", null, null }, // no match + new Object[] { 13, null, "thud", null, null }, // no match + new Object[] { 14, null, "foo2", null, null }, // no match + new Object[] { 15, null, "bar2", null, null }, // no match + new Object[] { List.of(17, 18), null, "xyz", null, null }, // no match for multi-value id + new Object[] { null, null, "plugh", null, null } // no match for null id + ); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index c85738e056254..6796e338a6b50 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1451,6 +1451,11 @@ public enum Cap { * Bugfix for lookup join with Full Text Function */ LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION_BUGFIX, + + /** + * Lookup join with General Expression + */ + LOOKUP_JOIN_WITH_GENERAL_EXPRESSION, /** * FORK with remote indices */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index c6d12e6770318..0179dd6ee9f15 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -190,6 +190,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; +import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -231,6 +232,10 @@ public class Analyzer extends ParameterizedRuleExecutor 15 are included in leftFields + if (context.minimumVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION) && result instanceof UnresolvedAttribute == false) { + // Extract all left-side attributes from the expression and add them to leftJoinKeysToPopulate + // This handles general expressions that reference left-side fields (e.g., ABS(value) > 15) + for (Attribute attr : condition.references()) { + if (leftChildOutput.contains(attr)) { + // Check if we've already added this attribute (by NameId to avoid duplicates) + boolean alreadyAdded = leftJoinKeysToPopulate.stream().anyMatch(a -> a.id().equals(attr.id())); + if (alreadyAdded == false) { + leftJoinKeysToPopulate.add(attr); + } + } + } + } + return result; } - private Expression handleRightOnlyPushableFilter(Expression condition, AttributeSet rightChildOutput) { + private Expression handleRightOnlyPushableFilter(Expression condition, AttributeSet rightChildOutput, AnalyzerContext context) { if (isCompletelyRightSideAndTranslatable(condition, rightChildOutput)) { // The condition is completely on the right side and is translation aware, so it can be (potentially) pushed down return condition; } else { + // Check if general expressions are enabled + if (context.minimumVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION)) { + // General expressions are enabled, allow the condition + return condition; + } // The condition cannot be used in the join on clause for now // It is not a binary comparison between left and right attributes // It is not using fields from the right side only and translation aware @@ -807,7 +836,13 @@ private Join resolveLookupJoin(LookupJoin join, AnalyzerContext context) { JoinConfig config = join.config(); // for now, support only (LEFT) USING clauses JoinType type = config.type(); - + if (context.minimumVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION) == false + && (join.config().leftFields().isEmpty() || join.config().rightFields().isEmpty())) { + throw new ParsingException( + Source.EMPTY, + "JOIN ON clause with expressions must contain at least one condition relating the left index and the lookup index" + ); + } // rewrite the join into an equi-join between the field with the same name between left and right if (type == JoinTypes.LEFT) { // the lookup cannot be resolved, bail out diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index dcfdd03fed9c1..1f2c2fab5a275 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -40,6 +40,7 @@ import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.FilterOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OutputOperator; import org.elasticsearch.compute.operator.ProjectOperator; @@ -74,20 +75,32 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders; +import org.elasticsearch.xpack.esql.planner.Layout; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.Executor; import java.util.function.Function; import java.util.stream.IntStream; @@ -339,23 +352,35 @@ private void doLookup(T request, CancellableTask task, ActionListener request.source.source().getColumnNumber(), request.source.text() ); + // creates a layout builder and appends Docs and Positions fields + Layout.Builder builder = createLookupLayoutBuilder(); + // add the main query operator (the Lucene search) LookupEnrichQueryGenerator queryList = queryList(request, shardContext.executionContext, aliasFilter, inputBlock, warnings); - var queryOperator = new EnrichQuerySourceOperator( - driverContext.blockFactory(), - EnrichQuerySourceOperator.DEFAULT_MAX_PAGE_SIZE, + EnrichQuerySourceOperator queryOperator = createQueryOperator( queryList, - new IndexedByShardIdFromSingleton<>(shardContext.context), - 0, - warnings + shardContext, + warnings, + driverContext, + builder, + request.extractFields, + releasables ); - releasables.add(queryOperator); - List operators = new ArrayList<>(); - if (request.extractFields.isEmpty() == false) { - var extractFieldsOperator = extractFieldsOperator(shardContext.context, driverContext, request.extractFields); - releasables.add(extractFieldsOperator); - operators.add(extractFieldsOperator); - } + + // get the fields from the right side, as specified in extractFields + // Also extract additional right-side fields that are referenced in post-join filters but not in extractFields + extractRightFields(queryList, request, shardContext, driverContext, builder, releasables, operators); + + // get the left side fields that are needed for filter application + // we read them from the input page and populate in the output page + extractLeftFields(queryList, request, builder, driverContext, releasables, operators); + + // add the filters to be executed after the join + // all fields needed have been populated above + addPostJoinFilterOperator(queryList, shardContext, driverContext, builder, releasables, operators); + + // append the finish pages operator at the end + // it should remove the fields that are not needed in the output (e.g. the left side fields extracted for filtering) operators.add(finishPages); /* @@ -417,6 +442,284 @@ public void onFailure(Exception e) { } } + /** + * Field for DocID in lookup operations. Contains a DocVector. + */ + private static final EsField LOOKUP_DOC_ID_FIELD = new EsField( + "$$DocID$$", + DataType.DOC_DATA_TYPE, + Map.of(), + false, + EsField.TimeSeriesFieldType.NONE + ); + + /** + * Field for Positions in lookup operations. Contains an IntBlock of positions. + */ + private static final EsField LOOKUP_POSITIONS_FIELD = new EsField( + "$$Positions$$", + DataType.INTEGER, + Map.of(), + false, + EsField.TimeSeriesFieldType.NONE + ); + + /** + * Creates a Layout.Builder for lookup operations with Docs and Positions fields. + */ + private static Layout.Builder createLookupLayoutBuilder() { + Layout.Builder builder = new Layout.Builder(); + // append the docsIds and positions to the layout + builder.append(new FieldAttribute(Source.EMPTY, null, null, LOOKUP_DOC_ID_FIELD.getName(), LOOKUP_DOC_ID_FIELD)); + builder.append(new FieldAttribute(Source.EMPTY, null, null, LOOKUP_POSITIONS_FIELD.getName(), LOOKUP_POSITIONS_FIELD)); + return builder; + } + + /** + * Creates the query operator for lookup operations and sets up the initial layout. + * Adds the operator to releasables and appends extractFields to the builder. + */ + private EnrichQuerySourceOperator createQueryOperator( + LookupEnrichQueryGenerator queryList, + LookupShardContext shardContext, + Warnings warnings, + DriverContext driverContext, + Layout.Builder builder, + List extractFields, + List releasables + ) { + var queryOperator = new EnrichQuerySourceOperator( + driverContext.blockFactory(), + EnrichQuerySourceOperator.DEFAULT_MAX_PAGE_SIZE, + queryList, + new IndexedByShardIdFromSingleton<>(shardContext.context), + 0, + warnings + ); + releasables.add(queryOperator); + builder.append(extractFields); + return queryOperator; + } + + /** + * Extracts right-side fields from the lookup index and creates the extractFields operator. + * Also extracts additional right-side fields that are referenced in post-join filters but not in extractFields. + */ + private void extractRightFields( + LookupEnrichQueryGenerator queryList, + T request, + LookupShardContext shardContext, + DriverContext driverContext, + Layout.Builder builder, + List releasables, + List operators + ) { + // Start with the original extractFields + List allExtractFields = new ArrayList<>(request.extractFields); + + // Collect additional right-side fields referenced in post-join filters but not in extractFields + collectAdditionalRightFieldsForFilters(queryList, request, builder, allExtractFields); + + // Create a single operator for all extract fields + if (allExtractFields.isEmpty() == false) { + var extractFieldsOperator = extractFieldsOperator(shardContext.context, driverContext, allExtractFields); + releasables.add(extractFieldsOperator); + operators.add(extractFieldsOperator); + } + } + + /** + * Collects additional right-side fields that are referenced in post-join filters but not in extractFields. + * These fields are added to allExtractFields and the layout builder. + */ + private void collectAdditionalRightFieldsForFilters( + LookupEnrichQueryGenerator queryList, + T request, + Layout.Builder builder, + List allExtractFields + ) { + if (queryList instanceof PostJoinFilterable postJoinFilterable) { + List postJoinFilterExpressions = postJoinFilterable.getPostJoinFilter(); + if (postJoinFilterExpressions.isEmpty() == false) { + LookupFromIndexService.TransportRequest lookupRequest = (LookupFromIndexService.TransportRequest) request; + // Build a set of extractFields NameIDs + Set extractFieldNameIds = new HashSet<>(); + for (NamedExpression extractField : request.extractFields) { + extractFieldNameIds.add(extractField.id()); + } + // Collect right-side field NameIDs from EsRelation in rightPreJoinPlan + Set rightSideFieldNameIds = collectRightSideFieldNameIds(lookupRequest); + + // Collect right-side attributes referenced in post-join filters but not in extractFields + Set addedNameIds = new HashSet<>(); + for (Expression filterExpr : postJoinFilterExpressions) { + for (Attribute attr : filterExpr.references()) { + NameId nameId = attr.id(); + // If it's a right-side field but not in extractFields, we need to extract it + if (rightSideFieldNameIds.contains(nameId) && extractFieldNameIds.contains(nameId) == false) { + if (addedNameIds.contains(nameId) == false) { + allExtractFields.add(attr); + builder.append(attr); + addedNameIds.add(nameId); + } + } + } + } + } + } + } + + /** + * Collects right-side field NameIDs from EsRelation in the rightPreJoinPlan. + * Similar to collectLeftSideFieldsToBroadcast, but collects right-side fields instead. + */ + private static Set collectRightSideFieldNameIds(LookupFromIndexService.TransportRequest request) { + Set rightSideFieldNameIds = new HashSet<>(); + if (request.getRightPreJoinPlan() instanceof FragmentExec fragmentExec) { + fragmentExec.fragment().forEachDown(EsRelation.class, esRelation -> { + for (Attribute attr : esRelation.output()) { + rightSideFieldNameIds.add(attr.id()); + } + }); + } + return rightSideFieldNameIds; + } + + /** + * Extracts left-side fields that need to be broadcast and creates the matchFields operator. + * Collects left-side fields from post-join filter expressions and broadcasts them. + */ + private void extractLeftFields( + LookupEnrichQueryGenerator queryList, + T request, + Layout.Builder builder, + DriverContext driverContext, + List releasables, + List operators + ) { + // Collect all left-side fields that will be added to the layout so we can broadcast them + List allLeftFieldsToBroadcast = collectLeftSideFieldsToBroadcast(queryList, request, builder); + // Add all left-side fields to the Page so they're available when evaluating post-join filters + // We broadcast them using the Positions block (channel 1), similar to RightChunkedLeftJoin + // The order must match the layout order + if (allLeftFieldsToBroadcast.isEmpty() == false) { + Operator matchFieldsOperator = broadcastMatchFieldsOperator(driverContext, request.inputPage, allLeftFieldsToBroadcast); + releasables.add(matchFieldsOperator); + operators.add(matchFieldsOperator); + } + } + + /** + * Adds a post-join filter operator if the queryList has post-join filter expressions. + */ + private void addPostJoinFilterOperator( + LookupEnrichQueryGenerator queryList, + LookupShardContext shardContext, + DriverContext driverContext, + Layout.Builder builder, + List releasables, + List operators + ) { + if (queryList instanceof PostJoinFilterable postJoinFilterable) { + List postJoinFilterExpressions = postJoinFilterable.getPostJoinFilter(); + if (postJoinFilterExpressions.isEmpty() == false) { + Expression combinedFilter = Predicates.combineAnd(postJoinFilterExpressions); + Operator postJoinFilter = filterExecOperator(combinedFilter, shardContext.context, driverContext, builder); + if (postJoinFilter != null) { + releasables.add(postJoinFilter); + operators.add(postJoinFilter); + } + } + } + } + + private Operator filterExecOperator( + Expression filterExpression, + EsPhysicalOperationProviders.ShardContext shardContext, + DriverContext driverContext, + Layout.Builder builder + ) { + if (filterExpression == null) { + return null; + } + + var evaluatorFactory = EvalMapper.toEvaluator( + FoldContext.small()/*is this correct*/, + filterExpression, + builder.build(), + new IndexedByShardIdFromSingleton<>(shardContext) + ); + var filterOperatorFactory = new FilterOperator.FilterOperatorFactory(evaluatorFactory); + return filterOperatorFactory.get(driverContext); + } + + /** + * Collects left-side fields from post-join filter expressions that need to be broadcast. + * Adds these fields to the layout builder and returns the list of MatchConfigs to broadcast. + */ + private List collectLeftSideFieldsToBroadcast(LookupEnrichQueryGenerator queryList, T request, Layout.Builder builder) { + List allLeftFieldsToBroadcast = new ArrayList<>(); + // Extract left-side fields from post-join filter expressions to determine what needs to be broadcast + if (queryList instanceof PostJoinFilterable postJoinFilterable) { + List postJoinFilterExpressions = postJoinFilterable.getPostJoinFilter(); + if (postJoinFilterExpressions.isEmpty() == false) { + LookupFromIndexService.TransportRequest lookupRequest = (LookupFromIndexService.TransportRequest) request; + // Build a set of extractFields NameIDs to avoid adding duplicates + Set extractFieldNameIds = new HashSet<>(); + for (NamedExpression extractField : request.extractFields) { + extractFieldNameIds.add(extractField.id()); + } + // Collect right-side field NameIDs from EsRelation in rightPreJoinPlan + Set rightSideFieldNameIds = new HashSet<>(); + if (lookupRequest.getRightPreJoinPlan() instanceof FragmentExec fragmentExec) { + fragmentExec.fragment().forEachDown(EsRelation.class, esRelation -> { + for (Attribute attr : esRelation.output()) { + rightSideFieldNameIds.add(attr.id()); + } + }); + } + + // Track which NameIDs we've already added to avoid duplicates + Set addedNameIds = new HashSet<>(); + // Traverse filter expressions and match attributes to MatchConfigs by NameID + // Exclude right-side fields and fields already in extractFields + for (Expression filterExpr : postJoinFilterExpressions) { + for (Attribute attr : filterExpr.references()) { + NameId nameId = attr.id(); + // Skip if right-side field, already in extractFields, or already found a match for + if (rightSideFieldNameIds.contains(nameId) + || extractFieldNameIds.contains(nameId) + || addedNameIds.contains(nameId)) { + continue; + } + // Find the corresponding MatchConfig for this attribute + // we do match by just name + // we made sure the same attribute is not on the right side with the checks above + for (MatchConfig matchField : lookupRequest.getMatchFields()) { + if (attr.equals(matchField.fieldName())) { + builder.append(attr); + allLeftFieldsToBroadcast.add(matchField); + addedNameIds.add(nameId); + break; + } + } + + } + } + } + } + return allLeftFieldsToBroadcast; + } + + /** + * Creates an operator that broadcasts matchFields from the inputPage to each output Page + * using the Positions block at the specified channel to determine which left-hand row each right-hand row corresponds to. + * This is similar to how RightChunkedLeftJoin broadcasts left-hand blocks. + */ + private Operator broadcastMatchFieldsOperator(DriverContext driverContext, Page inputPage, List matchFields) { + return new BroadcastMatchFieldsOperator(driverContext, inputPage, matchFields, 1); + } + private static Operator extractFieldsOperator( EsPhysicalOperationProviders.ShardContext shardContext, DriverContext driverContext, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/BroadcastMatchFieldsOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/BroadcastMatchFieldsOperator.java new file mode 100644 index 0000000000000..054e497cc27db --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/BroadcastMatchFieldsOperator.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.enrich; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.AbstractPageMappingOperator; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.util.List; + +/** + * Operator that broadcasts matchFields from the inputPage to each output Page + * using the Positions block at the specified channel to determine which left-hand row each right-hand row corresponds to. + * This is similar to how RightChunkedLeftJoin broadcasts left-hand blocks. + */ +public class BroadcastMatchFieldsOperator extends AbstractPageMappingOperator { + private final DriverContext driverContext; + private final Page inputPage; + private final List matchFields; + private final int positionsChannel; + + public BroadcastMatchFieldsOperator(DriverContext driverContext, Page inputPage, List matchFields, int positionsChannel) { + this.driverContext = driverContext; + this.inputPage = inputPage; + this.matchFields = matchFields; + this.positionsChannel = positionsChannel; + } + + @Override + protected Page process(Page page) { + // Extract Positions block from the specified channel + IntBlock positionsBlock = page.getBlock(positionsChannel); + IntVector positions = positionsBlock.asVector(); + + Block[] newBlocks = new Block[matchFields.size()]; + BlockFactory blockFactory = driverContext.blockFactory(); + int positionCount = positionsBlock.getPositionCount(); + + if (positions == null) { + // Multivalued positions - use the first value index for each position + for (int i = 0; i < matchFields.size(); i++) { + MatchConfig matchField = matchFields.get(i); + Block inputBlock = inputPage.getBlock(matchField.channel()); + Block.Builder builder = PlannerUtils.toElementType(matchField.type()).newBlockBuilder(positionCount, blockFactory); + for (int p = 0; p < positionCount; p++) { + int firstValueIndex = positionsBlock.getFirstValueIndex(p); + int valueCount = positionsBlock.getValueCount(p); + if (valueCount > 0) { + // Use the first position value + int pos = positionsBlock.getInt(firstValueIndex); + builder.copyFrom(inputBlock, pos, pos + 1); + } else { + builder.appendNull(); + } + } + newBlocks[i] = builder.build(); + } + } else { + // Single-valued positions - extract position array + int[] positionArray = new int[positions.getPositionCount()]; + for (int i = 0; i < positions.getPositionCount(); i++) { + positionArray[i] = positions.getInt(i); + } + // Filter/broadcast matchFields from inputPage + // We need to copy into new blocks using driverContext.blockFactory() to avoid thread safety issues + // We manually copy selected positions instead of using filter() to avoid accessing the wrong thread's circuit breaker + for (int i = 0; i < matchFields.size(); i++) { + MatchConfig matchField = matchFields.get(i); + Block inputBlock = inputPage.getBlock(matchField.channel()); + // Manually copy selected positions into a new block using driverContext's block factory + Block.Builder builder = PlannerUtils.toElementType(matchField.type()).newBlockBuilder(positionArray.length, blockFactory); + for (int pos : positionArray) { + builder.copyFrom(inputBlock, pos, pos + 1); + } + newBlocks[i] = builder.build(); + } + } + + // Use appendBlocks to create a new Page with matchFields appended + // This handles reference counting automatically + return page.appendBlocks(newBlocks); + } + + @Override + public String toString() { + return "BroadcastMatchFieldsOperator[matchFields=" + matchFields.size() + ", positionsChannel=" + positionsChannel + "]"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java index 934dd94770e73..9f026fa901f0e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/ExpressionQueryList.java @@ -9,6 +9,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.compute.data.Block; @@ -24,6 +25,7 @@ import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -38,7 +40,9 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION; import static org.elasticsearch.xpack.esql.enrich.AbstractLookupService.termQueryList; @@ -56,12 +60,15 @@ * It is used for field-based join when the join is on more than one field or there is a preJoinFilter * 2. Expression-based join: The join conditions are based on a complex expression that can involve multiple fields and operators. */ -public class ExpressionQueryList implements LookupEnrichQueryGenerator { +public class ExpressionQueryList implements LookupEnrichQueryGenerator, PostJoinFilterable { private final List queryLists; private final List lucenePushableFilters = new ArrayList<>(); private final SearchExecutionContext context; private final AliasFilter aliasFilter; private final LucenePushdownPredicates lucenePushdownPredicates; + private final Set rightSideFieldNameIds; + private List postJoinFilter; + private int inputPagePositionCount = -1; private ExpressionQueryList( List queryLists, @@ -77,9 +84,23 @@ private ExpressionQueryList( SearchContextStats.from(List.of(context)), new EsqlFlags(clusterService.getClusterSettings()) ); + this.rightSideFieldNameIds = collectRightSideFieldNameIds(rightPreJoinPlan); + postJoinFilter = new ArrayList<>(); buildPreJoinFilter(rightPreJoinPlan, clusterService); } + private static Set collectRightSideFieldNameIds(PhysicalPlan rightPreJoinPlan) { + Set rightSideFieldNameIds = new HashSet<>(); + if (rightPreJoinPlan != null) { + rightPreJoinPlan.forEachDown(EsSourceExec.class, esSourceExec -> { + for (Attribute attr : esSourceExec.output()) { + rightSideFieldNameIds.add(attr.id()); + } + }); + } + return rightSideFieldNameIds; + } + /** * Creates a new {@link ExpressionQueryList} for a field-based join. * A field-based join is a join where the join conditions are based on the equality of fields from the left and right datasets. @@ -146,19 +167,83 @@ private void buildJoinOnForExpressionJoin( ClusterService clusterService, Warnings warnings ) { + this.inputPagePositionCount = inputPage.getPositionCount(); List expressions = Predicates.splitAnd(joinOnConditions); - for (Expression expr : expressions) { + + // Split expressions into left-only, right-only, and mixed + // Anything not in right-side fields is left-side + List leftOnlyExpressions = new ArrayList<>(); + List rightOnlyExpressions = new ArrayList<>(); + List mixedExpressions = new ArrayList<>(); + splitExpressionsBySide(expressions, leftOnlyExpressions, rightOnlyExpressions, mixedExpressions); + + // Process mixed expressions - try as left-right binary comparison first + // If that fails, add to post-join filter + for (Expression expr : mixedExpressions) { boolean applied = applyAsLeftRightBinaryComparison(expr, matchFields, inputPage, clusterService, warnings); if (applied == false) { - applied = applyAsRightSidePushableFilter(expr); + postJoinFilter.add(expr); } + } + + // Process right-only expressions as right-side pushable filters + // If that fails, add to post-join filter + for (Expression expr : rightOnlyExpressions) { + boolean applied = applyAsRightSidePushableFilter(expr, matchFields); if (applied == false) { - throw new IllegalArgumentException("Cannot apply join condition: " + expr); + postJoinFilter.add(expr); + } + } + + // Process left-only expressions as post-join filters + postJoinFilter.addAll(leftOnlyExpressions); + } + + private void splitExpressionsBySide( + List expressions, + List leftOnlyExpressions, + List rightOnlyExpressions, + List mixedExpressions + ) { + for (Expression expr : expressions) { + List allAttributes = new ArrayList<>(); + expr.forEachDown(Attribute.class, allAttributes::add); + + boolean hasLeftSide = false; + boolean hasRightSide = false; + + for (Attribute attr : allAttributes) { + NameId nameId = attr.id(); + if (rightSideFieldNameIds.contains(nameId)) { + hasRightSide = true; + } else { + hasLeftSide = true; + } + } + + if (hasLeftSide && hasRightSide) { + mixedExpressions.add(expr); + } else if (hasLeftSide) { + leftOnlyExpressions.add(expr); + } else { + rightOnlyExpressions.add(expr); } } } - private boolean applyAsRightSidePushableFilter(Expression filter) { + private boolean applyAsRightSidePushableFilter(Expression filter, List matchFields) { + // Check if any attribute in the filter expression tree is from the left side + // We need to traverse the entire expression tree, not just top-level references, + // because some functions may have attributes nested in their children + List allAttributes = new ArrayList<>(); + filter.forEachDown(Attribute.class, allAttributes::add); + for (Attribute attr : allAttributes) { + if (rightSideFieldNameIds.contains(attr.id()) == false) { + // This filter references a left-side attribute, so it cannot be pushed to Lucene + return false; + } + } + // All attributes are from the right side, check if it's translatable if (filter instanceof TranslationAware translationAware) { if (TranslationAware.Translatable.YES.equals(translationAware.translatable(lucenePushdownPredicates))) { QueryBuilder queryBuilder = translationAware.asQuery(lucenePushdownPredicates, TRANSLATOR_HANDLER).toQueryBuilder(); @@ -176,6 +261,35 @@ private boolean applyAsRightSidePushableFilter(Expression filter) { return false; } + /** + * Reorients a binary comparison so that the left side is from the input page and the right side is from the lookup index. + * Returns the comparison as-is if already correctly oriented, or a swapped version if needed. + * Returns null if both attributes are from the same side (can't be reoriented). + */ + private EsqlBinaryComparison reorientBinaryComparison(EsqlBinaryComparison binaryComparison) { + if (binaryComparison.left() instanceof Attribute leftAttr && binaryComparison.right() instanceof Attribute rightAttr) { + // Determine which attribute is from the right side (lookup index) + boolean leftIsRightSide = rightSideFieldNameIds.contains(leftAttr.id()); + boolean rightIsRightSide = rightSideFieldNameIds.contains(rightAttr.id()); + + // We need exactly one attribute from the right side and one from the left side + if (leftIsRightSide == rightIsRightSide) { + // Both are from the same side, can't process as left-right comparison + return null; + } + + if (rightIsRightSide) { + // Original orientation is correct: left is from input, right is from lookup + return binaryComparison; + } else { + // Need to swap: original left is from lookup, original right is from input + // Swap the comparison and flip the operator if needed + return (EsqlBinaryComparison) binaryComparison.swapLeftAndRight(); + } + } + return null; + } + private boolean applyAsLeftRightBinaryComparison( Expression expr, List matchFields, @@ -183,17 +297,27 @@ private boolean applyAsLeftRightBinaryComparison( ClusterService clusterService, Warnings warnings ) { - if (expr instanceof EsqlBinaryComparison binaryComparison - && binaryComparison.left() instanceof Attribute leftAttribute - && binaryComparison.right() instanceof Attribute rightAttribute) { - // the left side comes from the page that was sent to the lookup node - // the right side is the field from the lookup index - // check if the left side is in the matchFields - // if it is its corresponding page is the corresponding number in inputPage + if (expr instanceof EsqlBinaryComparison binaryComparison) { + // Reorient the comparison so that left is from input page and right is from lookup index + EsqlBinaryComparison orientedComparison = reorientBinaryComparison(binaryComparison); + if (orientedComparison == null) { + // Can't reorient (both attributes from same side) + return false; + } + + // After reorientation, left is from input page and right is from lookup index + Attribute leftAttribute = (Attribute) orientedComparison.left(); + Attribute rightAttribute = (Attribute) orientedComparison.right(); + + // The left side comes from the page that was sent to the lookup node + // The right side is the field from the lookup index + // Check if the left side is in the matchFields + // If it is its corresponding page is the corresponding number in inputPage + // Compare by attribute (equals compares by NameId) to ensure we match the same attribute instance Block block = null; DataType dataType = null; for (int i = 0; i < matchFields.size(); i++) { - if (matchFields.get(i).fieldName().equals(leftAttribute.name())) { + if (matchFields.get(i).fieldName().equals(leftAttribute)) { block = inputPage.getBlock(i); dataType = matchFields.get(i).type(); break; @@ -204,7 +328,7 @@ private boolean applyAsLeftRightBinaryComparison( // special handle Equals operator // TermQuery is faster than BinaryComparisonQueryList, as it does less work per row // so here we reuse the existing logic from field based join to build a termQueryList for Equals - if (binaryComparison instanceof Equals) { + if (orientedComparison instanceof Equals) { QueryList termQueryForEquals = termQueryList(rightFieldType, context, aliasFilter, block, dataType).onlySingleValues( warnings, "LOOKUP JOIN encountered multi-value" @@ -216,7 +340,7 @@ private boolean applyAsLeftRightBinaryComparison( rightFieldType, context, block, - binaryComparison, + orientedComparison, clusterService, aliasFilter, warnings @@ -284,6 +408,11 @@ public Query getQuery(int position) { for (Query preJoinFilter : lucenePushableFilters) { builder.add(preJoinFilter, BooleanClause.Occur.FILTER); } + // If builder is empty (no queryLists and no lucenePushableFilters), + // we need to fetch all rows so post-join filters can evaluate general expression join conditions + if (queryLists.isEmpty() && lucenePushableFilters.isEmpty()) { + return new MatchAllDocsQuery(); + } return builder.build(); } @@ -295,6 +424,14 @@ public Query getQuery(int position) { */ @Override public int getPositionCount() { + if (queryLists.isEmpty()) { + // When all conditions are post-join filters or lucenePushableFilters, + // we need to return the input page position count + if (inputPagePositionCount < 0) { + throw new IllegalStateException("Input page position count not set"); + } + return inputPagePositionCount; + } int positionCount = queryLists.get(0).getPositionCount(); for (QueryList queryList : queryLists) { if (queryList.getPositionCount() != positionCount) { @@ -308,4 +445,9 @@ public int getPositionCount() { } return positionCount; } + + @Override + public List getPostJoinFilter() { + return postJoinFilter; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java index 53ebd1d89d3b4..ca04cd08a79ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperator.java @@ -184,7 +184,7 @@ private List uniqueMatchFieldsByName(List matchFields) return matchFields; } List uniqueFields = new ArrayList<>(); - Set seenFieldNames = new HashSet<>(); + Set seenFieldNames = new HashSet<>(); for (MatchConfig matchField : matchFields) { if (seenFieldNames.add(matchField.fieldName())) { uniqueFields.add(matchField); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java index 380ed83948fd8..0d1121c939510 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexService.java @@ -33,10 +33,14 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.action.EsqlQueryAction; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.plan.physical.FilterExec; @@ -46,10 +50,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION; + /** * {@link LookupFromIndexService} performs lookup against a Lookup index for * a given input page. See {@link AbstractLookupService} for how it works @@ -118,7 +126,7 @@ protected LookupEnrichQueryGenerator queryList( for (int i = 0; i < request.matchFields.size(); i++) { MatchConfig matchField = request.matchFields.get(i); QueryList q = termQueryList( - context.getFieldType(matchField.fieldName()), + context.getFieldType(matchField.fieldName().name()), context, aliasFilter, request.inputPage.getBlock(matchField.channel()), @@ -151,6 +159,35 @@ private static PhysicalPlan localLookupNodePlanning(PhysicalPlan physicalPlan) { return physicalPlan instanceof FragmentExec fragmentExec ? LocalMapper.INSTANCE.map(fragmentExec.fragment()) : null; } + /** + * Fixes MatchConfig fieldNames for older versions by finding matching attributes from joinOnConditions. + * For older transport versions, MatchConfig was serialized with just a string field name, but now we + * deserialize it as a NamedExpression. This method finds the actual attribute instances from joinOnConditions + * and updates MatchConfig to use those attributes (ensuring NameId consistency). + */ + private static void fixMatchConfigFieldNamesForOlderVersions(Expression joinOnConditions, List matchFields) { + if (joinOnConditions != null && matchFields != null) { + Map attributesByName = new HashMap<>(); + joinOnConditions.forEachDown(EsqlBinaryComparison.class, binaryComparison -> { + // Check left side + if (binaryComparison.left() instanceof Attribute leftAttr) { + attributesByName.put(leftAttr.name(), leftAttr); + } + }); + + // Update MatchConfig fieldNames to use matching attributes from joinOnConditions + for (int i = 0; i < matchFields.size(); i++) { + MatchConfig matchConfig = matchFields.get(i); + String fieldName = matchConfig.fieldName().name(); + Attribute matchingAttribute = attributesByName.get(fieldName); + if (matchingAttribute != null) { + // Create new MatchConfig with the matching attribute + matchFields.set(i, new MatchConfig(matchingAttribute, matchConfig.channel(), matchConfig.type())); + } + } + } + } + @Override protected LookupResponse createLookupResponse(List pages, BlockFactory blockFactory) throws IOException { return new LookupResponse(pages, blockFactory); @@ -244,9 +281,17 @@ static TransportRequest readFrom(StreamInput in, BlockFactory blockFactory) thro matchFields = planIn.readCollectionAsList(MatchConfig::new); } else { String matchField = in.readString(); + FieldAttribute fieldName = new FieldAttribute( + Source.EMPTY, + null, + null, + matchField, + new EsField(matchField, inputDataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + ; // For older versions, we only support a single match field. matchFields = new ArrayList<>(1); - matchFields.add(new MatchConfig(matchField, 0, inputDataType)); + matchFields.add(new MatchConfig(fieldName, 0, inputDataType)); } var source = Source.readFrom(planIn); // Source.readFrom() requires the query from the Configuration passed to PlanStreamInput. @@ -263,6 +308,10 @@ static TransportRequest readFrom(StreamInput in, BlockFactory blockFactory) thro if (in.getTransportVersion().supports(ESQL_LOOKUP_JOIN_ON_EXPRESSION)) { joinOnConditions = planIn.readOptionalNamedWriteable(Expression.class); } + + if (in.getTransportVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION) == false) { + fixMatchConfigFieldNamesForOlderVersions(joinOnConditions, matchFields); + } TransportRequest result = new TransportRequest( sessionId, shardId, @@ -287,6 +336,10 @@ public List getMatchFields() { return matchFields; } + public PhysicalPlan getRightPreJoinPlan() { + return rightPreJoinPlan; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -315,7 +368,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { // older versions only support a single match field, we already checked this above when writing the datatype // send the field name of the first and only match field here - out.writeString(matchFields.get(0).fieldName()); + out.writeString(matchFields.get(0).fieldName().name()); } source.writeTo(planOut); if (out.getTransportVersion().supports(ESQL_LOOKUP_JOIN_SOURCE_TEXT)) { @@ -336,7 +389,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override protected String extraDescription() { return " ,match_fields=" - + matchFields.stream().map(MatchConfig::fieldName).collect(Collectors.joining(", ")) + + matchFields.stream().map(x -> x.fieldName().name()).collect(Collectors.joining(", ")) + ", right_pre_join_plan=" + (rightPreJoinPlan == null ? "null" : rightPreJoinPlan.toString()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MatchConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MatchConfig.java index ed3da0d2da49e..88b40f58a426f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MatchConfig.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MatchConfig.java @@ -10,12 +10,20 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.Layout; import java.io.IOException; +import java.util.Map; import java.util.Objects; +import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION; + /** * Configuration for a field used in the join condition of a LOOKUP JOIN or ENRICH operation. *

@@ -35,32 +43,53 @@ * page sent to the lookup node. */ public final class MatchConfig implements Writeable { - private final String fieldName; + private final NamedExpression fieldName; private final int channel; private final DataType type; - public MatchConfig(String fieldName, int channel, DataType type) { + public MatchConfig(NamedExpression fieldName, int channel, DataType type) { this.fieldName = fieldName; this.channel = channel; this.type = type; } - public MatchConfig(String fieldName, Layout.ChannelAndType input) { + public MatchConfig(NamedExpression fieldName, Layout.ChannelAndType input) { this(fieldName, input.channel(), input.type()); } public MatchConfig(StreamInput in) throws IOException { - this(in.readString(), in.readInt(), DataType.readFrom(in)); + PlanStreamInput planIn = (PlanStreamInput) in; + if (in.getTransportVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION)) { + this.fieldName = planIn.readNamedWriteable(NamedExpression.class); + this.channel = in.readInt(); + this.type = DataType.readFrom(in); + } else { + // Old format: fieldName (string), channel (int), type (DataType) + String fieldNameString = in.readString(); + this.channel = in.readInt(); + this.type = DataType.readFrom(in); + this.fieldName = new FieldAttribute( + Source.EMPTY, + null, + null, + fieldNameString, + new EsField(fieldNameString, this.type, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + } } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(fieldName); + if (out.getTransportVersion().onOrAfter(ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION)) { + out.writeNamedWriteable(fieldName); + } else { + out.writeString(fieldName.name()); + } out.writeInt(channel); type.writeTo(out); } - public String fieldName() { + public NamedExpression fieldName() { return fieldName; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/PostJoinFilterable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/PostJoinFilterable.java new file mode 100644 index 0000000000000..5b2d31bcdc5c3 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/PostJoinFilterable.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.enrich; + +import org.elasticsearch.xpack.esql.core.expression.Expression; + +import java.util.List; + +/** + * An interface for a join operator that needs to have a filter applied after the join happens + * For now we use this for applying filters that are not translatable after a lookup join + */ +public interface PostJoinFilterable { + List getPostJoinFilter(); +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 1552108310480..a0b1a76c6c727 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -878,12 +878,6 @@ private JoinInfo processExpressionBasedJoin(List expressions, EsqlBa for (var f : expressions) { addJoinExpression(f, joinFields, joinExpressions, ctx); } - if (joinFields.isEmpty()) { - throw new ParsingException( - source(ctx), - "JOIN ON clause with expressions must contain at least one condition relating the left index and the lookup index" - ); - } return new JoinInfo(joinFields, joinExpressions); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 3410f8be0c027..048d5e304157a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -317,7 +317,23 @@ public boolean equals(Object obj) { @Override public void postAnalysisVerification(Failures failures) { - for (int i = 0; i < config.leftFields().size(); i++) { + // When joinOnConditions is present, leftFields and rightFields may have different sizes + // because general expressions can reference additional left-side fields that aren't in the join keys + if (config.leftFields().size() != config.rightFields().size()) { + if (config.joinOnConditions() == null) { + failures.add( + fail( + this, + "JOIN left fields count [{}] does not match right fields count [{}]", + config.leftFields().size(), + config.rightFields().size() + ) + ); + } + return; + } + int minSize = Math.min(config.leftFields().size(), config.rightFields().size()); + for (int i = 0; i < minSize; i++) { Attribute leftField = config.leftFields().get(i); Attribute rightField = config.rightFields().get(i); if (comparableTypes(leftField, rightField) == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 39dd5ea59b094..95a434c303860 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -135,10 +135,12 @@ import org.elasticsearch.xpack.esql.session.EsqlCCSUtils; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.IntStream; @@ -763,11 +765,16 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan ); } String indexName = indexSplit[1]; - if (join.leftFields().size() != join.rightFields().size()) { + // When joinOnConditions is present, leftFields can have more fields than rightFields + // because general expressions can reference additional left-side fields (e.g., ABS(value) > 15) + // In this case, we only create MatchConfig entries for equi-join fields (those with corresponding rightFields) + int equiJoinFieldCount = Math.min(join.leftFields().size(), join.rightFields().size()); + if (join.joinOnConditions() == null && join.leftFields().size() != join.rightFields().size()) { throw new IllegalArgumentException("can't plan [" + join + "]: mismatching left and right field count"); } - List matchFields = new ArrayList<>(join.leftFields().size()); - for (int i = 0; i < join.leftFields().size(); i++) { + List matchFields = new ArrayList<>(equiJoinFieldCount); + Set matchFieldIds = new HashSet<>(); + for (int i = 0; i < equiJoinFieldCount; i++) { TypedAttribute left = (TypedAttribute) join.leftFields().get(i); FieldAttribute right = (FieldAttribute) join.rightFields().get(i); Layout.ChannelAndType input = source.layout.get(left.id()); @@ -777,11 +784,11 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan // TODO: Using exactAttribute was supposed to handle TEXT fields with KEYWORD subfields - but we don't allow these in lookup // indices, so the call to exactAttribute looks redundant now. - String fieldName = right.exactAttribute().fieldName().string(); + Attribute matchFieldsAttribute = right.exactAttribute(); // we support 2 types of joins: Field name joins and Expression joins // for Field name join, we do not ship any join on expression. - // we built the Lucene query on the field name that is passed in the MatchConfig.fieldName + // we built the Lucene query on the field name that is passed in the MatchConfig.matchFieldsAttribute // so for Field name we need to pass the attribute name from the right side, because that is needed to build the query // For expression joins, we pass an expression such as left_id > right_id. // So in this case we pass in left_id as the field name, because that is what we are shipping to the lookup node @@ -791,9 +798,42 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan // e.g. LOOKUP JOIN ON left_id < right_id_1 and left_id >= right_id_2 // we want to be able to optimize this in the future and only ship the left_id once if (join.isOnJoinExpression()) { - fieldName = left.name(); + matchFieldsAttribute = left; + } + matchFields.add(new MatchConfig(matchFieldsAttribute, input)); + matchFieldIds.add(left.id()); + } + // Extract additional left-side fields referenced in joinOnConditions that aren't already in matchFields + // These are needed for evaluating general expressions like ABS(value) > 15 or MATCH(description, "test") + // We need MatchConfig entries for ALL left-side fields referenced in joinOnConditions + if (join.joinOnConditions() != null) { + // Iterate through ALL attributes referenced in joinOnConditions to find left-side fields + // This ensures we catch fields like 'description' in MATCH(description, "test") even if + // they're not explicitly in join.leftFields() + for (Attribute attr : join.joinOnConditions().references()) { + // Skip if already in matchFields + if (matchFieldIds.contains(attr.id())) { + continue; + } + // Check if this attribute is in the source layout (left side) + Layout.ChannelAndType input = source.layout.get(attr.id()); + if (input != null) { + // Check if it's not a right-side field + boolean isRightSide = false; + for (Attribute rightField : join.rightFields()) { + if (rightField.id().equals(attr.id())) { + isRightSide = true; + break; + } + } + // If it's in the source layout and not a right-side field, it's a left-side field + if (isRightSide == false) { + // Create MatchConfig with the left field name (since there's no corresponding right field) + matchFields.add(new MatchConfig(attr, input)); + matchFieldIds.add(attr.id()); + } + } } - matchFields.add(new MatchConfig(fieldName, input)); } return source.with( new LookupFromIndexOperator.Factory( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 8bfc09773993e..71e8d7ba3472a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -5653,6 +5653,48 @@ public void testSubqueryWithFullTextFunctionInMainQuery() { assertEquals("sample_data", subqueryIndex.indexPattern()); } + public void testLookupJoinExpressionOnlyRightFilter() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue( + "requires LOOKUP JOIN ON boolean expression capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + ); + String query = """ + from test + | rename languages as languages_left + | lookup join languages_lookup ON salary > 1000 + """; + + ParsingException e = expectThrows(ParsingException.class, () -> analyze(query)); + assertThat( + e.getMessage(), + containsString( + "JOIN ON clause with expressions must contain " + "at least one condition relating the left index and the lookup index" + ) + ); + } + + public void testLookupJoinExpressionMixed() { + assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + assumeTrue( + "requires LOOKUP JOIN ON boolean expression capability", + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + ); + String query = """ + from test + | rename languages as languages_left + | lookup join languages_lookup ON languages_left == language_code or salary > 1000 + """; + + ParsingException e = expectThrows(ParsingException.class, () -> analyze(query)); + assertThat( + e.getMessage(), + containsString( + "JOIN ON clause with expressions must contain at least " + "one condition relating the left index and the lookup index" + ) + ); + } + public void testLookupJoinOnFieldNotAnywhereElse() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 73027022ea030..0558c31e0a86c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -163,42 +163,6 @@ public void testJoinOnConstant() { ); } - public void testLookupJoinExpressionMixed() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); - assumeTrue( - "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() - ); - String queryString = """ - from test - | rename languages as languages_left - | lookup join languages_lookup ON languages_left == language_code or salary > 1000 - """; - - assertEquals( - "3:32: JOIN ON clause with expressions must contain at least one condition relating the left index and the lookup index", - error(queryString) - ); - } - - public void testLookupJoinExpressionOnlyRightFilter() { - assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); - assumeTrue( - "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() - ); - String queryString = """ - from test - | rename languages as languages_left - | lookup join languages_lookup ON salary > 1000 - """; - - assertEquals( - "3:32: JOIN ON clause with expressions must contain at least one condition relating the left index and the lookup index", - error(queryString) - ); - } - public void testLookupJoinExpressionFieldBasePlusRightFilterAnd() { assumeTrue("requires LOOKUP JOIN capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); assumeTrue( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index cabb14ab3877b..620f3f0075399 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -53,6 +53,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.testAnalyzerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION; +import static org.elasticsearch.xpack.esql.analysis.Analyzer.ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.TEXT_EMBEDDING_INFERENCE_ID; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultLookupResolution; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -2340,7 +2341,10 @@ public void testLookupJoinDataTypeMismatch() { assertEquals( "1:87: JOIN left field [language_code] of type [KEYWORD] is incompatible with right field [language_code] of type [INTEGER]", - error("FROM test | EVAL language_code = languages::keyword | LOOKUP JOIN languages_lookup ON language_code") + error( + "FROM test | EVAL language_code = languages::keyword | LOOKUP JOIN languages_lookup ON language_code", + ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION + ) ); } @@ -2364,7 +2368,7 @@ public void testLookupJoinExpressionAmbiguousRight() { public void testLookupJoinExpressionRightNotPushable() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() ); String queryString = """ from test @@ -2372,16 +2376,13 @@ public void testLookupJoinExpressionRightNotPushable() { | lookup join languages_lookup ON languages_left == language_code and abs(salary) > 1000 """; - assertEquals( - "3:71: Unsupported join filter expression:abs(salary) > 1000", - error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION) - ); + query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION); } public void testLookupJoinExpressionConstant() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() ); String queryString = """ from test @@ -2389,41 +2390,37 @@ public void testLookupJoinExpressionConstant() { | lookup join languages_lookup ON false and languages_left == language_code """; - assertEquals("3:35: Unsupported join filter expression:false", error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION)); + query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION); } public void testLookupJoinExpressionTranslatableButFromLeft() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() ); String queryString = """ from test | rename languages as languages_left - | lookup join languages_lookup ON languages_left == language_code and languages_left == "English" + | eval languages_left_str = to_string(languages_left) + | lookup join languages_lookup ON languages_left == language_code and languages_left_str == language_name """; - assertEquals( - "3:71: Unsupported join filter expression:languages_left == \"English\"", - error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION) - ); + query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION); } public void testLookupJoinExpressionTranslatableButMixedLeftRight() { assumeTrue( "requires LOOKUP JOIN ON boolean expression capability", - EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_FULL_TEXT_FUNCTION.isEnabled() + EsqlCapabilities.Cap.LOOKUP_JOIN_WITH_GENERAL_EXPRESSION.isEnabled() ); String queryString = """ from test | rename languages as languages_left - | lookup join languages_lookup ON languages_left == language_code and CONCAT(languages_left, language_code) == "English" + | eval language_name_left = "English" + | lookup join languages_lookup ON languages_left == language_code and CONCAT(language_name_left, language_name) == "English" """; - assertEquals( - "3:71: Unsupported join filter expression:CONCAT(languages_left, language_code) == \"English\"", - error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION) - ); + query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION); } public void testLookupJoinExpressionComplexFormula() { @@ -2434,13 +2431,11 @@ public void testLookupJoinExpressionComplexFormula() { String queryString = """ from test | rename languages as languages_left - | lookup join languages_lookup ON languages_left == language_code AND STARTSWITH(languages_left, language_code) + | eval languages_left_str = to_string(languages_left) + | lookup join languages_lookup ON languages_left == language_code AND starts_with(languages_left_str, language_name) """; - assertEquals( - "3:71: Unsupported join filter expression:STARTSWITH(languages_left, language_code)", - error(queryString, ESQL_LOOKUP_JOIN_FULL_TEXT_FUNCTION) - ); + query(queryString, ESQL_LOOKUP_JOIN_GENERAL_EXPRESSION); } public void testLookupJoinExpressionAmbiguousLeft() { @@ -3348,6 +3343,13 @@ private void query(String query, Analyzer analyzer) { analyzer.analyze(parser.createStatement(query)); } + private void query(String query, TransportVersion transportVersion) { + MutableAnalyzerContext mutableContext = (MutableAnalyzerContext) defaultAnalyzer.context(); + try (var restore = mutableContext.setTemporaryTransportVersionOnOrAfter(transportVersion)) { + query(query); + } + } + private String error(String query) { return error(query, defaultAnalyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java index bd1e40917cbad..361535b928b11 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/LookupFromIndexOperatorTests.java @@ -64,12 +64,12 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; @@ -89,7 +89,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -105,6 +104,88 @@ public class LookupFromIndexOperatorTests extends AsyncOperatorTestCase { private static final int LOOKUP_SIZE = 1000; private static final int LESS_THAN_VALUE = 40; + + // Precreate all attributes statically to ensure NameId matching + private static final FieldAttribute MATCH0_LEFT_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match0_left", + new EsField("match0_left", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute MATCH1_LEFT_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match1_left", + new EsField("match1_left", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute MATCH0_RIGHT_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match0_right", + new EsField("match0_right", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute MATCH1_RIGHT_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match1_right", + new EsField("match1_right", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute MATCH0_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match0", + new EsField("match0", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute MATCH1_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "match1", + new EsField("match1", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute LINT_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "lint", + new EsField("lint", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + private static final FieldAttribute LKWD_ATTR = new FieldAttribute( + Source.EMPTY, + null, + null, + "lkwd", + new EsField("lkwd", DataType.KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + + // Index all attributes by name for easy lookup - built dynamically from attribute names + private static final Map ATTRIBUTES_BY_NAME = Map.ofEntries( + Map.entry(MATCH0_LEFT_ATTR.name(), MATCH0_LEFT_ATTR), + Map.entry(MATCH1_LEFT_ATTR.name(), MATCH1_LEFT_ATTR), + Map.entry(MATCH0_RIGHT_ATTR.name(), MATCH0_RIGHT_ATTR), + Map.entry(MATCH1_RIGHT_ATTR.name(), MATCH1_RIGHT_ATTR), + Map.entry(MATCH0_ATTR.name(), MATCH0_ATTR), + Map.entry(MATCH1_ATTR.name(), MATCH1_ATTR), + Map.entry(LINT_ATTR.name(), LINT_ATTR), + Map.entry(LKWD_ATTR.name(), LKWD_ATTR) + ); + + /** + * Gets a FieldAttribute by name. Throws IllegalArgumentException if not found. + */ + private static FieldAttribute getAttribute(String name) { + FieldAttribute attr = ATTRIBUTES_BY_NAME.get(name); + if (attr == null) { + throw new IllegalArgumentException("Attribute not found: " + name); + } + return attr; + } + private final ThreadPool threadPool = threadPool(); private final Directory lookupIndexDirectory = newDirectory(); private final List releasables = new ArrayList<>(); @@ -238,41 +319,56 @@ protected Operator.OperatorFactory simple(SimpleOptions options) { int maxOutstandingRequests = 1; DataType inputDataType = DataType.LONG; String lookupIndex = "idx"; - List loadFields = List.of( - new ReferenceAttribute(Source.EMPTY, "lkwd", DataType.KEYWORD), - new ReferenceAttribute(Source.EMPTY, "lint", DataType.INTEGER) - ); List matchFields = new ArrayList<>(); String suffix = (operation == null) ? "" : ("_left"); for (int i = 0; i < numberOfJoinColumns; i++) { String matchField = "match" + i + suffix; - matchFields.add(new MatchConfig(matchField, i, inputDataType)); + matchFields.add(new MatchConfig(getAttribute(matchField), i, inputDataType)); } + + // Build right-side attributes for EsRelation using precreated static attributes + List rightSideAttributes = new ArrayList<>(); + if (operation == null) { + // Field-based join: use match0, match1 + rightSideAttributes.add(MATCH0_ATTR); + if (numberOfJoinColumns >= 2) { + rightSideAttributes.add(MATCH1_ATTR); + } + } else { + // Expression-based join: use match0_right, match1_right + rightSideAttributes.add(MATCH0_RIGHT_ATTR); + if (numberOfJoinColumns >= 2) { + rightSideAttributes.add(MATCH1_RIGHT_ATTR); + } + } + rightSideAttributes.add(LINT_ATTR); + + // Build loadFields - only include fields that should be in the final output + // Right-side match fields are extracted separately by collectAdditionalRightFieldsForFilters + // for filter evaluation, but they shouldn't be in the final output + List loadFields = new ArrayList<>(); + loadFields.add(LKWD_ATTR); + loadFields.add(LINT_ATTR); + Expression joinOnExpression = null; - FragmentExec rightPlanWithOptionalPreJoinFilter = buildLessThanFilter(LESS_THAN_VALUE); + // Pass all right-side attributes to buildLessThanFilter so the EsRelation has all of them + // This ensures NameId matching when we collect right-side field NameIds + FragmentExec rightPlanWithOptionalPreJoinFilter = buildLessThanFilter(LESS_THAN_VALUE, LINT_ATTR, rightSideAttributes); if (operation != null) { List conditions = new ArrayList<>(); for (int i = 0; i < numberOfJoinColumns; i++) { - String matchFieldLeft = "match" + i + "_left"; - String matchFieldRight = "match" + i + "_right"; - FieldAttribute left = new FieldAttribute( - Source.EMPTY, - matchFieldLeft, - new EsField(matchFieldLeft, inputDataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) - ); - FieldAttribute right = new FieldAttribute( - Source.EMPTY, - matchFieldRight, - new EsField(matchFieldRight.replace("left", "right"), inputDataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) - ); + // Use precreated static attributes directly + FieldAttribute left = (i == 0) ? MATCH0_LEFT_ATTR : MATCH1_LEFT_ATTR; + FieldAttribute right = (i == 0) ? MATCH0_RIGHT_ATTR : MATCH1_RIGHT_ATTR; conditions.add(operation.buildNewInstance(Source.EMPTY, left, right)); } if (applyRightFilterAsJoinOnFilter) { if (rightPlanWithOptionalPreJoinFilter instanceof FragmentExec fragmentExec && fragmentExec.fragment() instanceof Filter filterPlan) { conditions.add(filterPlan.condition()); - rightPlanWithOptionalPreJoinFilter = null; + EsRelation esRelation = (EsRelation) filterPlan.child(); + rightPlanWithOptionalPreJoinFilter = new FragmentExec(esRelation); } } joinOnExpression = Predicates.combineAnd(conditions); @@ -293,14 +389,9 @@ protected Operator.OperatorFactory simple(SimpleOptions options) { ); } - private FragmentExec buildLessThanFilter(int value) { - FieldAttribute filterAttribute = new FieldAttribute( - Source.EMPTY, - "lint", - new EsField("lint", DataType.INTEGER, Collections.emptyMap(), true, EsField.TimeSeriesFieldType.NONE) - ); - Expression lessThan = new LessThan(Source.EMPTY, filterAttribute, new Literal(Source.EMPTY, value, DataType.INTEGER)); - EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), List.of()); + private FragmentExec buildLessThanFilter(int value, FieldAttribute lintAttribute, List rightSideAttributes) { + EsRelation esRelation = new EsRelation(Source.EMPTY, "test", IndexMode.LOOKUP, Map.of(), Map.of(), Map.of(), rightSideAttributes); + Expression lessThan = new LessThan(Source.EMPTY, lintAttribute, new Literal(Source.EMPTY, value, DataType.INTEGER)); Filter filter = new Filter(Source.EMPTY, esRelation, lessThan); return new FragmentExec(filter); } @@ -326,15 +417,23 @@ public void testSimpleDescription() { protected Matcher expectedToStringOfSimple() { StringBuilder sb = new StringBuilder(); String suffix = (operation == null) ? "" : ("_left"); - sb.append("LookupOperator\\[index=idx load_fields=\\[lkwd\\{r}#\\d+, lint\\{r}#\\d+] "); + // load_fields now use FieldAttributes (showing {f}) instead of ReferenceAttributes (showing {r}) + sb.append("LookupOperator\\[index=idx load_fields=\\[lkwd\\{f}#\\d+, lint\\{f}#\\d+] "); for (int i = 0; i < numberOfJoinColumns; i++) { - // match_field=match_left (index first, then suffix) - sb.append("input_type=LONG match_field=match").append(i).append(suffix).append(" inputChannel=").append(i).append(" "); + // match_field includes the full attribute representation (e.g., match0_left{f}#\d+) + sb.append("input_type=LONG match_field=match") + .append(i) + .append(suffix) + .append("\\{f}#\\d+ inputChannel=") + .append(i) + .append(" "); } if (applyRightFilterAsJoinOnFilter && operation != null) { - // When applyRightFilterAsJoinOnFilter is true, right_pre_join_plan should be null - sb.append("right_pre_join_plan=null"); + // When applyRightFilterAsJoinOnFilter is true, we keep the EsRelation structure but remove the filter + // The FragmentExec wraps just the EsRelation (no Filter) + sb.append("right_pre_join_plan=FragmentExec\\[filter=null, estimatedRowSize=\\d+, reducer=\\[\\], fragment=\\[<>\\n") + .append("EsRelation\\[test]\\[LOOKUP]\\[.*]<>\\]\\]"); } else { // Accept either the legacy physical plan rendering (FilterExec/EsQueryExec) or the new FragmentExec rendering sb.append("right_pre_join_plan=(?:"); @@ -351,7 +450,7 @@ protected Matcher expectedToStringOfSimple() { .append("Filter\\[lint\\{f}#\\d+ < ") .append(LESS_THAN_VALUE) .append("\\[INTEGER]]\\n") - .append("\\\\_EsRelation\\[test]\\[LOOKUP]\\[\\]<>\\]\\]"); + .append("\\\\_EsRelation\\[test]\\[LOOKUP]\\[.*]<>\\]\\]"); sb.append(")"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MatchConfigSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MatchConfigSerializationTests.java index ce9924a4b320e..b2f76aa54c82d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MatchConfigSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MatchConfigSerializationTests.java @@ -17,15 +17,24 @@ */ import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.esql.SerializationTestUtils; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.ExpressionWritables; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.session.Configuration; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; @@ -38,6 +47,15 @@ public void initConfig() { this.config = randomConfiguration(); } + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + // Include NamedExpression entries so MatchConfig can serialize/deserialize NamedExpression + List entries = new ArrayList<>(); + entries.addAll(ExpressionWritables.namedExpressions()); + entries.addAll(ExpressionWritables.attributes()); + return new NamedWriteableRegistry(entries); + } + @Override protected Writeable.Reader instanceReader() { return MatchConfig::new; @@ -53,7 +71,14 @@ private MatchConfig randomMatchConfig() { String name = randomAlphaOfLengthBetween(1, 100); int channel = randomInt(); DataType type = randomValueOtherThanMany(t -> false == t.supportedVersion().supportedLocally(), () -> randomFrom(DataType.types())); - return new MatchConfig(name, channel, type); + FieldAttribute attribute = new FieldAttribute( + Source.EMPTY, + null, + null, + name, + new EsField(name, type, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + return new MatchConfig(attribute, channel, type); } @Override @@ -65,8 +90,15 @@ private MatchConfig mutateMatchConfig(MatchConfig instance) { int i = randomIntBetween(1, 3); return switch (i) { case 1 -> { - String name = randomValueOtherThan(instance.fieldName(), () -> randomAlphaOfLengthBetween(1, 100)); - yield new MatchConfig(name, instance.channel(), instance.type()); + String name = randomValueOtherThan(instance.fieldName().name(), () -> randomAlphaOfLengthBetween(1, 100)); + FieldAttribute attribute = new FieldAttribute( + Source.EMPTY, + null, + null, + name, + new EsField(name, instance.type(), Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + yield new MatchConfig(attribute, instance.channel(), instance.type()); } case 2 -> { int channel = randomValueOtherThan(instance.channel(), () -> randomInt()); @@ -82,7 +114,13 @@ private MatchConfig mutateMatchConfig(MatchConfig instance) { @Override protected MatchConfig copyInstance(MatchConfig instance, TransportVersion version) throws IOException { return copyInstance(instance, getNamedWriteableRegistry(), (out, v) -> v.writeTo(new PlanStreamOutput(out, config)), in -> { - PlanStreamInput pin = new PlanStreamInput(in, in.namedWriteableRegistry(), config); + // Use TestNameIdMapper to preserve NameIds during serialization/deserialization + PlanStreamInput pin = new PlanStreamInput( + in, + in.namedWriteableRegistry(), + config, + new SerializationTestUtils.TestNameIdMapper() + ); return new MatchConfig(pin); }, version); }