Skip to content

Commit 2ffa01f

Browse files
julian-elasticjulian-elastic
committed
Full Text Functions support for Lookup Join
1 parent 3fcd348 commit 2ffa01f

File tree

7 files changed

+456
-147
lines changed

7 files changed

+456
-147
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -736,6 +736,9 @@ id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:int
736736
14 | Nina | foo2 | omicron | 15000
737737
;
738738

739+
740+
741+
739742
lookupJoinExpressionOnUnionTypes
740743
required_capability: join_lookup_v12
741744
required_capability: lookup_join_on_boolean_expression
@@ -758,3 +761,206 @@ apps | 2 | French
758761
apps_short | 1 | English
759762
apps_short | 2 | French
760763
;
764+
765+
lookupJoinWithGreaterThanCondition
766+
required_capability: join_lookup_v12
767+
required_capability: lookup_join_on_boolean_expression_v2
768+
769+
FROM multi_column_joinable
770+
| RENAME id_int AS id_left, is_active_bool AS is_active_left
771+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND other2 > 10000
772+
| KEEP id_left, name_str, extra1, other1, other2
773+
| SORT id_left, name_str, extra1, other1, other2
774+
;
775+
776+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND other2 > 10000] failed, treating result as null. Only first 20 failures recorded.
777+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
778+
779+
id_left:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
780+
1 | null | foo | null | null
781+
[1, 19, 21] | null | zyx | null | null
782+
2 | null | bar | null | null
783+
3 | null | baz | null | null
784+
4 | null | qux | null | null
785+
5 | null | quux | null | null
786+
6 | null | corge | null | null
787+
7 | null | grault | null | null
788+
8 | Hank | garply | lambda | 11000
789+
9 | null | waldo | null | null
790+
10 | null | fred | null | null
791+
12 | Liam | xyzzy | nu | 13000
792+
13 | Mia | thud | xi | 14000
793+
14 | Nina | foo2 | omicron | 15000
794+
15 | null | bar2 | null | null
795+
[17, 18] | null | xyz | null | null
796+
null | null | plugh | null | null
797+
;
798+
799+
lookupJoinWithLikeCondition
800+
required_capability: join_lookup_v12
801+
required_capability: lookup_join_on_boolean_expression_v2
802+
803+
FROM multi_column_joinable
804+
| RENAME id_int AS id_left, is_active_bool AS is_active_left
805+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND other1 like "*ta"
806+
| KEEP id_left, name_str, extra1, other1, other2
807+
| SORT id_left, name_str, extra1, other1, other2
808+
;
809+
810+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND other1 like \"*ta\"] failed, treating result as null. Only first 20 failures recorded.
811+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
812+
813+
id_left:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
814+
1 | Alice | foo | beta | 2000
815+
[1, 19, 21] | null | zyx | null | null
816+
2 | null | bar | null | null
817+
3 | Charlie | baz | delta | 4000
818+
4 | David | qux | zeta | 6000
819+
5 | Eve | quux | eta | 7000
820+
5 | Eve | quux | theta | 8000
821+
6 | null | corge | iota | 9000
822+
7 | null | grault | null | null
823+
8 | null | garply | null | null
824+
9 | null | waldo | null | null
825+
10 | null | fred | null | null
826+
12 | null | xyzzy | null | null
827+
13 | null | thud | null | null
828+
14 | null | foo2 | null | null
829+
15 | null | bar2 | null | null
830+
[17, 18] | null | xyz | null | null
831+
null | null | plugh | null | null
832+
;
833+
834+
lookupJoinWithOrOfLikeGt
835+
required_capability: join_lookup_v12
836+
required_capability: lookup_join_on_boolean_expression_v2
837+
838+
FROM multi_column_joinable
839+
| RENAME id_int AS id_left, is_active_bool AS is_active_left
840+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND (other1 like "*ta" OR other2 > 10000)
841+
| KEEP id_left, name_str, extra1, other1, other2
842+
| SORT id_left, name_str, extra1, other1, other2
843+
;
844+
845+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON id_int == id_left and is_active_left == is_active_bool AND (other1 like \"*ta\" OR other2 > 10000)] failed, treating result as null. Only first 20 failures recorded.
846+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
847+
848+
id_left:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
849+
1 | Alice | foo | beta | 2000
850+
[1, 19, 21] | null | zyx | null | null
851+
2 | null | bar | null | null
852+
3 | Charlie | baz | delta | 4000
853+
4 | David | qux | zeta | 6000
854+
5 | Eve | quux | eta | 7000
855+
5 | Eve | quux | theta | 8000
856+
6 | null | corge | iota | 9000
857+
7 | null | grault | null | null
858+
8 | Hank | garply | lambda | 11000
859+
9 | null | waldo | null | null
860+
10 | null | fred | null | null
861+
12 | Liam | xyzzy | nu | 13000
862+
13 | Mia | thud | xi | 14000
863+
14 | Nina | foo2 | omicron | 15000
864+
15 | null | bar2 | null | null
865+
[17, 18] | null | xyz | null | null
866+
null | null | plugh | null | null
867+
;
868+
869+
lookupJoinExpressionWithMatch
870+
required_capability: join_lookup_v12
871+
required_capability: lookup_join_on_boolean_expression_v2
872+
FROM multi_column_joinable
873+
| RENAME id_int AS id_left, is_active_bool AS is_active_left
874+
| LOOKUP JOIN multi_column_joinable_lookup ON MATCH(other1, "beta") AND id_int == id_left and is_active_left == is_active_bool
875+
| WHERE other2 IS NOT NULL
876+
| KEEP id_left, name_str, extra1, other1, other2
877+
| SORT id_left, name_str, extra1, other1, other2
878+
;
879+
880+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON MATCH(other1, \"beta\") AND id_int == id_left and is_active_left == is_active_bool] failed, treating result as null. Only first 20 failures recorded.
881+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
882+
883+
id_left:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
884+
1 | Alice | foo | beta | 2000
885+
;
886+
887+
lookupJoinOnSameFieldTwiceWithOrNot
888+
required_capability: join_lookup_v12
889+
required_capability: lookup_join_on_boolean_expression_v2
890+
891+
FROM multi_column_joinable
892+
| RENAME id_int AS id_left, name_str AS name_left, is_active_bool AS is_active_left, ip_addr AS ip_addr_left
893+
| LOOKUP JOIN multi_column_joinable_lookup ON (other2 < 12000 OR NOT (other1 != "omicron" AND other1 != "nu")) AND id_left == id_int AND name_left == name_str AND id_left < other2
894+
| KEEP id_left, name_left, extra1, other1, other2
895+
| SORT id_left, name_left, extra1, other1, other2
896+
;
897+
898+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON (other2 < 12000 OR NOT (other1 != \"omicron\" AND other1 != \"nu\")) AND id_left == id_int AND name_left == name_str AND id_left < other2] failed, treating result as null. Only first 20 failures recorded.
899+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
900+
901+
id_left:integer | name_left:keyword | extra1:keyword | other1:keyword | other2:integer
902+
1 | Alice | foo | alpha | 1000
903+
1 | Alice | foo | beta | 2000
904+
[1, 19, 21] | Sophia | zyx | null | null
905+
2 | Bob | bar | gamma | 3000
906+
3 | Charlie | baz | delta | 4000
907+
3 | Charlie | baz | epsilon | 5000
908+
4 | David | qux | zeta | 6000
909+
5 | Eve | quux | eta | 7000
910+
5 | Eve | quux | theta | 8000
911+
6 | null | corge | null | null
912+
7 | Grace | grault | kappa | 10000
913+
8 | Hank | garply | lambda | 11000
914+
9 | Ivy | waldo | null | null
915+
10 | John | fred | null | null
916+
12 | Liam | xyzzy | nu | 13000
917+
13 | Mia | thud | null | null
918+
14 | Nina | foo2 | omicron | 15000
919+
15 | Oscar | bar2 | null | null
920+
[17, 18] | Olivia | xyz | null | null
921+
null | Kate | plugh | null | null
922+
;
923+
924+
925+
lookupJoinOnSameFieldWithPushableRightFilterAfter
926+
required_capability: join_lookup_v12
927+
required_capability: lookup_join_on_boolean_expression_v2
928+
929+
FROM multi_column_joinable
930+
| RENAME id_int AS id_left, name_str AS name_left, is_active_bool AS is_active_left, ip_addr AS ip_addr_left
931+
| LOOKUP JOIN multi_column_joinable_lookup ON (other2 < 12000 OR NOT (other1 != "omicron" AND other1 != "nu")) AND id_left == id_int AND name_left == name_str AND id_left < other2
932+
| WHERE other1 like ("a*", "b*", "o*")
933+
| KEEP id_left, name_left, extra1, other1, other2
934+
| SORT id_left, name_left, extra1, other1, other2
935+
;
936+
937+
warning:Line 3:3: evaluation of [LOOKUP JOIN multi_column_joinable_lookup ON (other2 < 12000 OR NOT (other1 != \"omicron\" AND other1 != \"nu\")) AND id_left == id_int AND name_left == name_str AND id_left < other2] failed, treating result as null. Only first 20 failures recorded.
938+
warning:Line 3:3: java.lang.IllegalArgumentException: LOOKUP JOIN encountered multi-value
939+
940+
id_left:integer | name_left:keyword | extra1:keyword | other1:keyword | other2:integer
941+
1 | Alice | foo | alpha | 1000
942+
1 | Alice | foo | beta | 2000
943+
14 | Nina | foo2 | omicron | 15000
944+
;
945+
946+
twoLookupJoinsInSameQueryOtherFilter
947+
required_capability: join_lookup_v12
948+
required_capability: lookup_join_on_boolean_expression_v2
949+
950+
FROM multi_column_joinable
951+
| WHERE id_int == 1
952+
| RENAME id_int AS id_left, name_str AS name_left
953+
| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND name_left == name_str
954+
| RENAME other1 AS other1_from_first_join, id_int AS id_from_first_join, name_str AS name_from_first_join
955+
| LOOKUP JOIN multi_column_joinable_lookup ON id_left == id_int AND other1_from_first_join != other1 AND other1 like ("a*", "c*")
956+
| KEEP id_left, name_left, other1_from_first_join, other1
957+
| SORT id_left, name_left, other1_from_first_join, other1
958+
;
959+
960+
warning:Line 2:9: evaluation of [id_int == 1] failed, treating result as null. Only first 20 failures recorded.
961+
warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
962+
963+
id_left:integer | name_left:keyword | other1_from_first_join:keyword | other1:keyword
964+
1 | Alice | alpha | null
965+
1 | Alice | beta | alpha
966+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1561,6 +1561,11 @@ public enum Cap {
15611561
INLINE_STATS_SUPPORTS_REMOTE(INLINESTATS_V11.enabled),
15621562

15631563
INLINE_STATS_WITH_UNION_TYPES_IN_STUB_RELATION(INLINE_STATS.enabled),
1564+
/*
1565+
* Lookup join with Full Text Function or other Lucene Pushable condition
1566+
* to be applied to the lookup index used
1567+
*/
1568+
LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION_V2,
15641569

15651570
/**
15661571
* Support TS command in non-snapshot builds

0 commit comments

Comments
 (0)