Skip to content

Commit d24dab3

Browse files
julian-elasticjulian-elastic
committed
Get basic case with translatable filters to work
1 parent bcde6be commit d24dab3

File tree

12 files changed

+430
-88
lines changed

12 files changed

+430
-88
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5149,3 +5149,181 @@ null | null | bar2 | null | null
51495149
null | null | corge | null | null
51505150
null | null | fred | null | null
51515151
;
5152+
5153+
5154+
lookupJoinWithPushableFilterOnLeft
5155+
required_capability: join_lookup_v12
5156+
required_capability: lookup_join_on_multiple_fields
5157+
5158+
FROM multi_column_joinable
5159+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5160+
| WHERE other2 > 5000
5161+
| KEEP id_int, name_str, extra1, other1, other2
5162+
| SORT id_int, name_str, extra1, other1, other2
5163+
| LIMIT 20
5164+
;
5165+
5166+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5167+
4 | David | qux | zeta | 6000
5168+
5 | Eve | quux | eta | 7000
5169+
5 | Eve | quux | theta | 8000
5170+
6 | null | corge | iota | 9000
5171+
7 | Grace | grault | kappa | 10000
5172+
8 | Hank | garply | lambda | 11000
5173+
12 | Liam | xyzzy | nu | 13000
5174+
13 | Mia | thud | xi | 14000
5175+
14 | Nina | foo2 | omicron | 15000
5176+
;
5177+
5178+
lookupJoinWithTwoPushableFiltersOnLeft
5179+
required_capability: join_lookup_v12
5180+
required_capability: lookup_join_on_multiple_fields
5181+
5182+
FROM multi_column_joinable
5183+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5184+
| WHERE other2 > 5000
5185+
| WHERE other1 like "*ta"
5186+
| KEEP id_int, name_str, extra1, other1, other2
5187+
| SORT id_int, name_str, extra1, other1, other2
5188+
| LIMIT 20
5189+
;
5190+
5191+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5192+
4 | David | qux | zeta | 6000
5193+
5 | Eve | quux | eta | 7000
5194+
5 | Eve | quux | theta | 8000
5195+
6 | null | corge | iota | 9000
5196+
;
5197+
5198+
lookupJoinWithMixLeftAndRightFilters
5199+
required_capability: join_lookup_v12
5200+
required_capability: lookup_join_on_multiple_fields
5201+
5202+
FROM multi_column_joinable
5203+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5204+
| WHERE other2 > 5000 AND (extra1 == "qux" OR extra1 == "foo2") AND other1 like ("*ta", "*ron")
5205+
| KEEP id_int, name_str, extra1, other1, other2
5206+
| SORT id_int, name_str, extra1, other1, other2
5207+
| LIMIT 20
5208+
;
5209+
5210+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5211+
4 | David | qux | zeta | 6000
5212+
14 | Nina | foo2 | omicron | 15000
5213+
;
5214+
5215+
lookupJoinWithMixLeftAndRightFiltersNotPushableToLucene
5216+
required_capability: join_lookup_v12
5217+
required_capability: lookup_join_on_multiple_fields
5218+
5219+
FROM multi_column_joinable
5220+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5221+
| WHERE ABS(other2) > 5000 AND (extra1 == "qux" OR extra1 == "foo2") AND other1 like ("*ta", "*ron")
5222+
| KEEP id_int, name_str, extra1, other1, other2
5223+
| SORT id_int, name_str, extra1, other1, other2
5224+
| LIMIT 20
5225+
;
5226+
5227+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5228+
4 | David | qux | zeta | 6000
5229+
14 | Nina | foo2 | omicron | 15000
5230+
;
5231+
5232+
5233+
lookupJoinWithMixJoinAndNonJoinColumnsNotPushable
5234+
required_capability: join_lookup_v12
5235+
required_capability: lookup_join_on_multiple_fields
5236+
5237+
FROM multi_column_joinable
5238+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5239+
| WHERE ABS(other2) > id_int + 5000
5240+
| KEEP id_int, name_str, extra1, other1, other2
5241+
| SORT id_int, name_str, extra1, other1, other2
5242+
| LIMIT 20
5243+
;
5244+
5245+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5246+
4 | David | qux | zeta | 6000
5247+
5 | Eve | quux | eta | 7000
5248+
5 | Eve | quux | theta | 8000
5249+
6 | null | corge | iota | 9000
5250+
7 | Grace | grault | kappa | 10000
5251+
8 | Hank | garply | lambda | 11000
5252+
12 | Liam | xyzzy | nu | 13000
5253+
13 | Mia | thud | xi | 14000
5254+
14 | Nina | foo2 | omicron | 15000
5255+
;
5256+
5257+
5258+
lookupJoinWithMixJoinAndNonJoinColumnsPushable
5259+
required_capability: join_lookup_v12
5260+
required_capability: lookup_join_on_multiple_fields
5261+
5262+
FROM multi_column_joinable
5263+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5264+
| WHERE other2 > id_int + 5000
5265+
| KEEP id_int, name_str, extra1, other1, other2
5266+
| SORT id_int, name_str, extra1, other1, other2
5267+
| LIMIT 20
5268+
;
5269+
5270+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5271+
4 | David | qux | zeta | 6000
5272+
5 | Eve | quux | eta | 7000
5273+
5 | Eve | quux | theta | 8000
5274+
6 | null | corge | iota | 9000
5275+
7 | Grace | grault | kappa | 10000
5276+
8 | Hank | garply | lambda | 11000
5277+
12 | Liam | xyzzy | nu | 13000
5278+
13 | Mia | thud | xi | 14000
5279+
14 | Nina | foo2 | omicron | 15000
5280+
;
5281+
5282+
lookupJoinWithJoinAttrFilter
5283+
required_capability: join_lookup_v12
5284+
required_capability: lookup_join_on_multiple_fields
5285+
5286+
FROM multi_column_joinable
5287+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5288+
| WHERE id_int > 7
5289+
| KEEP id_int, name_str, extra1, other1, other2
5290+
| SORT id_int, name_str, extra1, other1, other2
5291+
| LIMIT 20
5292+
;
5293+
5294+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5295+
8 | Hank | garply | lambda | 11000
5296+
9 | null | waldo | null | null
5297+
10 | null | fred | null | null
5298+
12 | Liam | xyzzy | nu | 13000
5299+
13 | Mia | thud | xi | 14000
5300+
14 | Nina | foo2 | omicron | 15000
5301+
15 | null | bar2 | null | null
5302+
;
5303+
5304+
5305+
lookupJoinWithExpressionOfOtherFields
5306+
required_capability: join_lookup_v12
5307+
required_capability: lookup_join_on_multiple_fields
5308+
5309+
FROM multi_column_joinable
5310+
| LOOKUP JOIN multi_column_joinable_lookup ON id_int, is_active_bool
5311+
| WHERE ABS(other2) > LENGTH(other1)*1000 + 2000
5312+
| KEEP id_int, name_str, extra1, other1, other2
5313+
| SORT id_int, name_str, extra1, other1, other2
5314+
| LIMIT 20
5315+
/*
5316+
NEED TO DEBUG WHY THE FILTER IS NOT PUSHED TO THE RIGHT SIDE OF THE LOOKUP JOIN
5317+
*/
5318+
;
5319+
5320+
id_int:integer | name_str:keyword | extra1:keyword | other1:keyword | other2:integer
5321+
5 | Eve | quux | eta | 7000
5322+
5 | Eve | quux | theta | 8000
5323+
6 | null | corge | iota | 9000
5324+
7 | Grace | grault | kappa | 10000
5325+
8 | Hank | garply | lambda | 11000
5326+
12 | Liam | xyzzy | nu | 13000
5327+
13 | Mia | thud | xi | 14000
5328+
14 | Nina | foo2 | omicron | 15000
5329+
;

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupFromIndexIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,8 @@ private void runLookup(List<DataType> keyTypes, PopulateIndices populateIndices)
329329
"lookup",
330330
"lookup",
331331
List.of(new Alias(Source.EMPTY, "l", new ReferenceAttribute(Source.EMPTY, "l", DataType.LONG))),
332-
Source.EMPTY
332+
Source.EMPTY,
333+
null
333334
);
334335
DriverContext driverContext = driverContext();
335336
try (

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
3939
import org.elasticsearch.compute.operator.Driver;
4040
import org.elasticsearch.compute.operator.DriverContext;
41+
import org.elasticsearch.compute.operator.FilterOperator;
4142
import org.elasticsearch.compute.operator.Operator;
4243
import org.elasticsearch.compute.operator.OutputOperator;
4344
import org.elasticsearch.compute.operator.ProjectOperator;
@@ -73,10 +74,15 @@
7374
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
7475
import org.elasticsearch.xpack.esql.core.expression.Alias;
7576
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
77+
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
7678
import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
7779
import org.elasticsearch.xpack.esql.core.tree.Source;
7880
import org.elasticsearch.xpack.esql.core.type.DataType;
81+
import org.elasticsearch.xpack.esql.core.type.EsField;
82+
import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
83+
import org.elasticsearch.xpack.esql.plan.physical.FilterExec;
7984
import org.elasticsearch.xpack.esql.planner.EsPhysicalOperationProviders;
85+
import org.elasticsearch.xpack.esql.planner.Layout;
8086
import org.elasticsearch.xpack.esql.planner.PlannerUtils;
8187
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
8288

@@ -346,13 +352,36 @@ private void doLookup(T request, CancellableTask task, ActionListener<List<Page>
346352
warnings
347353
);
348354
releasables.add(queryOperator);
349-
355+
Layout.Builder builder = new Layout.Builder();
356+
// append the docsIds and positions to the layout
357+
builder.append(
358+
// this looks wrong, what is the datatype for the Docs? It says DocVector but it is not a DataType
359+
new FieldAttribute(Source.EMPTY, "Docs", new EsField("Docs", DataType.DOC_DATA_TYPE, Collections.emptyMap(), false))
360+
);
361+
builder.append(
362+
new FieldAttribute(Source.EMPTY, "Positions", new EsField("Positions", DataType.INTEGER, Collections.emptyMap(), false))
363+
);
350364
List<Operator> operators = new ArrayList<>();
351365
if (request.extractFields.isEmpty() == false) {
352366
var extractFieldsOperator = extractFieldsOperator(shardContext.context, driverContext, request.extractFields);
367+
builder.append(request.extractFields);
353368
releasables.add(extractFieldsOperator);
354369
operators.add(extractFieldsOperator);
355370
}
371+
if (queryList instanceof PostJoinFilterable postJoinFilterable) {
372+
FilterExec filterExec = postJoinFilterable.getPostJoinFilter();
373+
Operator inputOperator;
374+
if (operators.isEmpty() == false) {
375+
inputOperator = operators.getLast();
376+
} else {
377+
inputOperator = queryOperator;
378+
}
379+
Operator postJoinFilter = filterExecOperator(filterExec, inputOperator, shardContext.context, driverContext, builder);
380+
if (postJoinFilter != null) {
381+
releasables.add(postJoinFilter);
382+
operators.add(postJoinFilter);
383+
}
384+
}
356385
operators.add(finishPages);
357386

358387
/*
@@ -414,6 +443,27 @@ public void onFailure(Exception e) {
414443
}
415444
}
416445

446+
private Operator filterExecOperator(
447+
FilterExec filterExec,
448+
Operator inputOperator, // not needed?
449+
EsPhysicalOperationProviders.ShardContext shardContext,
450+
DriverContext driverContext,
451+
Layout.Builder builder
452+
) {
453+
if (filterExec == null) {
454+
return null;
455+
}
456+
457+
var evaluatorFactory = EvalMapper.toEvaluator(
458+
FoldContext.small()/*is this correct*/,
459+
filterExec.condition(),
460+
builder.build(),
461+
List.of(shardContext)
462+
);
463+
var filterOperatorFactory = new FilterOperator.FilterOperatorFactory(evaluatorFactory);
464+
return filterOperatorFactory.get(driverContext);
465+
}
466+
417467
private static Operator extractFieldsOperator(
418468
EsPhysicalOperationProviders.ShardContext shardContext,
419469
DriverContext driverContext,

0 commit comments

Comments
 (0)