Change Parser to allow AND of expressions

Author: julian-elastic

.idea/runConfigurations/Debug_Elasticsearch__node_2_.xml

@@ -359,7 +359,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS = def(9_150_0_00);
     public static final TransportVersion ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER = def(9_151_0_00);
     public static final TransportVersion INFERENCE_API_DISABLE_EIS_RATE_LIMITING = def(9_152_0_00);
-    public static final TransportVersion ESQL_LOOKUP_JOIN_ON_EXPRESSION = def(9_143_0_00);
+    public static final TransportVersion ESQL_LOOKUP_JOIN_ON_EXPRESSION = def(9_153_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

.idea/runConfigurations/Debug_Elasticsearch__node_3_.xml

@@ -461,5 +461,5 @@ private IntFunction<Query> shapeQuery() {
         }
     }
 
-    protected record OnlySingleValueParams(Warnings warnings, String multiValueWarningMessage) {}
+    public record OnlySingleValueParams(Warnings warnings, String multiValueWarningMessage) {}
 }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -30,7 +30,11 @@ matchBooleanExpression
 
 valueExpression
     : operatorExpression                                                                      #valueExpressionDefault
-    | left=operatorExpression comparisonOperator right=operatorExpression                     #comparison
+    | comparisonExpression                                                                    #comparison
+    ;
+
+comparisonExpression
+    : left=operatorExpression comparisonOperator right=operatorExpression
     ;
 
 operatorExpression

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java

@@ -18,11 +18,8 @@ joinTarget
     ;
 
 joinCondition
-    : ON joinPredicate (COMMA joinPredicate)*
-    ;
-
-joinPredicate
-    : valueExpression
+    : ON qualifiedName (COMMA qualifiedName)*                                #fieldBasedLookupJoin
+    | ON comparisonExpression (AND comparisonExpression)*                    #expressionBasedLookupJoin
     ;
 
 

x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join-expression.csv-spec

@@ -39,9 +39,16 @@ public BinaryComparisonQueryList(
         Block block,
         EsqlBinaryComparison binaryComparison,
         ClusterService clusterService,
-        AliasFilter aliasFilter
+        AliasFilter aliasFilter,
+        Warnings warnings
     ) {
-        super(field, searchExecutionContext, aliasFilter, block, null);
+        super(
+            field,
+            searchExecutionContext,
+            aliasFilter,
+            block,
+            new OnlySingleValueParams(warnings, "LOOKUP JOIN encountered multi-value")
+        );
         // swap left and right if the field is on the right
         // We get a filter in the form left_expr >= right_expr
         // here we will swap it to right_expr <= left_expr
@@ -62,9 +69,6 @@ public QueryList onlySingleValues(Warnings warnings, String multiValueWarningMes
 
     @Override
     public Query doGetQuery(int position, int firstValueIndex, int valueCount) {
-        if (valueCount == 0) {
-            return null;
-        }
         Object value = blockValueReader.apply(firstValueIndex);
         // create a new comparison with the value from the block as a literal
         EsqlBinaryComparison comparison = binaryComparison.getFunctionType()

x-pack/plugin/esql/src/main/antlr/parser/Expression.g4

@@ -12,6 +12,7 @@
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.operator.Warnings;
 import org.elasticsearch.compute.operator.lookup.LookupEnrichQueryGenerator;
 import org.elasticsearch.compute.operator.lookup.QueryList;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -56,20 +57,21 @@ public ExpressionQueryList(
         PhysicalPlan rightPreJoinPlan,
         ClusterService clusterService,
         LookupFromIndexService.TransportRequest request,
-        AliasFilter aliasFilter
+        AliasFilter aliasFilter,
+        Warnings warnings
     ) {
         if (queryLists.size() < 2 && (rightPreJoinPlan instanceof FilterExec == false) && request.getJoinOnConditions() == null) {
             throw new IllegalArgumentException("ExpressionQueryList must have at least two QueryLists or a pre-join filter");
         }
         this.queryLists = queryLists;
         this.context = context;
         this.aliasFilter = aliasFilter;
-        buildJoinOnConditions(request, clusterService);
+        buildJoinOnConditions(request, clusterService, warnings);
         buildPreJoinFilter(rightPreJoinPlan, clusterService);
 
     }
 
-    private void buildJoinOnConditions(LookupFromIndexService.TransportRequest request, ClusterService clusterService) {
+    private void buildJoinOnConditions(LookupFromIndexService.TransportRequest request, ClusterService clusterService, Warnings warnings) {
         // we support 2 modes of operation:
         // Join on fields
         // Join on AND of binary comparisons
@@ -108,7 +110,8 @@ private void buildJoinOnConditions(LookupFromIndexService.TransportRequest reque
                                             block,
                                             binaryComparison,
                                             clusterService,
-                                            aliasFilter
+                                            aliasFilter,
+                                            warnings
                                         )
                                     );
                                     matched = true;

x-pack/plugin/esql/src/main/antlr/parser/Join.g4

@@ -128,7 +128,7 @@ protected LookupEnrichQueryGenerator queryList(
         if (queryLists.size() == 1 && physicalPlan instanceof FilterExec == false && request.joinOnConditions == null) {
             return queryLists.getFirst();
         }
-        return new ExpressionQueryList(queryLists, context, physicalPlan, clusterService, request, aliasFilter);
+        return new ExpressionQueryList(queryLists, context, physicalPlan, clusterService, request, aliasFilter, warnings);
 
     }