Adding support for casting on the field type and allowing multi-index queries with different types

Author: carlosdelest

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -52,6 +52,11 @@
 public class CsvTestsDataLoader {
     private static final int BULK_DATA_SIZE = 100_000;
     private static final TestsDataset EMPLOYEES = new TestsDataset("employees", "mapping-default.json", "employees.csv").noSubfields();
+    private static final TestsDataset EMPLOYEES_INCOMPATIBLE = new TestsDataset(
+        "employees_incompatible",
+        "mapping-default-incompatible.json",
+        "employees_incompatible.csv"
+    ).noSubfields();
     private static final TestsDataset HOSTS = new TestsDataset("hosts");
     private static final TestsDataset APPS = new TestsDataset("apps");
     private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short"));
@@ -98,6 +103,7 @@ public class CsvTestsDataLoader {
 
     public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
         Map.entry(EMPLOYEES.indexName, EMPLOYEES),
+        Map.entry(EMPLOYEES_INCOMPATIBLE.indexName, EMPLOYEES_INCOMPATIBLE),
         Map.entry(HOSTS.indexName, HOSTS),
         Map.entry(APPS.indexName, APPS),
         Map.entry(APPS_SHORT.indexName, APPS_SHORT),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv

@@ -0,0 +1,80 @@
+{
+    "properties" : {
+        "emp_no" : {
+            "type" : "long"
+        },
+        "first_name" : {
+            "type" : "text"
+        },
+        "last_name" : {
+            "type" : "text"
+        },
+        "gender" : {
+            "type" : "text"
+        },
+        "birth_date": {
+            "type" : "date"
+        },
+        "hire_date": {
+            "type" : "date"
+        },
+        "salary" : {
+            "type" : "long"
+        },
+        "languages" : {
+            "type" : "byte",
+            "fields": {
+                "long": {
+                    "type": "long"
+                },
+                "short": {
+                    "type": "short"
+                },
+                "int": {
+                    "type": "integer"
+                }
+            }
+        },
+        "height": {
+            "type" : "float",
+            "fields" : {
+                "double" : {
+                  "type" : "double"
+                },
+                "scaled_float": {
+                    "type": "scaled_float",
+                    "scaling_factor": 100
+                },
+                "half_float": {
+                    "type": "half_float"
+                }
+            }
+        },
+        "still_hired": {
+            "type" : "keyword"
+        },
+        "avg_worked_seconds" : {
+            "type" : "unsigned_long"
+        },
+        "job_positions" : {
+            "type" : "text"
+        },
+        "is_rehired" : {
+            "type" : "text"
+        },
+        "salary_change": {
+            "type": "float",
+            "fields": {
+                "int": {
+                   "type": "integer"
+                },
+                "long": {
+                    "type": "long"
+                },
+                "keyword": {
+                    "type" : "keyword"
+                }
+            }
+        }
+    }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json

@@ -449,3 +449,19 @@ emp_no:integer | height:double
 10048          | 2.0            
 10098          | 2.0     
 ;
+
+testMatchMultipleFieldTypes
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible 
+| where match(emp_no::int, 10005) 
+| eval emp_as_int = emp_no::int 
+| eval name_as_kw = first_name::keyword 
+| keep emp_as_int, name_as_kw
+;
+
+emp_as_int:integer | name_as_kw:keyword
+10005              | Kyoichi        
+10005              | Kyoichi      
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

@@ -469,3 +469,20 @@ emp_no:integer | height:double
 10048          | 2.0            
 10098          | 2.0     
 ;
+
+
+testMatchMultipleFieldTypes
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible 
+| where emp_no::int : 10005 
+| eval emp_as_int = emp_no::int 
+| eval name_as_kw = first_name::keyword 
+| keep emp_as_int, name_as_kw
+;
+
+emp_as_int:integer | name_as_kw:keyword
+10005              | Kyoichi        
+10005              | Kyoichi      
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

@@ -78,7 +78,7 @@ regexBooleanExpression
     ;
 
 matchBooleanExpression
-    : fieldExp=qualifiedName COLON matchQuery=constant (CAST_OP dataType)?
+    : fieldExp=qualifiedName (CAST_OP fieldType=dataType)? COLON matchQuery=constant (CAST_OP queryType=dataType)?
     ;
 
 valueExpression

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -927,12 +927,19 @@ String unresolvedAttributeNameInParam(ParserRuleContext ctx, Expression param) {
     @Override
     public Expression visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) {
         final Expression matchQueryExpression;
-        if (ctx.dataType() != null) {
-            matchQueryExpression = castToType(source(ctx), ctx.matchQuery, ctx.dataType());
+        if (ctx.queryType != null) {
+            matchQueryExpression = castToType(source(ctx), ctx.matchQuery, ctx.queryType);
         } else {
             matchQueryExpression = expression(ctx.matchQuery);
         }
 
-        return new Match(source(ctx), expression(ctx.fieldExp), matchQueryExpression);
+        final Expression matchFieldExpression;
+        if (ctx.fieldType != null) {
+            matchFieldExpression = castToType(source(ctx), ctx.fieldExp, ctx.fieldType);
+        } else {
+            matchFieldExpression = expression(ctx.fieldExp);
+        }
+
+        return new Match(source(ctx), matchFieldExpression, matchQueryExpression);
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp

@@ -33,6 +33,7 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.TermsQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
@@ -531,7 +532,17 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) {
     public static class MatchFunctionTranslator extends ExpressionTranslator<Match> {
         @Override
         protected Query asQuery(Match match, TranslatorHandler handler) {
-            return new MatchQuery(match.source(), ((FieldAttribute) match.field()).name(), match.queryAsObject());
+            Expression field = match.field();
+            if (field instanceof FieldAttribute fieldAttribute) {
+                String fieldName = fieldAttribute.name();
+                if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) {
+                    // If we have multiple field types, we allow the query to be done, but getting the underlying field name
+                    fieldName = multiTypeEsField.getName();
+                }
+                return new MatchQuery(match.source(), fieldName, match.queryAsObject());
+            }
+
+            throw new IllegalArgumentException("Match must have a field attribute as the first argument");
         }
     }