Additional tests and checks for using multi index fields with different types

Author: carlosdelest

x-pack/plugin/esql/qa/testFixtures/src/main/resources/employees_incompatible.csv

@@ -1,4 +1,4 @@
-birth_date:date     ,emp_no:long,first_name:text,gender:text,hire_date:date,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.float:float,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword
+birth_date:date_nanos     ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword
 1953-09-02T00:00:00Z,10001,Georgi     ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello     ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19]
 1964-06-02T00:00:00Z,10002,Bezalel    ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel      ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17]
 1959-12-03T00:00:00Z,10003,Parto      ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford     ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82]

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic.json

@@ -21,6 +21,9 @@
         "_meta_field": {
             "type" : "keyword"
         },
+        "hire_date": {
+          "type": "date"
+        },
         "job": {
             "type": "text",
             "fields": {

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-default-incompatible.json

@@ -16,7 +16,7 @@
             "type" : "date"
         },
         "hire_date": {
-            "type" : "date"
+            "type" : "date_nanos"
         },
         "salary" : {
             "type" : "long"
@@ -60,7 +60,7 @@
             "type" : "text"
         },
         "is_rehired" : {
-            "type" : "text"
+            "type" : "keyword"
         },
         "salary_change": {
             "type": "float",

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

@@ -450,7 +450,7 @@ emp_no:integer | height:double
 10098          | 2.0     
 ;
 
-testMatchMultipleFieldTypes
+testMatchMultipleFieldTypesIntLong
 required_capability: match_function
 required_capability: match_additional_types
 
@@ -465,3 +465,86 @@ emp_as_int:integer | name_as_kw:keyword
 10005              | Kyoichi        
 10005              | Kyoichi      
 ;
+
+testMatchMultipleFieldTypesKeywordText
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible 
+| where match(first_name::keyword, "Kazuhito") 
+| eval first_name_kwd = first_name::keyword 
+| keep first_name_kwd
+;
+
+first_name_kwd:keyword
+Kazuhito
+Kazuhito      
+;
+
+testMatchMultipleFieldTypesDoubleFloat
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where match(height::double, 2.03) 
+| eval height_dbl = height::double 
+| eval emp_no = emp_no::int 
+| keep emp_no, height_dbl
+;
+ignoreOrder:true
+
+emp_no:integer | height_dbl:double    
+10001          | 2.0299999713897705
+10090          | 2.0299999713897705
+10001          | 2.03              
+10090          | 2.03  
+;
+
+testMatchMultipleFieldTypesBooleanKeyword
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where match(still_hired::keyword, "true") and height.scaled_float == 2.08
+| eval still_hired_bool = still_hired::boolean
+| keep still_hired_bool
+;
+
+still_hired_bool:boolean    
+true
+true
+true
+true
+;
+
+testMatchMultipleFieldTypesLongUnsignedLong
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where match(avg_worked_seconds::unsigned_long, 200296405)
+| eval avg_worked_seconds_ul = avg_worked_seconds::unsigned_long
+| keep avg_worked_seconds_ul
+;
+
+avg_worked_seconds_ul:unsigned_long
+200296405            
+200296405            
+;
+
+testMatchMultipleFieldTypesDateNanosDate
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where match(hire_date::datetime, "1986-06-26T00:00:00.000Z")
+| eval hire_date_nanos = hire_date::date_nanos
+| keep hire_date_nanos
+;
+
+hire_date_nanos:date_nanos
+1986-06-26T00:00:00.000Z            
+1986-06-26T00:00:00.000Z
+;
+
+

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

@@ -470,7 +470,6 @@ emp_no:integer | height:double
 10098          | 2.0     
 ;
 
-
 testMatchMultipleFieldTypes
 required_capability: match_function
 required_capability: match_additional_types
@@ -486,3 +485,86 @@ emp_as_int:integer | name_as_kw:keyword
 10005              | Kyoichi        
 10005              | Kyoichi      
 ;
+
+
+testMatchMultipleFieldTypesKeywordText
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible 
+| where first_name::keyword : "Kazuhito" 
+| eval first_name_kwd = first_name::keyword 
+| keep first_name_kwd
+;
+
+first_name_kwd:keyword
+Kazuhito
+Kazuhito      
+;
+
+testMatchMultipleFieldTypesDoubleFloat
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where height::double : 2.03 
+| eval height_dbl = height::double 
+| eval emp_no = emp_no::int 
+| keep emp_no, height_dbl
+;
+ignoreOrder:true
+
+emp_no:integer | height_dbl:double    
+10001          | 2.0299999713897705
+10090          | 2.0299999713897705
+10001          | 2.03              
+10090          | 2.03  
+;
+
+testMatchMultipleFieldTypesBooleanKeyword
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where still_hired::keyword : "true" and height.scaled_float == 2.08
+| eval still_hired_bool = still_hired::boolean
+| keep still_hired_bool
+;
+
+still_hired_bool:boolean    
+true
+true
+true
+true
+;
+
+testMatchMultipleFieldTypesLongUnsignedLong
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible  
+| where avg_worked_seconds::unsigned_long : 200296405 
+| eval avg_worked_seconds_ul = avg_worked_seconds::unsigned_long 
+| keep avg_worked_seconds_ul
+;
+
+avg_worked_seconds_ul:unsigned_long
+200296405            
+200296405            
+;
+
+testMatchMultipleFieldTypesDateNanosDate
+required_capability: match_function
+required_capability: match_additional_types
+
+from employees,employees_incompatible
+| where hire_date::datetime : "1986-06-26T00:00:00.000Z"
+| eval hire_date_nanos = hire_date::date_nanos
+| keep hire_date_nanos
+;
+
+hire_date_nanos:date_nanos
+1986-06-26T00:00:00.000Z            
+1986-06-26T00:00:00.000Z
+;
+

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter;
 
@@ -177,7 +178,12 @@ protected TypeResolution checkParamCompatibility() {
 
     @Override
     public void validate(Failures failures) {
-        if (field instanceof FieldAttribute == false) {
+        Expression fieldExpression = field();
+        // Field may be converted to other data type (field_name :: data_type), so we need to check the original field
+        if (fieldExpression instanceof AbstractConvertFunction convertFunction) {
+            fieldExpression = convertFunction.field();
+        }
+        if (fieldExpression instanceof FieldAttribute == false) {
             failures.add(
                 Failure.fail(
                     field,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java

@@ -38,6 +38,7 @@
 import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
 import org.elasticsearch.xpack.esql.expression.function.fulltext.QueryString;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesUtils;
@@ -532,8 +533,12 @@ private static RangeQuery translate(Range r, TranslatorHandler handler) {
     public static class MatchFunctionTranslator extends ExpressionTranslator<Match> {
         @Override
         protected Query asQuery(Match match, TranslatorHandler handler) {
-            Expression field = match.field();
-            if (field instanceof FieldAttribute fieldAttribute) {
+            Expression fieldExpression = match.field();
+            // Field may be converted to other data type (field_name :: data_type), so we need to check the original field
+            if (fieldExpression instanceof AbstractConvertFunction convertFunction) {
+                fieldExpression = convertFunction.field();
+            }
+            if (fieldExpression instanceof FieldAttribute fieldAttribute) {
                 String fieldName = fieldAttribute.name();
                 if (fieldAttribute.field() instanceof MultiTypeEsField multiTypeEsField) {
                     // If we have multiple field types, we allow the query to be done, but getting the underlying field name

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java

@@ -1546,6 +1546,32 @@ public void testMultipleMatchFilterPushdown() {
         assertThat(actualLuceneQuery.toString(), is(expectedLuceneQuery.toString()));
     }
 
+    /**
+     * Expects
+     * LimitExec[1000[INTEGER]]
+     * \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12
+     *   \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12
+     *     \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen]
+     *       \_EsQueryExec[test], indexMode[standard], query[{"match":{"emp_no":{"query":123456}}}][_doc{f}#14],
+     *       limit[1000], sort[] estimatedRowSize[332]
+     */
+    public void testMatchWithFieldCasting() {
+        String query = """
+            from test
+            | where emp_no::long : 123456
+            """;
+        var plan = plannerOptimizer.plan(query);
+
+        var limit = as(plan, LimitExec.class);
+        var exchange = as(limit.child(), ExchangeExec.class);
+        var project = as(exchange.child(), ProjectExec.class);
+        var fieldExtract = as(project.child(), FieldExtractExec.class);
+        var queryExec = as(fieldExtract.child(), EsQueryExec.class);
+        var queryBuilder = as(queryExec.query(), MatchQueryBuilder.class);
+        assertThat(queryBuilder.fieldName(), is("emp_no"));
+        assertThat(queryBuilder.value(), is(123456));
+    }
+
     private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) {
         return FilterTests.singleValueQuery(query, inner, fieldName, source);
     }