Merge remote-tracking branch 'felixbarny/bulk-meta' into bulk-meta

Author: felixbarny

docs/changelog/138681.yaml

@@ -0,0 +1,5 @@
+pr: 138681
+summary: Field caps to support `project_routing` also in the body of the request
+area: Search
+type: enhancement
+issues: []

muted-tests.yml

@@ -439,6 +439,9 @@ tests:
 - class: org.elasticsearch.xpack.unsignedlong.UnsignedLongSyntheticSourceNativeArrayIntegrationTests
   method: testSynthesizeArrayRandom
   issue: https://github.com/elastic/elasticsearch/issues/138684
+- class: org.elasticsearch.index.mapper.HalfFloatSyntheticSourceNativeArrayIntegrationTests
+  method: testSynthesizeArrayRandom
+  issue: https://github.com/elastic/elasticsearch/issues/138730
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -280,7 +280,10 @@ public ResolvedIndexExpressions getResolvedIndexExpressions() {
         return resolvedIndexExpressions;
     }
 
-    public void projectRouting(String projectRouting) {
+    public void projectRouting(@Nullable String projectRouting) {
+        if (this.projectRouting != null) {
+            throw new IllegalArgumentException("project_routing is already set to [" + this.projectRouting + "]");
+        }
         this.projectRouting = projectRouting;
     }
 

server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java

@@ -107,12 +107,14 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
     private static final ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
     private static final ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings");
     private static final ParseField FIELDS_FIELD = new ParseField("fields");
+    private static final ParseField PROJECT_ROUTING = new ParseField("project_routing");
 
     private static final ObjectParser<FieldCapabilitiesRequest, Void> PARSER = new ObjectParser<>("field_caps_request");
 
     static {
         PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseTopLevelQuery(p), INDEX_FILTER_FIELD);
         PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD);
         PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields), FIELDS_FIELD);
+        PARSER.declareString(FieldCapabilitiesRequest::projectRouting, PROJECT_ROUTING);
     }
 }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -380,6 +380,9 @@ private Object valueMapper(CsvTestUtils.Type type, Object value) {
         if (value == null) {
             return "null";
         }
+        if (value instanceof CsvTestUtils.Range) {
+            return value;
+        }
         if (type == CsvTestUtils.Type.GEO_POINT || type == CsvTestUtils.Type.CARTESIAN_POINT) {
             // Point tests are failing in clustered integration tests because of tiny precision differences at very small scales
             if (value instanceof String wkt) {
@@ -421,6 +424,17 @@ private Object valueMapper(CsvTestUtils.Type type, Object value) {
                 }
             }
         }
+        if (type == CsvTestUtils.Type.DOUBLE || type == CsvTestUtils.Type.INTEGER || type == CsvTestUtils.Type.LONG) {
+            if (value instanceof List<?> vs) {
+                return vs.stream().map(v -> valueMapper(type, v)).toList();
+            } else if (type == CsvTestUtils.Type.DOUBLE) {
+                return ((Number) value).doubleValue();
+            } else if (type == CsvTestUtils.Type.INTEGER) {
+                return ((Number) value).intValue();
+            } else if (type == CsvTestUtils.Type.LONG) {
+                return ((Number) value).longValue();
+            }
+        }
         return value.toString();
     }
 

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java

@@ -236,7 +236,7 @@ public static void assertData(
 
                     var transformedExpected = valueTransformer.apply(expectedType, expectedValue);
                     var transformedActual = valueTransformer.apply(expectedType, actualValue);
-                    if (Objects.equals(transformedExpected, transformedActual) == false) {
+                    if (equals(transformedExpected, transformedActual) == false) {
                         dataFailures.add(new DataFailure(row, column, transformedExpected, transformedActual));
                     }
                     if (dataFailures.size() > 10) {
@@ -275,6 +275,24 @@ public static void assertData(
         }
     }
 
+    private static boolean equals(Object expected, Object actual) {
+        if (expected instanceof List<?> expectedList && actual instanceof List<?> actualList) {
+            if (expectedList.size() != actualList.size()) {
+                return false;
+            }
+            for (int i = 0; i < expectedList.size(); i++) {
+                if (equals(expectedList.get(i), actualList.get(i)) == false) {
+                    return false;
+                }
+            }
+            return true;
+        } else if (expected instanceof CsvTestUtils.Range expectedRange) {
+            return expectedRange.includes(actual);
+        } else {
+            return Objects.equals(expected, actual);
+        }
+    }
+
     private static void dataFailure(
         String description,
         List<DataFailure> dataFailures,

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java

@@ -82,6 +82,13 @@ public final class CsvTestUtils {
     public static final String COMMA_ESCAPING_REGEX = "(?<!\\" + ESCAPE_CHAR + "),";
     public static final String ESCAPED_COMMA_SEQUENCE = ESCAPE_CHAR + ",";
 
+    public record Range(Object lowerBound, Object upperBound) {
+        @SuppressWarnings("unchecked")
+        <T extends Comparable<T>> boolean includes(Object value) {
+            return ((T) value).compareTo((T) lowerBound) >= 0 && ((T) value).compareTo((T) upperBound) <= 0;
+        }
+    }
+
     private CsvTestUtils() {}
 
     public static boolean isEnabled(String testName, String instructions, Version version) {
@@ -617,7 +624,16 @@ Object convert(String value) {
             if (value == null) {
                 return null;
             }
-            return converter.apply(value);
+            if (Number.class.isAssignableFrom(clazz) && value.contains("..")) {
+                // Numbers of the form "lower..upper" are parsed to a Range, indicating that
+                // the expected value is within that range.
+                int separator = value.indexOf("..");
+                Object lowerBound = converter.apply(value.substring(0, separator).trim());
+                Object upperBound = converter.apply(value.substring(separator + 2).trim());
+                return new Range(lowerBound, upperBound);
+            } else {
+                return converter.apply(value);
+            }
         }
 
         Class<?> clazz() {

x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample.csv-spec

@@ -5,8 +5,6 @@
 // they also assert the value of MV_COUNT(VALUES(...)), which is not adjusted for
 // the sampling and therefore gives the size of the sample.
 // All ranges are very loose, so that the tests should practically never fail.
-// The range checks are done in ES|QL, resulting in one boolean value (is_expected),
-// because the CSV tests don't support such assertions.
 
 row
 required_capability: sample_v3
@@ -40,14 +38,10 @@ required_capability: sample_v3
 FROM employees
   | SAMPLE 0.5
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no), sum_emp_no = SUM(emp_no)
-  | EVAL is_expected = count >= 10 AND count <= 90 AND
-                       avg_emp_no > 10010 AND avg_emp_no < 10090 AND
-                       sum_emp_no > 10*10010 AND sum_emp_no < 90*10090
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double | sum_emp_no:long
+10..90     | 10010.0..10090.0  | 100100..908100
 ;
 
 
@@ -58,13 +52,10 @@ FROM employees
   | SAMPLE 0.5
   | WHERE emp_no > 10050 
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no)
-  | EVAL is_expected = count >= 2 AND count <= 48 AND
-                       avg_emp_no > 10055 AND avg_emp_no < 10095
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double
+2..48      | 10055..10095
 ;
 
 
@@ -75,13 +66,10 @@ FROM employees
   | WHERE emp_no <= 10050 
   | SAMPLE 0.5
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no)
-  | EVAL is_expected = count >= 2 AND count <= 48 AND
-                       avg_emp_no > 10005 AND avg_emp_no < 10045
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double
+2..48      | 10005..10045
 ;
 
 
@@ -92,13 +80,10 @@ FROM employees
   | SAMPLE 0.5
   | SORT emp_no
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no)
-  | EVAL is_expected = count >= 10 AND count <= 90 AND
-                       avg_emp_no > 10010 AND avg_emp_no < 10090
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double
+10..90     | 10010..10090
 ;
 
 
@@ -109,13 +94,10 @@ FROM employees
   | SORT emp_no
   | SAMPLE 0.5
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no)
-  | EVAL is_expected = count >= 10 AND count <= 90 AND
-                       avg_emp_no > 10010 AND avg_emp_no < 10090
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double
+10..90     | 10010..10090
 ;
 
 
@@ -126,12 +108,10 @@ FROM employees
   | SAMPLE 0.5
   | LIMIT 10
   | STATS count = COUNT(emp_no)
-  | EVAL is_expected = count == 10
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long
+10
 ;
 
 
@@ -142,12 +122,10 @@ FROM employees
   | LIMIT 50
   | SAMPLE 0.5
   | STATS count = COUNT(emp_no)
-  | EVAL is_expected = count >= 2 AND count <= 48
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long
+2..48
 ;
 
 
@@ -160,12 +138,11 @@ ROW x = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27
   | MV_EXPAND y
   | STATS count = COUNT() BY x
   | STATS counts = VALUES(count)
-  | EVAL is_expected = MV_COUNT(counts) == 1 AND MV_MIN(counts) == 2
-  | KEEP is_expected
+
 ;
 
-is_expected:boolean 
-true
+counts:long
+2
 ;
 
 
@@ -177,13 +154,11 @@ ROW x = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27
   | MV_EXPAND y
   | SAMPLE 0.85
   | STATS count = COUNT() BY x
-  | STATS counts = VALUES(count)
-  | EVAL is_expected = MV_COUNT(counts) == 2 AND MV_MIN(counts) == 1 AND MV_MAX(counts) == 2
-  | KEEP is_expected
+  | STATS counts = MV_SORT(VALUES(count))
 ;
 
-is_expected:boolean 
-true
+counts:long
+[1,2]
 ;
 
 
@@ -195,13 +170,10 @@ FROM employees
   | SAMPLE 0.8
   | SAMPLE 0.9
   | STATS count = COUNT(), avg_emp_no = AVG(emp_no)
-  | EVAL is_expected = count >= 10 AND count <= 90 AND
-                       avg_emp_no > 10010 AND avg_emp_no < 10090
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_emp_no:double
+10..90     | 10010..10090
 ;
 
 
@@ -213,13 +185,10 @@ FROM employees
   | STATS avg_salary = AVG(salary) BY job_positions
   | SAMPLE 0.8
   | STATS count = COUNT(), avg_avg_salary = AVG(avg_salary)
-  | EVAL is_expected = count >= 1 AND count <= 16 AND
-                       avg_avg_salary > 25000 AND avg_avg_salary < 75000
-  | KEEP is_expected
 ;
 
-is_expected:boolean
-true
+count:long | avg_avg_salary:double
+1..16      | 25000..75000
 ;
 
 
@@ -251,6 +220,7 @@ emp_no:integer
 sampleStatsEval
 required_capability: fix_no_columns
 required_capability: sample_v3
+
 FROM employees | SAMPLE 0.5 | LIMIT 10 | STATS count = COUNT() | EVAL is_expected = count > 0;
 
 count:long | is_expected:boolean