Merge branch 'main' into mtv32

Author: jdconrad

docs/changelog/135897.yaml

@@ -0,0 +1,5 @@
+pr: 135897
+summary: Apply source excludes early when retrieving the `_inference_fields`
+area: Search
+type: bug
+issues: []

docs/changelog/135901.yaml

@@ -0,0 +1,5 @@
+pr: 135901
+summary: Add support for dot-separated attribute names (e.g. `foo.bar`) and for parameters (e.g. `??my_param`) in FUSE GROUP BY
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -504,9 +504,6 @@ tests:
 - class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT
   method: testRepositoryAnalysis
   issue: https://github.com/elastic/elasticsearch/issues/134853
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlinestats
-  issue: https://github.com/elastic/elasticsearch/issues/134854
 - class: org.elasticsearch.xpack.esql.action.CrossClusterCancellationIT
   method: testCancelSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/134865
@@ -522,9 +519,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   method: test {csv-spec:stats.CountDistinctWithConditions}
   issue: https://github.com/elastic/elasticsearch/issues/134993
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryInlineStats
-  issue: https://github.com/elastic/elasticsearch/issues/135032
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:fork.ForkBeforeStatsWithWhere}
   issue: https://github.com/elastic/elasticsearch/issues/135041
@@ -678,6 +672,42 @@ tests:
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testOperationBasedRecovery {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/135944
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRomanianAnalyzerBWC {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135948
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testEmptyShard {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135951
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRollover {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135952
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testRecovery {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135953
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testNewReplicas {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135956
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testHistoryUUIDIsAdded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135957
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testTurnOffTranslogRetentionAfterUpgraded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135958
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSoftDeletes {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135959
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClusterState {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135960
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrink {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135962
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSearchTimeSeriesMode {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135963
+- class: org.elasticsearch.datastreams.DataStreamIndexSettingsProviderTests
+  method: testGetAdditionalIndexSettings
+  issue: https://github.com/elastic/elasticsearch/issues/135972
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -426,6 +426,14 @@ private static Boolean shouldExcludeVectorsFromSourceExplicit(FetchSourceContext
 
     public static boolean shouldExcludeInferenceFieldsFromSource(IndexSettings indexSettings, FetchSourceContext fetchSourceContext) {
         var explicit = shouldExcludeInferenceFieldsFromSourceExplicit(fetchSourceContext);
+        var filter = fetchSourceContext != null ? fetchSourceContext.filter() : null;
+        if (filter != null) {
+            if (filter.isPathFiltered(InferenceMetadataFieldsMapper.NAME, true)) {
+                return true;
+            } else if (filter.isExplicitlyIncluded(InferenceMetadataFieldsMapper.NAME)) {
+                return false;
+            }
+        }
         return explicit != null ? explicit : INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING.get(indexSettings.getSettings());
     }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.lookup.SourceFilter;
 
+import java.util.HashMap;
 import java.util.Map;
 
 public final class FetchSourcePhase implements FetchSubPhase {
@@ -99,6 +100,10 @@ private Source replaceInferenceMetadataFields(SearchHit hit, Source source) {
                     return source;
                 }
                 var newSource = source.source();
+                if (newSource instanceof HashMap == false) {
+                    // the map is not mutable
+                    newSource = new HashMap<>(newSource);
+                }
                 newSource.put(InferenceMetadataFieldsMapper.NAME, field.getValues().get(0));
                 return Source.fromMap(newSource, source.sourceContentType());
             }

x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Evaluator.java

@@ -42,4 +42,11 @@
      * into a warning and turn into a null value.
      */
     Class<? extends Exception>[] warnExceptions() default {};
+
+    /**
+     * Automatically evaluate each position to null if it is null on all the
+     * blocks. Setting this to {@code false} requires block parameters, like
+     * {@code static boolean process(@Position int position, LongBlock field1, LongBlock field2)}
+     */
+    boolean allNullsIsNull() default true;
 }

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java

@@ -51,13 +51,15 @@ public class EvaluatorImplementer {
     private final ProcessFunction processFunction;
     private final ClassName implementation;
     private final boolean processOutputsMultivalued;
+    private final boolean allNullsIsNull;
 
     public EvaluatorImplementer(
         Elements elements,
         javax.lang.model.util.Types types,
         ExecutableElement processFunction,
         String extraName,
-        List<TypeMirror> warnExceptions
+        List<TypeMirror> warnExceptions,
+        boolean allNullsIsNull
     ) {
         this.declarationType = (TypeElement) processFunction.getEnclosingElement();
         this.processFunction = new ProcessFunction(types, processFunction, warnExceptions);
@@ -66,7 +68,8 @@ public EvaluatorImplementer(
             elements.getPackageOf(declarationType).toString(),
             declarationType.getSimpleName() + extraName + "Evaluator"
         );
-        this.processOutputsMultivalued = this.processFunction.hasBlockType && (this.processFunction.builderArg != null);
+        this.processOutputsMultivalued = this.processFunction.hasBlockType;
+        this.allNullsIsNull = allNullsIsNull;
     }
 
     public JavaFile sourceFile() {
@@ -131,7 +134,7 @@ private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
         builder.addParameter(SOURCE, "source");
         builder.addStatement("this.source = source");
-        processFunction.args.stream().forEach(a -> a.implementCtor(builder));
+        processFunction.args.forEach(a -> a.implementCtor(builder));
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         builder.addStatement("this.driverContext = driverContext");
         return builder.build();
@@ -140,15 +143,15 @@ private MethodSpec ctor() {
     private MethodSpec eval() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("eval").addAnnotation(Override.class);
         builder.addModifiers(Modifier.PUBLIC).returns(BLOCK).addParameter(PAGE, "page");
-        processFunction.args.stream().forEach(a -> a.evalToBlock(builder));
+        processFunction.args.forEach(a -> a.evalToBlock(builder));
         String invokeBlockEval = invokeRealEval(true);
         if (processOutputsMultivalued) {
             builder.addStatement(invokeBlockEval);
         } else {
-            processFunction.args.stream().forEach(a -> a.resolveVectors(builder, invokeBlockEval));
+            processFunction.args.forEach(a -> a.resolveVectors(builder, invokeBlockEval));
             builder.addStatement(invokeRealEval(false));
         }
-        processFunction.args.stream().forEach(a -> a.closeEvalToBlock(builder));
+        processFunction.args.forEach(a -> a.closeEvalToBlock(builder));
         return builder.build();
     }
 
@@ -157,7 +160,7 @@ private String invokeRealEval(boolean blockStyle) {
 
         String params = processFunction.args.stream()
             .map(a -> a.paramName(blockStyle))
-            .filter(a -> a != null)
+            .filter(Objects::nonNull)
             .collect(Collectors.joining(", "));
         if (params.length() > 0) {
             builder.append(", ");
@@ -189,23 +192,23 @@ private MethodSpec realEval(boolean blockStyle) {
             buildFromFactory(builderType)
         );
         {
-            processFunction.args.stream().forEach(a -> {
+            processFunction.args.forEach(a -> {
                 if (a.paramName(blockStyle) != null) {
                     builder.addParameter(a.dataType(blockStyle), a.paramName(blockStyle));
                 }
             });
 
-            processFunction.args.stream().forEach(a -> a.createScratch(builder));
+            processFunction.args.forEach(a -> a.createScratch(builder));
 
             builder.beginControlFlow("position: for (int p = 0; p < positionCount; p++)");
             {
                 if (blockStyle) {
                     if (processOutputsMultivalued == false) {
-                        processFunction.args.stream().forEach(a -> a.skipNull(builder));
-                    } else {
+                        processFunction.args.forEach(a -> a.skipNull(builder));
+                    } else if (allNullsIsNull) {
                         builder.addStatement("boolean allBlocksAreNulls = true");
                         // allow block type inputs to be null
-                        processFunction.args.stream().forEach(a -> a.allBlocksAreNull(builder));
+                        processFunction.args.forEach(a -> a.allBlocksAreNull(builder));
 
                         builder.beginControlFlow("if (allBlocksAreNulls)");
                         {
@@ -214,25 +217,30 @@ private MethodSpec realEval(boolean blockStyle) {
                         }
                         builder.endControlFlow();
                     }
+                } else {
+                    assert allNullsIsNull : "allNullsIsNull == false is only supported for block style.";
                 }
-                processFunction.args.stream().forEach(a -> a.read(builder, blockStyle));
+                processFunction.args.forEach(a -> a.read(builder, blockStyle));
 
                 StringBuilder pattern = new StringBuilder();
                 List<Object> args = new ArrayList<>();
                 pattern.append("$T.$N(");
                 args.add(declarationType);
                 args.add(processFunction.function.getSimpleName());
-                processFunction.args.stream().forEach(a -> {
-                    if (args.size() > 2) {
-                        pattern.append(", ");
-                    }
-                    a.buildInvocation(pattern, args, blockStyle);
-                });
+                pattern.append(processFunction.args.stream().map(argument -> {
+                    var invocation = new StringBuilder();
+                    argument.buildInvocation(invocation, args, blockStyle);
+                    return invocation.toString();
+                }).collect(Collectors.joining(", ")));
                 pattern.append(")");
                 String builtPattern;
                 if (processFunction.builderArg == null) {
-                    builtPattern = vectorize ? "result.$L(p, " + pattern + ")" : "result.$L(" + pattern + ")";
-                    args.add(0, processFunction.appendMethod());
+                    if (vectorize) {
+                        builtPattern = "result.$L(p, " + pattern + ")";
+                    } else {
+                        builtPattern = "result.$L(" + pattern + ")";
+                    }
+                    args.addFirst(processFunction.appendMethod());
                 } else {
                     builtPattern = pattern.toString();
                 }
@@ -246,7 +254,7 @@ private MethodSpec realEval(boolean blockStyle) {
                     String catchPattern = "catch ("
                         + processFunction.warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | "))
                         + " e)";
-                    builder.nextControlFlow(catchPattern, processFunction.warnExceptions.stream().map(m -> TypeName.get(m)).toArray());
+                    builder.nextControlFlow(catchPattern, processFunction.warnExceptions.stream().map(TypeName::get).toArray());
                     builder.addStatement("warnings().registerException(e)");
                     builder.addStatement("result.appendNull()");
                     builder.endControlFlow();
@@ -282,7 +290,7 @@ private TypeSpec factory() {
         builder.addModifiers(Modifier.STATIC);
 
         builder.addField(SOURCE, "source", Modifier.PRIVATE, Modifier.FINAL);
-        processFunction.args.stream().forEach(a -> a.declareFactoryField(builder));
+        processFunction.args.forEach(a -> a.declareFactoryField(builder));
 
         builder.addMethod(processFunction.factoryCtor());
         builder.addMethod(processFunction.factoryGet(implementation));
@@ -371,7 +379,7 @@ MethodSpec factoryCtor() {
             MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
             builder.addParameter(SOURCE, "source");
             builder.addStatement("this.source = source");
-            args.stream().forEach(a -> a.implementFactoryCtor(builder));
+            args.forEach(a -> a.implementFactoryCtor(builder));
             return builder.build();
         }
 

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java

@@ -82,7 +82,8 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                                 env.getTypeUtils(),
                                 (ExecutableElement) evaluatorMethod,
                                 evaluatorAnn.extraName(),
-                                warnExceptionsTypes
+                                warnExceptionsTypes,
+                                evaluatorAnn.allNullsIsNull()
                             ).sourceFile(),
                             env
                         );

x-pack/plugin/esql/qa/testFixtures/src/main/resources/fuse.csv-spec

@@ -281,6 +281,33 @@ _score:double | new_fork:keyword  | _id:keyword
 0.01587       | fork1             | 26
 ;
 
+fuseWithRrfAndDiscriminatorColumnWithDots
+required_capability: fork_v9
+required_capability: fuse_v6
+required_capability: semantic_text_field_caps
+required_capability: metadata_score
+required_capability: dots_in_fuse
+
+FROM books METADATA _id, _index, _score
+| FORK ( WHERE title:"Tolkien" | SORT _score, _id DESC | LIMIT 3)
+       ( WHERE author:"Tolkien" | SORT _score, _id DESC | LIMIT 3)
+| EVAL new.fork = _fork
+| DROP _fork
+| FUSE rrf GROUP BY new.fork
+| SORT _score DESC, _id, _index
+| EVAL new.fork = mv_sort(new.fork)
+| EVAL _score = round(_score, 5)
+| KEEP _score, new.fork, _id
+;
+
+_score:double | new.fork:keyword  | _id:keyword
+0.03279       | [fork1, fork2]    | 4
+0.01613       | fork1             | 56
+0.01613       | fork2             | 60
+0.01587       | fork2             | 1
+0.01587       | fork1             | 26
+;
+
 fuseWithRrfAndKeyColumns
 required_capability: fork_v9
 required_capability: fuse_v6

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterAsyncQueryStopIT.java

@@ -314,9 +314,13 @@ public void testStopQueryInlineStats() throws Exception {
                 assertThat(asyncResponse.columns().size(), equalTo(2));
                 AtomicInteger i = new AtomicInteger(0);
                 asyncResponse.values().forEachRemaining(row -> {
-                    // We will have all rows here but total will be null since we stopped the inline stats before it could complete
-                    assertThat(row.next(), equalTo(null));
                     var v = row.next();
+                    // The sum could be null, if the stats did not manage to compute anything before being stopped
+                    // Or it could be 45L if it managed to add 0-9
+                    if (v != null) {
+                        assertThat((long) v, equalTo(45L));
+                    }
+                    v = row.next();
                     if (v != null) {
                         assertThat((long) v, lessThanOrEqualTo(10L));
                     }