Merge branch 'main' into build/windows_upload

Author: rjernst

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -22,9 +22,11 @@ public static TransportVersionDefinition fromString(Path file, String contents,
 
         String idsLine = null;
         if (contents.isEmpty() == false) {
-            String[] lines = contents.split(System.lineSeparator());
+            // Regardless of whether windows newlines exist (they could be added by git), we split on line feed.
+            // All we care about skipping lines with the comment character, so the remaining \r won't matter
+            String[] lines = contents.split("\n");
             for (String line : lines) {
-                line = line.replaceAll("\\s+", "");
+                line = line.strip();
                 if (line.startsWith("#") == false) {
                     idsLine = line;
                     break;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUpperBound.java

@@ -24,9 +24,11 @@ public static TransportVersionUpperBound fromString(Path file, String contents)
         String branch = filename.substring(slashIndex == -1 ? 0 : (slashIndex + 1), filename.length() - 4);
 
         String idsLine = null;
-        String[] lines = contents.split(System.lineSeparator());
+        // Regardless of whether windows newlines exist (they could be added by git), we split on line feed.
+        // All we care about skipping lines with the comment character, so the remaining \r won't matter
+        String[] lines = contents.split("\n");
         for (String line : lines) {
-            line = line.replaceAll("\\s+", "");
+            line = line.strip();
             if (line.startsWith("#") == false) {
                 idsLine = line;
                 break;

muted-tests.yml

@@ -690,6 +690,33 @@ tests:
 - class: org.elasticsearch.upgrades.FullClusterRestartIT
   method: testHistoryUUIDIsAdded {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/135957
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testTurnOffTranslogRetentionAfterUpgraded {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135958
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSoftDeletes {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135959
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testClusterState {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135960
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testShrink {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135962
+- class: org.elasticsearch.upgrades.FullClusterRestartIT
+  method: testSearchTimeSeriesMode {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/135963
+- class: org.elasticsearch.datastreams.DataStreamIndexSettingsProviderTests
+  method: testGetAdditionalIndexSettings
+  issue: https://github.com/elastic/elasticsearch/issues/135972
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=indices.get_sample/10_basic/Test get sample for index with no sample config}
+  issue: https://github.com/elastic/elasticsearch/issues/135975
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
+  method: testStopQueryInlineStats
+  issue: https://github.com/elastic/elasticsearch/issues/135032
+- class: org.elasticsearch.xpack.esql.plan.logical.local.ImmediateLocalSupplierTests
+  method: testEqualsAndHashcode
+  issue: https://github.com/elastic/elasticsearch/issues/135977
 
 # Examples:
 #

x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Evaluator.java

@@ -42,4 +42,11 @@
      * into a warning and turn into a null value.
      */
     Class<? extends Exception>[] warnExceptions() default {};
+
+    /**
+     * Automatically evaluate each position to null if it is null on all the
+     * blocks. Setting this to {@code false} requires block parameters, like
+     * {@code static boolean process(@Position int position, LongBlock field1, LongBlock field2)}
+     */
+    boolean allNullsIsNull() default true;
 }

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java

@@ -51,13 +51,15 @@ public class EvaluatorImplementer {
     private final ProcessFunction processFunction;
     private final ClassName implementation;
     private final boolean processOutputsMultivalued;
+    private final boolean allNullsIsNull;
 
     public EvaluatorImplementer(
         Elements elements,
         javax.lang.model.util.Types types,
         ExecutableElement processFunction,
         String extraName,
-        List<TypeMirror> warnExceptions
+        List<TypeMirror> warnExceptions,
+        boolean allNullsIsNull
     ) {
         this.declarationType = (TypeElement) processFunction.getEnclosingElement();
         this.processFunction = new ProcessFunction(types, processFunction, warnExceptions);
@@ -66,7 +68,8 @@ public EvaluatorImplementer(
             elements.getPackageOf(declarationType).toString(),
             declarationType.getSimpleName() + extraName + "Evaluator"
         );
-        this.processOutputsMultivalued = this.processFunction.hasBlockType && (this.processFunction.builderArg != null);
+        this.processOutputsMultivalued = this.processFunction.hasBlockType;
+        this.allNullsIsNull = allNullsIsNull;
     }
 
     public JavaFile sourceFile() {
@@ -131,7 +134,7 @@ private MethodSpec ctor() {
         MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
         builder.addParameter(SOURCE, "source");
         builder.addStatement("this.source = source");
-        processFunction.args.stream().forEach(a -> a.implementCtor(builder));
+        processFunction.args.forEach(a -> a.implementCtor(builder));
         builder.addParameter(DRIVER_CONTEXT, "driverContext");
         builder.addStatement("this.driverContext = driverContext");
         return builder.build();
@@ -140,15 +143,15 @@ private MethodSpec ctor() {
     private MethodSpec eval() {
         MethodSpec.Builder builder = MethodSpec.methodBuilder("eval").addAnnotation(Override.class);
         builder.addModifiers(Modifier.PUBLIC).returns(BLOCK).addParameter(PAGE, "page");
-        processFunction.args.stream().forEach(a -> a.evalToBlock(builder));
+        processFunction.args.forEach(a -> a.evalToBlock(builder));
         String invokeBlockEval = invokeRealEval(true);
         if (processOutputsMultivalued) {
             builder.addStatement(invokeBlockEval);
         } else {
-            processFunction.args.stream().forEach(a -> a.resolveVectors(builder, invokeBlockEval));
+            processFunction.args.forEach(a -> a.resolveVectors(builder, invokeBlockEval));
             builder.addStatement(invokeRealEval(false));
         }
-        processFunction.args.stream().forEach(a -> a.closeEvalToBlock(builder));
+        processFunction.args.forEach(a -> a.closeEvalToBlock(builder));
         return builder.build();
     }
 
@@ -157,7 +160,7 @@ private String invokeRealEval(boolean blockStyle) {
 
         String params = processFunction.args.stream()
             .map(a -> a.paramName(blockStyle))
-            .filter(a -> a != null)
+            .filter(Objects::nonNull)
             .collect(Collectors.joining(", "));
         if (params.length() > 0) {
             builder.append(", ");
@@ -189,23 +192,23 @@ private MethodSpec realEval(boolean blockStyle) {
             buildFromFactory(builderType)
         );
         {
-            processFunction.args.stream().forEach(a -> {
+            processFunction.args.forEach(a -> {
                 if (a.paramName(blockStyle) != null) {
                     builder.addParameter(a.dataType(blockStyle), a.paramName(blockStyle));
                 }
             });
 
-            processFunction.args.stream().forEach(a -> a.createScratch(builder));
+            processFunction.args.forEach(a -> a.createScratch(builder));
 
             builder.beginControlFlow("position: for (int p = 0; p < positionCount; p++)");
             {
                 if (blockStyle) {
                     if (processOutputsMultivalued == false) {
-                        processFunction.args.stream().forEach(a -> a.skipNull(builder));
-                    } else {
+                        processFunction.args.forEach(a -> a.skipNull(builder));
+                    } else if (allNullsIsNull) {
                         builder.addStatement("boolean allBlocksAreNulls = true");
                         // allow block type inputs to be null
-                        processFunction.args.stream().forEach(a -> a.allBlocksAreNull(builder));
+                        processFunction.args.forEach(a -> a.allBlocksAreNull(builder));
 
                         builder.beginControlFlow("if (allBlocksAreNulls)");
                         {
@@ -214,25 +217,30 @@ private MethodSpec realEval(boolean blockStyle) {
                         }
                         builder.endControlFlow();
                     }
+                } else {
+                    assert allNullsIsNull : "allNullsIsNull == false is only supported for block style.";
                 }
-                processFunction.args.stream().forEach(a -> a.read(builder, blockStyle));
+                processFunction.args.forEach(a -> a.read(builder, blockStyle));
 
                 StringBuilder pattern = new StringBuilder();
                 List<Object> args = new ArrayList<>();
                 pattern.append("$T.$N(");
                 args.add(declarationType);
                 args.add(processFunction.function.getSimpleName());
-                processFunction.args.stream().forEach(a -> {
-                    if (args.size() > 2) {
-                        pattern.append(", ");
-                    }
-                    a.buildInvocation(pattern, args, blockStyle);
-                });
+                pattern.append(processFunction.args.stream().map(argument -> {
+                    var invocation = new StringBuilder();
+                    argument.buildInvocation(invocation, args, blockStyle);
+                    return invocation.toString();
+                }).collect(Collectors.joining(", ")));
                 pattern.append(")");
                 String builtPattern;
                 if (processFunction.builderArg == null) {
-                    builtPattern = vectorize ? "result.$L(p, " + pattern + ")" : "result.$L(" + pattern + ")";
-                    args.add(0, processFunction.appendMethod());
+                    if (vectorize) {
+                        builtPattern = "result.$L(p, " + pattern + ")";
+                    } else {
+                        builtPattern = "result.$L(" + pattern + ")";
+                    }
+                    args.addFirst(processFunction.appendMethod());
                 } else {
                     builtPattern = pattern.toString();
                 }
@@ -246,7 +254,7 @@ private MethodSpec realEval(boolean blockStyle) {
                     String catchPattern = "catch ("
                         + processFunction.warnExceptions.stream().map(m -> "$T").collect(Collectors.joining(" | "))
                         + " e)";
-                    builder.nextControlFlow(catchPattern, processFunction.warnExceptions.stream().map(m -> TypeName.get(m)).toArray());
+                    builder.nextControlFlow(catchPattern, processFunction.warnExceptions.stream().map(TypeName::get).toArray());
                     builder.addStatement("warnings().registerException(e)");
                     builder.addStatement("result.appendNull()");
                     builder.endControlFlow();
@@ -282,7 +290,7 @@ private TypeSpec factory() {
         builder.addModifiers(Modifier.STATIC);
 
         builder.addField(SOURCE, "source", Modifier.PRIVATE, Modifier.FINAL);
-        processFunction.args.stream().forEach(a -> a.declareFactoryField(builder));
+        processFunction.args.forEach(a -> a.declareFactoryField(builder));
 
         builder.addMethod(processFunction.factoryCtor());
         builder.addMethod(processFunction.factoryGet(implementation));
@@ -371,7 +379,7 @@ MethodSpec factoryCtor() {
             MethodSpec.Builder builder = MethodSpec.constructorBuilder().addModifiers(Modifier.PUBLIC);
             builder.addParameter(SOURCE, "source");
             builder.addStatement("this.source = source");
-            args.stream().forEach(a -> a.implementFactoryCtor(builder));
+            args.forEach(a -> a.implementFactoryCtor(builder));
             return builder.build();
         }
 

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorProcessor.java

@@ -82,7 +82,8 @@ public boolean process(Set<? extends TypeElement> set, RoundEnvironment roundEnv
                                 env.getTypeUtils(),
                                 (ExecutableElement) evaluatorMethod,
                                 evaluatorAnn.extraName(),
-                                warnExceptionsTypes
+                                warnExceptionsTypes,
+                                evaluatorAnn.allNullsIsNull()
                             ).sourceFile(),
                             env
                         );

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java

@@ -53,6 +53,7 @@ protected boolean supportsSourceFieldMapping() {
 
     @Before
     public void configureChunks() throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         boolean smallChunks = randomBoolean();
         Request request = new Request("PUT", "/_cluster/settings");
         XContentBuilder builder = JsonXContent.contentBuilder().startObject().startObject("persistent");

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -122,9 +122,11 @@ protected EsqlSpecTestCase(
     }
 
     private static boolean dataLoaded = false;
+    protected static boolean testClustersOk = true;
 
     @Before
     public void setup() throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         boolean supportsLookup = supportsIndexModeLookup();
         boolean supportsSourceMapping = supportsSourceFieldMapping();
         boolean supportsInferenceTestService = supportsInferenceTestService();
@@ -140,6 +142,9 @@ public void setup() throws IOException {
 
     @AfterClass
     public static void wipeTestData() throws IOException {
+        if (testClustersOk == false) {
+            return;
+        }
         try {
             dataLoaded = false;
             adminClient().performRequest(new Request("DELETE", "/*"));
@@ -162,11 +167,25 @@ public final void test() throws Throwable {
             shouldSkipTest(testName);
             doTest();
         } catch (Exception e) {
+            ensureTestClustersAreOk(e);
             throw reworkException(e);
         }
     }
 
+    protected void ensureTestClustersAreOk(Exception failure) {
+        try {
+            ensureHealth(client(), "", (request) -> {
+                request.addParameter("wait_for_status", "yellow");
+                request.addParameter("level", "shards");
+            });
+        } catch (Exception inner) {
+            testClustersOk = false;
+            failure.addSuppressed(inner);
+        }
+    }
+
     protected void shouldSkipTest(String testName) throws IOException {
+        assumeTrue("test clusters were broken", testClustersOk);
         if (requiresInferenceEndpoint()) {
             assumeTrue("Inference test service needs to be supported", supportsInferenceTestService());
         }
@@ -374,7 +393,9 @@ protected boolean preserveClusterUponCompletion() {
 
     @After
     public void assertRequestBreakerEmptyAfterTests() throws Exception {
-        assertRequestBreakerEmpty();
+        if (testClustersOk) {
+            assertRequestBreakerEmpty();
+        }
     }
 
     public static void assertRequestBreakerEmpty() throws Exception {