Merge remote-tracking branch 'origin/main' into lucene_snapshot_10_3

Author: romseygeek

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayIndexOfBenchmark.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package org.elasticsearch.benchmark.bytes;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+@Warmup(iterations = 4, time = 1)
+@Measurement(iterations = 5, time = 1)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@Fork(value = 1)
+public class BytesArrayIndexOfBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    static final byte MARKER = (byte) '\n';
+
+    @Param(value = { "64", "127", "128", "4096", "16384", "65536", "1048576" })
+    public int size;
+
+    BytesArray bytesArray;
+
+    @Setup
+    public void setup() {
+        byte[] bytes = new byte[size];
+        bytes[bytes.length - 1] = MARKER;
+        bytesArray = new BytesArray(bytes, 0, bytes.length);
+    }
+
+    @Benchmark
+    public int indexOf() {
+        return bytesArray.indexOf(MARKER, 0);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int indexOfPanama() {
+        return bytesArray.indexOf(MARKER, 0);
+    }
+
+    @Benchmark
+    public int withOffsetIndexOf() {
+        return bytesArray.indexOf(MARKER, 1);
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public int withOffsetIndexPanama() {
+        return bytesArray.indexOf(MARKER, 1);
+    }
+}

benchmarks/src/test/java/org/elasticsearch/benchmark/bytes/BytesArrayIndexOfBenchmarkTests.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.bytes;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.ESTestCase;
+import org.openjdk.jmh.annotations.Param;
+
+import java.util.Arrays;
+
+public class BytesArrayIndexOfBenchmarkTests extends ESTestCase {
+
+    final int size;
+
+    public BytesArrayIndexOfBenchmarkTests(int size) {
+        this.size = size;
+    }
+
+    public void testIndexOf() {
+        var bench = new BytesArrayIndexOfBenchmark();
+        bench.size = size;
+        bench.setup();
+        assertEquals(size - 1, bench.indexOf());
+        assertEquals(size - 1, bench.indexOfPanama());
+        assertEquals(size - 1, bench.withOffsetIndexOf());
+        assertEquals(size - 1, bench.withOffsetIndexPanama());
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parametersFactory() {
+        try {
+            var params = BytesArrayIndexOfBenchmark.class.getField("size").getAnnotationsByType(Param.class)[0].value();
+            return () -> Arrays.stream(params).map(Integer::parseInt).map(i -> new Object[] { i }).iterator();
+        } catch (NoSuchFieldException e) {
+            throw new AssertionError(e);
+        }
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -125,6 +125,9 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             tasks.named('generateTransportVersion') {
                 currentUpperBoundName = '9.2'
             }
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.2'
+            }
         """
         referableAndReferencedTransportVersion("existing_91", "8012000")
         referableAndReferencedTransportVersion("older_92", "8122000")

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -277,4 +277,21 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
             "[myserver/src/main/resources/transport/definitions/referable/existing_92.csv]"
         )
     }
+
+    def "primary id checks skipped on release branch"() {
+        given:
+        file("myserver/build.gradle") << """
+            tasks.named('validateTransportVersionResources') {
+                currentUpperBoundName = '9.1'
+            }
+        """
+        referableAndReferencedTransportVersion("some_tv", "8125000")
+        transportVersionUpperBound("9.2", "some_tv", "8125000")
+
+        when:
+        def result = gradleRunner("validateTransportVersionResources").build()
+
+        then:
+        result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionResourcesPlugin.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.gradle.internal.conventions.VersionPropertiesPlugin;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitTaskPlugin;
+import org.gradle.api.Action;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.Directory;
@@ -23,7 +24,6 @@
 
 import java.util.Map;
 import java.util.Properties;
-import java.util.function.Consumer;
 
 public class TransportVersionResourcesPlugin implements Plugin<Project> {
 
@@ -70,6 +70,7 @@ public void apply(Project project) {
                 t.getReferencesFiles().setFrom(tvReferencesConfig);
                 t.getShouldValidateDensity().convention(true);
                 t.getShouldValidatePrimaryIdNotPatch().convention(true);
+                t.getCurrentUpperBoundName().convention(currentVersion.getMajor() + "." + currentVersion.getMinor());
             });
         project.getTasks().named(PrecommitPlugin.PRECOMMIT_TASK_NAME).configure(t -> t.dependsOn(validateTask));
 
@@ -83,7 +84,7 @@ public void apply(Project project) {
             t.into(resourceRoot + "/definitions", c -> c.from(generateManifestTask));
         });
 
-        Consumer<GenerateTransportVersionDefinitionTask> generationConfiguration = t -> {
+        Action<GenerateTransportVersionDefinitionTask> generationConfiguration = t -> {
             t.setGroup(taskGroup);
             t.getReferencesFiles().setFrom(tvReferencesConfig);
             t.getIncrement().convention(1000);
@@ -92,17 +93,17 @@ public void apply(Project project) {
 
         var generateDefinitionsTask = project.getTasks()
             .register("generateTransportVersion", GenerateTransportVersionDefinitionTask.class, t -> {
-                generationConfiguration.accept(t);
                 t.setDescription("(Re)generates a transport version definition file");
             });
+        generateDefinitionsTask.configure(generationConfiguration);
         validateTask.configure(t -> t.mustRunAfter(generateDefinitionsTask));
 
         var resolveConflictTask = project.getTasks()
             .register("resolveTransportVersionConflict", GenerateTransportVersionDefinitionTask.class, t -> {
-                generationConfiguration.accept(t);
                 t.setDescription("Resolve merge conflicts in transport version internal state files");
                 t.getResolveConflict().set(true);
             });
+        resolveConflictTask.configure(generationConfiguration);
         validateTask.configure(t -> t.mustRunAfter(resolveConflictTask));
 
         var generateInitialTask = project.getTasks()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionResourcesTask.java

@@ -60,6 +60,13 @@ public Path getResourcesDir() {
     @Input
     public abstract Property<Boolean> getShouldValidatePrimaryIdNotPatch();
 
+    /**
+     * The name of the upper bounds file which will be used at runtime on the current branch. Normally
+     * this equates to VersionProperties.getElasticsearchVersion().
+     */
+    @Input
+    public abstract Property<String> getCurrentUpperBoundName();
+
     private record IdAndDefinition(TransportVersionId id, TransportVersionDefinition definition) {}
 
     private static final Pattern NAME_FORMAT = Pattern.compile("[a-z0-9_]+");
@@ -76,6 +83,7 @@ public void validateTransportVersions() throws IOException {
         Map<String, TransportVersionDefinition> allDefinitions = collectAllDefinitions(referableDefinitions, unreferableDefinitions);
         Map<Integer, List<IdAndDefinition>> idsByBase = collectIdsByBase(allDefinitions.values());
         Map<String, TransportVersionUpperBound> upperBounds = resources.getUpperBounds();
+        boolean onReleaseBranch = checkIfDefinitelyOnReleaseBranch(upperBounds);
 
         for (var definition : referableDefinitions.values()) {
             validateNamedDefinition(definition, referencedNames);
@@ -93,7 +101,9 @@ public void validateTransportVersions() throws IOException {
             validateUpperBound(upperBound, allDefinitions, idsByBase);
         }
 
-        validatePrimaryIds(resources, upperBounds, allDefinitions);
+        if (onReleaseBranch == false) {
+            validatePrimaryIds(resources, upperBounds, allDefinitions);
+        }
     }
 
     private Map<String, TransportVersionDefinition> collectAllDefinitions(
@@ -245,7 +255,7 @@ private void validateUpperBound(
         }
 
         TransportVersionUpperBound existingUpperBound = getResources().get().getUpperBoundFromUpstream(upperBound.name());
-        if (existingUpperBound != null) {
+        if (existingUpperBound != null && getShouldValidatePrimaryIdNotPatch().get()) {
             if (upperBound.definitionId().patch() != 0 && upperBound.definitionId().base() != existingUpperBound.definitionId().base()) {
                 throwUpperBoundFailure(
                     upperBound,
@@ -318,6 +328,15 @@ private void validatePrimaryIds(
         );
     }
 
+    private boolean checkIfDefinitelyOnReleaseBranch(Map<String, TransportVersionUpperBound> upperBounds) {
+        // only want to look at definitions <= the current upper bound.
+        // TODO: we should filter all of the upper bounds/definitions that are validated by this, not just in this method
+        String currentUpperBoundName = getCurrentUpperBoundName().get();
+        TransportVersionUpperBound currentUpperBound = upperBounds.get(currentUpperBoundName);
+
+        return upperBounds.values().stream().anyMatch(u -> u.definitionId().complete() > currentUpperBound.definitionId().complete());
+    }
+
     private void throwDefinitionFailure(TransportVersionDefinition definition, String message) {
         Path relativePath = getResources().get().getDefinitionPath(definition);
         throw new VerificationException("Transport version definition file [" + relativePath + "] " + message);

build-tools-internal/src/main/resources/fips_java.security

@@ -49,3 +49,6 @@ jdk.xml.dsig.secureValidationPolicy=\
     noRetrievalMethodLoops
 jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep;\
   java.base/java.security.KeyRep$Type;java.base/javax.crypto.spec.SecretKeySpec;!*
+
+# Needed since JDK25 to match the default config in the out-of-the-box java.security
+jdk.includeInExceptions=hostInfoExclSocket

build-tools-internal/src/main/resources/fips_java_oracle.security

@@ -50,3 +50,6 @@ jdk.xml.dsig.secureValidationPolicy=\
     noRetrievalMethodLoops
 jceks.key.serialFilter = java.base/java.lang.Enum;java.base/java.security.KeyRep;\
   java.base/java.security.KeyRep$Type;java.base/javax.crypto.spec.SecretKeySpec;!*
+
+# Needed since JDK25 to match the default config in the out-of-the-box java.security
+jdk.includeInExceptions=hostInfoExclSocket

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 9.2.0
 lucene            = 10.3.0
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 24.0.2+12@fdc5d0102fe0414db21410ad5834341f
+bundled_jdk = 25+36@bd75d5f9689641da8e1daabeccb5528b
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java

@@ -49,6 +49,7 @@
 import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode;
 import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.startsWith;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -314,7 +315,7 @@ public void testNodeSelector() throws Exception {
                     } catch (ConnectException e) {
                         // Windows isn't consistent here. Sometimes the message is even null!
                         if (false == System.getProperty("os.name").startsWith("Windows")) {
-                            assertEquals("Connection refused", e.getMessage());
+                            assertThat(e.getMessage(), startsWith("Connection refused"));
                         }
                     }
                 } else {