Merge branch 'main' into limit-in-flight-requests

Author: elasticmachine

.buildkite/pipelines/periodic.yml

@@ -584,7 +584,7 @@ steps:
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
       diskSizeGb: 250
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/scripts/lucene-snapshot/update-branch.sh

@@ -7,21 +7,12 @@ if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then
   exit 1
 fi
 
-if [[ "$BUILDKITE_BRANCH" == "lucene_snapshot_10" ]]; then
-  UPSTREAM="main"
-elif [[ "$BUILDKITE_BRANCH" == "lucene_snapshot" ]]; then
-  UPSTREAM="8.x"
-else
-  echo "Error: unknown branch: $BUILDKITE_BRANCH"
-  exit 1
-fi
-
-echo --- Updating "$BUILDKITE_BRANCH" branch with "$UPSTREAM"
+echo --- Updating "$BUILDKITE_BRANCH" branch with main
 
 git config --global user.name elasticsearchmachine
 git config --global user.email '[email protected]'
 
 git checkout "$BUILDKITE_BRANCH"
-git fetch origin "$UPSTREAM"
-git merge --no-edit "origin/$UPSTREAM"
+git fetch origin main
+git merge --no-edit origin/main
 git push origin "$BUILDKITE_BRANCH"

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -22,6 +22,9 @@
 package org.elasticsearch.benchmark.tdigest;
 
 import org.elasticsearch.tdigest.Sort;
+import org.elasticsearch.tdigest.arrays.TDigestDoubleArray;
+import org.elasticsearch.tdigest.arrays.TDigestIntArray;
+import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -35,7 +38,6 @@
 import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Warmup;
 
-import java.util.Arrays;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
@@ -49,7 +51,7 @@
 @State(Scope.Thread)
 public class SortBench {
     private final int size = 100000;
-    private final double[] values = new double[size];
+    private final TDigestDoubleArray values = WrapperTDigestArrays.INSTANCE.newDoubleArray(size);
 
     @Param({ "0", "1", "-1" })
     public int sortDirection;
@@ -58,22 +60,22 @@ public class SortBench {
     public void setup() {
         Random prng = new Random(999983);
         for (int i = 0; i < size; i++) {
-            values[i] = prng.nextDouble();
+            values.set(i, prng.nextDouble());
         }
         if (sortDirection > 0) {
-            Arrays.sort(values);
+            values.sort();
         } else if (sortDirection < 0) {
-            Arrays.sort(values);
-            Sort.reverse(values, 0, values.length);
+            values.sort();
+            Sort.reverse(values, 0, values.size());
         }
     }
 
     @Benchmark
-    public void quicksort() {
-        int[] order = new int[size];
+    public void stableSort() {
+        TDigestIntArray order = WrapperTDigestArrays.INSTANCE.newIntArray(size);
         for (int i = 0; i < size; i++) {
-            order[i] = i;
+            order.set(i, i);
         }
-        Sort.sort(order, values, null, values.length);
+        Sort.stableSort(order, values, values.size());
     }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -21,9 +21,9 @@
 
 package org.elasticsearch.benchmark.tdigest;
 
-import org.elasticsearch.tdigest.AVLTreeDigest;
 import org.elasticsearch.tdigest.MergingDigest;
 import org.elasticsearch.tdigest.TDigest;
+import org.elasticsearch.tdigest.arrays.WrapperTDigestArrays;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -61,13 +61,19 @@ public enum TDigestFactory {
         MERGE {
             @Override
             TDigest create(double compression) {
-                return new MergingDigest(compression, (int) (10 * compression));
+                return new MergingDigest(WrapperTDigestArrays.INSTANCE, compression, (int) (10 * compression));
             }
         },
         AVL_TREE {
             @Override
             TDigest create(double compression) {
-                return new AVLTreeDigest(compression);
+                return TDigest.createAvlTreeDigest(WrapperTDigestArrays.INSTANCE, compression);
+            }
+        },
+        HYBRID {
+            @Override
+            TDigest create(double compression) {
+                return TDigest.createHybridDigest(WrapperTDigestArrays.INSTANCE, compression);
             }
         };
 
@@ -77,7 +83,7 @@ TDigest create(double compression) {
     @Param({ "100", "300" })
     double compression;
 
-    @Param({ "MERGE", "AVL_TREE" })
+    @Param({ "MERGE", "AVL_TREE", "HYBRID" })
     TDigestFactory tdigestFactory;
 
     @Param({ "NORMAL", "GAUSSIAN" })

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
+distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -168,7 +168,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,CLDR',
+              '-Djava.locale.providers=CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -93,7 +93,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,CLDR");
+                    test.systemProperty("java.locale.providers", "CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -59,7 +59,6 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.nativeaccess",
         "org.elasticsearch.plugin",
         "org.elasticsearch.plugin.analysis",
-        "org.elasticsearch.pluginclassloader",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
         "org.elasticsearch.simdvec",

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -22,7 +22,9 @@
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.JavaCompile;
+import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.external.javadoc.CoreJavadocOptions;
 import org.gradle.jvm.tasks.Jar;
 import org.gradle.jvm.toolchain.JavaLanguageVersion;
 import org.gradle.jvm.toolchain.JavaToolchainService;
@@ -73,8 +75,10 @@ public void apply(Project project) {
             List<Integer> mainVersions = findSourceVersions(project);
             List<String> mainSourceSets = new ArrayList<>();
             mainSourceSets.add(SourceSet.MAIN_SOURCE_SET_NAME);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.MAIN_SOURCE_SET_NAME), 21);
             List<String> testSourceSets = new ArrayList<>(mainSourceSets);
             testSourceSets.add(SourceSet.TEST_SOURCE_SET_NAME);
+            configurePreviewFeatures(project, javaExtension.getSourceSets().getByName(SourceSet.TEST_SOURCE_SET_NAME), 21);
             for (int javaVersion : mainVersions) {
                 String mainSourceSetName = SourceSet.MAIN_SOURCE_SET_NAME + javaVersion;
                 SourceSet mainSourceSet = addSourceSet(project, javaExtension, mainSourceSetName, mainSourceSets, javaVersion);
@@ -124,11 +128,8 @@ private SourceSet addSourceSet(
             compileTask.setSourceCompatibility(Integer.toString(javaVersion));
             CompileOptions compileOptions = compileTask.getOptions();
             compileOptions.getRelease().set(javaVersion);
-            compileOptions.getCompilerArgs().add("--enable-preview");
-            compileOptions.getCompilerArgs().add("-Xlint:-preview");
-
-            compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
         });
+        configurePreviewFeatures(project, sourceSet, javaVersion);
 
         // Since we configure MRJAR sourcesets to allow preview apis, class signatures for those
         // apis are not known by forbidden apis, so we must ignore all missing classes. We could, in theory,
@@ -142,6 +143,21 @@ private SourceSet addSourceSet(
         return sourceSet;
     }
 
+    private void configurePreviewFeatures(Project project, SourceSet sourceSet, int javaVersion) {
+        project.getTasks().withType(JavaCompile.class).named(sourceSet.getCompileJavaTaskName()).configure(compileTask -> {
+            CompileOptions compileOptions = compileTask.getOptions();
+            compileOptions.getCompilerArgs().add("--enable-preview");
+            compileOptions.getCompilerArgs().add("-Xlint:-preview");
+
+            compileTask.doLast(t -> { stripPreviewFromFiles(compileTask.getDestinationDirectory().getAsFile().get().toPath()); });
+        });
+        project.getTasks().withType(Javadoc.class).named(name -> name.equals(sourceSet.getJavadocTaskName())).configureEach(javadocTask -> {
+            CoreJavadocOptions options = (CoreJavadocOptions) javadocTask.getOptions();
+            options.addBooleanOption("-enable-preview", true);
+            options.addStringOption("-release", String.valueOf(javaVersion));
+        });
+    }
+
     private void configureSourceSetInJar(Project project, SourceSet sourceSet, int javaVersion) {
         var jarTask = project.getTasks().withType(Jar.class).named(JavaPlugin.JAR_TASK_NAME);
         jarTask.configure(task -> task.into("META-INF/versions/" + javaVersion, copySpec -> copySpec.from(sourceSet.getOutput())));

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.10
+8.10.1