Merge branch 'main' of https://github.com/elastic/elasticsearch into support_multi_inlinestats_logicalPlan_approach

Author: astefan

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -56,20 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
 
 cat <<EOF | buildkite-agent pipeline upload
 steps:
-    - label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
-      timeout_in_minutes: 300
-      agents:
-        provider: gcp
-        image: family/elasticsearch-ubuntu-2004
-        machineType: n1-standard-32
-        buildDirectory: /dev/shm/bk
-        preemptible: true
-      retry:
-        automatic:
-          - exit_status: "-1"
-            limit: 3
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
+  - group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
+    steps:
+      - label: "bc-upgrade-tests-part{{matrix.PART}}"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}}
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        matrix:
+          setup:
+            PART: ["1", "2", "3", "4", "5", "6"]
 EOF

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -24,22 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
 
 echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
 
-cat <<EOF | buildkite-agent pipeline upload
+cat << EOF | buildkite-agent pipeline upload
 steps:
-    - label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
-      timeout_in_minutes: 300
-      agents:
-        provider: gcp
-        image: family/elasticsearch-ubuntu-2004
-        machineType: n1-standard-32
-        buildDirectory: /dev/shm/bk
-        preemptible: true
-      retry:
-        automatic:
-          - exit_status: "-1"
-            limit: 3
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
+    - group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
+      steps:
+        - label: "pr-upgrade-part-{{matrix.PART}}"
+          command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}}
+          timeout_in_minutes: 300
+          agents:
+            provider: gcp
+            image: family/elasticsearch-ubuntu-2004
+            machineType: n1-standard-32
+            buildDirectory: /dev/shm/bk
+          matrix:
+            setup:
+              PART: ["1", "2", "3", "4", "5", "6"]
 EOF

.github/workflows/docs-build.yml

@@ -16,4 +16,4 @@ jobs:
       deployments: write
       id-token: write
       contents: read
-      pull-requests: read
+      pull-requests: write

benchmarks/src/main/java/org/elasticsearch/benchmark/search/query/range/DateFieldMapperDocValuesSkipperBenchmark.java

@@ -21,14 +21,14 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -295,7 +295,7 @@ public void rangeQueryWithDocValuesSkipper(final Blackhole bh) throws IOExceptio
     /**
      * Runs the actual Lucene range query, optionally combining a {@link LongPoint} index query
      * with doc values ({@link SortedNumericDocValuesField}) via {@link IndexOrDocValuesQuery},
-     * and then wrapping it with an {@link XIndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
+     * and then wrapping it with an {@link IndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
      *
      * @param searcher            the Lucene {@link IndexSearcher}
      * @param rangeStartTimestamp lower bound of the timestamp range
@@ -316,7 +316,7 @@ private long rangeQuery(final IndexSearcher searcher, long rangeStartTimestamp,
             )
             : SortedNumericDocValuesField.newSlowRangeQuery(TIMESTAMP_FIELD, rangeStartTimestamp, rangeEndTimestamp);
 
-        final Query query = new XIndexSortSortedNumericDocValuesRangeQuery(
+        final Query query = new IndexSortSortedNumericDocValuesRangeQuery(
             TIMESTAMP_FIELD,
             rangeStartTimestamp,
             rangeEndTimestamp,

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.internal;
 
 import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
-import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
@@ -28,14 +27,14 @@ public void apply(final Project project) {
         var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
 
         depsInfo.configure(t -> {
-            t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
-            t.setCompileOnlyConfiguration(
-                project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
-            );
-            t.getConventionMapping().map("mappings", () -> {
-                var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
-                return depLic.get().getMappings();
-            });
+            var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+            t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
+            t.getClasspath().from(runtimeConfiguration);
+            var compileOnlyConfiguration = project.getConfigurations()
+                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+            t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
+            t.getClasspath().from(compileOnlyConfiguration);
+
         });
         Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
         dependenciesInfoFilesConfiguration.setCanBeResolved(false);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -11,19 +11,22 @@
 
 import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
 import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.Dependency;
-import org.gradle.api.artifacts.DependencySet;
-import org.gradle.api.artifacts.ModuleVersionIdentifier;
+import org.gradle.api.artifacts.ArtifactCollection;
 import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.internal.ConventionTask;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.TaskAction;
@@ -34,6 +37,7 @@
 import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -51,31 +55,58 @@
  *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
  * </ul>
  */
-public class DependenciesInfoTask extends ConventionTask {
+public abstract class DependenciesInfoTask extends ConventionTask {
 
-    private final DirectoryProperty licensesDir;
+    @Inject
+    public abstract ProviderFactory getProviderFactory();
 
-    @OutputFile
-    private File outputFile;
+    /**
+     * We have to use ArtifactCollection instead of ResolvedArtifactResult here as we're running
+     * into a an issue in Gradle: https://github.com/gradle/gradle/issues/27582
+     */
 
-    private LinkedHashMap<String, String> mappings;
+    @Internal
+    abstract Property<ArtifactCollection> getRuntimeArtifacts();
 
-    public Configuration getRuntimeConfiguration() {
-        return runtimeConfiguration;
+    @Input
+    public Provider<Set<ModuleComponentIdentifier>> getRuntimeModules() {
+        return mapToModuleComponentIdentifiers(getRuntimeArtifacts().get());
     }
 
-    public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
-        this.runtimeConfiguration = runtimeConfiguration;
-    }
+    @Internal
+    abstract Property<ArtifactCollection> getCompileOnlyArtifacts();
 
-    public Configuration getCompileOnlyConfiguration() {
-        return compileOnlyConfiguration;
+    @Input
+    public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
+        return mapToModuleComponentIdentifiers(getCompileOnlyArtifacts().get());
     }
 
-    public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
-        this.compileOnlyConfiguration = compileOnlyConfiguration;
+    /**
+     * We need to track file inputs here from the configurations we inspect to ensure we dont miss any
+     * artifact transforms that might be applied and fail due to missing task dependency to jar
+     * generating tasks.
+     * */
+    @InputFiles
+    abstract ConfigurableFileCollection getClasspath();
+
+    private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
+        return getProviderFactory().provider(
+            () -> artifacts.getArtifacts()
+                .stream()
+                .map(r -> r.getId())
+                .filter(id -> id instanceof ModuleComponentIdentifier)
+                .map(id -> (ModuleComponentIdentifier) id)
+                .collect(Collectors.toSet())
+        );
     }
 
+    private final DirectoryProperty licensesDir;
+
+    @OutputFile
+    private File outputFile;
+
+    private LinkedHashMap<String, String> mappings;
+
     /**
      * Directory to read license files
      */
@@ -102,17 +133,6 @@ public void setOutputFile(File outputFile) {
         this.outputFile = outputFile;
     }
 
-    /**
-     * Dependencies to gather information from.
-     */
-    @InputFiles
-    private Configuration runtimeConfiguration;
-    /**
-     * We subtract compile-only dependencies.
-     */
-    @InputFiles
-    private Configuration compileOnlyConfiguration;
-
     @Inject
     public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFactory, ProviderFactory providerFactory) {
         this.licensesDir = objectFactory.directoryProperty();
@@ -123,22 +143,18 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
 
     @TaskAction
     public void generateDependenciesInfo() throws IOException {
-        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
-        // we have to resolve the transitive dependencies and create a group:artifactId:version map
-
-        final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
-            .getResolvedArtifacts()
-            .stream()
-            .map(r -> {
-                ModuleVersionIdentifier id = r.getModuleVersion().getId();
-                return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
-            })
-            .collect(Collectors.toSet());
 
+        final Set<String> compileOnlyIds = getCompileOnlyModules().map(
+            set -> set.stream()
+                .map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
+                .collect(Collectors.toSet())
+        ).get();
         final StringBuilder output = new StringBuilder();
-        for (final Dependency dep : runtimeDependencies) {
+        Map<String, String> mappings = getMappings().get();
+        for (final ModuleComponentIdentifier dep : getRuntimeModules().get()) {
             // we do not need compile-only dependencies here
-            if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
+            String moduleName = dep.getModuleIdentifier().getName();
+            if (compileOnlyIds.contains(dep.getGroup() + ":" + moduleName + ":" + dep.getVersion())) {
                 continue;
             }
 
@@ -147,25 +163,20 @@ public void generateDependenciesInfo() throws IOException {
                 continue;
             }
 
-            final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
-            final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
-            getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
+            final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
+            final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
+            getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
 
             final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
-            output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
+            output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
         Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
     @Input
-    public LinkedHashMap<String, String> getMappings() {
-        return mappings;
-    }
-
-    public void setMappings(LinkedHashMap<String, String> mappings) {
-        this.mappings = mappings;
-    }
+    @Optional
+    public abstract MapProperty<String, String> getMappings();
 
     /**
      * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>

build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt

@@ -61,11 +61,3 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
 
 @defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
 org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
-
-# This is a temporary patch as there is a low level Lucene bug in certain scenarios
-# this should be fixed in the new Lucene release 10.3+
-org.apache.lucene.document.LongField#newExactQuery(java.lang.String, long) @ Use org.elasticsearch.lucene.document.NumericField#newExactLongQuery(java.lang.String, long) instead.
-org.apache.lucene.document.LongField#newRangeQuery(java.lang.String, long, long) @ Use org.elasticsearch.lucene.document.NumericField#newRangeLongQuery(java.lang.String, long, long) instead.
-org.apache.lucene.document.IntField#newExactQuery(java.lang.String, int) @ Use org.elasticsearch.lucene.document.NumericField#newExactIntQuery(java.lang.String, int) instead.
-org.apache.lucene.document.IntField#newRangeQuery(java.lang.String, int, int) @ Use org.elasticsearch.lucene.document.NumericField#newRangeIntQuery(java.lang.String, int, int) instead.
-org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery @ use org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery instead.

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.1.0
-lucene            = 10.2.1
+lucene            = 10.2.2
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90