Merge remote-tracking branch 'es/main' into tests/TSDBDocValuesFormatSingleNodeTests

Author: martijnvg

AGENTS.md

@@ -8,7 +8,7 @@
 - **Cursor/Copilot rules**: None provided in repo; follow this guide plus CONTRIBUTING.md.
 
 ## Build & Run Commands
-- Refer to CONTRIBUTING.md & TESTING.asciidoc for comprehensive build/test instructions.
+- Refer to BUILDING.md, CONTRIBUTING.md & TESTING.asciidoc for comprehensive build/test instructions.
 
 ## Verification & Lint Tasks
 - `./gradlew spotlessJavaCheck` / `spotlessApply` (or `:server:spotlessJavaCheck`): enforce formatter profile in `build-conventions/formatterConfig.xml`.
@@ -23,7 +23,7 @@ The repository is organized into several key directories:
 *   `docs`: Project documentation.
 *   `distribution`: Logic for building distribution packages.
 *   `x-pack`: Additional code modules and plugins under Elastic License.
-*   `build-conventions`, `build-tools`, `build-tools-internal`: Gradle build logic.
+*   `build-conventions`, `build-tools`, `build-tools-internal`: Gradle build logic. Refer to BUILDING.md for details on how these are structured and used.
 
 ## Testing Cheatsheet
 - Standard suite: `./gradlew test` (respects cached results; add `-Dtests.timestamp=$(date +%s)` to bypass caches when reusing seeds).
@@ -38,7 +38,7 @@ The repository is organized into several key directories:
 - Yaml REST tests: `./gradlew ":rest-api-spec:yamlRestTest" --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=<relative_test_file_path>}"`
 - Use the Elasticsearch testing framework where possible for unit and yaml tests and be consistent in style with other elasticsearch tests.
 - Use real classes over mocks or stubs for unit tests, unless the real class is complex then either a simplified subclass should be created within the test or, as a last resort, a mock or stub can be used. Unit tests must be as close to real-world scenarios as possible.
-- Ensure mocks or stubs are well-documented and clearly indicate why they were necessary. 
+- Ensure mocks or stubs are well-documented and clearly indicate why they were necessary.
 
 ### Test Types
 - Unit Tests: Preferred. Extend `ESTestCase`.

BUILDING.md

@@ -27,9 +27,60 @@ These build tools are also used by the `elasticsearch-hadoop` project maintained
 
 ### `build-tools-internal`
 
-This project contains all Elasticsearch project specific build logic that is not meant to be shared
+This project contains all Elasticsearch project specific build logic not meant to be shared
 with other internal or external projects.
 
+## Third-Party Gradle Plugins
+
+The Elasticsearch build uses several third-party Gradle plugins. All versions are centralized in
+`/gradle/build.versions.toml` (version catalog).
+
+### Build Infrastructure Plugins
+
+| Plugin ID | Purpose |
+|-----------|---------|
+| `com.gradle.develocity` | Enables build scans and integration with Gradle Enterprise at `gradle-enterprise.elastic.co`. Provides build performance metrics, failure diagnostics, and CI integration for Buildkite. |
+| `com.gradleup.nmcp.aggregation` | Aggregates all Maven artifacts from projects applying `elasticsearch.publish` for publishing to Maven Central via DRA infrastructure. |
+
+### Packaging & Distribution Plugins
+
+| Plugin ID | Purpose |
+|-----------|---------|
+| `com.netflix.nebula.ospackage-base` | Creates DEB and RPM Linux packages for Elasticsearch distribution. Handles package metadata, install/remove scripts, file permissions, and package signing. |
+| `com.gradleup.shadow` | Creates fat JARs (uber-jars) by merging dependencies into a single JAR. Used for standalone CLI tools (`plugin-cli`, `sql-cli`) and the JDBC driver. |
+
+### Code Quality & Precommit Plugins
+
+| Plugin ID | Purpose |
+|-----------|---------|
+| `de.thetaphi:forbiddenapis` | Static bytecode analysis that detects invocations of forbidden API methods. Ensures code doesn't use unsafe or deprecated JDK APIs. Integrated via `ForbiddenApisPrecommitPlugin`. |
+| `org.apache.rat:apache-rat` | License header validation (Apache RAT - Release Audit Tool). Ensures all source files have proper license headers. |
+| `com.diffplug.spotless` | Code formatting enforcement using Eclipse JDT formatter. Provides `spotlessJavaCheck` and `spotlessApply` tasks. Configuration in `build-conventions/formatterConfig.xml`. |
+
+### Build Metadata & Tooling Plugins
+
+| Plugin ID | Purpose |
+|-----------|---------|
+| `com.netflix.nebula:gradle-info-plugin` | Automatically includes build metadata (git info, build time, Java version) in JAR manifests. Applied as `nebula.info-broker`, `nebula.info-basic`, `nebula.info-java`, `nebula.info-jar`. |
+| `com.avast.gradle:docker-compose` | Manages Docker Compose environments for integration testing fixtures. Used by `TestFixturesPlugin` for test infrastructure (AWS, Azure, GCS, HDFS mocks). |
+| `org.jetbrains.gradle.plugin.idea-ext` | Enhanced IntelliJ IDEA project configuration. Customizes IDE settings, JUnit configurations, and post-sync tasks. |
+
+## Internal Plugin Selection Guide
+
+When creating a new subproject, choose the appropriate Elasticsearch plugin based on your project type:
+
+| Project Type | Plugin to Apply | Example Projects |
+|--------------|-----------------|------------------|
+| Core library | `elasticsearch.build` | `server`, `libs/*` |
+| Module shipped with ES | `elasticsearch.internal-es-plugin` | `modules/*` |
+| External plugin | `elasticsearch.internal-es-plugin` + `elasticsearch.publish` | `plugins/*` |
+| X-Pack plugin | `elasticsearch.internal-es-plugin` + `elasticsearch.publish` | `x-pack/plugin/*` |
+| YAML REST tests | `elasticsearch.internal-yaml-rest-test` | modules/plugins with REST APIs |
+| Java REST tests | `elasticsearch.internal-java-rest-test` | modules/plugins needing Java test flexibility |
+| Cluster integration tests | `elasticsearch.internal-cluster-test` | Projects testing cluster behavior |
+| BWC/upgrade tests | `elasticsearch.bwc-test` or `elasticsearch.fwc-test` | `qa/rolling-upgrade`, `qa/full-cluster-restart` |
+| Standalone QA project | `elasticsearch.standalone-rest-test` | `qa/*` subprojects |
+
 ## Build guidelines
 
 This is an intentionally small set of guidelines to build users and authors
@@ -161,16 +212,25 @@ When adding or updating dependencies, ensure that any required transitive depend
 2. Added as direct dependencies if they're actually used by our code
 3. Properly excluded if they're not needed
 
-#### Custom plugin and task implementations
+#### Custom Gradle plugin and Gradle task implementations
 
-Build logic that is used across multiple subprojects should be considered to be moved into a Gradle plugin with according Gradle task implementation.
-Elasticsearch specific build logic is located in the `build-tools-internal` subproject including integration tests.
+Build logic that is used across multiple subprojects should be considered to be
+moved into a Gradle plugin with according Gradle task implementation.
+Elasticsearch specific build logic is located in the `build-tools-internal`
+subproject including integration tests.
 
 - Gradle plugins and Tasks should be written in Java
 - We use a groovy and spock for setting up Gradle integration tests.
-  (see https://github.com/elastic/elasticsearch/blob/main/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy)
+- For each Gradle Plugin and Gradle Task implementation (e.g.
+  org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin) we aim for
+  a dedicated
+    - unit test class containing all relaed unit tests in src/test (e.g. GlobalBuildInfoPluginSpec) for basic unit testing of
+      the plugin logic and
+    - integration test containing all related Gradle TestKit based integration tests in src/integTest (e.g. GlobalBuildInfoPluginFuncTest) for testing the
+      plugin in a real Gradle build based on
+      src/main/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy)
 
-#### Declaring tasks
+#### Declaring Gradle tasks
 
 The Elasticsearch build makes use of the [task avoidance API](https://docs.gradle.org/current/userguide/task_configuration_avoidance.html) to keep the configuration time of the build low.
 
@@ -244,6 +304,69 @@ dependencies {
 }
 ```
 
+#### Configuring precommit tasks
+
+Several precommit tasks support project-specific configuration. Use the task avoidance API when configuring them.
+
+##### thirdPartyAudit
+
+Configure missing class and violation ignores when third-party dependencies use optional APIs:
+
+```groovy
+tasks.named("thirdPartyAudit").configure {
+  // Ignore classes that are optional dependencies of our dependencies
+  ignoreMissingClasses(
+    'javax.servlet.ServletContextEvent',
+    'org.apache.log.Logger'
+  )
+
+  // Ignore known-safe internal API usage in dependencies
+  ignoreViolations(
+    'com.google.common.hash.Striped64'
+  )
+}
+```
+
+##### dependencyLicenses
+
+Map related artifacts to a single license when dependencies are published as multiple JARs:
+
+```groovy
+tasks.named("dependencyLicenses").configure {
+  mapping from: /lucene-.*/, to: 'lucene'
+  mapping from: /netty-.*/, to: 'netty'
+  mapping from: /jackson-.*/, to: 'jackson'
+}
+```
+
+##### forbiddenPatterns
+
+Exclude files that legitimately contain patterns detected as forbidden:
+
+```groovy
+tasks.named("forbiddenPatterns").configure {
+  exclude '**/*.key'     // Test certificates
+  exclude '**/*.p12'     // PKCS12 keystores
+  exclude '**/*.json'    // Test data files
+}
+```
+
+#### Configuring REST resources
+
+Projects with REST APIs should declare which specs and tests they need:
+
+```groovy
+restResources {
+  restApi {
+    include '_common', 'cluster', 'indices', 'your_api_name'
+  }
+  restTests {
+    includeCore 'your_api_tests'  // For core APIs
+    includeXpack 'your_xpack_tests'  // For X-Pack APIs
+  }
+}
+```
+
 ## FAQ
 
 ### How do I test a development version of a third party dependency?
@@ -314,7 +437,7 @@ by JitPack in the background before we can resolve the adhoc built dependency.
 
 > [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
-not want to ship unreleased libraries into our releases.
+> not want to ship unreleased libraries into our releases.
 
 #### How to use a custom third party artifact?
 
@@ -352,7 +475,7 @@ allprojects {
 
 > [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
-flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
+> flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
-that supports meta-data, it will use the second repository to provide the module.
+> that supports meta-data, it will use the second repository to provide the module.
 > Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)

benchmarks/src/main/java/org/elasticsearch/benchmark/Utils.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.common.logging.NodeNamePatternConverter;
+
+public final class Utils {
+
+    private Utils() {
+        // utility class
+    }
+
+    static {
+        LogConfigurator.setClusterName("elasticsearch-benchmark");
+        LogConfigurator.setNodeName("test");
+    }
+
+    public static void configureBenchmarkLogging() {
+        LogConfigurator.loadLog4jPlugins();
+        NodeNamePatternConverter.setGlobalNodeName("benchmark");
+        LogConfigurator.configureESLogging();
+    }
+
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -10,7 +10,7 @@
 package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.license.XPackLicenseState;
@@ -70,7 +70,7 @@
 public class QueryPlanningBenchmark {
 
     static {
-        LogConfigurator.configureESLogging();
+        Utils.configureBenchmarkLogging();
     }
 
     private PlanTelemetry telemetry;

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/TopNBenchmark.java

@@ -10,9 +10,9 @@
 package org.elasticsearch.benchmark._nightly.esql;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
@@ -57,9 +57,6 @@
 @State(Scope.Thread)
 @Fork(1)
 public class TopNBenchmark {
-    static {
-        LogConfigurator.configureESLogging();
-    }
 
     private static final BlockFactory blockFactory = BlockFactory.getInstance(
         new NoopCircuitBreaker("noop"),
@@ -80,7 +77,7 @@ public class TopNBenchmark {
     private static final String AND = "_and_";
 
     static {
-        LogConfigurator.configureESLogging();
+        Utils.configureBenchmarkLogging();
         // Smoke test all the expected values and force loading subclasses more like prod
         selfTest();
     }
@@ -159,6 +156,7 @@ private static Operator operator(String data, int topCount, boolean sortedInput)
             encoders,
             sortOrders,
             8 * 1024,
+            Long.MAX_VALUE,
             sortedInput ? TopNOperator.InputOrdering.SORTED : TopNOperator.InputOrdering.NOT_SORTED,
             minCompetitive // This is optional, but doesn't add much overhead either way
         );

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -23,9 +23,9 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -93,8 +93,9 @@
 @State(Scope.Thread)
 @Fork(1)
 public class ValuesSourceReaderBenchmark {
+
     static {
-        LogConfigurator.configureESLogging();
+        Utils.configureBenchmarkLogging();
     }
 
     private static final String[] SUPPORTED_LAYOUTS = new String[] { "in_order", "shuffled", "shuffled_singles" };
@@ -317,7 +318,8 @@ public void benchmark() {
                 throw new UnsupportedOperationException("can't load _source here");
             }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             reuseColumnLoaders,
-            0
+            0,
+            PlannerSettings.SOURCE_RESERVATION_FACTOR.getDefault(Settings.EMPTY)
         );
         long sum = 0;
         for (Page page : pages) {

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/BytesArrayIndexOfBenchmark.java

@@ -8,8 +8,8 @@
  */
 package org.elasticsearch.benchmark.bytes;
 
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -33,7 +33,7 @@
 public class BytesArrayIndexOfBenchmark {
 
     static {
-        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+        Utils.configureBenchmarkLogging();
     }
 
     static final byte MARKER = (byte) '\n';

benchmarks/src/main/java/org/elasticsearch/benchmark/common/util/CodePointCountBenchmark.java

@@ -11,7 +11,7 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.UnicodeUtil;
-import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.simdvec.ESVectorUtil;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -37,7 +37,7 @@
 public class CodePointCountBenchmark {
 
     static {
-        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+        Utils.configureBenchmarkLogging();
     }
 
     @Param({ "1", "5", "10", "20", "50", "100", "1000" })

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -10,8 +10,8 @@
 package org.elasticsearch.benchmark.compute.operator;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.benchmark.Utils;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.AggregatorMode;
@@ -115,7 +115,7 @@ public class AggregatorBenchmark {
     private static final String CONSTANT_FALSE = "constant_false";
 
     static {
-        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+        Utils.configureBenchmarkLogging();
 
         // Smoke test all the expected values and force loading subclasses more like prod
         if (false == "true".equals(System.getProperty("skipSelfTest"))) {