Merge branch 'main' into cps-transform-6

Author: prwhelan

.github/CODEOWNERS

@@ -99,3 +99,28 @@ server/src/main/java/org/elasticsearch/common/xcontent @elastic/es-core-infra
 # Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security
+
+# Docs team
+docs/README.md @elastic/core-docs
+docs/Versions.asciidoc @elastic/core-docs
+docs/*.yml @elastic/core-docs
+docs/community-clients @elastic/core-docs
+docs/release-notes @elastic/core-docs
+docs/extend @elastic/admin-docs
+docs/java-rest @elastic/dev-docs
+docs/reference @elastic/core-docs
+docs/reference/aggregations @elastic/dev-docs
+docs/reference/community-contributed @elastic/dev-docs
+docs/reference/elasticsearch/command-line-tools @elastic/admin-docs
+docs/reference/elasticsearch/index-lifecycle-actions @elastic/dev-docs
+docs/reference/elasticsearch/mapping-reference @elastic/dev-docs
+docs/reference/elasticsearch/rest-apis @elastic/dev-docs
+docs/reference/elasticsearch/elasticsearch-audit-events.md @elastic/admin-docs
+docs/reference/elasticsearch/jvm-settings.md @elastic/admin-docs
+docs/reference/elasticsearch/roles.md @elastic/admin-docs
+docs/reference/enrich-processor @elastic/admin-docs
+docs/reference/query-languages @elastic/dev-docs
+docs/reference/scripting-languages @elastic/admin-docs
+docs/reference/search-connectors @elastic/dev-docs
+docs/reference/setup/install/docker @elastic/admin-docs
+docs/reference/text-analysis @elastic/dev-docs

.gitignore

@@ -2,6 +2,9 @@
 # claude
 .claude
 
+# cursor
+.cursorrules
+
 # JetBrainsAI/Junie
 .junie/
 

AGENTS.md

@@ -0,0 +1,98 @@
+# Elasticsearch
+
+## Toolchain Snapshot
+- **Java**: JDK 25 via `JAVA_HOME`; use the bundled Gradle wrapper (`./gradlew`).
+- **Build tooling**: Gradle composite build with `build-conventions`, `build-tools`, and `build-tools-internal`; Docker is required for some packaging/tests.
+- **OS packages**: Packaging and QA jobs expect ephemeral hosts; do not run packaging suites on your workstation.
+- **Security**: Default dev clusters enable security; use `elastic-admin:elastic-password` or disable with `-Dtests.es.xpack.security.enabled=false`.
+- **Cursor/Copilot rules**: None provided in repo; follow this guide plus CONTRIBUTING.md.
+
+## Build & Run Commands
+- Refer to CONTRIBUTING.md & TESTING.asciidoc for comprehensive build/test instructions.
+
+## Verification & Lint Tasks
+- `./gradlew spotlessJavaCheck` / `spotlessApply` (or `:server:spotlessJavaCheck`): enforce formatter profile in `build-conventions/formatterConfig.xml`.
+
+## Project Structure
+The repository is organized into several key directories:
+*   `server`: The core Elasticsearch server.
+*   `modules`: Features shipped with Elasticsearch by default.
+*   `plugins`: Officially supported plugins.
+*   `libs`: Internal libraries used by other parts of the project.
+*   `qa`: Integration and multi-version tests.
+*   `docs`: Project documentation.
+*   `distribution`: Logic for building distribution packages.
+*   `x-pack`: Additional code modules and plugins under Elastic License.
+*   `build-conventions`, `build-tools`, `build-tools-internal`: Gradle build logic.
+
+## Testing Cheatsheet
+- Standard suite: `./gradlew test` (respects cached results; add `-Dtests.timestamp=$(date +%s)` to bypass caches when reusing seeds).
+- Single project: `./gradlew :server:test` (or other subproject path).
+- Single class: `./gradlew :server:test --tests org.elasticsearch.package.ClassName`.
+- Single package: `./gradlew :server:test --tests 'org.elasticsearch.package.*'`.
+- Single method / repeated runs: `./gradlew :server:test --tests org.elasticsearch.package.ClassName.methodName -Dtests.iters=N`.
+- Deterministic seed: append `-Dtests.seed=DEADBEEF` (each method uses derived seeds).
+- JVM tuning knobs: `-Dtests.jvms=8`, `-Dtests.heap.size=4G`, `-Dtests.jvm.argline="-verbose:gc"`, `-Dtests.output=always`, etc.
+- Debugging: append `--debug-jvm` to the Gradle test task and attach a debugger on port 5005.
+- CI reproductions: copy the `REPRODUCE WITH` line from CI logs; it includes project path, seed, and JVM flags.
+- Yaml REST tests: `./gradlew ":rest-api-spec:yamlRestTest" --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=<relative_test_file_path>}"`
+- Use the Elasticsearch testing framework where possible for unit and yaml tests and be consistent in style with other elasticsearch tests.
+- Use real classes over mocks or stubs for unit tests, unless the real class is complex then either a simplified subclass should be created within the test or, as a last resort, a mock or stub can be used. Unit tests must be as close to real-world scenarios as possible.
+- Ensure mocks or stubs are well-documented and clearly indicate why they were necessary. 
+
+### Test Types
+- Unit Tests: Preferred. Extend `ESTestCase`.
+- Single Node: Extend `ESSingleNodeTestCase` (lighter than full integ test).
+- Integration: Extend `ESIntegTestCase`.
+- REST API: Extend `ESRestTestCase` or `ESClientYamlSuiteTestCase`. **YAML based REST tests are preferred** for integration/API testing.
+
+## Dependency Hygiene
+- Never add a dependency without checking for existing alternatives in the repo.
+
+## Formatting & Imports
+- Absolutely no wildcard imports; keep existing import order and avoid reordering untouched lines.
+
+## Types, Generics, and Suppressions
+- Prefer type-safe constructs; avoid raw types and unchecked casts.
+- If suppressing warnings, scope `@SuppressWarnings` narrowly (ideally a single statement or method).
+- Document non-obvious casts or type assumptions via Javadoc/comments for reviewers.
+
+## Naming Conventions
+- REST handlers typically use the `Rest*Action` pattern; transport-layer handlers mirror them with `Transport*Action` classes.
+- REST classes expose routes via `RestHandler#routes`; when adding endpoints ensure naming matches existing REST/Transport patterns to aid discoverability.
+- Transport `ActionType` strings encode scope (`indices:data/read/...`, `cluster:admin/...`, etc.); align new names with these conventions to integrate with privilege resolution.
+
+## Logging & Error Handling
+- Elasticsearch should prefer its own logger `org.elasticsearch.logging.LogManager` & `org.elasticsearch.logging.Logger`; declare `private static final Logger logger = LogManager.getLogger(Class.class)`.
+- Always use parameterized logging (`logger.debug("operation [{}]", value)`); never build strings via concatenation.
+- Wrap expensive log-message construction in `() -> Strings.format(...)` suppliers when logging at `TRACE`/`DEBUG` to avoid unnecessary work.
+- Log levels:
+  - `TRACE`: highly verbose developer diagnostics; usually read alongside code.
+  - `DEBUG`: detailed production troubleshooting; ensure volume is bounded.
+  - `INFO`: default-enabled operational milestones; prefer factual language.
+  - `WARN`: actionable problems users must investigate; include context and, if needed, exception stack traces.
+  - `ERROR`: reserve for unrecoverable states (e.g., storage health failures); prefer `WARN` otherwise.
+- Only log client-caused exceptions when the cluster admin can act on them; otherwise rely on API responses.
+- Tests can assert logging via `MockLog` for complex flows.
+
+## Javadoc & Comments
+- New packages/classes/public or abstract methods require Javadoc explaining the "why" rather than the implementation details.
+- Avoid documenting trivial getters/setters; focus on behavior, preconditions, or surprises.
+- For tests, Javadoc can describe scenario setup/expectations to aid future contributors.
+- Do not remove existing comments from code unless the code is also being removed or the comment has become incorrect.
+
+## License Headers
+- Default header (outside `x-pack`): Elastic License 2.0, SSPL v1, or AGPL v3—they are already codified at the top of Java files; copy from existing sources.
+- Files under `x-pack` require the Elastic License 2.0-only header; IDEs configured per CONTRIBUTING.md can insert correct text automatically.
+
+## Best Practices for Automation Agents
+- Never edit unrelated files; keep diffs tightly scoped to the task at hand.
+- Prefer Gradle tasks over ad-hoc scripts.
+- When scripting CLI sequences, leverage `gradlew` task.
+- Unrecognized changes: assume other agent; keep going; focus your changes. If it causes issues, stop + ask user.
+- Do not add "Co-Authored-By" lines to commit messages. commit messages should adhere to the 50/72 rule: use a maximum of 50 columns for the commit summary
+
+## Backwards compatibility
+- For changes to a `Writeable` implementation (`writeTo` and constructor from `StreamInput`), add a new `public static final <UNIQUE_DESCRIPTIVE_NAME> = TransportVersion.fromName("<unique_descriptive_name>")` and use it in the new code paths. Confirm the backport branches and then generate a new version file with `./gradlew generateTransportVersion`.
+
+Stay aligned with `CONTRIBUTING.md`, `BUILDING.md`, and `TESTING.asciidoc`; this AGENTS guide summarizes—but does not replace—those authoritative docs.

CONTRIBUTING.md

@@ -274,10 +274,8 @@ Please follow these formatting guidelines:
   consistency.
 * Note that Javadoc and block comments i.e. `/* ... */` are not formatted,
   but line comments i.e. `// ...` are.
-* Negative boolean expressions must use the form `foo == false` instead of
-  `!foo` for better readability of the code. This is enforced via
-  Checkstyle. Conversely, you should not write e.g. `if (foo == true)`, but
-  just `if (foo)`.
+* Negative boolean expressions should use the form `foo == false` instead of
+  `!foo` where this improves readability. Conversely, you should not write e.g. `if (foo == true)`, but just `if (foo)`.
 
 #### Editor / IDE support
 

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -43,7 +43,7 @@
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.lucene.AlwaysReferencedIndexedByShardId;
 import org.elasticsearch.compute.lucene.IndexedByShardIdFromSingleton;
-import org.elasticsearch.compute.lucene.LuceneSourceOperator;
+import org.elasticsearch.compute.lucene.query.LuceneSourceOperator;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorStatus;
 import org.elasticsearch.compute.operator.DriverContext;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy

@@ -152,8 +152,10 @@ class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest {
         """
 
         when:
-        def result1 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
-        def result2 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
+        // Avoid invoking cleanTest* on Windows; it can flake due to file handle timing when deleting build/test-results/**/binary.
+        // We still need two consecutive task executions to observe two different seeds while reusing the configuration cache.
+        def result1 = gradleRunner("test", "test2", "--rerun-tasks").build()
+        def result2 = gradleRunner("test", "test2", "--rerun-tasks").build()
 
         then:
         def seeds1 = result1.output.findAll(/(?m)TESTSEED=\[([^\]]+)\]/) { it[1] }

build-tools-internal/src/main/resources/checkstyle.xml

@@ -187,18 +187,6 @@
       <message key="descendant.token.max" value="Do not check for equality with 'true', since it is implied" />
     </module>
 
-    <!-- Forbid using '!' for logical negations in favour of checking against 'false' explicitly. -->
-    <!-- This is only reported in the IDE for now because there are many violations -->
-    <module name="DescendantToken">
-        <property name="id" value="BooleanNegation" />
-        <property name="tokens" value="EXPR"/>
-        <property name="limitedTokens" value="LNOT"/>
-        <property name="maximumNumber" value="0"/>
-        <message
-            key="descendant.token.max"
-            value="Do not negate boolean expressions with '!', but check explicitly with '== false' as it is more explicit"/>
-    </module>
-
     <module name="NeedBraces">
         <!-- We have many generated equals classes with LITERAL_IF and no braces, so we don't check IF or ELSE -->
         <property name="tokens" value="LITERAL_DO,LITERAL_FOR,LITERAL_WHILE"/>

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 9.4.0
 lucene            = 10.3.2
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 25.0.1+8@2fbf10d8c78e40bd87641c434705079d
+bundled_jdk = 25.0.2+10@b1e0dfa218384cb9959bdcb897162d4e
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.20.0
@@ -24,6 +24,7 @@ cuvs_java           = 25.12.0
 ldapsdk             = 7.0.3
 
 antlr4            = 4.13.1
+iceberg           = 1.10.1
 # bouncy castle version for non-fips. fips jars use a different version
 bouncycastle=1.79
 # used by security and idp (need to be in sync due to cross-dependency in testing)

dev-tools/prometheus-local/README.md

@@ -0,0 +1,38 @@
+# Local Prometheus -> Elasticsearch remote_write
+
+This setup runs Prometheus and Kibana in Docker and starts Elasticsearch from source.
+Prometheus scrapes itself and forwards samples to Elasticsearch via the Prometheus remote write endpoint.
+
+## Files
+
+- `prometheus.yml` configures `scrape_configs` and `remote_write`.
+- `docker-compose.yml` runs Prometheus on port `9090` and Kibana on port `5601`.
+
+## Start
+
+Start Elasticsearch from source and ensure it listens on a non-loopback interface
+so that Prometheus in Docker can reach it.
+
+```bash
+./gradlew run --configuration-cache \
+    -Dtests.es.http.host=0.0.0.0 \
+    -Dtests.es.xpack.ml.enabled=false \
+    -Drun.license_type=trial \
+    -Dtests.heap.size=4G \
+    -Dtests.jvm.argline="-da -dsa -Dio.netty.leakDetection.level=simple"
+```
+
+```bash
+cd dev-tools/prometheus-local
+docker compose up -d
+```
+
+## Verify
+
+- Prometheus UI: `http://localhost:9090`
+- Targets page: `http://localhost:9090/targets` (the `prometheus` job should be `UP`)
+- Kibana UI: `http://localhost:5601`
+- Remote write health in Prometheus UI (example query):
+  - `rate(prometheus_remote_storage_samples_total[1m])`
+- Query PromQL in Kibana Discover (example query):
+  - `PROMQL step=1m rate(prometheus_remote_storage_samples_total[1m])`

dev-tools/prometheus-local/docker-compose.yml

@@ -0,0 +1,56 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: es-local-prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.enable-lifecycle"
+    restart: unless-stopped
+  kibana_settings:
+    image: curlimages/curl:latest
+    container_name: kibana-settings
+    restart: 'no'
+    command:
+      - /bin/sh
+      - -c
+      - |
+          echo "Setup kibana_system password"
+          start_time=$$(date +%s)
+          timeout=60
+          until curl -s -u "elastic:password" -X POST "http://host.docker.internal:9200/_security/user/kibana_system/_password" \
+            -H "Content-Type: application/json" \
+            -d "{\"password\":\"password\"}" | grep -q "^{}"; do
+            if [ $$(($$(date +%s) - $$start_time)) -ge $$timeout ]; then
+              echo "Error: Elasticsearch timeout while setting kibana_system password"
+              exit 1
+            fi
+            sleep 2
+          done
+  kibana:
+    depends_on:
+      kibana_settings:
+        condition: service_completed_successfully
+    image: docker.elastic.co/kibana/kibana:9.4.0-SNAPSHOT
+    container_name: es-local-kibana
+    ports:
+      - "5601:5601"
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_HOSTS: http://host.docker.internal:9200
+      ELASTICSEARCH_USERNAME: kibana_system
+      ELASTICSEARCH_PASSWORD: password
+      SERVER_HOST: "0.0.0.0"
+      XPACK_SECURITY_ENABLED: "false"
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: "prometheus-local-eso-encryption-key-32"
+      ELASTICSEARCH_PUBLICBASEURL: http://host.docker.internal:9200
+      XPACK_SPACES_DEFAULTSOLUTION: "oblt"
+    restart: unless-stopped
+
+volumes:
+  prometheus_data: