Merge remote-tracking branch 'upstream/main' into invalid-grok-500

* upstream/main: (58 commits)
  ESQL: mark LOOKUP JOIN as ExecutesOn.Any by default (elastic#133064)
  Fix 404s in REST API landing page (elastic#133086)
  Fix release tests for OptimizerVerificationTests (elastic#133100)
  Make Glob non-recursive (elastic#132798)
  Update ES|QL function list for release versions (elastic#133096)
  Split transport version func test into abstract base (elastic#133035)
  Omit project ID from snapshot metrics (elastic#133098)
  Mute org.elasticsearch.xpack.esql.analysis.AnalyzerTests testNoDenseVectorFailsForMagnitude elastic#133013
  Mute org.elasticsearch.xpack.esql.optimizer.OptimizerVerificationTests testRemoteEnrichAfterCoordinatorOnlyPlans elastic#133015
  Mute org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT test {p0=search/160_exists_query/Test exists query on _id field} elastic#133097
  Rename initial to unreferenced in transport versions (elastic#133082)
  Rename exception type header (elastic#133045)
  ESQL: Pluggable tests for Operator status (elastic#132876)
  ESQL: Mark new signatures in MIN and MAX (elastic#132980)
  Don't try to serialize half-baked cluster info (elastic#132756)
  migrate ml_rollover_legacy_indices transport version (elastic#133008)
  Enable `exclude_source_vectors` by default for new indices (elastic#131907)
  Expose APIs needed by flush during translog replay (elastic#132960)
  Change reporting_user role to leverage reserved kibana privileges (elastic#132766)
  Update TasksIT for batched execution (elastic#132762)
  ...

Author: szybia

.gitignore

@@ -69,7 +69,7 @@ testfixtures_shared/
 # Generated
 checkstyle_ide.xml
 x-pack/plugin/esql/src/main/generated-src/generated/
-server/src/main/resources/transport/defined/manifest.txt
+server/src/main/resources/transport/definitions/manifest.txt
 
 # JEnv
 .java-version

BUILDING.md

@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major
+java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+
+```
+./gradlew clean test -Druntime.java=26-ea
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +280,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -92,7 +92,8 @@ public void setup() {
         var fields = 10_000;
         var mapping = LinkedHashMap.<String, EsField>newLinkedHashMap(fields);
         for (int i = 0; i < fields; i++) {
-            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true));
+            // We're creating a standard index, so none of these fields should be marked as dimensions.
+            mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
         }
 
         var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -212,7 +212,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 FieldAttribute timestamp = new FieldAttribute(
                     Source.EMPTY,
                     "timestamp",
-                    new EsField("timestamp", DataType.DATETIME, Map.of(), true)
+                    new EsField("timestamp", DataType.DATETIME, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
                 );
                 yield EvalMapper.toEvaluator(
                     FOLD_CONTEXT,
@@ -321,19 +321,35 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
     }
 
     private static FieldAttribute longField() {
-        return new FieldAttribute(Source.EMPTY, "long", new EsField("long", DataType.LONG, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "long",
+            new EsField("long", DataType.LONG, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute doubleField() {
-        return new FieldAttribute(Source.EMPTY, "double", new EsField("double", DataType.DOUBLE, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "double",
+            new EsField("double", DataType.DOUBLE, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute intField() {
-        return new FieldAttribute(Source.EMPTY, "int", new EsField("int", DataType.INTEGER, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "int",
+            new EsField("int", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static FieldAttribute keywordField() {
-        return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
+        return new FieldAttribute(
+            Source.EMPTY,
+            "keyword",
+            new EsField("keyword", DataType.KEYWORD, Map.of(), true, EsField.TimeSeriesFieldType.NONE)
+        );
     }
 
     private static Configuration configuration() {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
-import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
     private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
+    private static final String CATALOG_EA_VERSION = "25-ea+30"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
 
+    def setup() {
+        configurationCacheCompatible = false // JDK class references configurations which break configuration cache
+    }
+
     @Unroll
-    def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
+    def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
-        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
+        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
         def mockedContent = filebytes(jdkVendor, platform)
         buildFile.text = """
             plugins {
@@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         where:
         platform  | arch      | jdkVendor       | jdkVersion           | distributionVersion   | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null                  | "bin/java"               | "(jdk 11)"
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null                  | "bin/java"               | "(jdk 15)"
+        "linux"   | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "darwin"  | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | ""                    | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | ""                    | "bin/java"               | "(jdk 15)"
         "darwin"  | "aarch64" | VENDOR_ZULU     | AZUL_JDK_VERSION_8   | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
     }
 
@@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static String urlPath(final String vendor,
                                   final String version,
                                   final String platform,
-                                  final String arch = 'x64') {
-        if (vendor.equals(VENDOR_ADOPTIUM)) {
+                                  final String arch = 'x64',
+                                  final String distributedVersion = '') {
+        final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
+
+        if (distributedVersion.equals("ea")) {
+            def effectivePlatform = isMac(platform) ? "macos" : platform;
+            def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
+            return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
+        } else if (vendor.equals(VENDOR_ADOPTIUM)) {
             final String module = isMac(platform) ? "mac" : platform;
             return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             final String effectivePlatform = isMac(platform) ? "macos" : platform;
-            final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
             return "/java/GA/" + versionPath + "/GPL/" + filename;

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.TaskOutcome
+
+class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
+    def javaResource(String project, String path, String content) {
+        file("${project}/src/main/resources/${path}").withWriter { writer ->
+            writer << content
+        }
+    }
+
+    def javaSource(String project, String packageName, String className, String imports, String content) {
+        String packageSlashes = packageName.replace('.', '/')
+        file("${project}/src/main/java/${packageSlashes}/${className}.java").withWriter { writer ->
+            writer << """
+                package ${packageName};
+                ${imports}
+                public class ${className} {
+                    ${content}
+                }
+            """
+        }
+    }
+
+    def namedTransportVersion(String name, String ids) {
+        javaResource("myserver", "transport/definitions/named/" + name + ".csv", ids)
+    }
+
+    def unreferencedTransportVersion(String name, String id) {
+        javaResource("myserver", "transport/definitions/unreferenced/" + name + ".csv", id)
+    }
+
+    def definedAndUsedTransportVersion(String name, String ids) {
+        return definedAndUsedTransportVersion(name, ids, "Test${name.capitalize()}")
+    }
+
+    def definedAndUsedTransportVersion(String name, String ids, String classname) {
+        javaSource("myserver", "org.elasticsearch", classname, "", """
+            static final TransportVersion usage = TransportVersion.fromName("${name}");
+        """)
+        namedTransportVersion(name, ids)
+    }
+
+    def latestTransportVersion(String branch, String name, String id) {
+        javaResource("myserver", "transport/latest/" + branch + ".csv","${name},${id}")
+    }
+
+    def validateReferencesFails(String project) {
+        return gradleRunner(":${project}:validateTransportVersionReferences").buildAndFail()
+    }
+
+    def validateDefinitionsFails() {
+        return gradleRunner(":myserver:validateTransportVersionDefinitions").buildAndFail()
+    }
+
+    def assertReferencesFailure(BuildResult result, String project, String expectedOutput) {
+        result.task(":${project}:validateTransportVersionReferences").outcome == TaskOutcome.FAILED
+        assertOutputContains(result.output, expectedOutput)
+    }
+
+    def assertDefinitionsFailure(BuildResult result, String expectedOutput) {
+        result.task(":myserver:validateTransportVersionDefinitions").outcome == TaskOutcome.FAILED
+        assertOutputContains(result.output, expectedOutput)
+    }
+
+    def setup() {
+        configurationCacheCompatible = false
+        internalBuild()
+        settingsFile << """
+            include ':myserver'
+            include ':myplugin'
+        """
+        file("gradle.properties") << """
+            org.elasticsearch.transport.definitionsProject=:myserver
+        """
+
+        file("myserver/build.gradle") << """
+            apply plugin: 'java-library'
+            apply plugin: 'elasticsearch.transport-version-references'
+            apply plugin: 'elasticsearch.transport-version-resources'
+        """
+        namedTransportVersion("existing_91", "8012000")
+        namedTransportVersion("existing_92", "8123000,8012001")
+        unreferencedTransportVersion("initial_9_0_0", "8000000")
+        latestTransportVersion("9.2", "existing_92", "8123000")
+        latestTransportVersion("9.1", "existing_92", "8012001")
+        // a mock version of TransportVersion, just here so we can compile Dummy.java et al
+        javaSource("myserver", "org.elasticsearch", "TransportVersion", "", """
+            public static TransportVersion fromName(String name) {
+                return null;
+            }
+        """)
+        javaSource("myserver", "org.elasticsearch", "Dummy", "", """
+            static final TransportVersion existing91 = TransportVersion.fromName("existing_91");
+            static final TransportVersion existing92 = TransportVersion.fromName("existing_92");
+        """)
+
+        file("myplugin/build.gradle") << """
+            apply plugin: 'java-library'
+            apply plugin: 'elasticsearch.transport-version-references'
+
+            dependencies {
+                implementation project(":myserver")
+            }
+        """
+
+        setupLocalGitRepo()
+        execute("git checkout -b main")
+        execute("git checkout -b test")
+    }
+}