[9.0] Initial support for unmapped fields (Plus a few others) (#119886, #121261, #122607)

.buildkite/hooks/pre-command

@@ -44,6 +44,9 @@ export GRADLE_BUILD_CACHE_USERNAME
 GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
 export GRADLE_BUILD_CACHE_PASSWORD
 
+DEVELOCITY_ACCESS_KEY="gradle-enterprise.elastic.co=$(vault read -field=accesskey secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)"
+export DEVELOCITY_ACCESS_KEY
+
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 

.buildkite/pipelines/dra-workflow.yml

@@ -20,4 +20,5 @@ steps:
       env:
         DRA_WORKFLOW: staging
         USE_PROD_DOCKER_CREDENTIALS: "true"
+        VERSION_QUALIFIER: "${VERSION_QUALIFIER}"
     if: build.env('DRA_WORKFLOW') == 'staging'

.buildkite/pipelines/intake.template.yml

@@ -96,7 +96,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 8.* 7.17"
+    branches: "main 9.* 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.5", "8.17.3", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -76,7 +76,7 @@ steps:
             ES_VERSION:
               - "9.0.0"
             ES_COMMIT:
-              - "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
+              - "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump / head of combat-lucene-10-0-0
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -97,7 +97,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
-    branches: "main 8.* 7.17"
+    branches: "main 9.* 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/periodic-packaging.yml

@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.4
+      - label: "{{matrix.image}} / 8.16.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,10 +301,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.4
+          BWC_VERSION: 8.16.5
 
-      - label: "{{matrix.image}} / 8.17.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.2
+      - label: "{{matrix.image}} / 8.17.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,7 +317,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.2
+          BWC_VERSION: 8.17.3
 
       - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.4#bwcTest
+      - label: 8.16.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.4
+          BWC_VERSION: 8.16.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.2#bwcTest
+      - label: 8.17.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.2
+          BWC_VERSION: 8.17.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.5", "8.17.3", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.4", "8.17.2", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.5", "8.17.3", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -15,7 +15,7 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.4"
-  - "8.17.2"
+  - "8.16.5"
+  - "8.17.3"
   - "8.18.0"
   - "9.0.0"

.ci/init.gradle

@@ -1,29 +1,24 @@
+final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
+final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
+
 gradle.settingsEvaluated { settings ->
   settings.pluginManager.withPlugin("com.gradle.develocity") {
     settings.develocity {
-      server = 'https://gradle-enterprise.elastic.co'
+      server = "https://gradle-enterprise.elastic.co"
     }
-  }
-}
-
-final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
-final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
-
-if (buildCacheUrl) {
-  gradle.settingsEvaluated { settings ->
-    settings.buildCache {
-      local {
-        // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
-        enabled = false
-      }
-      remote(HttpBuildCache) {
-        url = buildCacheUrl
-        push = buildCachePush
-        credentials {
-          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME")
-          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD")
+    if (buildCacheUrl) {
+      settings.buildCache {
+        local {
+          // Disable the local build cache in CI since we use ephemeral workers and it incurs an IO penalty
+          enabled = false
+        }
+        remote(settings.develocity.buildCache) {
+          enabled = true
+          push = buildCachePush
         }
       }
     }
   }
 }
+
+

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "8.16.4"
-  - "8.17.2"
+  - "8.16.5"
+  - "8.17.3"
   - "8.18.0"
   - "9.0.0"

.github/CODEOWNERS

@@ -49,6 +49,7 @@ distribution/docker/src @elastic/es-delivery
 # Core/Infra
 distribution/tools @elastic/es-core-infra
 libs/core @elastic/es-core-infra
+libs/entitlement @elastic/es-core-infra
 libs/logging @elastic/es-core-infra
 libs/native @elastic/es-core-infra
 libs/plugin-analysis-api @elastic/es-core-infra

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -155,7 +155,7 @@ private static Operator operator(DriverContext driverContext, String grouping, S
 
         if (grouping.equals("none")) {
             return new AggregationOperator(
-                List.of(supplier(op, dataType, filter, 0).aggregatorFactory(AggregatorMode.SINGLE).apply(driverContext)),
+                List.of(supplier(op, dataType, filter).aggregatorFactory(AggregatorMode.SINGLE, List.of(0)).apply(driverContext)),
                 driverContext
             );
         }
@@ -182,33 +182,33 @@ private static Operator operator(DriverContext driverContext, String grouping, S
             default -> throw new IllegalArgumentException("unsupported grouping [" + grouping + "]");
         };
         return new HashAggregationOperator(
-            List.of(supplier(op, dataType, filter, groups.size()).groupingAggregatorFactory(AggregatorMode.SINGLE)),
+            List.of(supplier(op, dataType, filter).groupingAggregatorFactory(AggregatorMode.SINGLE, List.of(groups.size()))),
             () -> BlockHash.build(groups, driverContext.blockFactory(), 16 * 1024, false),
             driverContext
         );
     }
 
-    private static AggregatorFunctionSupplier supplier(String op, String dataType, String filter, int dataChannel) {
+    private static AggregatorFunctionSupplier supplier(String op, String dataType, String filter) {
         return filtered(switch (op) {
-            case COUNT -> CountAggregatorFunction.supplier(List.of(dataChannel));
+            case COUNT -> CountAggregatorFunction.supplier();
             case COUNT_DISTINCT -> switch (dataType) {
-                case LONGS -> new CountDistinctLongAggregatorFunctionSupplier(List.of(dataChannel), 3000);
-                case DOUBLES -> new CountDistinctDoubleAggregatorFunctionSupplier(List.of(dataChannel), 3000);
+                case LONGS -> new CountDistinctLongAggregatorFunctionSupplier(3000);
+                case DOUBLES -> new CountDistinctDoubleAggregatorFunctionSupplier(3000);
                 default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]");
             };
             case MAX -> switch (dataType) {
-                case LONGS -> new MaxLongAggregatorFunctionSupplier(List.of(dataChannel));
-                case DOUBLES -> new MaxDoubleAggregatorFunctionSupplier(List.of(dataChannel));
+                case LONGS -> new MaxLongAggregatorFunctionSupplier();
+                case DOUBLES -> new MaxDoubleAggregatorFunctionSupplier();
                 default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]");
             };
             case MIN -> switch (dataType) {
-                case LONGS -> new MinLongAggregatorFunctionSupplier(List.of(dataChannel));
-                case DOUBLES -> new MinDoubleAggregatorFunctionSupplier(List.of(dataChannel));
+                case LONGS -> new MinLongAggregatorFunctionSupplier();
+                case DOUBLES -> new MinDoubleAggregatorFunctionSupplier();
                 default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]");
             };
             case SUM -> switch (dataType) {
-                case LONGS -> new SumLongAggregatorFunctionSupplier(List.of(dataChannel));
-                case DOUBLES -> new SumDoubleAggregatorFunctionSupplier(List.of(dataChannel));
+                case LONGS -> new SumLongAggregatorFunctionSupplier();
+                case DOUBLES -> new SumDoubleAggregatorFunctionSupplier();
                 default -> throw new IllegalArgumentException("unsupported data type [" + dataType + "]");
             };
             default -> throw new IllegalArgumentException("unsupported op [" + op + "]");

branches.json

@@ -13,9 +13,6 @@
     {
       "branch": "8.17"
     },
-    {
-      "branch": "8.x"
-    },
     {
       "branch": "7.17"
     }

build-conventions/settings.gradle

@@ -6,6 +6,11 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
+
 rootProject.name = 'build-conventions'
 
 dependencyResolutionManagement {

build-tools-internal/settings.gradle

@@ -8,6 +8,10 @@ pluginManagement {
     includeBuild "../build-tools"
 }
 
+plugins {
+  id "com.gradle.develocity" version "3.18.1"
+}
+
 dependencyResolutionManagement {
     versionCatalogs {
         buildLibs {

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -32,7 +32,9 @@ develocity {
     // Automatically publish scans from Elasticsearch CI
     if (onCI) {
       publishing.onlyIf { true }
-      server = 'https://gradle-enterprise.elastic.co'
+      if(server.isPresent() == false) {
+        server = 'https://gradle-enterprise.elastic.co'
+      }
     } else if( server.isPresent() == false) {
       publishing.onlyIf { false }
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -22,7 +22,7 @@ public enum DockerBase {
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
-    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:bd401704a162a7937cd1015f755ca9da9aba0fdf967fc6bf90bf8d3f6b2eb557",
+    WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:d74b1fda6b7fee2c90b410df258e005c049e0672fe16d79d00e58f14fb69f90b",
         "-wolfi",
         "apk"
     ),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -58,7 +58,6 @@ public class ElasticsearchJavaBasePlugin implements Plugin<Project> {
 
     @Override
     public void apply(Project project) {
-        project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
         // make sure the global build info plugin is applied to the root project
         project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
         buildParams = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/MrjarPlugin.java

@@ -163,6 +163,7 @@ private void addJar(Project project, SourceSet sourceSet, int javaVersion) {
         project.getConfigurations().register("java" + javaVersion);
         TaskProvider<Jar> jarTask = project.getTasks().register("java" + javaVersion + "Jar", Jar.class, task -> {
             task.from(sourceSet.getOutput());
+            task.getArchiveClassifier().set("java" + javaVersion);
         });
         project.getArtifacts().add("java" + javaVersion, jarTask);
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -93,7 +93,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-security-with-mustache");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:xpack-prefix-rest-compat");
         map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-geoip:qa:file-based-update");
-        map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-ec2:qa:amazon-ec2");
         map.put(LegacyRestTestBasePlugin.class, ":plugins:discovery-gce:qa:gce");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
         map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java

@@ -60,6 +60,7 @@
 import static org.gradle.api.JavaVersion.VERSION_21;
 import static org.gradle.api.JavaVersion.VERSION_22;
 import static org.gradle.api.JavaVersion.VERSION_23;
+import static org.gradle.api.JavaVersion.VERSION_24;
 
 @CacheableTask
 public abstract class ThirdPartyAuditTask extends DefaultTask {
@@ -341,8 +342,12 @@ private String runForbiddenAPIsCli() throws IOException {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
             spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath());
-            // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
-            if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
+            // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23/24 for now, and just the vector module.
+            if (isJavaVersion(VERSION_20)
+                || isJavaVersion(VERSION_21)
+                || isJavaVersion(VERSION_22)
+                || isJavaVersion(VERSION_23)
+                || isJavaVersion(VERSION_24)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
             }
             spec.jvmArgs("-Xmx1g");