Merge branch 'main' of github.com:elastic/elasticsearch into ml-eis-integration

Author: jonathan-buttner

.buildkite/pipelines/periodic-packaging.template.yml

@@ -7,19 +7,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

.buildkite/pipelines/periodic-packaging.yml

@@ -8,19 +8,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -7,19 +7,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8
@@ -90,7 +85,6 @@ steps:
           setup:
             image:
               - amazonlinux-2023
-              - amazonlinux-2
         agents:
           provider: aws
           imagePrefix: elasticsearch-{{matrix.image}}

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -10,19 +10,14 @@ steps:
         matrix:
           setup:
             image:
-              - debian-11
               - debian-12
               - opensuse-leap-15
-              - oraclelinux-7
               - oraclelinux-8
-              - sles-12
               - sles-15
-              - ubuntu-1804
               - ubuntu-2004
               - ubuntu-2204
               - rocky-8
               - rocky-9
-              - rhel-7
               - rhel-8
               - rhel-9
               - almalinux-8

benchmarks/src/main/java/org/elasticsearch/benchmark/search/fetch/subphase/FetchSourcePhaseBenchmark.java

@@ -63,7 +63,7 @@ public void setup() throws IOException {
         );
         includesSet = Set.of(fetchContext.includes());
         excludesSet = Set.of(fetchContext.excludes());
-        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(includesSet, excludesSet, false);
+        parserConfig = XContentParserConfiguration.EMPTY.withFiltering(null, includesSet, excludesSet, false);
     }
 
     private BytesReference read300BytesExample() throws IOException {

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/FilterContentBenchmark.java

@@ -170,7 +170,7 @@ private XContentParserConfiguration buildParseConfig(boolean matchDotsInFieldNam
             includes = null;
             excludes = filters;
         }
-        return XContentParserConfiguration.EMPTY.withFiltering(includes, excludes, matchDotsInFieldNames);
+        return XContentParserConfiguration.EMPTY.withFiltering(null, includes, excludes, matchDotsInFieldNames);
     }
 
     private BytesReference filter(XContentParserConfiguration contentParserConfiguration) throws IOException {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -252,6 +252,20 @@ private List<Version> getReleased() {
             .toList();
     }
 
+    public List<Version> getReadOnlyIndexCompatible() {
+        // Lucene can read indices in version N-2
+        int compatibleMajor = currentVersion.getMajor() - 2;
+        return versions.stream().filter(v -> v.getMajor() == compatibleMajor).sorted(Comparator.naturalOrder()).toList();
+    }
+
+    public void withLatestReadOnlyIndexCompatible(Consumer<Version> versionAction) {
+        var compatibleVersions = getReadOnlyIndexCompatible();
+        if (compatibleVersions == null || compatibleVersions.isEmpty()) {
+            throw new IllegalStateException("No read-only compatible version found.");
+        }
+        versionAction.accept(compatibleVersions.getLast());
+    }
+
     /**
      * Return versions of Elasticsearch which are index compatible with the current version.
      */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -13,8 +13,8 @@
  * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch.
  */
 public enum DockerBase {
-    // "latest" here is intentional, since the image name specifies "8"
-    DEFAULT("docker.elastic.co/ubi8/ubi-minimal:latest", "", "microdnf"),
+    // "latest" here is intentional, since the image name specifies "9"
+    DEFAULT("docker.elastic.co/ubi9/ubi-minimal:latest", "", "microdnf"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.gradle.test.GradleTestPolicySetupPlugin;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
 import org.gradle.api.Action;
+import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
@@ -112,7 +113,6 @@ public void execute(Task t) {
             test.jvmArgs(
                 "-Xmx" + System.getProperty("tests.heap.size", "512m"),
                 "-Xms" + System.getProperty("tests.heap.size", "512m"),
-                "-Djava.security.manager=allow",
                 "-Dtests.testfeatures.enabled=true",
                 "--add-opens=java.base/java.util=ALL-UNNAMED",
                 // TODO: only open these for mockito when it is modularized
@@ -127,6 +127,13 @@ public void execute(Task t) {
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));
+            test.getJvmArgumentProviders().add(() -> {
+                if (test.getJavaVersion().compareTo(JavaVersion.VERSION_23) <= 0) {
+                    return List.of("-Djava.security.manager=allow");
+                } else {
+                    return List.of();
+                }
+            });
 
             String argline = System.getProperty("tests.jvm.argline");
             if (argline != null) {

build-tools-internal/version.properties

@@ -17,6 +17,8 @@ jna               = 5.12.1
 netty             = 4.1.115.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
+awsv1sdk            = 1.12.270
+awsv2sdk            = 2.28.13
 
 antlr4            = 4.13.1
 # bouncy castle version for non-fips. fips jars use a different version
@@ -33,7 +35,7 @@ commonscodec      = 1.15
 protobuf          = 3.25.5
 
 # test dependencies
-randomizedrunner  = 2.8.0
+randomizedrunner  = 2.8.2
 junit             = 4.13.2
 junit5            = 5.7.1
 hamcrest          = 2.1