Merge remote-tracking branch 'upstream/main' into 04212025/PauseIndexingES11516

refresh

Author: ankikuma

.buildkite/scripts/dra-workflow.trigger.sh

@@ -2,10 +2,34 @@
 
 set -euo pipefail
 
-echo "steps:"
-
 source .buildkite/scripts/branches.sh
 
+# We use that filtering to keep different schedule for different branches
+if [ -n "${INCLUDED_BRANCHES:-}" ]; then
+  # If set, only trigger the pipeline for the specified branches
+  IFS=',' read -r -a BRANCHES <<< "${INCLUDED_BRANCHES}"
+elif [ -n "${EXCLUDED_BRANCHES:-}" ]; then
+  # If set, listed branches will be excluded from the list of branches in branches.json
+  IFS=',' read -r -a EXCLUDED_BRANCHES <<< "${EXCLUDED_BRANCHES}"
+  FILTERED_BRANCHES=()
+    for BRANCH in "${BRANCHES[@]}"; do
+      EXCLUDE=false
+      for EXCLUDED_BRANCH in "${EXCLUDED_BRANCHES[@]}"; do
+        if [ "$BRANCH" == "$EXCLUDED_BRANCH" ]; then
+          EXCLUDE=true
+          break
+        fi
+      done
+      if [ "$EXCLUDE" = false ]; then
+        FILTERED_BRANCHES+=("$BRANCH")
+      fi
+    done
+    BRANCHES=("${FILTERED_BRANCHES[@]}")
+fi
+
+
+echo "steps:"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java

@@ -123,7 +123,7 @@ private LogicalPlan plan(String query) {
     }
 
     @Benchmark
-    public void run(Blackhole blackhole) {
+    public void manyFields(Blackhole blackhole) {
         blackhole.consume(plan("FROM test | LIMIT 10"));
     }
 }

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=fba8464465835e74f7270bbf43d6d8a8d7709ab0a43ce1aa3323f73e9aa0c612
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-all.zip
+distributionSha256Sum=efe9a3d147d948d7528a9887fa35abcf24ca1a43ad06439996490f77569b02d1
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestApiTask.java

@@ -25,8 +25,7 @@
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
-import org.gradle.api.tasks.util.PatternSet;
-import org.gradle.internal.Factory;
+import org.gradle.api.tasks.util.internal.PatternSetFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -65,14 +64,14 @@ public class CopyRestApiTask extends DefaultTask {
     @Inject
     public CopyRestApiTask(
         ProjectLayout projectLayout,
-        Factory<PatternSet> patternSetFactory,
+        PatternSetFactory patternSetFactory,
         FileSystemOperations fileSystemOperations,
         ObjectFactory objectFactory
     ) {
         this.include = objectFactory.listProperty(String.class);
         this.outputResourceDir = objectFactory.directoryProperty();
         this.additionalYamlTestsDir = objectFactory.directoryProperty();
-        this.patternSet = patternSetFactory.create();
+        this.patternSet = patternSetFactory.createPatternSet();
         this.projectLayout = projectLayout;
         this.fileSystemOperations = fileSystemOperations;
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/CopyRestTestsTask.java

@@ -29,8 +29,7 @@
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
-import org.gradle.api.tasks.util.PatternSet;
-import org.gradle.internal.Factory;
+import org.gradle.api.tasks.util.internal.PatternSetFactory;
 
 import java.io.File;
 import java.util.Map;
@@ -66,25 +65,25 @@ public abstract class CopyRestTestsTask extends DefaultTask {
     private final ProjectLayout projectLayout;
     private final FileSystemOperations fileSystemOperations;
 
-    @Inject
-    public abstract FileOperations getFileOperations();
-
     @Inject
     public CopyRestTestsTask(
         ProjectLayout projectLayout,
-        Factory<PatternSet> patternSetFactory,
+        PatternSetFactory patternSetFactory,
         FileSystemOperations fileSystemOperations,
         ObjectFactory objectFactory
     ) {
         this.includeCore = objectFactory.listProperty(String.class);
         this.includeXpack = objectFactory.listProperty(String.class);
         this.outputResourceDir = objectFactory.directoryProperty();
-        this.corePatternSet = patternSetFactory.create();
-        this.xpackPatternSet = patternSetFactory.create();
+        this.corePatternSet = patternSetFactory.createPatternSet();
+        this.xpackPatternSet = patternSetFactory.createPatternSet();
         this.projectLayout = projectLayout;
         this.fileSystemOperations = fileSystemOperations;
     }
 
+    @Inject
+    public abstract FileOperations getFileOperations();
+
     @Input
     public ListProperty<String> getIncludeCore() {
         return includeCore;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -58,7 +58,7 @@
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
 import org.gradle.api.tasks.util.PatternSet;
-import org.gradle.internal.Factory;
+import org.gradle.api.tasks.util.internal.PatternSetFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -98,25 +98,23 @@ public abstract class RestCompatTestTransformTask extends DefaultTask {
     // PatternFilterable -> list of full test names and reasons. Needed for 1 pattern may include many tests and reasons
     private final Map<PatternFilterable, List<Pair<String, String>>> skippedTestByTestNameTransformations = new HashMap<>();
 
-    @Inject
-    protected Factory<PatternSet> getPatternSetFactory() {
-        throw new UnsupportedOperationException();
-    }
-
     @Inject
     public RestCompatTestTransformTask(FileSystemOperations fileSystemOperations, ObjectFactory objectFactory) {
         this.fileSystemOperations = fileSystemOperations;
         this.compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
         this.sourceDirectory = objectFactory.directoryProperty();
         this.outputDirectory = objectFactory.directoryProperty();
-        this.testPatternSet = getPatternSetFactory().create();
+        this.testPatternSet = getPatternSetFactory().createPatternSet();
         this.testPatternSet.include("/*" + "*/*.yml"); // concat these strings to keep build from thinking this is invalid javadoc
         // always inject compat headers
         headers.put("Content-Type", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion);
         headers.put("Accept", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion);
         getTransformations().add(new InjectHeaders(headers, Sets.newHashSet(RestCompatTestTransformTask::doesNotHaveCatOperation)));
     }
 
+    @Inject
+    protected abstract PatternSetFactory getPatternSetFactory();
+
     private static boolean doesNotHaveCatOperation(ObjectNode doNodeValue) {
         final Iterator<String> fieldNamesIterator = doNodeValue.fieldNames();
         while (fieldNamesIterator.hasNext()) {
@@ -144,7 +142,7 @@ public void skipTest(String fullTestName, String reason) {
             );
         }
 
-        PatternSet skippedPatternSet = getPatternSetFactory().create();
+        PatternSet skippedPatternSet = getPatternSetFactory().createPatternSet();
         // create file patterns for all a1/a2/a3/b.yml possibilities.
         for (int i = testParts.length - 1; i > 1; i--) {
             final String lastPart = testParts[i];
@@ -158,7 +156,7 @@ public void skipTest(String fullTestName, String reason) {
     }
 
     public void skipTestsByFilePattern(String filePattern, String reason) {
-        PatternSet skippedPatternSet = getPatternSetFactory().create();
+        PatternSet skippedPatternSet = getPatternSetFactory().createPatternSet();
         skippedPatternSet.include(filePattern);
         skippedTestByFilePatternTransformations.put(skippedPatternSet, reason);
     }

build-tools-internal/src/main/resources/checkstyle.xml

@@ -68,6 +68,11 @@
 
     <!-- Unused imports are forbidden -->
     <module name="UnusedImports" />
+    <module name="SuppressionCommentFilter">
+      <property name="offCommentFormat" value="begin generated imports"/>
+      <property name="onCommentFormat" value="end generated imports"/>
+      <property name="checkFormat" value="UnusedImports"/>
+    </module>
 
     <!-- Non-inner classes must be in files that match their names. -->
     <module name="OuterTypeFilename" />

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-8.12.1
+8.14

build-tools/src/main/java/org/elasticsearch/gradle/test/SystemPropertyCommandLineArgumentProvider.java

@@ -8,7 +8,6 @@
  */
 package org.elasticsearch.gradle.test;
 
-import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.Input;
 import org.gradle.process.CommandLineArgumentProvider;
 
@@ -20,10 +19,6 @@
 public class SystemPropertyCommandLineArgumentProvider implements CommandLineArgumentProvider {
     private final Map<String, Object> systemProperties = new LinkedHashMap<>();
 
-    public void systemProperty(String key, Provider<Object> value) {
-        systemProperties.put(key, (Supplier<String>) () -> String.valueOf(value.get()));
-    }
-
     public void systemProperty(String key, Supplier<String> value) {
         systemProperties.put(key, value);
     }

distribution/docker/src/docker/Dockerfile.ess-fips

@@ -96,15 +96,6 @@ COPY fips/resources/fips_java.policy /usr/share/elasticsearch/config/fips_java.p
 
 WORKDIR /usr/share/elasticsearch/config
 
-## Add fips specific JVM options
-RUN cat <<EOF > /usr/share/elasticsearch/config/jvm.options.d/fips.options
--Djavax.net.ssl.keyStoreType=BCFKS
--Dorg.bouncycastle.fips.approved_only=true
--Djava.security.properties=config/fips_java.security
--Djava.security.policy=config/fips_java.policy
-EOF
-
-
 ################################################################################
 # Build stage 2 (the actual Elasticsearch image):
 #
@@ -136,6 +127,10 @@ ENV ELASTIC_CONTAINER=true
 WORKDIR /usr/share/elasticsearch
 
 COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch
+COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
+COPY --from=builder --chown=0:0 /opt /opt
+
+ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive
 ENV PATH=/usr/share/elasticsearch/bin:\$PATH
 ENV SHELL=/bin/bash
 COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
@@ -159,6 +154,28 @@ RUN chmod g=u /etc/passwd && \\
 
 RUN ln -sf /etc/ssl/certs/java/cacerts /usr/share/elasticsearch/jdk/lib/security/cacerts
 
+# Convert cacerts (PKCS12) to BCFKS format using POSIX-compatible shell syntax
+RUN printf "\\n" | jdk/bin/keytool -importkeystore \
+  -srckeystore /usr/share/elasticsearch/jdk/lib/security/cacerts \
+  -srcstoretype PKCS12 \
+  -destkeystore config/cacerts.bcfks \
+  -deststorepass passwordcacert \
+  -deststoretype BCFKS \
+  -providerclass org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider \
+  -providerpath lib/bc-fips-1.0.2.5.jar \
+  -destprovidername BCFIPS
+
+
+## Add fips specific JVM options
+RUN cat <<EOF > /usr/share/elasticsearch/config/jvm.options.d/fips.options
+-Djavax.net.ssl.keyStoreType=BCFKS
+-Dorg.bouncycastle.fips.approved_only=true
+-Djava.security.properties=config/fips_java.security
+-Djava.security.policy=config/fips_java.policy
+-Djavax.net.ssl.trustStore=config/cacerts.bcfks
+-Djavax.net.ssl.trustStorePassword=passwordcacert
+EOF
+
 EXPOSE 9200 9300
 
 LABEL org.label-schema.build-date="${build_date}" \\
@@ -196,11 +213,6 @@ CMD ["/app/elasticsearch.sh"]
 
 USER 1000:0
 
-COPY --from=builder --chown=0:0 /opt /opt
-ENV ES_PLUGIN_ARCHIVE_DIR=/opt/plugins/archive
-WORKDIR /usr/share/elasticsearch
-COPY --from=builder --chown=0:0 /fips/libs/*.jar /usr/share/elasticsearch/lib/
-
 ################################################################################
 # End of multi-stage Dockerfile
 ################################################################################