Merge branch 'main' into bk-cost-opt-spot

Author: brianseeders

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -305,8 +305,8 @@ steps:
         env:
           BWC_VERSION: 7.16.3
 
-      - label: "{{matrix.image}} / 7.17.20 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.20
+      - label: "{{matrix.image}} / 7.17.21 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.21
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -319,7 +319,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 7.17.20
+          BWC_VERSION: 7.17.21
 
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1
@@ -529,8 +529,8 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
-      - label: "{{matrix.image}} / 8.13.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.1
+      - label: "{{matrix.image}} / 8.13.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -543,7 +543,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.1
+          BWC_VERSION: 8.13.3
 
       - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0

.buildkite/pipelines/periodic.template.yml

@@ -66,7 +66,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -97,7 +96,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             BWC_VERSION: $BWC_LIST
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 7.17.20 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.20#bwcTest
+      - label: 7.17.21 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.21#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           preemptible: true
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 7.17.20
+          BWC_VERSION: 7.17.21
         retry:
           automatic:
             - exit_status: '-1'
@@ -591,8 +591,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.13.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.1#bwcTest
+      - label: 8.13.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -601,7 +601,7 @@ steps:
           preemptible: true
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.13.1
+          BWC_VERSION: 8.13.3
         retry:
           automatic:
             - exit_status: '-1'
@@ -666,7 +666,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -694,7 +694,6 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
             GRADLE_TASK:
               - checkPart1
               - checkPart2
@@ -725,8 +724,7 @@ steps:
               - graalvm-ce17
               - openjdk17
               - openjdk21
-              - openjdk22
-            BWC_VERSION: ["7.17.20", "8.13.1", "8.14.0"]
+            BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,7 +16,7 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.2"
   - "7.16.3"
-  - "7.17.20"
+  - "7.17.21"
   - "8.0.1"
   - "8.1.3"
   - "8.2.3"
@@ -30,5 +30,5 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.4"
   - "8.12.2"
-  - "8.13.1"
+  - "8.13.3"
   - "8.14.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.20"
-  - "8.13.1"
+  - "7.17.21"
+  - "8.13.3"
   - "8.14.0"

.gitignore

@@ -68,4 +68,4 @@ testfixtures_shared/
 
 # Generated
 checkstyle_ide.xml
-x-pack/plugin/esql/gen/
+x-pack/plugin/esql/src/main/generated-src/generated/

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -25,6 +25,7 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.model.ObjectFactory;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.IgnoreEmptyDirectories;
@@ -39,21 +40,22 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
-import org.gradle.api.model.ObjectFactory;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.Serializable;
 import java.io.Writer;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
-import java.io.Serializable;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
 
 /**
  * Checks files for license headers..
@@ -232,7 +234,7 @@ private ClaimStatistic toXmlReportFile(ReportConfiguration config, Writer writer
 
     private static List<String> unapprovedFiles(File xmlReportFile) {
         try {
-            NodeList resourcesNodes = DocumentBuilderFactory.newInstance()
+            NodeList resourcesNodes = createXmlDocumentBuilderFactory()
                 .newDocumentBuilder()
                 .parse(xmlReportFile)
                 .getElementsByTagName("resource");
@@ -249,6 +251,21 @@ private static List<String> unapprovedFiles(File xmlReportFile) {
         }
     }
 
+    private static DocumentBuilderFactory createXmlDocumentBuilderFactory() throws ParserConfigurationException {
+        final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+        dbf.setXIncludeAware(false);
+        dbf.setIgnoringComments(true);
+        dbf.setExpandEntityReferences(false);
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+        dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, "");
+        dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+        dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
+        dbf.setFeature("http://xml.org/sax/features/external-general-entities", false);
+        dbf.setFeature("http://xml.org/sax/features/external-parameter-entities", false);
+        dbf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
+        return dbf;
+    }
+
     private static List<Element> elementList(NodeList resourcesNodes) {
         List<Element> nodeList = new ArrayList<>(resourcesNodes.getLength());
         for (int idx = 0; idx < resourcesNodes.getLength(); idx++) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/doc/DocsTestPluginFuncTest.groovy

@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.doc
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest
+import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin
+import org.gradle.testkit.runner.TaskOutcome
+
+class DocsTestPluginFuncTest extends AbstractGradleInternalPluginFuncTest {
+    Class<? extends PrecommitPlugin> pluginClassUnderTest = DocsTestPlugin.class
+
+    def setup() {
+        File docDir = new File(projectDir, 'doc');
+        docDir.mkdirs()
+        addSampleDoc(docDir)
+        buildApiRestrictionsDisabled = true
+    configurationCacheCompatible = false;
+        buildFile << """
+tasks.named('listSnippets') {
+   docs = fileTree('doc')
+}
+
+tasks.named('listConsoleCandidates') {
+   docs = fileTree('doc')
+}
+"""
+    }
+
+    def "can list snippets"() {
+        when:
+        def result = gradleRunner("listSnippets").build()
+        then:
+        result.task(":listSnippets").outcome == TaskOutcome.SUCCESS
+        assertOutputContains(result.output, """
+> Task :listSnippets
+mapper-annotated-text.asciidoc[37:39](Painless)
+mapper-annotated-text.asciidoc[42:44](js)
+mapper-annotated-text.asciidoc[51:69](console)// TEST[setup:seats]
+""")
+    }
+
+    def "can console candidates"() {
+        when:
+        def result = gradleRunner("listConsoleCandidates").build()
+        then:
+        result.task(":listConsoleCandidates").outcome == TaskOutcome.SUCCESS
+        assertOutputContains(result.output, """
+> Task :listConsoleCandidates
+mapper-annotated-text.asciidoc[42:44](js)
+""")
+    }
+
+    void addSampleDoc(File docFolder) {
+        new File(docFolder, "mapper-annotated-text.asciidoc").text = """
+[[painless-filter-context]]
+=== Filter context
+
+Use a Painless script as a {ref}/query-dsl-script-query.html[filter] in a
+query to include and exclude documents.
+
+
+*Variables*
+
+`params` (`Map`, read-only)::
+        User-defined parameters passed in as part of the query.
+
+`doc` (`Map`, read-only)::
+        Contains the fields of the current document where each field is a
+        `List` of values.
+
+*Return*
+
+`boolean`::
+        Return `true` if the current document should be returned as a result of
+        the query, and `false` otherwise.
+
+
+*API*
+
+The standard <<painless-api-reference-shared, Painless API>> is available.
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+This script finds all unsold documents that cost less than \$25.
+
+[source,Painless]
+----
+doc['sold'].value == false && doc['cost'].value < 25
+----
+
+[source,js]
+----
+curl 'hello world'
+----
+
+Defining `cost` as a script parameter enables the cost to be configured
+in the script query request. For example, the following request finds
+all available theatre seats for evening performances that are under \$25.
+
+[source,console]
+----
+GET seats/_search
+{
+  "query": {
+    "bool": {
+      "filter": {
+        "script": {
+          "script": {
+            "source": "doc['sold'].value == false && doc['cost'].value < params.cost",
+            "params": {
+              "cost": 25
+            }
+          }
+        }
+      }
+    }
+  }
+}
+----
+// TEST[setup:seats]
+"""
+    }
+}