Merge remote-tracking branch 'upstream/main' into inference_metadata_fields

Author: jimczi

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -24,12 +24,17 @@ allprojects {
   }
 }
 
+interface Injected {
+  @Inject FileSystemOperations getFs()
+}
+
 // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE
 if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin)
 
   def elasticsearchProject = locateElasticsearchWorkspace(gradle)
 
+  def rootFolder = project.rootDir
   tasks.register('configureIdeCheckstyle') {
     group = 'ide'
     description = 'Generated a suitable checkstyle config for IDEs'
@@ -39,10 +44,10 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     String checkstyleConfig = "${resources}/checkstyle.xml"
     String checkstyleSuppressions = "${resources}/checkstyle_suppressions.xml"
     String checkstyleIdeFragment = "${resources}/checkstyle_ide_fragment.xml"
-    String checkstyleIdeConfig = "${rootDir}/checkstyle_ide.xml"
+    String checkstyleIdeConfig = "${rootFolder}/checkstyle_ide.xml"
 
     String checkstylePluginConfigTemplate = "${resources}/checkstyle-idea.xml"
-    String checkstylePluginConfig = "${rootDir}/.idea/checkstyle-idea.xml"
+    String checkstylePluginConfig = "${rootFolder}/.idea/checkstyle-idea.xml"
 
     inputs.files(
       file(checkstyleConfig),
@@ -53,31 +58,33 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       file(checkstyleIdeConfig),
       file(checkstylePluginConfig)
     )
+    def injected = project.objects.newInstance(Injected)
 
+    def projectFolder = project.layout.projectDirectory.asFile
     doLast {
       // Configure the IntelliJ Checkstyle plugin by copying a standard file. We don't simply commit
       // the result to version control, because the plugin has a habit of modifying the file and
       // replacing the `$PROJECT_DIR$` placeholders, which developers must then revert.
-      project.copy {
+      injected.fs.copy {
         from(checkstylePluginConfigTemplate)
-        into("${rootDir}/.idea")
+        into("${rootFolder}/.idea")
         expand(jarLocation: buildConventionsJar, configLocation: checkstyleIdeConfig)
       }
 
       // Create an IDE-specific checkstyle config by first copying the standard config
       Files.copy(
-        Paths.get(file(checkstyleConfig).getPath()),
-        Paths.get(file(checkstyleIdeConfig).getPath()),
+        Paths.get(new File(checkstyleConfig).getPath()),
+        Paths.get(new File(checkstyleIdeConfig).getPath()),
         StandardCopyOption.REPLACE_EXISTING
       )
 
       // There are some rules that we only want to enable in an IDE. These
       // are extracted to a separate file, and merged into the IDE-specific
       // Checkstyle config.
-      Node xmlFragment = parseXml(checkstyleIdeFragment)
+      Node xmlFragment = IdeaXmlUtil.parseXml(checkstyleIdeFragment)
 
       // Edit the copy so that IntelliJ can copy with it
-      modifyXml(checkstyleIdeConfig, { xml ->
+      IdeaXmlUtil.modifyXml(checkstyleIdeConfig, { xml ->
         // Add all the nodes from the fragment file
         Node treeWalker = xml.module.find { it.'@name' == 'TreeWalker' }
         xmlFragment.module.each { treeWalker.append(it) }
@@ -103,7 +110,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Configures the appropriate JVM for Gradle'
 
     doLast {
-      modifyXml('.idea/gradle.xml') { xml ->
+      IdeaXmlUtil.modifyXml('.idea/gradle.xml') { xml ->
         def gradleSettings = xml.component.find { it.'@name' == 'GradleSettings' }.option[0].GradleProjectSettings
         // Remove configured JVM option to force IntelliJ to use the project JDK for Gradle
         gradleSettings.option.findAll { it.'@name' == 'gradleJvm' }.each { it.parent().remove(it) }
@@ -127,7 +134,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enable per-module *.iml files'
 
     doLast {
-      modifyXml('.idea/misc.xml') {xml ->
+      IdeaXmlUtil.modifyXml('.idea/misc.xml') {xml ->
         def externalStorageConfig = xml.component.find { it.'@name' == 'ExternalStorageConfigurationManager' }
         if (externalStorageConfig) {
           xml.remove(externalStorageConfig)
@@ -142,13 +149,13 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
     description = 'Enables preview features on native library module'
     dependsOn tasks.named("enableExternalConfiguration")
 
-    ext {
-      enablePreview = { moduleFile, languageLevel ->
-        modifyXml(moduleFile) { xml ->
+//    ext {
+      def enablePreview = { moduleFile, languageLevel ->
+        IdeaXmlUtil.modifyXml(moduleFile) { xml ->
           xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = languageLevel
         }
       }
-    }
+//    }
 
     doLast {
       enablePreview('.idea/modules/libs/native/elasticsearch.libs.native.main.iml', 'JDK_21_PREVIEW')
@@ -278,33 +285,37 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
  * @param preface optional front matter to add after the XML declaration
  * but before the XML document, e.g. a doctype or comment
  */
-void modifyXml(Object path, Action<? super Node> action, String preface = null) {
-  if (project.file(path).exists()) {
-    Node xml = parseXml(path)
-    action.execute(xml)
-
-    File xmlFile = project.file(path)
-    xmlFile.withPrintWriter { writer ->
-      def printer = new XmlNodePrinter(writer)
-      printer.namespaceAware = true
-      printer.preserveWhitespace = true
-      writer.write("<?xml version=\"1.0\"?>\n")
-
-      if (preface != null) {
-        writer.write(preface)
+
+class IdeaXmlUtil {
+  static Node parseXml(Object xmlPath) {
+    File xmlFile = new File(xmlPath)
+    XmlParser xmlParser = new XmlParser(false, true, true)
+    xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
+    Node xml = xmlParser.parse(xmlFile)
+    return xml
+  }
+
+  static void modifyXml(Object xmlPath, Action<? super Node> action, String preface = null) {
+    File xmlFile = new File(xmlPath)
+    if (xmlFile.exists()) {
+      Node xml = parseXml(xmlPath)
+      action.execute(xml)
+
+      xmlFile.withPrintWriter { writer ->
+        def printer = new XmlNodePrinter(writer)
+        printer.namespaceAware = true
+        printer.preserveWhitespace = true
+        writer.write("<?xml version=\"1.0\"?>\n")
+
+        if (preface != null) {
+          writer.write(preface)
+        }
+        printer.print(xml)
       }
-      printer.print(xml)
     }
   }
 }
 
-Node parseXml(Object path) {
-  File xmlFile = project.file(path)
-  XmlParser xmlParser = new XmlParser(false, true, true)
-  xmlParser.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
-  Node xml = xmlParser.parse(xmlFile)
-  return xml
-}
 
 Pair<File, IncludedBuild> locateElasticsearchWorkspace(Gradle gradle) {
   if (gradle.parent == null) {

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -1,21 +1,17 @@
 ---
 apiVersion: v1
-
 # The repository name in registry1, excluding /ironbank/
 name: "elastic/elasticsearch/elasticsearch"
-
 # List of tags to push for the repository in registry1
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
 - "${version}"
 - "latest"
-
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.4"
-
+  BASE_TAG: "9.5"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"
@@ -34,7 +30,6 @@ labels:
   mil.dso.ironbank.image.type: "commercial"
   # Product the image belongs to for grouping multiple images
   mil.dso.ironbank.product.name: "elasticsearch"
-
 # List of resources to make available to the offline build context
 resources:
   - filename: "elasticsearch-${version}-linux-x86_64.tar.gz"
@@ -47,7 +42,6 @@ resources:
     validation:
       type: "sha256"
       value: "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c"
-
 # List of project maintainers
 maintainers:
   - name: "Mark Vieira"

docs/changelog/118603.yaml

@@ -0,0 +1,6 @@
+pr: 118603
+summary: Allow DATE_PARSE to read the timezones
+area: ES|QL
+type: bug
+issues:
+ - 117680

docs/reference/esql/esql-limitations.asciidoc

@@ -112,7 +112,7 @@ it is necessary to use the search function, like <<esql-match>>, in a <<esql-whe
 directly after the <<esql-from>> source command, or close enough to it.
 Otherwise, the query will fail with a validation error.
 Another limitation is that any <<esql-where>> command containing a full-text search function
-cannot also use disjunctions (`OR`).
+cannot also use disjunctions (`OR`) unless all functions used in the OR clauses are full-text functions themselves.
 
 For example, this query is valid:
 
@@ -139,6 +139,15 @@ FROM books
 | WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
+However this query will succeed because it uses full text functions on both `OR` clauses:
+
+[source,esql]
+----
+FROM books
+| WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
+----
+
+
 Note that, because of <<esql-limitations-text-fields,the way {esql} treats `text` values>>,
 any queries on `text` fields that do not explicitly use the full-text functions,
 <<esql-match>> or <<esql-qstr>>, will behave as if the fields are actually `keyword` fields:

docs/reference/esql/functions/description/match.asciidoc

@@ -6,11 +6,13 @@
 ++++
 
 Full text functions are used to search for text in fields.
-<<analysis, Text analysiss>> is used to analyze the query before it is searched.
+<<analysis, Text analysis>> is used to analyze the query before it is searched.
 
 Full text functions can be used to match <<esql-multivalued-fields,multivalued fields>>.
 A multivalued field that contains a value that matches a full text query is considered to match the query.
 
+Full text functions are significantly more performant for text search use cases on large data sets than using pattern matching or regular expressions with `LIKE` or `RLIKE`
+
 See <<esql-limitations-full-text-search,full text search limitations>> for information on the limitations of full text search.
 
 {esql} supports these full-text search functions:

docs/reference/esql/functions/kibana/definition/match.json

@@ -7,7 +7,7 @@ the input table for which the provided condition evaluates to `true`.
 
 [TIP]
 ====
-In case of value exclusions, fields with `null` values will be excluded from search results. 
+In case of value exclusions, fields with `null` values will be excluded from search results.
 In this context a `null` means either there is an explicit `null` value in the document or there is no value at all.
 For example: `WHERE field != "value"` will be interpreted as `WHERE field != "value" AND field IS NOT NULL`.
 ====
@@ -58,6 +58,26 @@ For a complete list of all functions, refer to <<esql-functions>>.
 
 include::../functions/predicates.asciidoc[tag=body]
 
+For matching text, you can use <<esql-search-functions,full text search functions>> like `MATCH`.
+
+Use <<esql-match,`MATCH`>> to perform a <<query-dsl-match-query,match query>> on a specified field.
+
+Match can be used on text fields, as well as other field types like boolean, dates, and numeric types.
+
+[source.merge.styled,esql]
+----
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field]
+----
+[%header.monospaced.styled,format=dsv,separator=|]
+|===
+include::{esql-specs}/match-function.csv-spec[tag=match-with-field-result]
+|===
+
+[TIP]
+====
+You can also use the shorthand <<esql-search-operators,match operator>> `:` instead of `MATCH`.
+====
+
 include::../functions/like.asciidoc[tag=body]
 
 include::../functions/rlike.asciidoc[tag=body]

docs/reference/esql/functions/kibana/docs/match.md

@@ -172,9 +172,8 @@ The API returns the following response:
                             "attributes": {},
                             "roles": [...],
                             "version": "8.10.0",
-                            "min_index_version": 8000099,
-                            "min_read_only_index_version": 7000099,
-                            "max_index_version": 9004000
+                            "min_index_version": 7000099,
+                            "max_index_version": 8100099
                         },
                         "allocation_id": "2iNySv_OQVePRX-yaRH_lQ", <4>
                         "allocation" : "primary|replica|unused" <5>
@@ -194,7 +193,6 @@ The API returns the following response:
 // TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/]
 // TESTRESPONSE[s/"8.10.0"/\$node_version/]
 // TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/]
-// TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/]
 // TESTRESPONSE[s/"max_index_version": 8100099/"max_index_version": $body.$_path/]