Add support or RLIKE (LIST)

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -56,20 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
 
 cat <<EOF | buildkite-agent pipeline upload
 steps:
-    - label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
-      timeout_in_minutes: 300
-      agents:
-        provider: gcp
-        image: family/elasticsearch-ubuntu-2004
-        machineType: n1-standard-32
-        buildDirectory: /dev/shm/bk
-        preemptible: true
-      retry:
-        automatic:
-          - exit_status: "-1"
-            limit: 3
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
+  - group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
+    steps:
+      - label: "bc-upgrade-tests-part{{matrix.PART}}"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}}
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        matrix:
+          setup:
+            PART: ["1", "2", "3", "4", "5", "6"]
 EOF

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -24,22 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
 
 echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
 
-cat <<EOF | buildkite-agent pipeline upload
+cat << EOF | buildkite-agent pipeline upload
 steps:
-    - label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
-      command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
-      timeout_in_minutes: 300
-      agents:
-        provider: gcp
-        image: family/elasticsearch-ubuntu-2004
-        machineType: n1-standard-32
-        buildDirectory: /dev/shm/bk
-        preemptible: true
-      retry:
-        automatic:
-          - exit_status: "-1"
-            limit: 3
-            signal_reason: none
-          - signal_reason: agent_stop
-            limit: 3
+    - group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
+      steps:
+        - label: "pr-upgrade-part-{{matrix.PART}}"
+          command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}}
+          timeout_in_minutes: 300
+          agents:
+            provider: gcp
+            image: family/elasticsearch-ubuntu-2004
+            machineType: n1-standard-32
+            buildDirectory: /dev/shm/bk
+          matrix:
+            setup:
+              PART: ["1", "2", "3", "4", "5", "6"]
 EOF

.github/workflows/docs-build.yml

@@ -16,4 +16,4 @@ jobs:
       deployments: write
       id-token: write
       contents: read
-      pull-requests: read
+      pull-requests: write

benchmarks/src/main/java/org/elasticsearch/benchmark/search/query/range/DateFieldMapperDocValuesSkipperBenchmark.java

@@ -21,14 +21,14 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -295,7 +295,7 @@ public void rangeQueryWithDocValuesSkipper(final Blackhole bh) throws IOExceptio
     /**
      * Runs the actual Lucene range query, optionally combining a {@link LongPoint} index query
      * with doc values ({@link SortedNumericDocValuesField}) via {@link IndexOrDocValuesQuery},
-     * and then wrapping it with an {@link XIndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
+     * and then wrapping it with an {@link IndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
      *
      * @param searcher            the Lucene {@link IndexSearcher}
      * @param rangeStartTimestamp lower bound of the timestamp range
@@ -316,7 +316,7 @@ private long rangeQuery(final IndexSearcher searcher, long rangeStartTimestamp,
             )
             : SortedNumericDocValuesField.newSlowRangeQuery(TIMESTAMP_FIELD, rangeStartTimestamp, rangeEndTimestamp);
 
-        final Query query = new XIndexSortSortedNumericDocValuesRangeQuery(
+        final Query query = new IndexSortSortedNumericDocValuesRangeQuery(
             TIMESTAMP_FIELD,
             rangeStartTimestamp,
             rangeEndTimestamp,

build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt

@@ -61,11 +61,3 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
 
 @defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
 org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
-
-# This is a temporary patch as there is a low level Lucene bug in certain scenarios
-# this should be fixed in the new Lucene release 10.3+
-org.apache.lucene.document.LongField#newExactQuery(java.lang.String, long) @ Use org.elasticsearch.lucene.document.NumericField#newExactLongQuery(java.lang.String, long) instead.
-org.apache.lucene.document.LongField#newRangeQuery(java.lang.String, long, long) @ Use org.elasticsearch.lucene.document.NumericField#newRangeLongQuery(java.lang.String, long, long) instead.
-org.apache.lucene.document.IntField#newExactQuery(java.lang.String, int) @ Use org.elasticsearch.lucene.document.NumericField#newExactIntQuery(java.lang.String, int) instead.
-org.apache.lucene.document.IntField#newRangeQuery(java.lang.String, int, int) @ Use org.elasticsearch.lucene.document.NumericField#newRangeIntQuery(java.lang.String, int, int) instead.
-org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery @ use org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery instead.

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.1.0
-lucene            = 10.2.1
+lucene            = 10.2.2
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90

build.gradle

@@ -351,95 +351,45 @@ allprojects {
     }
   }
 
+  def splitForCI = { proj, partString ->
+    proj.tasks.register("check$partString") {
+      dependsOn 'check'
+      withReleaseBuild {
+        dependsOn 'assemble'
+      }
+    }
+
+    proj.tasks.addRule("Pattern: v<BWC_VERSION>#bwcTest$partString") { name ->
+      if(name.endsWith("#bwcTest$partString")) {
+        proj.project.getTasks().register(name) {
+          task -> task.dependsOn(proj.tasks.named { tskName -> tskName == (name - partString) })
+        }
+      }
+    }
+
+    proj.tasks.register("bcUpgradeTest$partString") {
+      dependsOn tasks.matching { it.name == 'bcUpgradeTest' }
+      withReleaseBuild {
+        dependsOn 'assemble'
+      }
+    }
+  }
+
   plugins.withId('lifecycle-base') {
     if (project.path.startsWith(":x-pack:")) {
       if (project.path.contains("security") || project.path.contains(":ml")) {
-        tasks.register('checkPart4') {
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart4") { name ->
-          if(name.endsWith("#bwcTestPart4")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part4") })
-            }
-          }
-        }
+        splitForCI(project, "Part4")
       } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
-        tasks.register('checkPart3') {
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart3") { name ->
-          if(name.endsWith("#bwcTestPart3")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part3") })
-            }
-          }
-        }
+        splitForCI(project, "Part3")
       } else if (project.path.contains("multi-node")) {
-        tasks.register('checkPart5') {
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart5") { name ->
-          if(name.endsWith("#bwcTestPart5")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part5") })
-            }
-          }
-        }
+        splitForCI(project, "Part5")
       } else {
-        tasks.register('checkPart2') {
-          dependsOn 'check'
-          withReleaseBuild {
-            dependsOn 'assemble'
-          }
-        }
-        tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart2") { name ->
-          if(name.endsWith("#bwcTestPart2")) {
-            project.getTasks().register(name) {
-              task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part2") })
-            }
-          }
-        }
+        splitForCI(project, "Part2")
       }
     } else if(project.path.startsWith(":qa:")) {
-      tasks.register('checkPart6') {
-        dependsOn 'check'
-        withReleaseBuild {
-          dependsOn 'assemble'
-        }
-      }
-      tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart6") { name ->
-        if(name.endsWith("#bwcTestPart6")) {
-          project.getTasks().register(name) {
-            task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part6") })
-          }
-        }
-      }
+      splitForCI(project, "Part6")
     } else {
-      tasks.register('checkPart1') {
-        dependsOn 'check'
-        withReleaseBuild {
-          dependsOn 'assemble'
-        }
-      }
-      tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart1") { name ->
-        if(name.endsWith("#bwcTestPart1")) {
-          project.getTasks().register(name) {
-            task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part1") })
-          }
-        }
-      }
+      splitForCI(project, "Part1")
     }
     tasks.register('functionalTests') {
       dependsOn 'check'

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.2.1
-:lucene_version_path:   10_2_1
+:lucene_version:        10.2.2
+:lucene_version_path:   10_2_2
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/129503.yaml

@@ -0,0 +1,6 @@
+pr: 129503
+summary: Pushdown Lookup Join past Project
+area: ES|QL
+type: enhancement
+issues:
+ - 119082

docs/changelog/129546.yaml

@@ -0,0 +1,13 @@
+pr: 129546
+summary: Upgrade to Lucene 10.2.2
+area: Search
+type: upgrade
+issues: []
+highlight:
+  title: Upgrade to lucene 10.2.2
+  body: |-
+    * Reduce NeighborArray on-heap memory during HNSW graph building
+    * Fix IndexSortSortedNumericDocValuesRangeQuery for integer sorting
+    * ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when used if the DoubleValuesSource needed scores
+    ----
+  notable: true

docs/changelog/129557.yaml

@@ -0,0 +1,5 @@
+pr: 129557
+summary: Pushdown for LIKE (LIST)
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/129600.yaml

@@ -0,0 +1,7 @@
+pr: 129600
+summary: Make flattened synthetic source concatenate object keys on scalar/object
+  mismatch
+area: Mapping
+type: bug
+issues:
+ - 122936

docs/reference/elasticsearch/mapping-reference/flattened.md

@@ -364,3 +364,36 @@ Will become (note the nested objects instead of the "flattened" array):
   }
 }
 ```
+
+Flattened fields allow for a key to contain both an object and a scalar value.
+For example, consider the following flattened field `flattened`:
+
+```console-result
+{
+  "flattened": {
+    "foo.bar": "10",
+    "foo": {
+      "bar": {
+        "baz": "20"
+      }
+    }
+  }
+}
+```
+
+Because `"foo.bar": "10"` is implicitly equivalent to `"foo": { "bar": "10" }`,
+`"bar"` has both a scalar value `"10"`, and an object value of `{ "baz": "20" }`.
+
+With synthetic source, to produce a valid JSON output, objects with such fields will appear differently in `_source`.
+For example, if the field is defined in an index configured with synthetic source, the value of `_source` would be:
+
+```console-result
+{
+  "flattened": {
+    "foo": {
+      "bar": "10",
+      "bar.baz": "20"
+    }
+  }
+}
+```

docs/reference/query-languages/esql/_snippets/commands/examples/change_point.csv-spec/changePointForDocs.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 ROW key=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]

docs/reference/query-languages/esql/_snippets/commands/examples/date.csv-spec/docsNowWhere.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 FROM sample_data

docs/reference/query-languages/esql/_snippets/commands/examples/docs-lookup-join.csv-spec/lookupJoinHostNameTwice.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 FROM system_metrics

docs/reference/query-languages/esql/_snippets/commands/examples/docs-lookup-join.csv-spec/lookupJoinServiceId.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 FROM app_logs

docs/reference/query-languages/esql/_snippets/commands/examples/docs-lookup-join.csv-spec/lookupJoinSourceIp.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 FROM firewall_logs

docs/reference/query-languages/esql/_snippets/commands/examples/docs-lookup-join.csv-spec/lookupJoinSourceIpWhere.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 FROM firewall_logs

docs/reference/query-languages/esql/_snippets/commands/examples/docs.csv-spec/basicDissect.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1"

docs/reference/query-languages/esql/_snippets/commands/examples/docs.csv-spec/basicGrok.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 [email protected] 42"

docs/reference/query-languages/esql/_snippets/commands/examples/docs.csv-spec/dissectWithToDatetime.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 ROW a = "2023-01-23T12:15:00.000Z - some text - 127.0.0.1"

docs/reference/query-languages/esql/_snippets/commands/examples/docs.csv-spec/grokWithConversionSuffix.md

@@ -1,4 +1,4 @@
-% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.
 
 ```esql
 ROW a = "2023-01-23T12:15:00.000Z 127.0.0.1 [email protected] 42"