Skip to content

Commit d144f83

Browse files
leemthompoleemthompo
authored
Merge branch 'main' into esql_docs_real_applies_to
2 parents 55ce8a8 + 085ef5c commit d144f83

File tree

81 files changed

+4269
-3784
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

81 files changed

+4269
-3784
lines changed

‎.buildkite/scripts/run-bc-upgrade-tests.sh‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,20 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
5656

5757
cat <<EOF | buildkite-agent pipeline upload
5858
steps:
59-
- label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
60-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
61-
timeout_in_minutes: 300
62-
agents:
63-
provider: gcp
64-
image: family/elasticsearch-ubuntu-2004
65-
machineType: n1-standard-32
66-
buildDirectory: /dev/shm/bk
67-
preemptible: true
68-
retry:
69-
automatic:
70-
- exit_status: "-1"
71-
limit: 3
72-
signal_reason: none
73-
- signal_reason: agent_stop
74-
limit: 3
59+
- group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
60+
steps:
61+
- label: "bc-upgrade-tests-part{{matrix.PART}}"
62+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
63+
timeout_in_minutes: 300
64+
agents:
65+
provider: gcp
66+
image: family/elasticsearch-ubuntu-2004
67+
machineType: n1-standard-32
68+
buildDirectory: /dev/shm/bk
69+
matrix:
70+
setup:
71+
PART: ["1", "2", "3", "4", "5", "6"]
7572
EOF

‎.buildkite/scripts/run-pr-upgrade-tests.sh‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,22 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
2424

2525
echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
2626

27-
cat <<EOF | buildkite-agent pipeline upload
27+
cat << EOF | buildkite-agent pipeline upload
2828
steps:
29-
- label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
30-
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
31-
timeout_in_minutes: 300
32-
agents:
33-
provider: gcp
34-
image: family/elasticsearch-ubuntu-2004
35-
machineType: n1-standard-32
36-
buildDirectory: /dev/shm/bk
37-
preemptible: true
38-
retry:
39-
automatic:
40-
- exit_status: "-1"
41-
limit: 3
42-
signal_reason: none
43-
- signal_reason: agent_stop
44-
limit: 3
29+
- group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
30+
steps:
31+
- label: "pr-upgrade-part-{{matrix.PART}}"
32+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
33+
timeout_in_minutes: 300
34+
agents:
35+
provider: gcp
36+
image: family/elasticsearch-ubuntu-2004
37+
machineType: n1-standard-32
38+
buildDirectory: /dev/shm/bk
39+
matrix:
40+
setup:
41+
PART: ["1", "2", "3", "4", "5", "6"]
4542
EOF

‎.github/workflows/docs-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ jobs:
1616
deployments: write
1717
id-token: write
1818
contents: read
19-
pull-requests: read
19+
pull-requests: write

‎benchmarks/src/main/java/org/elasticsearch/benchmark/search/query/range/DateFieldMapperDocValuesSkipperBenchmark.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@
2121
import org.apache.lucene.index.IndexWriterConfig;
2222
import org.apache.lucene.search.IndexOrDocValuesQuery;
2323
import org.apache.lucene.search.IndexSearcher;
24+
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
2425
import org.apache.lucene.search.Query;
2526
import org.apache.lucene.search.Sort;
2627
import org.apache.lucene.search.SortField;
2728
import org.apache.lucene.search.SortedNumericSortField;
2829
import org.apache.lucene.store.Directory;
2930
import org.apache.lucene.store.FSDirectory;
3031
import org.apache.lucene.util.BytesRef;
31-
import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
3232
import org.openjdk.jmh.annotations.Benchmark;
3333
import org.openjdk.jmh.annotations.BenchmarkMode;
3434
import org.openjdk.jmh.annotations.Fork;
@@ -295,7 +295,7 @@ public void rangeQueryWithDocValuesSkipper(final Blackhole bh) throws IOExceptio
295295
/**
296296
* Runs the actual Lucene range query, optionally combining a {@link LongPoint} index query
297297
* with doc values ({@link SortedNumericDocValuesField}) via {@link IndexOrDocValuesQuery},
298-
* and then wrapping it with an {@link XIndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
298+
* and then wrapping it with an {@link IndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
299299
*
300300
* @param searcher the Lucene {@link IndexSearcher}
301301
* @param rangeStartTimestamp lower bound of the timestamp range
@@ -316,7 +316,7 @@ private long rangeQuery(final IndexSearcher searcher, long rangeStartTimestamp,
316316
)
317317
: SortedNumericDocValuesField.newSlowRangeQuery(TIMESTAMP_FIELD, rangeStartTimestamp, rangeEndTimestamp);
318318

319-
final Query query = new XIndexSortSortedNumericDocValuesRangeQuery(
319+
final Query query = new IndexSortSortedNumericDocValuesRangeQuery(
320320
TIMESTAMP_FIELD,
321321
rangeStartTimestamp,
322322
rangeEndTimestamp,

‎build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,11 +61,3 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
6161

6262
@defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
6363
org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
64-
65-
# This is a temporary patch as there is a low level Lucene bug in certain scenarios
66-
# this should be fixed in the new Lucene release 10.3+
67-
org.apache.lucene.document.LongField#newExactQuery(java.lang.String, long) @ Use org.elasticsearch.lucene.document.NumericField#newExactLongQuery(java.lang.String, long) instead.
68-
org.apache.lucene.document.LongField#newRangeQuery(java.lang.String, long, long) @ Use org.elasticsearch.lucene.document.NumericField#newRangeLongQuery(java.lang.String, long, long) instead.
69-
org.apache.lucene.document.IntField#newExactQuery(java.lang.String, int) @ Use org.elasticsearch.lucene.document.NumericField#newExactIntQuery(java.lang.String, int) instead.
70-
org.apache.lucene.document.IntField#newRangeQuery(java.lang.String, int, int) @ Use org.elasticsearch.lucene.document.NumericField#newRangeIntQuery(java.lang.String, int, int) instead.
71-
org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery @ use org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery instead.

‎build-tools-internal/version.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
elasticsearch = 9.1.0
2-
lucene = 10.2.1
2+
lucene = 10.2.2
33

44
bundled_jdk_vendor = openjdk
55
bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90

‎build.gradle‎

Lines changed: 30 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -351,95 +351,45 @@ allprojects {
351351
}
352352
}
353353

354+
def splitForCI = { proj, partString ->
355+
proj.tasks.register("check$partString") {
356+
dependsOn 'check'
357+
withReleaseBuild {
358+
dependsOn 'assemble'
359+
}
360+
}
361+
362+
proj.tasks.addRule("Pattern: v<BWC_VERSION>#bwcTest$partString") { name ->
363+
if(name.endsWith("#bwcTest$partString")) {
364+
proj.project.getTasks().register(name) {
365+
task -> task.dependsOn(proj.tasks.named { tskName -> tskName == (name - partString) })
366+
}
367+
}
368+
}
369+
370+
proj.tasks.register("bcUpgradeTest$partString") {
371+
dependsOn tasks.matching { it.name == 'bcUpgradeTest' }
372+
withReleaseBuild {
373+
dependsOn 'assemble'
374+
}
375+
}
376+
}
377+
354378
plugins.withId('lifecycle-base') {
355379
if (project.path.startsWith(":x-pack:")) {
356380
if (project.path.contains("security") || project.path.contains(":ml")) {
357-
tasks.register('checkPart4') {
358-
dependsOn 'check'
359-
withReleaseBuild {
360-
dependsOn 'assemble'
361-
}
362-
}
363-
364-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart4") { name ->
365-
if(name.endsWith("#bwcTestPart4")) {
366-
project.getTasks().register(name) {
367-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part4") })
368-
}
369-
}
370-
}
381+
splitForCI(project, "Part4")
371382
} else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
372-
tasks.register('checkPart3') {
373-
dependsOn 'check'
374-
withReleaseBuild {
375-
dependsOn 'assemble'
376-
}
377-
}
378-
379-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart3") { name ->
380-
if(name.endsWith("#bwcTestPart3")) {
381-
project.getTasks().register(name) {
382-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part3") })
383-
}
384-
}
385-
}
383+
splitForCI(project, "Part3")
386384
} else if (project.path.contains("multi-node")) {
387-
tasks.register('checkPart5') {
388-
dependsOn 'check'
389-
withReleaseBuild {
390-
dependsOn 'assemble'
391-
}
392-
}
393-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart5") { name ->
394-
if(name.endsWith("#bwcTestPart5")) {
395-
project.getTasks().register(name) {
396-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part5") })
397-
}
398-
}
399-
}
385+
splitForCI(project, "Part5")
400386
} else {
401-
tasks.register('checkPart2') {
402-
dependsOn 'check'
403-
withReleaseBuild {
404-
dependsOn 'assemble'
405-
}
406-
}
407-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart2") { name ->
408-
if(name.endsWith("#bwcTestPart2")) {
409-
project.getTasks().register(name) {
410-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part2") })
411-
}
412-
}
413-
}
387+
splitForCI(project, "Part2")
414388
}
415389
} else if(project.path.startsWith(":qa:")) {
416-
tasks.register('checkPart6') {
417-
dependsOn 'check'
418-
withReleaseBuild {
419-
dependsOn 'assemble'
420-
}
421-
}
422-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart6") { name ->
423-
if(name.endsWith("#bwcTestPart6")) {
424-
project.getTasks().register(name) {
425-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part6") })
426-
}
427-
}
428-
}
390+
splitForCI(project, "Part6")
429391
} else {
430-
tasks.register('checkPart1') {
431-
dependsOn 'check'
432-
withReleaseBuild {
433-
dependsOn 'assemble'
434-
}
435-
}
436-
tasks.addRule("Pattern: v<BWC_VERSION>#bwcTestPart1") { name ->
437-
if(name.endsWith("#bwcTestPart1")) {
438-
project.getTasks().register(name) {
439-
task -> task.dependsOn(tasks.named { tskName -> tskName == (name - "Part1") })
440-
}
441-
}
442-
}
392+
splitForCI(project, "Part1")
443393
}
444394
tasks.register('functionalTests') {
445395
dependsOn 'check'

‎docs/Versions.asciidoc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11

22
include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
33

4-
:lucene_version: 10.2.1
5-
:lucene_version_path: 10_2_1
4+
:lucene_version: 10.2.2
5+
:lucene_version_path: 10_2_2
66
:jdk: 11.0.2
77
:jdk_major: 11
88
:build_type: tar

‎docs/changelog/129503.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129503
2+
summary: Pushdown Lookup Join past Project
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 119082

‎docs/changelog/129546.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pr: 129546
2+
summary: Upgrade to Lucene 10.2.2
3+
area: Search
4+
type: upgrade
5+
issues: []
6+
highlight:
7+
title: Upgrade to lucene 10.2.2
8+
body: |-
9+
* Reduce NeighborArray on-heap memory during HNSW graph building
10+
* Fix IndexSortSortedNumericDocValuesRangeQuery for integer sorting
11+
* ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when used if the DoubleValuesSource needed scores
12+
----
13+
notable: true

0 commit comments

Comments
 (0)