Skip to content

Commit d4fb250

Browse files
afoucretafoucret
committed
Merge branch 'main' of https://github.com/elastic/elasticsearch into rerank-refactoring
2 parents a2aa5ea + c709765 commit d4fb250

File tree

205 files changed

+4545
-1895
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

205 files changed

+4545
-1895
lines changed

‎.buildkite/pipelines/intake.template.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ steps:
7272
buildDirectory: /dev/shm/bk
7373
env:
7474
BWC_VERSION: "{{matrix.BWC_VERSION}}"
75+
- label: bc-upgrade
76+
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
7577
- group: lucene-compat
7678
steps:
7779
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/pipelines/intake.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ steps:
7272
buildDirectory: /dev/shm/bk
7373
env:
7474
BWC_VERSION: "{{matrix.BWC_VERSION}}"
75+
- label: bc-upgrade
76+
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
77+
agents:
78+
image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
79+
memory: "4G"
7580
- group: lucene-compat
7681
steps:
7782
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/scripts/run-bc-upgrade-tests.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
3+
#
4+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
5+
# or more contributor license agreements. Licensed under the "Elastic License
6+
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
7+
# Public License v 1"; you may not use this file except in compliance with, at
8+
# your election, the "Elastic License 2.0", the "GNU Affero General Public
9+
# License v3.0 only", or the "Server Side Public License, v 1".
10+
#
11+
12+
set -euo pipefail
13+
14+
echo "Selecting the most recent build from branch [$BUILDKITE_BRANCH]."
15+
16+
# Select the most recent build from the current branch.
17+
# We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last.
18+
# So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot.
19+
MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json |
20+
jq ".releases[] |
21+
select(.branch == \"$BUILDKITE_BRANCH\") |
22+
select(.active_release == true) |
23+
((.snapshots | to_entries | sort_by(.value.completed_at)) +
24+
(.build_candidates | to_entries | sort_by(.value.completed_at))) |
25+
last | .value.manifest_url")"
26+
27+
if [[ -z "$MANIFEST_URL" ]]; then
28+
echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
29+
echo "Skipping BC upgrade tests."
30+
exit 0
31+
fi
32+
33+
echo "Getting build manifest from [$MANIFEST_URL]"
34+
35+
# Note: we use eval to perform variable substitution for the curl arguments, and command substitution to
36+
# set the output variable. Double quotes are not enough in this case.
37+
MANIFEST="$(eval "curl -s $MANIFEST_URL")"
38+
if [[ -z "$MANIFEST" ]]; then
39+
echo "Cannot get the build manifest from [$MANIFEST_URL]"
40+
exit 1
41+
fi
42+
43+
CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
44+
45+
BC_VERSION=$(echo "$MANIFEST" | jq -r .version)
46+
BC_BUILD_ID=$(echo "$MANIFEST" | jq -r .build_id)
47+
BC_COMMIT_HASH=$(echo "$MANIFEST" | jq -r .projects.elasticsearch.commit_hash)
48+
49+
if [ "$CURRENT_VERSION-SNAPSHOT" != "$BC_VERSION" ]; then
50+
echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]."
51+
echo "Skipping BC upgrade tests."
52+
exit 0
53+
fi
54+
55+
echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH] with build id [$BC_BUILD_ID]."
56+
57+
cat <<EOF | buildkite-agent pipeline upload
58+
steps:
59+
- label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
60+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
61+
timeout_in_minutes: 300
62+
agents:
63+
provider: gcp
64+
image: family/elasticsearch-ubuntu-2004
65+
machineType: n1-standard-32
66+
buildDirectory: /dev/shm/bk
67+
preemptible: true
68+
retry:
69+
automatic:
70+
- exit_status: "-1"
71+
limit: 3
72+
signal_reason: none
73+
- signal_reason: agent_stop
74+
limit: 3
75+
EOF

‎.github/workflows/docs-preview-comment.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ jobs:
3535
!/(^|\/)_snippets\//i.test(f.filename)
3636
)
3737
.map(f => {
38-
let p = f.filename.replace(/\/index\.md$/i, '/');
39-
if (p === f.filename) p = p.replace(/\.md$/i, '');
38+
let p = f.filename.replace(/^docs\//, '').replace(/\/index\.md$/i, '/');
39+
if (p === f.filename.replace(/^docs\//, '')) p = p.replace(/\.md$/i, '');
4040
return `- [\`${f.filename}\`](${base}/${p})`;
4141
});
4242
if (!links.length) return; // nothing to do
@@ -68,4 +68,4 @@ jobs:
6868
issue_number: prNum,
6969
body
7070
});
71-
}
71+
}

‎CONTRIBUTING.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,18 @@ Alternative manual steps for IntelliJ.
203203
3. Navigate to the file `build-conventions/formatterConfig.xml`
204204
4. Click "OK"
205205

206+
#### Options
207+
208+
When importing to IntelliJ, we offer a few options that can be used to
209+
configure the behaviour of the import:
210+
211+
| Property | Description | Values (* = default) |
212+
|--------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------|
213+
| `org.elasticsearch.idea-configuration-cache` | Should IntelliJ enable the Gradle Configuration cache to speed up builds when generating run configs | *`true`, `false` |
214+
| `org.elasticsearch.idea-delegate-to-gradle` | Should IntelliJ use Gradle for all generated run / test configs or prompt each time | `true`, *`false` |
215+
216+
These options can be set anywhere on the Gradle config path including in `~/.gradle/gradle.properties`
217+
206218
### REST endpoint conventions
207219

208220
Elasticsearch typically uses singular nouns rather than plurals in URLs.

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,10 @@ public void scoreFromArray(Blackhole bh) throws IOException {
126126
in.readFloats(corrections, 0, corrections.length);
127127
int addition = Short.toUnsignedInt(in.readShort());
128128
float score = scorer.score(
129-
result,
129+
result.lowerInterval(),
130+
result.upperInterval(),
131+
result.quantizedComponentSum(),
132+
result.additionalCorrection(),
130133
VectorSimilarityFunction.EUCLIDEAN,
131134
centroidDp,
132135
corrections[0],
@@ -150,7 +153,10 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
150153
in.readFloats(corrections, 0, corrections.length);
151154
int addition = Short.toUnsignedInt(in.readShort());
152155
float score = scorer.score(
153-
result,
156+
result.lowerInterval(),
157+
result.upperInterval(),
158+
result.quantizedComponentSum(),
159+
result.additionalCorrection(),
154160
VectorSimilarityFunction.EUCLIDEAN,
155161
centroidDp,
156162
corrections[0],
@@ -175,7 +181,10 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
175181
in.readFloats(corrections, 0, corrections.length);
176182
int addition = Short.toUnsignedInt(in.readShort());
177183
float score = scorer.score(
178-
result,
184+
result.lowerInterval(),
185+
result.upperInterval(),
186+
result.quantizedComponentSum(),
187+
result.additionalCorrection(),
179188
VectorSimilarityFunction.EUCLIDEAN,
180189
centroidDp,
181190
corrections[0],
@@ -196,7 +205,16 @@ public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
196205
for (int j = 0; j < numQueries; j++) {
197206
in.seek(0);
198207
for (int i = 0; i < numVectors; i += 16) {
199-
scorer.scoreBulk(binaryQueries[j], result, VectorSimilarityFunction.EUCLIDEAN, centroidDp, scratchScores);
208+
scorer.scoreBulk(
209+
binaryQueries[j],
210+
result.lowerInterval(),
211+
result.upperInterval(),
212+
result.quantizedComponentSum(),
213+
result.additionalCorrection(),
214+
VectorSimilarityFunction.EUCLIDEAN,
215+
centroidDp,
216+
scratchScores
217+
);
200218
bh.consume(scratchScores);
201219
}
202220
}

‎build-tools-internal/src/main/groovy/elasticsearch.ide.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
180180
// this path is produced by the extractLibs task above
181181
String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
182182
def enableIdeaCC = providers.gradleProperty("org.elasticsearch.idea-configuration-cache").getOrElse("true").toBoolean()
183+
def delegateToGradle = providers.gradleProperty("org.elasticsearch.idea-delegate-to-gradle").getOrElse("false").toBoolean()
183184
idea {
184185
project {
185186
vcs = 'Git'
@@ -188,7 +189,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
188189
settings {
189190
delegateActions {
190191
delegateBuildRunToGradle = false
191-
testRunner = 'choose_per_test'
192+
testRunner = delegateToGradle ? 'gradle' : 'choose_per_test'
192193
}
193194
taskTriggers {
194195
afterSync tasks.named('configureIdeCheckstyle'),

‎docs/changelog/119967.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119967
2+
summary: Add `index_options` to `semantic_text` field mappings
3+
area: Mapping
4+
type: enhancement
5+
issues: [ ]

‎docs/changelog/129464.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
pr: 129464
2+
summary: Deprecate `indices.merge.scheduler.use_thread_pool` setting
3+
area: Engine
4+
type: deprecation
5+
issues: []
6+
deprecation:
7+
title: Deprecate `indices.merge.scheduler.use_thread_pool` setting
8+
area: Ingest
9+
details: This deprecates the `indices.merge.scheduler.use_thread_pool` node setting that was introduced in #120869. This setting should not normally be used, unless instructed so by engineering to get around temporary issues with the new threadpool-based merge scheduler.
10+
impact: There should be no impact to users since the setting was not released before its deprecation here (and is not documented).

‎docs/changelog/129507.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129507
2+
summary: Using a temp `IndexService` for template validation
3+
area: Indices APIs
4+
type: bug
5+
issues:
6+
- 129473

0 commit comments

Comments
 (0)