Skip to content

Commit 0b98494

Browse files
jonathan-buttnerjonathan-buttner
committed
Merge branch 'main' of github.com:elastic/elasticsearch into ml-cs-semantic-text
2 parents 9e7cb3f + 449ec6b commit 0b98494

File tree

545 files changed

+14641
-5724
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

545 files changed

+14641
-5724
lines changed

‎.buildkite/pipelines/intake.template.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ steps:
7272
buildDirectory: /dev/shm/bk
7373
env:
7474
BWC_VERSION: "{{matrix.BWC_VERSION}}"
75+
- label: bc-upgrade
76+
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
7577
- group: lucene-compat
7678
steps:
7779
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/pipelines/intake.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ steps:
7272
buildDirectory: /dev/shm/bk
7373
env:
7474
BWC_VERSION: "{{matrix.BWC_VERSION}}"
75+
- label: bc-upgrade
76+
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
77+
agents:
78+
image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
79+
memory: "4G"
7580
- group: lucene-compat
7681
steps:
7782
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

‎.buildkite/pipelines/pull-request/pr-upgrade.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
steps:
2+
- label: pr-upgrade
3+
command: ".buildkite/scripts/run-pr-upgrade-tests.sh"
4+
agents:
5+
image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
6+
memory: "4G"

‎.buildkite/scripts/release-tests.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,13 @@ export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
99
mkdir -p ${BEATS_DIR}
1010
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1111
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
12-
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
12+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
13+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
14+
15+
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1316
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
17+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
18+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
1419

1520
# Fetch ML artifacts
1621
export ML_IVY_REPO=$(mktemp -d)

‎.buildkite/scripts/run-bc-upgrade-tests.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
3+
#
4+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
5+
# or more contributor license agreements. Licensed under the "Elastic License
6+
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
7+
# Public License v 1"; you may not use this file except in compliance with, at
8+
# your election, the "Elastic License 2.0", the "GNU Affero General Public
9+
# License v3.0 only", or the "Server Side Public License, v 1".
10+
#
11+
12+
set -euo pipefail
13+
14+
echo "Selecting the most recent build from branch [$BUILDKITE_BRANCH]."
15+
16+
# Select the most recent build from the current branch.
17+
# We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last.
18+
# So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot.
19+
MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json |
20+
jq ".releases[] |
21+
select(.branch == \"$BUILDKITE_BRANCH\") |
22+
select(.active_release == true) |
23+
((.snapshots | to_entries | sort_by(.value.completed_at)) +
24+
(.build_candidates | to_entries | sort_by(.value.completed_at))) |
25+
last | .value.manifest_url")"
26+
27+
if [[ -z "$MANIFEST_URL" ]]; then
28+
echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
29+
echo "Skipping BC upgrade tests."
30+
exit 0
31+
fi
32+
33+
echo "Getting build manifest from [$MANIFEST_URL]"
34+
35+
# Note: we use eval to perform variable substitution for the curl arguments, and command substitution to
36+
# set the output variable. Double quotes are not enough in this case.
37+
MANIFEST="$(eval "curl -s $MANIFEST_URL")"
38+
if [[ -z "$MANIFEST" ]]; then
39+
echo "Cannot get the build manifest from [$MANIFEST_URL]"
40+
exit 1
41+
fi
42+
43+
CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
44+
45+
BC_VERSION=$(echo "$MANIFEST" | jq -r .version)
46+
BC_BUILD_ID=$(echo "$MANIFEST" | jq -r .build_id)
47+
BC_COMMIT_HASH=$(echo "$MANIFEST" | jq -r .projects.elasticsearch.commit_hash)
48+
49+
if [ "$CURRENT_VERSION-SNAPSHOT" != "$BC_VERSION" ]; then
50+
echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]."
51+
echo "Skipping BC upgrade tests."
52+
exit 0
53+
fi
54+
55+
echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH] with build id [$BC_BUILD_ID]."
56+
57+
cat <<EOF | buildkite-agent pipeline upload
58+
steps:
59+
- label: bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH
60+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
61+
timeout_in_minutes: 300
62+
agents:
63+
provider: gcp
64+
image: family/elasticsearch-ubuntu-2004
65+
machineType: n1-standard-32
66+
buildDirectory: /dev/shm/bk
67+
preemptible: true
68+
retry:
69+
automatic:
70+
- exit_status: "-1"
71+
limit: 3
72+
signal_reason: none
73+
- signal_reason: agent_stop
74+
limit: 3
75+
EOF

‎.buildkite/scripts/run-pr-upgrade-tests.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
3+
#
4+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
5+
# or more contributor license agreements. Licensed under the "Elastic License
6+
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
7+
# Public License v 1"; you may not use this file except in compliance with, at
8+
# your election, the "Elastic License 2.0", the "GNU Affero General Public
9+
# License v3.0 only", or the "Server Side Public License, v 1".
10+
#
11+
12+
set -euo pipefail
13+
14+
if [[ -z "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]]; then
15+
echo "Not a pull request, skipping PR upgrade tests."
16+
exit 0
17+
fi
18+
19+
# Identify the merge base of the current commit (branch) and the base branch of the pull request.
20+
# PR upgrade tests are run from the merge base to the current commit.
21+
BASE_COMMIT=$(git merge-base $BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
22+
23+
VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
24+
25+
echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
26+
27+
cat <<EOF | buildkite-agent pipeline upload
28+
steps:
29+
- label: pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH
30+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTest -Dtests.jvm.argline="-Des.serverless_transport=true"
31+
timeout_in_minutes: 300
32+
agents:
33+
provider: gcp
34+
image: family/elasticsearch-ubuntu-2004
35+
machineType: n1-standard-32
36+
buildDirectory: /dev/shm/bk
37+
preemptible: true
38+
retry:
39+
automatic:
40+
- exit_status: "-1"
41+
limit: 3
42+
signal_reason: none
43+
- signal_reason: agent_stop
44+
limit: 3
45+
EOF

‎.github/workflows/docs-preview-comment.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: "Docs preview comment"
2+
3+
on:
4+
pull_request_target:
5+
types: [opened, reopened, synchronize]
6+
7+
permissions:
8+
contents: read
9+
issues: write
10+
pull-requests: write
11+
12+
jobs:
13+
preview-links:
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Comment preview links for changed docs
17+
uses: actions/github-script@v7
18+
with:
19+
github-token: ${{ secrets.GITHUB_TOKEN }}
20+
script: |
21+
const pr = context.payload.pull_request;
22+
const prNum = pr.number;
23+
const owner = context.repo.owner;
24+
const repo = context.repo.repo;
25+
const base = `https://docs-v3-preview.elastic.dev/${owner}/${repo}/pull/${prNum}`;
26+
// 1) List all files in this PR
27+
const { data: files } = await github.rest.pulls.listFiles({
28+
owner, repo, pull_number: prNum
29+
});
30+
// 2) Filter to only added/modified .md files (skip removed and _snippets/)
31+
const links = files
32+
.filter(f =>
33+
f.status !== 'removed' &&
34+
/\.md$/i.test(f.filename) &&
35+
!/(^|\/)_snippets\//i.test(f.filename)
36+
)
37+
.map(f => {
38+
let p = f.filename.replace(/^docs\//, '').replace(/\/index\.md$/i, '/');
39+
if (p === f.filename.replace(/^docs\//, '')) p = p.replace(/\.md$/i, '');
40+
return `- [\`${f.filename}\`](${base}/${p})`;
41+
});
42+
if (!links.length) return; // nothing to do
43+
// 3) Build the comment body
44+
const body = [
45+
"🔍 **Preview links for changed docs:**",
46+
"",
47+
...links,
48+
"",
49+
"🔔 *The preview site may take up to **3 minutes** to finish building. These links will become live once it completes.*"
50+
].join("\n");
51+
// 4) Post or update a single bot comment
52+
const { data: comments } = await github.rest.issues.listComments({
53+
owner, repo, issue_number: prNum
54+
});
55+
const existing = comments.find(c =>
56+
c.user.type === 'Bot' &&
57+
c.body.startsWith("🔍 **Preview links for changed docs:**")
58+
);
59+
if (existing) {
60+
await github.rest.issues.updateComment({
61+
owner, repo,
62+
comment_id: existing.id,
63+
body
64+
});
65+
} else {
66+
await github.rest.issues.createComment({
67+
owner, repo,
68+
issue_number: prNum,
69+
body
70+
});
71+
}

‎CONTRIBUTING.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,18 @@ Alternative manual steps for IntelliJ.
203203
3. Navigate to the file `build-conventions/formatterConfig.xml`
204204
4. Click "OK"
205205

206+
#### Options
207+
208+
When importing to IntelliJ, we offer a few options that can be used to
209+
configure the behaviour of the import:
210+
211+
| Property | Description | Values (* = default) |
212+
|--------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------|
213+
| `org.elasticsearch.idea-configuration-cache` | Should IntelliJ enable the Gradle Configuration cache to speed up builds when generating run configs | *`true`, `false` |
214+
| `org.elasticsearch.idea-delegate-to-gradle` | Should IntelliJ use Gradle for all generated run / test configs or prompt each time | `true`, *`false` |
215+
216+
These options can be set anywhere on the Gradle config path including in `~/.gradle/gradle.properties`
217+
206218
### REST endpoint conventions
207219

208220
Elasticsearch typically uses singular nouns rather than plurals in URLs.

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ dependencies {
4141
}
4242
api(project(':libs:h3'))
4343
api(project(':modules:aggregations'))
44+
implementation project(':modules:mapper-extras');
4445
api(project(':x-pack:plugin:esql-core'))
4546
api(project(':x-pack:plugin:core'))
4647
api(project(':x-pack:plugin:esql'))

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
3030
import org.elasticsearch.index.similarity.SimilarityService;
3131
import org.elasticsearch.indices.IndicesModule;
32+
import org.elasticsearch.plugins.MapperPlugin;
3233
import org.elasticsearch.script.Script;
3334
import org.elasticsearch.script.ScriptCompiler;
3435
import org.elasticsearch.script.ScriptContext;
@@ -38,11 +39,16 @@
3839
import java.io.IOException;
3940
import java.io.UncheckedIOException;
4041
import java.util.Collections;
42+
import java.util.List;
4143
import java.util.Map;
4244

4345
public class MapperServiceFactory {
4446

4547
public static MapperService create(String mappings) {
48+
return create(mappings, Collections.emptyList());
49+
}
50+
51+
public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
4652
Settings settings = Settings.builder()
4753
.put("index.number_of_replicas", 0)
4854
.put("index.number_of_shards", 1)
@@ -51,7 +57,7 @@ public static MapperService create(String mappings) {
5157
.build();
5258
IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
5359
IndexSettings indexSettings = new IndexSettings(meta, settings);
54-
MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
60+
MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
5561

5662
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
5763
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

0 commit comments

Comments
 (0)