Skip to content

Commit ae7766c

Browse files
moschemosche
authored
Merge branch 'main' into entitlements/test_rule
2 parents 0cc6a66 + fc0ea64 commit ae7766c

File tree

76 files changed

+2894
-653
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+2894
-653
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int7ScorerBenchmark.java‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.benchmark.vector;
10+
11+
import org.apache.lucene.index.VectorSimilarityFunction;
12+
import org.apache.lucene.store.Directory;
13+
import org.apache.lucene.store.IOContext;
14+
import org.apache.lucene.store.IndexInput;
15+
import org.apache.lucene.store.IndexOutput;
16+
import org.apache.lucene.store.MMapDirectory;
17+
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
18+
import org.elasticsearch.common.logging.LogConfigurator;
19+
import org.elasticsearch.core.IOUtils;
20+
import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
21+
import org.elasticsearch.simdvec.ES92Int7VectorsScorer;
22+
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
23+
import org.openjdk.jmh.annotations.Benchmark;
24+
import org.openjdk.jmh.annotations.BenchmarkMode;
25+
import org.openjdk.jmh.annotations.Fork;
26+
import org.openjdk.jmh.annotations.Measurement;
27+
import org.openjdk.jmh.annotations.Mode;
28+
import org.openjdk.jmh.annotations.OutputTimeUnit;
29+
import org.openjdk.jmh.annotations.Param;
30+
import org.openjdk.jmh.annotations.Scope;
31+
import org.openjdk.jmh.annotations.Setup;
32+
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
34+
import org.openjdk.jmh.annotations.Warmup;
35+
import org.openjdk.jmh.infra.Blackhole;
36+
37+
import java.io.IOException;
38+
import java.nio.file.Files;
39+
import java.util.concurrent.ThreadLocalRandom;
40+
import java.util.concurrent.TimeUnit;
41+
42+
@BenchmarkMode(Mode.Throughput)
43+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
44+
@State(Scope.Benchmark)
45+
// first iteration is complete garbage, so make sure we really warmup
46+
@Warmup(iterations = 4, time = 1)
47+
// real iterations. not useful to spend tons of time here, better to fork more
48+
@Measurement(iterations = 5, time = 1)
49+
// engage some noise reduction
50+
@Fork(value = 1)
51+
public class Int7ScorerBenchmark {
52+
53+
static {
54+
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
55+
}
56+
57+
@Param({ "384", "782", "1024" })
58+
int dims;
59+
60+
int numVectors = 20 * ES92Int7VectorsScorer.BULK_SIZE;
61+
int numQueries = 5;
62+
63+
byte[] scratch;
64+
byte[][] binaryVectors;
65+
byte[][] binaryQueries;
66+
float[] scores = new float[ES92Int7VectorsScorer.BULK_SIZE];
67+
68+
ES92Int7VectorsScorer scorer;
69+
Directory dir;
70+
IndexInput in;
71+
72+
OptimizedScalarQuantizer.QuantizationResult queryCorrections;
73+
float centroidDp;
74+
75+
@Setup
76+
public void setup() throws IOException {
77+
binaryVectors = new byte[numVectors][dims];
78+
dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
79+
try (IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT)) {
80+
for (byte[] binaryVector : binaryVectors) {
81+
for (int i = 0; i < dims; i++) {
82+
// 4-bit quantization
83+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(128);
84+
}
85+
out.writeBytes(binaryVector, 0, binaryVector.length);
86+
ThreadLocalRandom.current().nextBytes(binaryVector);
87+
out.writeBytes(binaryVector, 0, 16); // corrections
88+
}
89+
}
90+
91+
queryCorrections = new OptimizedScalarQuantizer.QuantizationResult(
92+
ThreadLocalRandom.current().nextFloat(),
93+
ThreadLocalRandom.current().nextFloat(),
94+
ThreadLocalRandom.current().nextFloat(),
95+
Short.toUnsignedInt((short) ThreadLocalRandom.current().nextInt())
96+
);
97+
centroidDp = ThreadLocalRandom.current().nextFloat();
98+
99+
in = dir.openInput("vectors", IOContext.DEFAULT);
100+
binaryQueries = new byte[numVectors][dims];
101+
for (byte[] binaryVector : binaryVectors) {
102+
for (int i = 0; i < dims; i++) {
103+
// 7-bit quantization
104+
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(128);
105+
}
106+
}
107+
108+
scratch = new byte[dims];
109+
scorer = ESVectorizationProvider.getInstance().newES92Int7VectorsScorer(in, dims);
110+
}
111+
112+
@TearDown
113+
public void teardown() throws IOException {
114+
IOUtils.close(dir, in);
115+
}
116+
117+
@Benchmark
118+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
119+
public void scoreFromMemorySegment(Blackhole bh) throws IOException {
120+
for (int j = 0; j < numQueries; j++) {
121+
in.seek(0);
122+
for (int i = 0; i < numVectors; i++) {
123+
bh.consume(
124+
scorer.score(
125+
binaryQueries[j],
126+
queryCorrections.lowerInterval(),
127+
queryCorrections.upperInterval(),
128+
queryCorrections.quantizedComponentSum(),
129+
queryCorrections.additionalCorrection(),
130+
VectorSimilarityFunction.EUCLIDEAN,
131+
centroidDp
132+
)
133+
);
134+
}
135+
}
136+
}
137+
138+
@Benchmark
139+
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
140+
public void scoreFromMemorySegmentBulk(Blackhole bh) throws IOException {
141+
for (int j = 0; j < numQueries; j++) {
142+
in.seek(0);
143+
for (int i = 0; i < numVectors; i += ES91Int4VectorsScorer.BULK_SIZE) {
144+
scorer.scoreBulk(
145+
binaryQueries[j],
146+
queryCorrections.lowerInterval(),
147+
queryCorrections.upperInterval(),
148+
queryCorrections.quantizedComponentSum(),
149+
queryCorrections.additionalCorrection(),
150+
VectorSimilarityFunction.EUCLIDEAN,
151+
centroidDp,
152+
scores
153+
);
154+
for (float score : scores) {
155+
bh.consume(score);
156+
}
157+
}
158+
}
159+
}
160+
}

‎distribution/docker/src/docker/iron_bank/hardening_manifest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tags:
1111
# Build args passed to Dockerfile ARGs
1212
args:
1313
BASE_IMAGE: "redhat/ubi/ubi9"
14-
BASE_TAG: "9.5"
14+
BASE_TAG: "9.6"
1515
# Docker image labels
1616
labels:
1717
org.opencontainers.image.title: "elasticsearch"

‎docs/changelog/129662.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129662
2+
summary: "[Security] Add entity store and asset criticality index privileges to built\
3+
\ in Editor, Viewer and Kibana System roles"
4+
area: Authorization
5+
type: enhancement
6+
issues: []

‎docs/changelog/130658.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130658
2+
summary: Add support for weighted RRF in retrievers
3+
area: Relevance
4+
type: enhancement
5+
issues: []

‎docs/changelog/131261.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pr: 131261
2+
summary: Enable Failure Store for new logs-*-* data streams
3+
area: Data streams
4+
type: feature
5+
issues:
6+
- 131105
7+
highlight:
8+
title: Enable Failure Store for new logs data streams
9+
body: |-
10+
The [Failure Store](docs-content://manage-data/data-store/data-streams/failure-store.md) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a
11+
dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](docs-content://manage-data/data-store/data-streams/failure-store.md#set-up-failure-store-existing) for existing data streams.
12+
Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`.
13+
notable: true

‎docs/changelog/131937.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131937
2+
summary: Fix race condition in `RemoteClusterService.collectNodes()`
3+
area: Distributed
4+
type: bug
5+
issues: []

‎docs/changelog/132101.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎docs/release-notes/breaking-changes.md‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@ If you are migrating from a version prior to version 9.0, you must first upgrade
1212

1313
% ## Next version [elasticsearch-nextversion-breaking-changes]
1414

15-
```{applies_to}
16-
stack: coming 9.1.0
17-
```
1815
## 9.1.0 [elasticsearch-9.1.0-breaking-changes]
1916

2017
Discovery-Plugins:
@@ -119,7 +116,6 @@ Test the upgrade in a non-production environment. Adapt your configuration to th
119116
For more information, view [#126843](https://github.com/elastic/elasticsearch/pull/126843) (issue: [#120993](https://github.com/elastic/elasticsearch/issues/120993))
120117
:::
121118

122-
123119
## 9.0.4 [elasticsearch-9.0.4-breaking-changes]
124120

125121
No breaking changes in this version.

‎docs/release-notes/changelog-bundles/9.1.0.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
version: 9.1.0
2-
released: false
3-
generated: 2025-07-26T00:06:59.671585841Z
2+
released: true
3+
generated: 2025-07-29T11:34:08.731686399Z
44
changelogs:
55
- pr: 105773
66
summary: Inject an unfollow action before executing a downsample action in ILM

‎docs/release-notes/deprecations.md‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,6 @@ To give you insight into what deprecated features you’re using, {{es}}:
1616

1717
% ## Next version [elasticsearch-nextversion-deprecations]
1818

19-
```{applies_to}
20-
stack: coming 9.1.0
21-
```
2219
## 9.1.0 [elasticsearch-9.1.0-deprecations]
2320

2421
There are no deprecations associated with this release.

0 commit comments

Comments
 (0)