Skip to content

Commit b9cdea0

Browse files
iveraseiverase
authored
Merge branch 'main' into computeNeighbours
2 parents d4eea58 + 8f82f3f commit b9cdea0

File tree

42 files changed

+1246
-191
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1246
-191
lines changed

‎docs/changelog/133599.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 133599
2+
summary: Support Gemini thinking budget in inference API
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎muted-tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -525,9 +525,6 @@ tests:
525525
- class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
526526
method: test {csv-spec:spatial.ConvertFromStringParseError}
527527
issue: https://github.com/elastic/elasticsearch/issues/134104
528-
- class: org.elasticsearch.index.codec.vectors.DocIdsWriterTests
529-
method: testSorted
530-
issue: https://github.com/elastic/elasticsearch/issues/134106
531528
- class: org.elasticsearch.xpack.writeloadforecaster.WriteLoadForecasterIT
532529
method: testWriteLoadForecastIsOverriddenBySetting
533530
issue: https://github.com/elastic/elasticsearch/issues/133455

‎qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchErrorTraceIT.java‎

Lines changed: 6 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import org.elasticsearch.action.search.MultiSearchRequest;
1717
import org.elasticsearch.action.search.SearchRequest;
1818
import org.elasticsearch.client.Request;
19-
import org.elasticsearch.common.settings.Settings;
2019
import org.elasticsearch.common.util.CollectionUtils;
2120
import org.elasticsearch.plugins.Plugin;
2221
import org.elasticsearch.search.ErrorTraceHelper;
@@ -25,19 +24,15 @@
2524
import org.elasticsearch.test.MockLog;
2625
import org.elasticsearch.test.transport.MockTransportService;
2726
import org.elasticsearch.xcontent.XContentType;
28-
import org.junit.After;
29-
import org.junit.Before;
3027
import org.junit.BeforeClass;
3128

3229
import java.io.IOException;
3330
import java.nio.charset.Charset;
3431
import java.util.Collection;
35-
import java.util.function.BooleanSupplier;
3632

3733
import static org.elasticsearch.index.query.QueryBuilders.simpleQueryStringQuery;
3834

3935
public class SearchErrorTraceIT extends HttpSmokeTestCase {
40-
private BooleanSupplier hasStackTrace;
4136

4237
@Override
4338
protected Collection<Class<? extends Plugin>> nodePlugins() {
@@ -49,18 +44,6 @@ public static void setDebugLogLevel() {
4944
Configurator.setLevel(SearchService.class, Level.DEBUG);
5045
}
5146

52-
@Before
53-
public void setupMessageListener() {
54-
hasStackTrace = ErrorTraceHelper.setupErrorTraceListener(internalCluster());
55-
// TODO: make this test work with batched query execution by enhancing ErrorTraceHelper.setupErrorTraceListener
56-
updateClusterSettings(Settings.builder().put(SearchService.BATCHED_QUERY_PHASE.getKey(), false));
57-
}
58-
59-
@After
60-
public void resetSettings() {
61-
updateClusterSettings(Settings.builder().putNull(SearchService.BATCHED_QUERY_PHASE.getKey()));
62-
}
63-
6447
private void setupIndexWithDocs() {
6548
createIndex("test1", "test2");
6649
indexRandom(
@@ -86,7 +69,7 @@ public void testSearchFailingQueryErrorTraceDefault() throws IOException {
8669
}
8770
""");
8871
getRestClient().performRequest(searchRequest);
89-
assertFalse(hasStackTrace.getAsBoolean());
72+
ErrorTraceHelper.assertStackTraceCleared(internalCluster());
9073
}
9174

9275
public void testSearchFailingQueryErrorTraceTrue() throws IOException {
@@ -105,7 +88,7 @@ public void testSearchFailingQueryErrorTraceTrue() throws IOException {
10588
""");
10689
searchRequest.addParameter("error_trace", "true");
10790
getRestClient().performRequest(searchRequest);
108-
assertTrue(hasStackTrace.getAsBoolean());
91+
ErrorTraceHelper.assertStackTraceObserved(internalCluster());
10992
}
11093

11194
public void testSearchFailingQueryErrorTraceFalse() throws IOException {
@@ -124,7 +107,7 @@ public void testSearchFailingQueryErrorTraceFalse() throws IOException {
124107
""");
125108
searchRequest.addParameter("error_trace", "false");
126109
getRestClient().performRequest(searchRequest);
127-
assertFalse(hasStackTrace.getAsBoolean());
110+
ErrorTraceHelper.assertStackTraceCleared(internalCluster());
128111
}
129112

130113
public void testDataNodeLogsStackTrace() throws IOException {
@@ -173,7 +156,7 @@ public void testMultiSearchFailingQueryErrorTraceDefault() throws IOException {
173156
new NByteArrayEntity(requestBody, ContentType.create(contentType.mediaTypeWithoutParameters(), (Charset) null))
174157
);
175158
getRestClient().performRequest(searchRequest);
176-
assertFalse(hasStackTrace.getAsBoolean());
159+
ErrorTraceHelper.assertStackTraceCleared(internalCluster());
177160
}
178161

179162
public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException {
@@ -190,7 +173,7 @@ public void testMultiSearchFailingQueryErrorTraceTrue() throws IOException {
190173
);
191174
searchRequest.addParameter("error_trace", "true");
192175
getRestClient().performRequest(searchRequest);
193-
assertTrue(hasStackTrace.getAsBoolean());
176+
ErrorTraceHelper.assertStackTraceObserved(internalCluster());
194177
}
195178

196179
public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException {
@@ -207,8 +190,7 @@ public void testMultiSearchFailingQueryErrorTraceFalse() throws IOException {
207190
);
208191
searchRequest.addParameter("error_trace", "false");
209192
getRestClient().performRequest(searchRequest);
210-
211-
assertFalse(hasStackTrace.getAsBoolean());
193+
ErrorTraceHelper.assertStackTraceCleared(internalCluster());
212194
}
213195

214196
public void testDataNodeLogsStackTraceMultiSearch() throws IOException {

‎renovate.json‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
{
22
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
33
"extends": [
4-
"github>elastic/renovate-config:only-chainguard",
4+
"github>elastic/renovate-config:base",
5+
"github>elastic/renovate-config:ci-agent-images",
56
":disableDependencyDashboard"
67
],
78
"schedule": [
@@ -10,6 +11,7 @@
1011
"labels": [
1112
">non-issue",
1213
":Delivery/Packaging",
14+
":Delivery/Build",
1315
"Team:Delivery",
1416
"auto-merge-without-approval"
1517
],
@@ -20,5 +22,21 @@
2022
"8.19",
2123
"8.18",
2224
"7.17"
25+
],
26+
"packageRules": [
27+
{
28+
"description": "Don't manage dependencies by default",
29+
"matchPackageNames": [
30+
"*"
31+
],
32+
"enabled": false
33+
},
34+
{
35+
"description": "Explicitly enable Elastic Docker updates",
36+
"matchPackageNames": [
37+
"/docker.elastic.co/.*/"
38+
],
39+
"enabled": true
40+
}
2341
]
2442
}

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -87,17 +87,10 @@ static TransportVersion def(int id) {
8787
public static final TransportVersion V_8_16_1 = def(8_772_0_04);
8888
public static final TransportVersion V_8_16_5 = def(8_772_0_05);
8989
public static final TransportVersion V_8_16_6 = def(8_772_0_06);
90-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_16_7 = def(8_772_0_07);
9190
public static final TransportVersion V_8_17_0 = def(8_797_0_02);
9291
public static final TransportVersion V_8_17_3 = def(8_797_0_03);
9392
public static final TransportVersion V_8_17_4 = def(8_797_0_04);
9493
public static final TransportVersion V_8_17_5 = def(8_797_0_05);
95-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_6 = def(8_797_0_06);
96-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_7 = def(8_797_0_07);
97-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_8 = def(8_797_0_08);
98-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_9 = def(8_797_0_09);
99-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_10 = def(8_797_0_10);
100-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_11 = def(8_797_0_11);
10194
public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_0_00);
10295
public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_0_00);
10396
public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_0_00);
@@ -143,10 +136,6 @@ static TransportVersion def(int id) {
143136
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
144137
public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_18 = def(8_840_0_01);
145138
public static final TransportVersion V_8_18_0 = def(8_840_0_02);
146-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_1 = def(8_840_0_03);
147-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_2 = def(8_840_0_04);
148-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_3 = def(8_840_0_05);
149-
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_4 = def(8_840_0_06);
150139
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_7 = def(8_840_0_09);
151140
public static final TransportVersion INITIAL_ELASTICSEARCH_8_19 = def(8_841_0_00);
152141
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_841_0_01);
@@ -359,6 +348,7 @@ static TransportVersion def(int id) {
359348
public static final TransportVersion SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS = def(9_150_0_00);
360349
public static final TransportVersion ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER = def(9_151_0_00);
361350
public static final TransportVersion INFERENCE_API_DISABLE_EIS_RATE_LIMITING = def(9_152_0_00);
351+
public static final TransportVersion GEMINI_THINKING_BUDGET_ADDED = def(9_153_0_00);
362352

363353
/*
364354
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ public static final class NodeQueryResponse extends TransportResponse {
222222
private final SearchPhaseController.TopDocsStats topDocsStats;
223223
private final QueryPhaseResultConsumer.MergeResult mergeResult;
224224

225-
NodeQueryResponse(StreamInput in) throws IOException {
225+
public NodeQueryResponse(StreamInput in) throws IOException {
226226
this.results = in.readArray(i -> i.readBoolean() ? new QuerySearchResult(i) : i.readException(), Object[]::new);
227227
this.mergeResult = QueryPhaseResultConsumer.MergeResult.readFrom(in);
228228
this.topDocsStats = SearchPhaseController.TopDocsStats.readFrom(in);

‎server/src/main/java/org/elasticsearch/index/codec/vectors/DocIdsWriter.java‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -74,20 +74,39 @@ byte calculateBlockEncoding(IntToIntFunction docIds, int count, int blockSize) {
7474
if (count == 0) {
7575
return CONTINUOUS_IDS;
7676
}
77-
byte encoding = CONTINUOUS_IDS;
7877
int iterationLimit = count - blockSize + 1;
7978
int i = 0;
79+
int maxValue = 0;
80+
int maxMin2Max = 0;
81+
boolean continuousIds = true;
8082
for (; i < iterationLimit; i += blockSize) {
8183
int offset = i;
82-
encoding = (byte) Math.max(encoding, blockEncoding(d -> docIds.apply(offset + d), blockSize));
84+
var r = sortedAndMaxAndMin2Max(d -> docIds.apply(offset + d), blockSize);
85+
continuousIds &= r[0] == 1;
86+
maxValue = Math.max(maxValue, r[1]);
87+
maxMin2Max = Math.max(maxMin2Max, r[2]);
8388
}
8489
// check the tail
85-
if (i == count) {
86-
return encoding;
90+
if (i < count) {
91+
int offset = i;
92+
var r = sortedAndMaxAndMin2Max(d -> docIds.apply(offset + d), count - i);
93+
continuousIds &= r[0] == 1;
94+
maxValue = Math.max(maxValue, r[1]);
95+
maxMin2Max = Math.max(maxMin2Max, r[2]);
96+
}
97+
if (continuousIds) {
98+
return CONTINUOUS_IDS;
99+
} else if (maxMin2Max <= 0xFFFF) {
100+
return DELTA_BPV_16;
101+
} else {
102+
if (maxValue <= 0x1FFFFF) {
103+
return BPV_21;
104+
} else if (maxValue <= 0xFFFFFF) {
105+
return BPV_24;
106+
} else {
107+
return BPV_32;
108+
}
87109
}
88-
int offset = i;
89-
encoding = (byte) Math.max(encoding, blockEncoding(d -> docIds.apply(offset + d), count - i));
90-
return encoding;
91110
}
92111

93112
void writeDocIds(IntToIntFunction docIds, int count, byte encoding, DataOutput out) throws IOException {
@@ -197,7 +216,7 @@ private void write32(IntToIntFunction docIds, int count, int min, DataOutput out
197216
}
198217
}
199218

200-
private static byte blockEncoding(IntToIntFunction docIds, int count) {
219+
private static int[] sortedAndMaxAndMin2Max(IntToIntFunction docIds, int count) {
201220
// docs can be sorted either when all docs in a block have the same value
202221
// or when a segment is sorted
203222
boolean strictlySorted = true;
@@ -214,20 +233,7 @@ private static byte blockEncoding(IntToIntFunction docIds, int count) {
214233
}
215234

216235
int min2max = max - min + 1;
217-
if (strictlySorted && min2max == count) {
218-
return CONTINUOUS_IDS;
219-
}
220-
if (min2max <= 0xFFFF) {
221-
return DELTA_BPV_16;
222-
} else {
223-
if (max <= 0x1FFFFF) {
224-
return BPV_21;
225-
} else if (max <= 0xFFFFFF) {
226-
return BPV_24;
227-
} else {
228-
return BPV_32;
229-
}
230-
}
236+
return new int[] { (strictlySorted && min2max == count) ? 1 : 0, max, min2max };
231237
}
232238

233239
void writeDocIds(IntToIntFunction docIds, int count, DataOutput out) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -864,10 +864,10 @@ public String toString() {
864864
}
865865
}
866866

867-
public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader {
867+
public static class BytesRefsFromCustomBinaryBlockLoader extends DocValuesBlockLoader {
868868
private final String fieldName;
869869

870-
public BytesRefsFromBinaryBlockLoader(String fieldName) {
870+
public BytesRefsFromCustomBinaryBlockLoader(String fieldName) {
871871
this.fieldName = fieldName;
872872
}
873873

@@ -882,16 +882,14 @@ public AllReader reader(LeafReaderContext context) throws IOException {
882882
if (docValues == null) {
883883
return new ConstantNullsReader();
884884
}
885-
return new BytesRefsFromBinary(docValues);
885+
return new BytesRefsFromCustomBinary(docValues);
886886
}
887887
}
888888

889-
private static class BytesRefsFromBinary extends BlockDocValuesReader {
890-
private final BinaryDocValues docValues;
891-
private final ByteArrayStreamInput in = new ByteArrayStreamInput();
892-
private final BytesRef scratch = new BytesRef();
889+
abstract static class AbstractBytesRefsFromBinary extends BlockDocValuesReader {
890+
protected final BinaryDocValues docValues;
893891

894-
BytesRefsFromBinary(BinaryDocValues docValues) {
892+
AbstractBytesRefsFromBinary(BinaryDocValues docValues) {
895893
this.docValues = docValues;
896894
}
897895

@@ -911,7 +909,27 @@ public void read(int docId, BlockLoader.StoredFields storedFields, Builder build
911909
read(docId, (BytesRefBuilder) builder);
912910
}
913911

914-
private void read(int doc, BytesRefBuilder builder) throws IOException {
912+
@Override
913+
public int docId() {
914+
return docValues.docID();
915+
}
916+
917+
abstract void read(int docId, BytesRefBuilder builder) throws IOException;
918+
}
919+
920+
/**
921+
* Read BinaryDocValues encoded by {@link BinaryFieldMapper.CustomBinaryDocValuesField}
922+
*/
923+
static class BytesRefsFromCustomBinary extends AbstractBytesRefsFromBinary {
924+
private final ByteArrayStreamInput in = new ByteArrayStreamInput();
925+
private final BytesRef scratch = new BytesRef();
926+
927+
BytesRefsFromCustomBinary(BinaryDocValues docValues) {
928+
super(docValues);
929+
}
930+
931+
@Override
932+
void read(int doc, BytesRefBuilder builder) throws IOException {
915933
if (false == docValues.advanceExact(doc)) {
916934
builder.appendNull();
917935
return;
@@ -939,8 +957,28 @@ private void read(int doc, BytesRefBuilder builder) throws IOException {
939957
}
940958

941959
@Override
942-
public int docId() {
943-
return docValues.docID();
960+
public String toString() {
961+
return "BlockDocValuesReader.BytesCustom";
962+
}
963+
}
964+
965+
/**
966+
* Read BinaryDocValues with no additional structure in the BytesRefs.
967+
* Each BytesRef from the doc values maps directly to a value in the block loader.
968+
*/
969+
public static class BytesRefsFromBinary extends AbstractBytesRefsFromBinary {
970+
public BytesRefsFromBinary(BinaryDocValues docValues) {
971+
super(docValues);
972+
}
973+
974+
@Override
975+
void read(int doc, BytesRefBuilder builder) throws IOException {
976+
if (false == docValues.advanceExact(doc)) {
977+
builder.appendNull();
978+
return;
979+
}
980+
BytesRef bytes = docValues.binaryValue();
981+
builder.appendBytesRef(bytes);
944982
}
945983

946984
@Override

‎server/src/main/resources/transport/definitions/unreferable/initial_elasticsearch_8_16_7.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
8772007

‎server/src/main/resources/transport/definitions/unreferable/initial_elasticsearch_8_17_10.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
8797010

0 commit comments

Comments
 (0)