Skip to content

Commit 0402eef

Browse files
tteofilitteofili
committed
Merge branch 'main' of github.com:elastic/elasticsearch into knn_patience
2 parents 1a6ca1a + 7836826 commit 0402eef

File tree

12 files changed

+138
-74
lines changed

12 files changed

+138
-74
lines changed

‎docs/reference/elasticsearch/mapping-reference/semantic-text.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ Refer
231231
to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)
232232
to learn more about semantic search using `semantic_text`.
233233

234-
## Extracting Relevant Fragments from Semantic Text [semantic-text-highlighting]
234+
## Extracting relevant fragments from semantic text [semantic-text-highlighting]
235235

236236
You can extract the most relevant fragments from a semantic text field by using
237237
the [highlight parameter](/reference/elasticsearch/rest-apis/highlighting.md) in
@@ -295,8 +295,7 @@ automatic {{infer}} and a dedicated query so you don’t need to provide further
295295
details.
296296

297297
If you want to override those defaults and customize the embeddings that
298-
`semantic_text` indexes, you can do so by modifying <<semantic-text-params,
299-
parameters>>:
298+
`semantic_text` indexes, you can do so by modifying [parameters](#semantic-text-params):
300299

301300
- Use `index_options` to specify alternate index options such as specific
302301
`dense_vector` quantization methods
@@ -398,6 +397,6 @@ PUT test-index
398397
* `semantic_text` fields are not currently supported as elements
399398
of [nested fields](/reference/elasticsearch/mapping-reference/nested.md).
400399
* `semantic_text` fields can’t currently be set as part
401-
of [Dynamic templates](docs-content://manage-data/data-store/mapping/dynamic-templates.md).
400+
of [dynamic templates](docs-content://manage-data/data-store/mapping/dynamic-templates.md).
402401
* `semantic_text` fields are not supported with Cross-Cluster Search (CCS) or
403402
Cross-Cluster Replication (CCR).

‎muted-tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -524,9 +524,6 @@ tests:
524524
- class: org.elasticsearch.search.query.VectorIT
525525
method: testFilteredQueryStrategy
526526
issue: https://github.com/elastic/elasticsearch/issues/129517
527-
- class: org.elasticsearch.snapshots.SnapshotShutdownIT
528-
method: testSnapshotShutdownProgressTracker
529-
issue: https://github.com/elastic/elasticsearch/issues/129752
530527
- class: org.elasticsearch.xpack.security.SecurityRolesMultiProjectIT
531528
method: testUpdatingFileBasedRoleAffectsAllProjects
532529
issue: https://github.com/elastic/elasticsearch/issues/129775

‎qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import java.io.IOException;
2323
import java.nio.file.Path;
24+
import java.util.List;
2425
import java.util.Locale;
2526

2627
/**
@@ -35,7 +36,7 @@ record CmdLineArgs(
3536
KnnIndexTester.IndexType indexType,
3637
int numCandidates,
3738
int k,
38-
int nProbe,
39+
int[] nProbes,
3940
int ivfClusterSize,
4041
int overSamplingFactor,
4142
int hnswM,
@@ -88,7 +89,7 @@ static CmdLineArgs fromXContent(XContentParser parser) throws IOException {
8889
PARSER.declareString(Builder::setIndexType, INDEX_TYPE_FIELD);
8990
PARSER.declareInt(Builder::setNumCandidates, NUM_CANDIDATES_FIELD);
9091
PARSER.declareInt(Builder::setK, K_FIELD);
91-
PARSER.declareInt(Builder::setNProbe, N_PROBE_FIELD);
92+
PARSER.declareIntArray(Builder::setNProbe, N_PROBE_FIELD);
9293
PARSER.declareInt(Builder::setIvfClusterSize, IVF_CLUSTER_SIZE_FIELD);
9394
PARSER.declareInt(Builder::setOverSamplingFactor, OVER_SAMPLING_FACTOR_FIELD);
9495
PARSER.declareInt(Builder::setHnswM, HNSW_M_FIELD);
@@ -118,7 +119,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
118119
builder.field(INDEX_TYPE_FIELD.getPreferredName(), indexType.name().toLowerCase(Locale.ROOT));
119120
builder.field(NUM_CANDIDATES_FIELD.getPreferredName(), numCandidates);
120121
builder.field(K_FIELD.getPreferredName(), k);
121-
builder.field(N_PROBE_FIELD.getPreferredName(), nProbe);
122+
builder.field(N_PROBE_FIELD.getPreferredName(), nProbes);
122123
builder.field(IVF_CLUSTER_SIZE_FIELD.getPreferredName(), ivfClusterSize);
123124
builder.field(OVER_SAMPLING_FACTOR_FIELD.getPreferredName(), overSamplingFactor);
124125
builder.field(HNSW_M_FIELD.getPreferredName(), hnswM);
@@ -147,7 +148,7 @@ static class Builder {
147148
private KnnIndexTester.IndexType indexType = KnnIndexTester.IndexType.HNSW;
148149
private int numCandidates = 1000;
149150
private int k = 10;
150-
private int nProbe = 10;
151+
private int[] nProbes = new int[] { 10 };
151152
private int ivfClusterSize = 1000;
152153
private int overSamplingFactor = 1;
153154
private int hnswM = 16;
@@ -197,8 +198,8 @@ public Builder setK(int k) {
197198
return this;
198199
}
199200

200-
public Builder setNProbe(int nProbe) {
201-
this.nProbe = nProbe;
201+
public Builder setNProbe(List<Integer> nProbes) {
202+
this.nProbes = nProbes.stream().mapToInt(Integer::intValue).toArray();
202203
return this;
203204
}
204205

@@ -284,7 +285,7 @@ public CmdLineArgs build() {
284285
indexType,
285286
numCandidates,
286287
k,
287-
nProbe,
288+
nProbes,
288289
ivfClusterSize,
289290
overSamplingFactor,
290291
hnswM,

‎qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java‎

Lines changed: 58 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,15 @@ public static void main(String[] args) throws Exception {
172172
}
173173
}
174174
FormattedResults formattedResults = new FormattedResults();
175+
175176
for (CmdLineArgs cmdLineArgs : cmdLineArgsList) {
176-
Results result = new Results(cmdLineArgs.indexType().name().toLowerCase(Locale.ROOT), cmdLineArgs.numDocs());
177+
int[] nProbes = cmdLineArgs.indexType().equals(IndexType.IVF) && cmdLineArgs.numQueries() > 0
178+
? cmdLineArgs.nProbes()
179+
: new int[] { 0 };
180+
Results[] results = new Results[nProbes.length];
181+
for (int i = 0; i < nProbes.length; i++) {
182+
results[i] = new Results(cmdLineArgs.indexType().name().toLowerCase(Locale.ROOT), cmdLineArgs.numDocs());
183+
}
177184
logger.info("Running KNN index tester with arguments: " + cmdLineArgs);
178185
Codec codec = createCodec(cmdLineArgs);
179186
Path indexPath = PathUtils.get(formatIndexPath(cmdLineArgs));
@@ -192,19 +199,22 @@ public static void main(String[] args) throws Exception {
192199
throw new IllegalArgumentException("Index path does not exist: " + indexPath);
193200
}
194201
if (cmdLineArgs.reindex()) {
195-
knnIndexer.createIndex(result);
202+
knnIndexer.createIndex(results[0]);
196203
}
197204
if (cmdLineArgs.forceMerge()) {
198-
knnIndexer.forceMerge(result);
205+
knnIndexer.forceMerge(results[0]);
199206
} else {
200-
knnIndexer.numSegments(result);
207+
knnIndexer.numSegments(results[0]);
201208
}
202209
}
203210
if (cmdLineArgs.queryVectors() != null && cmdLineArgs.numQueries() > 0) {
204-
KnnSearcher knnSearcher = new KnnSearcher(indexPath, cmdLineArgs);
205-
knnSearcher.runSearch(result, cmdLineArgs.earlyTermination());
211+
for (int i = 0; i < results.length; i++) {
212+
int nProbe = nProbes[i];
213+
KnnSearcher knnSearcher = new KnnSearcher(indexPath, cmdLineArgs, nProbe);
214+
knnSearcher.runSearch(results[i], cmdLineArgs.earlyTermination());
215+
}
206216
}
207-
formattedResults.results.add(result);
217+
formattedResults.results.addAll(List.of(results));
208218
}
209219
logger.info("Results: \n" + formattedResults);
210220
}
@@ -218,13 +228,12 @@ public String toString() {
218228
return "No results available.";
219229
}
220230

231+
String[] indexingHeaders = { "index_type", "num_docs", "index_time(ms)", "force_merge_time(ms)", "num_segments" };
232+
221233
// Define column headers
222-
String[] headers = {
234+
String[] searchHeaders = {
223235
"index_type",
224-
"num_docs",
225-
"index_time(ms)",
226-
"force_merge_time(ms)",
227-
"num_segments",
236+
"n_probe",
228237
"latency(ms)",
229238
"net_cpu_time(ms)",
230239
"avg_cpu_count",
@@ -233,41 +242,58 @@ public String toString() {
233242
"visited" };
234243

235244
// Calculate appropriate column widths based on headers and data
236-
int[] widths = calculateColumnWidths(headers);
237245

238246
StringBuilder sb = new StringBuilder();
239247

240-
// Format and append header
241-
sb.append(formatRow(headers, widths));
242-
sb.append("\n");
248+
Results indexResult = results.get(0); // Assuming all results have the same index type and numDocs
249+
String[] indexData = {
250+
indexResult.indexType,
251+
Integer.toString(indexResult.numDocs),
252+
Long.toString(indexResult.indexTimeMS),
253+
Long.toString(indexResult.forceMergeTimeMS),
254+
Integer.toString(indexResult.numSegments) };
243255

244-
// Add separator line
245-
for (int width : widths) {
246-
sb.append("-".repeat(width)).append(" ");
247-
}
248-
sb.append("\n");
256+
printBlock(sb, indexingHeaders, new String[][] { indexData });
249257

258+
String[][] searchData = new String[results.size()][];
250259
// Format and append each row of data
251-
for (Results result : results) {
252-
String[] rowData = {
260+
for (int i = 0; i < results.size(); i++) {
261+
Results result = results.get(i);
262+
searchData[i] = new String[] {
253263
result.indexType,
254-
Integer.toString(result.numDocs),
255-
Long.toString(result.indexTimeMS),
256-
Long.toString(result.forceMergeTimeMS),
257-
Integer.toString(result.numSegments),
264+
Integer.toString(result.nProbe),
258265
String.format(Locale.ROOT, "%.2f", result.avgLatency),
259266
String.format(Locale.ROOT, "%.2f", result.netCpuTimeMS),
260267
String.format(Locale.ROOT, "%.2f", result.avgCpuCount),
261268
String.format(Locale.ROOT, "%.2f", result.qps),
262269
String.format(Locale.ROOT, "%.2f", result.avgRecall),
263270
String.format(Locale.ROOT, "%.2f", result.averageVisited) };
264-
sb.append(formatRow(rowData, widths));
265-
sb.append("\n");
271+
266272
}
267273

274+
printBlock(sb, searchHeaders, searchData);
275+
268276
return sb.toString();
269277
}
270278

279+
private void printBlock(StringBuilder sb, String[] headers, String[][] rows) {
280+
int[] widths = calculateColumnWidths(headers, rows);
281+
sb.append("\n");
282+
sb.append(formatRow(headers, widths));
283+
sb.append("\n");
284+
285+
// Add separator line
286+
for (int width : widths) {
287+
sb.append("-".repeat(width)).append(" ");
288+
}
289+
sb.append("\n");
290+
291+
for (String[] row : rows) {
292+
sb.append(formatRow(row, widths));
293+
sb.append("\n");
294+
}
295+
}
296+
271297
// Helper method to format a single row with proper column widths
272298
private String formatRow(String[] values, int[] widths) {
273299
StringBuilder row = new StringBuilder();
@@ -285,7 +311,7 @@ private String formatRow(String[] values, int[] widths) {
285311
}
286312

287313
// Calculate appropriate column widths based on headers and data
288-
private int[] calculateColumnWidths(String[] headers) {
314+
private int[] calculateColumnWidths(String[] headers, String[]... data) {
289315
int[] widths = new int[headers.length];
290316

291317
// Initialize widths with header lengths
@@ -294,20 +320,7 @@ private int[] calculateColumnWidths(String[] headers) {
294320
}
295321

296322
// Update widths based on data
297-
for (Results result : results) {
298-
String[] values = {
299-
result.indexType,
300-
Integer.toString(result.numDocs),
301-
Long.toString(result.indexTimeMS),
302-
Long.toString(result.forceMergeTimeMS),
303-
Integer.toString(result.numSegments),
304-
String.format(Locale.ROOT, "%.2f", result.avgLatency),
305-
String.format(Locale.ROOT, "%.2f", result.netCpuTimeMS),
306-
String.format(Locale.ROOT, "%.2f", result.avgCpuCount),
307-
String.format(Locale.ROOT, "%.2f", result.qps),
308-
String.format(Locale.ROOT, "%.2f", result.avgRecall),
309-
String.format(Locale.ROOT, "%.2f", result.averageVisited) };
310-
323+
for (String[] values : data) {
311324
for (int i = 0; i < values.length; i++) {
312325
widths[i] = Math.max(widths[i], values[i].length());
313326
}
@@ -323,6 +336,7 @@ static class Results {
323336
long indexTimeMS;
324337
long forceMergeTimeMS;
325338
int numSegments;
339+
int nProbe;
326340
double avgLatency;
327341
double qps;
328342
double avgRecall;

‎qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ class KnnSearcher {
9797
private final float overSamplingFactor;
9898
private final int searchThreads;
9999

100-
KnnSearcher(Path indexPath, CmdLineArgs cmdLineArgs) {
100+
KnnSearcher(Path indexPath, CmdLineArgs cmdLineArgs, int nProbe) {
101101
this.docPath = cmdLineArgs.docVectors();
102102
this.indexPath = indexPath;
103103
this.queryPath = cmdLineArgs.queryVectors();
@@ -112,7 +112,7 @@ class KnnSearcher {
112112
throw new IllegalArgumentException("numQueryVectors must be > 0");
113113
}
114114
this.efSearch = cmdLineArgs.numCandidates();
115-
this.nProbe = cmdLineArgs.nProbe();
115+
this.nProbe = nProbe;
116116
this.indexType = cmdLineArgs.indexType();
117117
this.searchThreads = cmdLineArgs.searchThreads();
118118
}
@@ -209,6 +209,7 @@ void runSearch(KnnIndexTester.Results finalResults, boolean earlyTermination) th
209209
}
210210
logger.info("checking results");
211211
int[][] nn = getOrCalculateExactNN(offsetByteSize);
212+
finalResults.nProbe = indexType == KnnIndexTester.IndexType.IVF ? nProbe : 0;
212213
finalResults.avgRecall = checkResults(resultIds, nn, topK);
213214
finalResults.qps = (1000f * numQueryVectors) / elapsed;
214215
finalResults.avgLatency = (float) elapsed / numQueryVectors;

‎server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,7 @@ private void startNewShardSnapshots(String localNodeId, SnapshotsInProgress.Entr
411411
entry.version(),
412412
entry.startTime()
413413
);
414+
snapshotStatus.updateStatusDescription("shard snapshot enqueuing to start");
414415
startShardSnapshotTaskRunner.enqueueTask(new ActionListener<>() {
415416
@Override
416417
public void onResponse(Releasable releasable) {
@@ -429,7 +430,6 @@ public void onFailure(Exception e) {
429430
assert false : wrapperException; // impossible
430431
}
431432
});
432-
snapshotStatus.updateStatusDescription("shard snapshot enqueued to start");
433433
}
434434

435435
// apply some backpressure by reserving one SNAPSHOT thread for the startup work

‎server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,10 @@ public void testDataStreamInfoToXContent() throws IOException {
6666
assertThat(lifecycleResult.get("retention_determined_by"), equalTo("max_global_retention"));
6767
Map<String, Map<String, Object>> settingsMap = (Map<String, Map<String, Object>>) resultMap.get("settings");
6868
assertThat(Settings.builder().loadFromMap(settingsMap).build(), equalTo(dataStreamInfo.getDataStream().getSettings()));
69-
Map<String, Object> mappingsMap = (Map<String, Object>) resultMap.get("mappings");
70-
assertThat(new CompressedXContent(mappingsMap), equalTo(dataStreamInfo.getDataStream().getMappings()));
69+
if (DataStream.LOGS_STREAM_FEATURE_FLAG) {
70+
Map<String, Object> mappingsMap = (Map<String, Object>) resultMap.get("mappings");
71+
assertThat(new CompressedXContent(mappingsMap), equalTo(dataStreamInfo.getDataStream().getMappings()));
72+
}
7173
}
7274
}
7375

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/RequestIndexFilteringIT.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
3939
import static org.hamcrest.Matchers.hasSize;
4040
import static org.hamcrest.Matchers.instanceOf;
41+
import static org.hamcrest.Matchers.is;
4142

4243
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
4344
public class RequestIndexFilteringIT extends RequestIndexFilteringTestCase {
@@ -91,14 +92,20 @@ protected String from(String... indexName) {
9192

9293
@Override
9394
public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException {
95+
return runEsql(requestObject, true);
96+
}
97+
98+
@Override
99+
public Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject, boolean checkPartialResults)
100+
throws IOException {
94101
if (requestObject.allowPartialResults() != null) {
95102
assumeTrue(
96103
"require allow_partial_results on local cluster",
97104
clusterHasCapability("POST", "/_query", List.of(), List.of("support_partial_results")).orElse(false)
98105
);
99106
}
100107
requestObject.includeCCSMetadata(true);
101-
return super.runEsql(requestObject);
108+
return super.runEsql(requestObject, checkPartialResults);
102109
}
103110

104111
@After
@@ -154,7 +161,30 @@ public void testIndicesDontExistRemote() throws IOException {
154161
indexTimestampData(docsTest1, "test1", "2024-11-26", "id1");
155162

156163
Map<String, Object> result = runEsql(
157-
timestampFilter("gte", "2020-01-01").query("FROM *:foo,*:test1 METADATA _index | SORT id1 | KEEP _index, id*")
164+
timestampFilter("gte", "2020-01-01").query("FROM *:foo,*:test1 METADATA _index | SORT id1 | KEEP _index, id*"),
165+
false
166+
);
167+
168+
// `foo` index doesn't exist, so the request will currently be successful, but with partial results
169+
var isPartial = result.get("is_partial");
170+
assertThat(isPartial, is(true));
171+
assertThat(
172+
result,
173+
matchesMap().entry(
174+
"_clusters",
175+
matchesMap().entry(
176+
"details",
177+
matchesMap().entry(
178+
"remote_cluster",
179+
matchesMap().entry(
180+
"failures",
181+
matchesList().item(
182+
matchesMap().entry("reason", matchesMap().entry("reason", "no such index [foo]").extraOk()).extraOk()
183+
)
184+
).extraOk()
185+
).extraOk()
186+
).extraOk()
187+
).extraOk()
158188
);
159189
@SuppressWarnings("unchecked")
160190
var columns = (List<List<Object>>) result.get("columns");

0 commit comments

Comments
 (0)