Skip to content

Commit bacc3fe

Browse files
jimczijimczi
committed
Merge remote-tracking branch 'upstream/main' into synthetic_vectors
2 parents 1d64c86 + 8b6acd0 commit bacc3fe

File tree

57 files changed

+810
-349
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+810
-349
lines changed

‎modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
209209
String auth = context.getHeader(AUTHORIZATION_HEADER);
210210
if (auth == null) {
211211
ElasticsearchSecurityException e = new ElasticsearchSecurityException("Authentication required", RestStatus.UNAUTHORIZED);
212-
e.addHeader("WWW-Authenticate", "Basic realm=auth-realm");
212+
e.addBodyHeader("WWW-Authenticate", "Basic realm=auth-realm");
213213
throw e;
214214
}
215215
if (false == REQUIRED_AUTH.equals(auth)) {

‎qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java‎

Lines changed: 42 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -177,9 +177,11 @@ public static void main(String[] args) throws Exception {
177177
int[] nProbes = cmdLineArgs.indexType().equals(IndexType.IVF) && cmdLineArgs.numQueries() > 0
178178
? cmdLineArgs.nProbes()
179179
: new int[] { 0 };
180+
String indexType = cmdLineArgs.indexType().name().toLowerCase(Locale.ROOT);
181+
Results indexResults = new Results(cmdLineArgs.docVectors().getFileName().toString(), indexType, cmdLineArgs.numDocs());
180182
Results[] results = new Results[nProbes.length];
181183
for (int i = 0; i < nProbes.length; i++) {
182-
results[i] = new Results(cmdLineArgs.indexType().name().toLowerCase(Locale.ROOT), cmdLineArgs.numDocs());
184+
results[i] = new Results(cmdLineArgs.docVectors().getFileName().toString(), indexType, cmdLineArgs.numDocs());
183185
}
184186
logger.info("Running KNN index tester with arguments: " + cmdLineArgs);
185187
Codec codec = createCodec(cmdLineArgs);
@@ -199,12 +201,12 @@ public static void main(String[] args) throws Exception {
199201
throw new IllegalArgumentException("Index path does not exist: " + indexPath);
200202
}
201203
if (cmdLineArgs.reindex()) {
202-
knnIndexer.createIndex(results[0]);
204+
knnIndexer.createIndex(indexResults);
203205
}
204206
if (cmdLineArgs.forceMerge()) {
205-
knnIndexer.forceMerge(results[0]);
207+
knnIndexer.forceMerge(indexResults);
206208
} else {
207-
knnIndexer.numSegments(results[0]);
209+
knnIndexer.numSegments(indexResults);
208210
}
209211
}
210212
if (cmdLineArgs.queryVectors() != null && cmdLineArgs.numQueries() > 0) {
@@ -214,24 +216,27 @@ public static void main(String[] args) throws Exception {
214216
knnSearcher.runSearch(results[i], cmdLineArgs.earlyTermination());
215217
}
216218
}
217-
formattedResults.results.addAll(List.of(results));
219+
formattedResults.queryResults.addAll(List.of(results));
220+
formattedResults.indexResults.add(indexResults);
218221
}
219222
logger.info("Results: \n" + formattedResults);
220223
}
221224

222225
static class FormattedResults {
223-
List<Results> results = new ArrayList<>();
226+
List<Results> indexResults = new ArrayList<>();
227+
List<Results> queryResults = new ArrayList<>();
224228

225229
@Override
226230
public String toString() {
227-
if (results.isEmpty()) {
231+
if (indexResults.isEmpty() && queryResults.isEmpty()) {
228232
return "No results available.";
229233
}
230234

231-
String[] indexingHeaders = { "index_type", "num_docs", "index_time(ms)", "force_merge_time(ms)", "num_segments" };
235+
String[] indexingHeaders = { "index_name", "index_type", "num_docs", "index_time(ms)", "force_merge_time(ms)", "num_segments" };
232236

233237
// Define column headers
234238
String[] searchHeaders = {
239+
"index_name",
235240
"index_type",
236241
"n_probe",
237242
"latency(ms)",
@@ -245,33 +250,34 @@ public String toString() {
245250

246251
StringBuilder sb = new StringBuilder();
247252

248-
Results indexResult = results.get(0); // Assuming all results have the same index type and numDocs
249-
String[] indexData = {
250-
indexResult.indexType,
251-
Integer.toString(indexResult.numDocs),
252-
Long.toString(indexResult.indexTimeMS),
253-
Long.toString(indexResult.forceMergeTimeMS),
254-
Integer.toString(indexResult.numSegments) };
255-
256-
printBlock(sb, indexingHeaders, new String[][] { indexData });
257-
258-
String[][] searchData = new String[results.size()][];
259-
// Format and append each row of data
260-
for (int i = 0; i < results.size(); i++) {
261-
Results result = results.get(i);
262-
searchData[i] = new String[] {
263-
result.indexType,
264-
Integer.toString(result.nProbe),
265-
String.format(Locale.ROOT, "%.2f", result.avgLatency),
266-
String.format(Locale.ROOT, "%.2f", result.netCpuTimeMS),
267-
String.format(Locale.ROOT, "%.2f", result.avgCpuCount),
268-
String.format(Locale.ROOT, "%.2f", result.qps),
269-
String.format(Locale.ROOT, "%.2f", result.avgRecall),
270-
String.format(Locale.ROOT, "%.2f", result.averageVisited) };
271-
253+
String[][] indexResultsArray = new String[indexResults.size()][];
254+
for (int i = 0; i < indexResults.size(); i++) {
255+
Results indexResult = indexResults.get(i);
256+
indexResultsArray[i] = new String[] {
257+
indexResult.indexName,
258+
indexResult.indexType,
259+
Integer.toString(indexResult.numDocs),
260+
Long.toString(indexResult.indexTimeMS),
261+
Long.toString(indexResult.forceMergeTimeMS),
262+
Integer.toString(indexResult.numSegments) };
263+
}
264+
printBlock(sb, indexingHeaders, indexResultsArray);
265+
String[][] queryResultsArray = new String[queryResults.size()][];
266+
for (int i = 0; i < queryResults.size(); i++) {
267+
Results queryResult = queryResults.get(i);
268+
queryResultsArray[i] = new String[] {
269+
queryResult.indexName,
270+
queryResult.indexType,
271+
Integer.toString(queryResult.nProbe),
272+
String.format(Locale.ROOT, "%.2f", queryResult.avgLatency),
273+
String.format(Locale.ROOT, "%.2f", queryResult.netCpuTimeMS),
274+
String.format(Locale.ROOT, "%.2f", queryResult.avgCpuCount),
275+
String.format(Locale.ROOT, "%.2f", queryResult.qps),
276+
String.format(Locale.ROOT, "%.2f", queryResult.avgRecall),
277+
String.format(Locale.ROOT, "%.2f", queryResult.averageVisited) };
272278
}
273279

274-
printBlock(sb, searchHeaders, searchData);
280+
printBlock(sb, searchHeaders, queryResultsArray);
275281

276282
return sb.toString();
277283
}
@@ -331,7 +337,7 @@ private int[] calculateColumnWidths(String[] headers, String[]... data) {
331337
}
332338

333339
static class Results {
334-
final String indexType;
340+
final String indexType, indexName;
335341
final int numDocs;
336342
long indexTimeMS;
337343
long forceMergeTimeMS;
@@ -344,7 +350,8 @@ static class Results {
344350
double netCpuTimeMS;
345351
double avgCpuCount;
346352

347-
Results(String indexType, int numDocs) {
353+
Results(String indexName, String indexType, int numDocs) {
354+
this.indexName = indexName;
348355
this.indexType = indexType;
349356
this.numDocs = numDocs;
350357
}

‎server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,8 @@ public void testPipelineOriginHeader() throws Exception {
322322
client().index(indexRequest).get();
323323
});
324324
IngestProcessorException ingestException = (IngestProcessorException) e.getCause();
325-
assertThat(ingestException.getHeader("processor_type"), equalTo(List.of("fail")));
326-
assertThat(ingestException.getHeader("pipeline_origin"), equalTo(List.of("3", "2", "1")));
325+
assertThat(ingestException.getBodyHeader("processor_type"), equalTo(List.of("fail")));
326+
assertThat(ingestException.getBodyHeader("pipeline_origin"), equalTo(List.of("3", "2", "1")));
327327
}
328328

329329
public void testPipelineProcessorOnFailure() throws Exception {

‎server/src/main/java/module-info.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,8 @@
431431
org.elasticsearch.search.SearchFeatures,
432432
org.elasticsearch.script.ScriptFeatures,
433433
org.elasticsearch.search.retriever.RetrieversFeatures,
434-
org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures;
434+
org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures,
435+
org.elasticsearch.ingest.IngestFeatures;
435436

436437
uses org.elasticsearch.plugins.internal.SettingsExtension;
437438
uses RestExtension;

‎server/src/main/java/org/elasticsearch/ElasticsearchException.java‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
122122
private static final Map<Integer, CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException>> ID_TO_SUPPLIER;
123123
private static final Map<Class<? extends ElasticsearchException>, ElasticsearchExceptionHandle> CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE;
124124
private final Map<String, List<String>> metadata = new HashMap<>();
125-
private final Map<String, List<String>> headers = new HashMap<>();
125+
private final Map<String, List<String>> bodyHeaders = new HashMap<>();
126+
private final Map<String, List<String>> httpHeaders = new HashMap<>();
126127

127128
/**
128129
* Construct a <code>ElasticsearchException</code> with the specified cause exception.
@@ -169,14 +170,14 @@ public ElasticsearchException(String msg, Throwable cause, Object... args) {
169170
public ElasticsearchException(StreamInput in) throws IOException {
170171
super(in.readOptionalString(), in.readException());
171172
readStackTrace(this, in);
172-
headers.putAll(in.readMapOfLists(StreamInput::readString));
173+
bodyHeaders.putAll(in.readMapOfLists(StreamInput::readString));
173174
metadata.putAll(in.readMapOfLists(StreamInput::readString));
174175
}
175176

176177
private void maybePutTimeoutHeader() {
177178
if (isTimeout()) {
178179
// see https://www.rfc-editor.org/rfc/rfc8941.html#section-4.1.9 for booleans in structured headers
179-
headers.put(TIMED_OUT_HEADER, List.of("?1"));
180+
bodyHeaders.put(TIMED_OUT_HEADER, List.of("?1"));
180181
}
181182
}
182183

@@ -220,42 +221,77 @@ protected Map<String, List<String>> getMetadata() {
220221
}
221222

222223
/**
223-
* Adds a new header with the given key.
224+
* Adds a new header with the given key that is part of the response body and http headers.
224225
* This method will replace existing header if a header with the same key already exists
225226
*/
226-
public void addHeader(String key, List<String> value) {
227+
public void addBodyHeader(String key, List<String> value) {
227228
// we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
228229
if (key.startsWith("es.")) {
229230
throw new IllegalArgumentException("exception headers must not start with [es.], found [" + key + "] instead");
230231
}
231-
this.headers.put(key, value);
232+
this.bodyHeaders.put(key, value);
232233
}
233234

234235
/**
235-
* Adds a new header with the given key.
236+
* Adds a new header with the given key that is part of the response body and http headers.
236237
* This method will replace existing header if a header with the same key already exists
237238
*/
238-
public void addHeader(String key, String... value) {
239-
addHeader(key, Arrays.asList(value));
239+
public void addBodyHeader(String key, String... value) {
240+
addBodyHeader(key, Arrays.asList(value));
240241
}
241242

242243
/**
243-
* Returns a set of all header keys on this exception
244+
* Returns a set of all body header keys on this exception
244245
*/
245-
public Set<String> getHeaderKeys() {
246-
return headers.keySet();
246+
public Set<String> getBodyHeaderKeys() {
247+
return bodyHeaders.keySet();
247248
}
248249

249250
/**
250-
* Returns the list of header values for the given key or {@code null} if no header for the
251+
* Returns the list of body header values for the given key or {@code null} if no header for the
251252
* given key exists.
252253
*/
253-
public List<String> getHeader(String key) {
254-
return headers.get(key);
254+
public List<String> getBodyHeader(String key) {
255+
return bodyHeaders.get(key);
255256
}
256257

257-
protected Map<String, List<String>> getHeaders() {
258-
return headers;
258+
protected Map<String, List<String>> getBodyHeaders() {
259+
return bodyHeaders;
260+
}
261+
262+
/**
263+
* Adds a new http header with the given key.
264+
* This method will replace existing http header if a header with the same key already exists
265+
*/
266+
public void addHttpHeader(String key, List<String> value) {
267+
this.httpHeaders.put(key, value);
268+
}
269+
270+
/**
271+
* Adds a new http header with the given key.
272+
* This method will replace existing http header if a header with the same key already exists
273+
*/
274+
public void addHttpHeader(String key, String... value) {
275+
this.httpHeaders.put(key, List.of(value));
276+
}
277+
278+
/**
279+
* Returns a set of all body header keys on this exception
280+
*/
281+
public Set<String> getHttpHeaderKeys() {
282+
return httpHeaders.keySet();
283+
}
284+
285+
/**
286+
* Returns the list of http header values for the given key or {@code null} if no header for the
287+
* given key exists.
288+
*/
289+
public List<String> getHttpHeader(String key) {
290+
return httpHeaders.get(key);
291+
}
292+
293+
protected Map<String, List<String>> getHttpHeaders() {
294+
return httpHeaders;
259295
}
260296

261297
/**
@@ -335,7 +371,7 @@ private static Writer<Throwable> createNestingFunction(int thisLevel, Runnable n
335371
protected void writeTo(StreamOutput out, Writer<Throwable> nestedExceptionsWriter) throws IOException {
336372
out.writeOptionalString(this.getMessage());
337373
nestedExceptionsWriter.write(out, this);
338-
out.writeMap(headers, StreamOutput::writeStringCollection);
374+
out.writeMap(bodyHeaders, StreamOutput::writeStringCollection);
339375
out.writeMap(metadata, StreamOutput::writeStringCollection);
340376
}
341377

@@ -384,7 +420,7 @@ protected XContentBuilder toXContent(XContentBuilder builder, Params params, int
384420
if (ex != this) {
385421
generateThrowableXContent(builder, params, this, nestedLevel);
386422
} else {
387-
innerToXContent(builder, params, this, headers, metadata, getCause(), nestedLevel);
423+
innerToXContent(builder, params, this, bodyHeaders, metadata, getCause(), nestedLevel);
388424
}
389425
return builder;
390426
}
@@ -581,7 +617,7 @@ public static ElasticsearchException innerFromXContent(XContentParser parser, bo
581617
e.addMetadata("es." + entry.getKey(), entry.getValue());
582618
}
583619
for (Map.Entry<String, List<String>> header : headers.entrySet()) {
584-
e.addHeader(header.getKey(), header.getValue());
620+
e.addBodyHeader(header.getKey(), header.getValue());
585621
}
586622

587623
// Adds root causes as suppressed exception. This way they are not lost

‎server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -115,22 +115,23 @@ private static XContentBuilder createSource(IndexRequest source, Exception excep
115115
// we can't instantiate it in tests, so we'll have to check for the headers directly.
116116
var ingestException = ExceptionsHelper.<ElasticsearchException>unwrapCausesAndSuppressed(
117117
exception,
118-
t -> t instanceof ElasticsearchException e && Sets.haveNonEmptyIntersection(e.getHeaderKeys(), INGEST_EXCEPTION_HEADERS)
118+
t -> t instanceof ElasticsearchException e
119+
&& Sets.haveNonEmptyIntersection(e.getBodyHeaderKeys(), INGEST_EXCEPTION_HEADERS)
119120
).orElse(null);
120121
if (ingestException != null) {
121-
if (ingestException.getHeaderKeys().contains(PIPELINE_ORIGIN_EXCEPTION_HEADER)) {
122-
List<String> pipelineOrigin = ingestException.getHeader(PIPELINE_ORIGIN_EXCEPTION_HEADER);
122+
if (ingestException.getBodyHeaderKeys().contains(PIPELINE_ORIGIN_EXCEPTION_HEADER)) {
123+
List<String> pipelineOrigin = ingestException.getBodyHeader(PIPELINE_ORIGIN_EXCEPTION_HEADER);
123124
Collections.reverse(pipelineOrigin);
124125
if (pipelineOrigin.isEmpty() == false) {
125126
builder.field("pipeline_trace", pipelineOrigin);
126127
builder.field("pipeline", pipelineOrigin.get(pipelineOrigin.size() - 1));
127128
}
128129
}
129-
if (ingestException.getHeaderKeys().contains(PROCESSOR_TAG_EXCEPTION_HEADER)) {
130-
builder.field("processor_tag", ingestException.getHeader(PROCESSOR_TAG_EXCEPTION_HEADER).get(0));
130+
if (ingestException.getBodyHeaderKeys().contains(PROCESSOR_TAG_EXCEPTION_HEADER)) {
131+
builder.field("processor_tag", ingestException.getBodyHeader(PROCESSOR_TAG_EXCEPTION_HEADER).get(0));
131132
}
132-
if (ingestException.getHeaderKeys().contains(PROCESSOR_TYPE_EXCEPTION_HEADER)) {
133-
builder.field("processor_type", ingestException.getHeader(PROCESSOR_TYPE_EXCEPTION_HEADER).get(0));
133+
if (ingestException.getBodyHeaderKeys().contains(PROCESSOR_TYPE_EXCEPTION_HEADER)) {
134+
builder.field("processor_type", ingestException.getBodyHeader(PROCESSOR_TYPE_EXCEPTION_HEADER).get(0));
134135
}
135136
}
136137
}

‎server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ static void executeDocument(
4848
pipeline.getVersion(),
4949
pipeline.getMetadata(),
5050
verbosePipelineProcessor,
51+
pipeline.getFieldAccessPattern(),
5152
pipeline.getDeprecated()
5253
);
5354
ingestDocument.executePipeline(verbosePipeline, (result, e) -> {

‎server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.elasticsearch.common.xcontent.XContentHelper;
2222
import org.elasticsearch.core.RestApiVersion;
2323
import org.elasticsearch.core.UpdateForV10;
24+
import org.elasticsearch.features.NodeFeature;
2425
import org.elasticsearch.index.VersionType;
2526
import org.elasticsearch.ingest.ConfigurationUtils;
2627
import org.elasticsearch.ingest.IngestDocument;
@@ -38,6 +39,7 @@
3839
import java.util.List;
3940
import java.util.Map;
4041
import java.util.Objects;
42+
import java.util.function.Predicate;
4143

4244
public class SimulatePipelineRequest extends LegacyActionRequest implements ToXContentObject {
4345
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(SimulatePipelineRequest.class);
@@ -154,15 +156,17 @@ static Parsed parse(
154156
Map<String, Object> config,
155157
boolean verbose,
156158
IngestService ingestService,
157-
RestApiVersion restApiVersion
159+
RestApiVersion restApiVersion,
160+
Predicate<NodeFeature> hasFeature
158161
) throws Exception {
159162
Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
160163
Pipeline pipeline = Pipeline.create(
161164
SIMULATED_PIPELINE_ID,
162165
pipelineConfig,
163166
ingestService.getProcessorFactories(),
164167
ingestService.getScriptService(),
165-
projectId
168+
projectId,
169+
hasFeature
166170
);
167171
List<IngestDocument> ingestDocumentList = parseDocs(config, restApiVersion);
168172
return new Parsed(pipeline, ingestDocumentList, verbose);

0 commit comments

Comments
 (0)