Skip to content

Commit 2489f5d

Browse files
fang-xing-esqlfang-xing-esql
authored
Merge branch 'main' into date-nanos-implicit-casting-behind-snapshot
2 parents 921eb11 + a69c484 commit 2489f5d

File tree

52 files changed

+539
-309
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+539
-309
lines changed

‎docs/reference/query-languages/esql/kibana/definition/commands/change_point.json‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/reference/query-languages/esql/kibana/definition/functions/categorize.json‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.elasticsearch.common.text.UTF8DecodingReader;
3636
import org.elasticsearch.common.unit.Fuzziness;
3737
import org.elasticsearch.index.IndexVersion;
38+
import org.elasticsearch.index.IndexVersions;
3839
import org.elasticsearch.index.analysis.IndexAnalyzers;
3940
import org.elasticsearch.index.analysis.NamedAnalyzer;
4041
import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -105,8 +106,15 @@ public static class Builder extends FieldMapper.Builder {
105106

106107
private final TextParams.Analyzers analyzers;
107108
private final boolean withinMultiField;
109+
private final boolean storedFieldInBinaryFormat;
108110

109-
public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
111+
public Builder(
112+
String name,
113+
IndexVersion indexCreatedVersion,
114+
IndexAnalyzers indexAnalyzers,
115+
boolean withinMultiField,
116+
boolean storedFieldInBinaryFormat
117+
) {
110118
super(name);
111119
this.indexCreatedVersion = indexCreatedVersion;
112120
this.analyzers = new TextParams.Analyzers(
@@ -116,6 +124,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
116124
indexCreatedVersion
117125
);
118126
this.withinMultiField = withinMultiField;
127+
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
119128
}
120129

121130
@Override
@@ -135,7 +144,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
135144
context.isSourceSynthetic(),
136145
meta.getValue(),
137146
withinMultiField,
138-
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()
147+
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(),
148+
storedFieldInBinaryFormat
139149
);
140150
return ft;
141151
}
@@ -155,8 +165,22 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
155165
}
156166
}
157167

168+
private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion indexCreatedVersion) {
169+
return indexCreatedVersion.onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES)
170+
|| indexCreatedVersion.between(
171+
IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BACKPORT_8_X,
172+
IndexVersions.UPGRADE_TO_LUCENE_10_0_0
173+
);
174+
}
175+
158176
public static final TypeParser PARSER = new TypeParser(
159-
(n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
177+
(n, c) -> new Builder(
178+
n,
179+
c.indexVersionCreated(),
180+
c.getIndexAnalyzers(),
181+
c.isWithinMultiField(),
182+
isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated())
183+
)
160184
);
161185

162186
public static class MatchOnlyTextFieldType extends StringFieldType {
@@ -167,6 +191,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType {
167191

168192
private final boolean withinMultiField;
169193
private final boolean hasCompatibleMultiFields;
194+
private final boolean storedFieldInBinaryFormat;
170195

171196
public MatchOnlyTextFieldType(
172197
String name,
@@ -175,14 +200,16 @@ public MatchOnlyTextFieldType(
175200
boolean isSyntheticSource,
176201
Map<String, String> meta,
177202
boolean withinMultiField,
178-
boolean hasCompatibleMultiFields
203+
boolean hasCompatibleMultiFields,
204+
boolean storedFieldInBinaryFormat
179205
) {
180206
super(name, true, false, false, tsi, meta);
181207
this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
182208
this.textFieldType = new TextFieldType(name, isSyntheticSource);
183209
this.originalName = isSyntheticSource ? name + "._original" : null;
184210
this.withinMultiField = withinMultiField;
185211
this.hasCompatibleMultiFields = hasCompatibleMultiFields;
212+
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
186213
}
187214

188215
public MatchOnlyTextFieldType(String name) {
@@ -193,6 +220,7 @@ public MatchOnlyTextFieldType(String name) {
193220
false,
194221
Collections.emptyMap(),
195222
false,
223+
false,
196224
false
197225
);
198226
}
@@ -451,7 +479,11 @@ protected BytesRef toBytesRef(Object v) {
451479
@Override
452480
public BlockLoader blockLoader(BlockLoaderContext blContext) {
453481
if (textFieldType.isSyntheticSource()) {
454-
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
482+
if (storedFieldInBinaryFormat) {
483+
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
484+
} else {
485+
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
486+
}
455487
}
456488
SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
457489
// MatchOnlyText never has norms, so we have to use the field names field
@@ -502,6 +534,7 @@ private String storedFieldNameForSyntheticSource() {
502534
private final boolean storeSource;
503535
private final FieldType fieldType;
504536
private final boolean withinMultiField;
537+
private final boolean storedFieldInBinaryFormat;
505538

506539
private MatchOnlyTextFieldMapper(
507540
String simpleName,
@@ -521,6 +554,7 @@ private MatchOnlyTextFieldMapper(
521554
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
522555
this.storeSource = storeSource;
523556
this.withinMultiField = builder.withinMultiField;
557+
this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat;
524558
}
525559

526560
@Override
@@ -530,7 +564,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
530564

531565
@Override
532566
public FieldMapper.Builder getMergeBuilder() {
533-
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
567+
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this);
534568
}
535569

536570
@Override
@@ -547,8 +581,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
547581
context.addToFieldNames(fieldType().name());
548582

549583
if (storeSource) {
550-
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
551-
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
584+
if (storedFieldInBinaryFormat) {
585+
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
586+
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
587+
} else {
588+
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string()));
589+
}
552590
}
553591
}
554592

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,10 @@
2626
import org.apache.lucene.tests.index.RandomIndexWriter;
2727
import org.apache.lucene.util.BytesRef;
2828
import org.elasticsearch.common.Strings;
29+
import org.elasticsearch.common.settings.Settings;
2930
import org.elasticsearch.core.Tuple;
3031
import org.elasticsearch.index.IndexSettings;
32+
import org.elasticsearch.index.IndexVersions;
3133
import org.elasticsearch.index.mapper.DocumentMapper;
3234
import org.elasticsearch.index.mapper.KeywordFieldMapper;
3335
import org.elasticsearch.index.mapper.LuceneDocument;
@@ -356,10 +358,14 @@ public void testStoreParameterDefaultsSyntheticSourceTextFieldIsMultiField() thr
356358
}
357359

358360
public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException {
359-
DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> {
361+
var mappings = mapping(b -> {
360362
b.startObject("field1").field("type", "match_only_text").endObject();
361363
b.startObject("field2").field("type", "match_only_text").endObject();
362-
})).documentMapper();
364+
});
365+
var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
366+
DocumentMapper mapper = createMapperService(IndexVersions.UPGRADE_TO_LUCENE_10_2_2, settings, () -> true, mappings)
367+
.documentMapper();
368+
363369
try (Directory directory = newDirectory()) {
364370
RandomIndexWriter iw = indexWriterForSyntheticSource(directory);
365371

‎modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexFromRemoteWithAuthTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void app
209209
String auth = context.getHeader(AUTHORIZATION_HEADER);
210210
if (auth == null) {
211211
ElasticsearchSecurityException e = new ElasticsearchSecurityException("Authentication required", RestStatus.UNAUTHORIZED);
212-
e.addHeader("WWW-Authenticate", "Basic realm=auth-realm");
212+
e.addBodyHeader("WWW-Authenticate", "Basic realm=auth-realm");
213213
throw e;
214214
}
215215
if (false == REQUIRED_AUTH.equals(auth)) {

‎server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,8 @@ public void testPipelineOriginHeader() throws Exception {
322322
client().index(indexRequest).get();
323323
});
324324
IngestProcessorException ingestException = (IngestProcessorException) e.getCause();
325-
assertThat(ingestException.getHeader("processor_type"), equalTo(List.of("fail")));
326-
assertThat(ingestException.getHeader("pipeline_origin"), equalTo(List.of("3", "2", "1")));
325+
assertThat(ingestException.getBodyHeader("processor_type"), equalTo(List.of("fail")));
326+
assertThat(ingestException.getBodyHeader("pipeline_origin"), equalTo(List.of("3", "2", "1")));
327327
}
328328

329329
public void testPipelineProcessorOnFailure() throws Exception {

‎server/src/main/java/org/elasticsearch/ElasticsearchException.java‎

Lines changed: 56 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
122122
private static final Map<Integer, CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException>> ID_TO_SUPPLIER;
123123
private static final Map<Class<? extends ElasticsearchException>, ElasticsearchExceptionHandle> CLASS_TO_ELASTICSEARCH_EXCEPTION_HANDLE;
124124
private final Map<String, List<String>> metadata = new HashMap<>();
125-
private final Map<String, List<String>> headers = new HashMap<>();
125+
private final Map<String, List<String>> bodyHeaders = new HashMap<>();
126+
private final Map<String, List<String>> httpHeaders = new HashMap<>();
126127

127128
/**
128129
* Construct a <code>ElasticsearchException</code> with the specified cause exception.
@@ -169,14 +170,14 @@ public ElasticsearchException(String msg, Throwable cause, Object... args) {
169170
public ElasticsearchException(StreamInput in) throws IOException {
170171
super(in.readOptionalString(), in.readException());
171172
readStackTrace(this, in);
172-
headers.putAll(in.readMapOfLists(StreamInput::readString));
173+
bodyHeaders.putAll(in.readMapOfLists(StreamInput::readString));
173174
metadata.putAll(in.readMapOfLists(StreamInput::readString));
174175
}
175176

176177
private void maybePutTimeoutHeader() {
177178
if (isTimeout()) {
178179
// see https://www.rfc-editor.org/rfc/rfc8941.html#section-4.1.9 for booleans in structured headers
179-
headers.put(TIMED_OUT_HEADER, List.of("?1"));
180+
bodyHeaders.put(TIMED_OUT_HEADER, List.of("?1"));
180181
}
181182
}
182183

@@ -220,42 +221,77 @@ protected Map<String, List<String>> getMetadata() {
220221
}
221222

222223
/**
223-
* Adds a new header with the given key.
224+
* Adds a new header with the given key that is part of the response body and http headers.
224225
* This method will replace existing header if a header with the same key already exists
225226
*/
226-
public void addHeader(String key, List<String> value) {
227+
public void addBodyHeader(String key, List<String> value) {
227228
// we need to enforce this otherwise bw comp doesn't work properly, as "es." was the previous criteria to split headers in two sets
228229
if (key.startsWith("es.")) {
229230
throw new IllegalArgumentException("exception headers must not start with [es.], found [" + key + "] instead");
230231
}
231-
this.headers.put(key, value);
232+
this.bodyHeaders.put(key, value);
232233
}
233234

234235
/**
235-
* Adds a new header with the given key.
236+
* Adds a new header with the given key that is part of the response body and http headers.
236237
* This method will replace existing header if a header with the same key already exists
237238
*/
238-
public void addHeader(String key, String... value) {
239-
addHeader(key, Arrays.asList(value));
239+
public void addBodyHeader(String key, String... value) {
240+
addBodyHeader(key, Arrays.asList(value));
240241
}
241242

242243
/**
243-
* Returns a set of all header keys on this exception
244+
* Returns a set of all body header keys on this exception
244245
*/
245-
public Set<String> getHeaderKeys() {
246-
return headers.keySet();
246+
public Set<String> getBodyHeaderKeys() {
247+
return bodyHeaders.keySet();
247248
}
248249

249250
/**
250-
* Returns the list of header values for the given key or {@code null} if no header for the
251+
* Returns the list of body header values for the given key or {@code null} if no header for the
251252
* given key exists.
252253
*/
253-
public List<String> getHeader(String key) {
254-
return headers.get(key);
254+
public List<String> getBodyHeader(String key) {
255+
return bodyHeaders.get(key);
255256
}
256257

257-
protected Map<String, List<String>> getHeaders() {
258-
return headers;
258+
protected Map<String, List<String>> getBodyHeaders() {
259+
return bodyHeaders;
260+
}
261+
262+
/**
263+
* Adds a new http header with the given key.
264+
* This method will replace existing http header if a header with the same key already exists
265+
*/
266+
public void addHttpHeader(String key, List<String> value) {
267+
this.httpHeaders.put(key, value);
268+
}
269+
270+
/**
271+
* Adds a new http header with the given key.
272+
* This method will replace existing http header if a header with the same key already exists
273+
*/
274+
public void addHttpHeader(String key, String... value) {
275+
this.httpHeaders.put(key, List.of(value));
276+
}
277+
278+
/**
279+
* Returns a set of all body header keys on this exception
280+
*/
281+
public Set<String> getHttpHeaderKeys() {
282+
return httpHeaders.keySet();
283+
}
284+
285+
/**
286+
* Returns the list of http header values for the given key or {@code null} if no header for the
287+
* given key exists.
288+
*/
289+
public List<String> getHttpHeader(String key) {
290+
return httpHeaders.get(key);
291+
}
292+
293+
protected Map<String, List<String>> getHttpHeaders() {
294+
return httpHeaders;
259295
}
260296

261297
/**
@@ -335,7 +371,7 @@ private static Writer<Throwable> createNestingFunction(int thisLevel, Runnable n
335371
protected void writeTo(StreamOutput out, Writer<Throwable> nestedExceptionsWriter) throws IOException {
336372
out.writeOptionalString(this.getMessage());
337373
nestedExceptionsWriter.write(out, this);
338-
out.writeMap(headers, StreamOutput::writeStringCollection);
374+
out.writeMap(bodyHeaders, StreamOutput::writeStringCollection);
339375
out.writeMap(metadata, StreamOutput::writeStringCollection);
340376
}
341377

@@ -384,7 +420,7 @@ protected XContentBuilder toXContent(XContentBuilder builder, Params params, int
384420
if (ex != this) {
385421
generateThrowableXContent(builder, params, this, nestedLevel);
386422
} else {
387-
innerToXContent(builder, params, this, headers, metadata, getCause(), nestedLevel);
423+
innerToXContent(builder, params, this, bodyHeaders, metadata, getCause(), nestedLevel);
388424
}
389425
return builder;
390426
}
@@ -581,7 +617,7 @@ public static ElasticsearchException innerFromXContent(XContentParser parser, bo
581617
e.addMetadata("es." + entry.getKey(), entry.getValue());
582618
}
583619
for (Map.Entry<String, List<String>> header : headers.entrySet()) {
584-
e.addHeader(header.getKey(), header.getValue());
620+
e.addBodyHeader(header.getKey(), header.getValue());
585621
}
586622

587623
// Adds root causes as suppressed exception. This way they are not lost

‎server/src/main/java/org/elasticsearch/action/bulk/FailureStoreDocumentConverter.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -115,22 +115,23 @@ private static XContentBuilder createSource(IndexRequest source, Exception excep
115115
// we can't instantiate it in tests, so we'll have to check for the headers directly.
116116
var ingestException = ExceptionsHelper.<ElasticsearchException>unwrapCausesAndSuppressed(
117117
exception,
118-
t -> t instanceof ElasticsearchException e && Sets.haveNonEmptyIntersection(e.getHeaderKeys(), INGEST_EXCEPTION_HEADERS)
118+
t -> t instanceof ElasticsearchException e
119+
&& Sets.haveNonEmptyIntersection(e.getBodyHeaderKeys(), INGEST_EXCEPTION_HEADERS)
119120
).orElse(null);
120121
if (ingestException != null) {
121-
if (ingestException.getHeaderKeys().contains(PIPELINE_ORIGIN_EXCEPTION_HEADER)) {
122-
List<String> pipelineOrigin = ingestException.getHeader(PIPELINE_ORIGIN_EXCEPTION_HEADER);
122+
if (ingestException.getBodyHeaderKeys().contains(PIPELINE_ORIGIN_EXCEPTION_HEADER)) {
123+
List<String> pipelineOrigin = ingestException.getBodyHeader(PIPELINE_ORIGIN_EXCEPTION_HEADER);
123124
Collections.reverse(pipelineOrigin);
124125
if (pipelineOrigin.isEmpty() == false) {
125126
builder.field("pipeline_trace", pipelineOrigin);
126127
builder.field("pipeline", pipelineOrigin.get(pipelineOrigin.size() - 1));
127128
}
128129
}
129-
if (ingestException.getHeaderKeys().contains(PROCESSOR_TAG_EXCEPTION_HEADER)) {
130-
builder.field("processor_tag", ingestException.getHeader(PROCESSOR_TAG_EXCEPTION_HEADER).get(0));
130+
if (ingestException.getBodyHeaderKeys().contains(PROCESSOR_TAG_EXCEPTION_HEADER)) {
131+
builder.field("processor_tag", ingestException.getBodyHeader(PROCESSOR_TAG_EXCEPTION_HEADER).get(0));
131132
}
132-
if (ingestException.getHeaderKeys().contains(PROCESSOR_TYPE_EXCEPTION_HEADER)) {
133-
builder.field("processor_type", ingestException.getHeader(PROCESSOR_TYPE_EXCEPTION_HEADER).get(0));
133+
if (ingestException.getBodyHeaderKeys().contains(PROCESSOR_TYPE_EXCEPTION_HEADER)) {
134+
builder.field("processor_type", ingestException.getBodyHeader(PROCESSOR_TYPE_EXCEPTION_HEADER).get(0));
134135
}
135136
}
136137
}

0 commit comments

Comments
 (0)