Skip to content

Commit 47fc38d

Browse files
prdoyleprdoyle
authored
Merge branch 'main' into simple-thread-properties
2 parents bcd0183 + 99c5398 commit 47fc38d

File tree

27 files changed

+489
-125
lines changed

27 files changed

+489
-125
lines changed

‎docs/changelog/122074.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
pr: 122074
2+
summary: If the Transform is configured to write to an alias as its destination index,
3+
when the delete_dest_index parameter is set to true, then the Delete API will now
4+
delete the write index backing the alias
5+
area: Transform
6+
type: bug
7+
issues:
8+
- 121913

‎muted-tests.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -320,9 +320,6 @@ tests:
320320
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
321321
method: testGetUsersWithProfileUid
322322
issue: https://github.com/elastic/elasticsearch/issues/121483
323-
- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
324-
method: testTransformLifecycle_RangeQueryThatMatchesNoShards
325-
issue: https://github.com/elastic/elasticsearch/issues/121480
326323
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
327324
method: testSuggestProfilesWithHint
328325
issue: https://github.com/elastic/elasticsearch/issues/121116
@@ -404,6 +401,9 @@ tests:
404401
issue: https://github.com/elastic/elasticsearch/issues/122377
405402
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.HdfsRepositoryAnalysisRestIT
406403
issue: https://github.com/elastic/elasticsearch/issues/122378
404+
- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
405+
method: testStatsMissingFieldWithStats
406+
issue: https://github.com/elastic/elasticsearch/issues/122327
407407

408408
# Examples:
409409
#

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ static TransportVersion def(int id) {
185185
public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION = def(9_005_0_00);
186186
public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE = def(9_006_0_00);
187187
public static final TransportVersion ESQL_PROFILE_ASYNC_NANOS = def(9_007_00_0);
188+
public static final TransportVersion ESQL_LOOKUP_JOIN_SOURCE_TEXT = def(9_008_0_00);
188189

189190
/*
190191
* STOP! READ THIS FIRST! No, really,
@@ -197,6 +198,8 @@ static TransportVersion def(int id) {
197198
* A new transport version should be added EVERY TIME a change is made to the serialization protocol of one or more classes. Each
198199
* transport version should only be used in a single merged commit (apart from the BwC versions copied from o.e.Version, ≤V_8_8_1).
199200
*
201+
* More information about versions and backporting at docs/internal/Versioning.md
202+
*
200203
* ADDING A TRANSPORT VERSION
201204
* To add a new transport version, add a new constant at the bottom of the list, above this comment. Don't add other lines,
202205
* comments, etc. The version id has the following layout:

‎server/src/main/java/org/elasticsearch/search/aggregations/InternalOrder.java‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -438,16 +438,7 @@ public static boolean isKeyDesc(BucketOrder order) {
438438
* @return {@code true} if the order matches, {@code false} otherwise.
439439
*/
440440
private static boolean isOrder(BucketOrder order, BucketOrder expected) {
441-
if (order == expected) {
442-
return true;
443-
} else if (order instanceof CompoundOrder) {
444-
// check if its a compound order with the first element that matches
445-
List<BucketOrder> orders = ((CompoundOrder) order).orderElements;
446-
if (orders.size() >= 1) {
447-
return isOrder(orders.get(0), expected);
448-
}
449-
}
450-
return false;
441+
return order == expected || (order instanceof CompoundOrder compoundOrder && compoundOrder.orderElements.getFirst() == expected);
451442
}
452443

453444
/**

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.common.io.stream.StreamInput;
1818
import org.elasticsearch.common.io.stream.StreamOutput;
1919
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
20+
import org.elasticsearch.common.util.concurrent.ThreadContext;
2021
import org.elasticsearch.compute.data.Page;
2122
import org.elasticsearch.core.TimeValue;
2223
import org.elasticsearch.index.seqno.LocalCheckpointTracker;
@@ -34,6 +35,11 @@
3435
* to reduce communication overhead and fetches a {@code Fetched} at a time.
3536
* It's the responsibility of subclasses to transform that {@code Fetched} into
3637
* output.
38+
* <p>
39+
* This operator will also take care of merging response headers from the thread context into the main thread,
40+
* which <b>must</b> be the one that closes this.
41+
* </p>
42+
*
3743
* @see #performAsync(Page, ActionListener)
3844
*/
3945
public abstract class AsyncOperator<Fetched> implements Operator {
@@ -45,6 +51,7 @@ public abstract class AsyncOperator<Fetched> implements Operator {
4551
private final DriverContext driverContext;
4652

4753
private final int maxOutstandingRequests;
54+
private final ResponseHeadersCollector responseHeadersCollector;
4855
private final LongAdder processNanos = new LongAdder();
4956

5057
private boolean finished = false;
@@ -66,9 +73,10 @@ public abstract class AsyncOperator<Fetched> implements Operator {
6673
*
6774
* @param maxOutstandingRequests the maximum number of outstanding requests
6875
*/
69-
public AsyncOperator(DriverContext driverContext, int maxOutstandingRequests) {
76+
public AsyncOperator(DriverContext driverContext, ThreadContext threadContext, int maxOutstandingRequests) {
7077
this.driverContext = driverContext;
7178
this.maxOutstandingRequests = maxOutstandingRequests;
79+
this.responseHeadersCollector = new ResponseHeadersCollector(threadContext);
7280
}
7381

7482
@Override
@@ -97,6 +105,7 @@ public void addInput(Page input) {
97105
});
98106
final long startNanos = System.nanoTime();
99107
performAsync(input, ActionListener.runAfter(listener, () -> {
108+
responseHeadersCollector.collect();
100109
driverContext.removeAsyncAction();
101110
processNanos.add(System.nanoTime() - startNanos);
102111
}));
@@ -172,6 +181,7 @@ public final void close() {
172181
finish();
173182
closed = true;
174183
discardResults();
184+
responseHeadersCollector.finish();
175185
doClose();
176186
}
177187

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/Warnings.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,23 @@ private Warnings(int lineNumber, int columnNumber, String sourceText, String fir
102102
}
103103

104104
public void registerException(Exception exception) {
105+
registerException(exception.getClass(), exception.getMessage());
106+
}
107+
108+
/**
109+
* Register an exception to be included in the warnings.
110+
* <p>
111+
* This overload avoids the need to instantiate the exception, which can be expensive.
112+
* Instead, it asks only the required pieces to build the warning.
113+
* </p>
114+
*/
115+
public void registerException(Class<? extends Exception> exceptionClass, String message) {
105116
if (addedWarnings < MAX_ADDED_WARNINGS) {
106117
if (addedWarnings == 0) {
107118
addWarning(first);
108119
}
109120
// location needs to be added to the exception too, since the headers are deduplicated
110-
addWarning(location + exception.getClass().getName() + ": " + exception.getMessage());
121+
addWarning(location + exceptionClass.getName() + ": " + message);
111122
addedWarnings++;
112123
}
113124
}

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.elasticsearch.index.query.SearchExecutionContext;
3737

3838
import java.io.IOException;
39+
import java.io.UncheckedIOException;
3940
import java.util.ArrayList;
4041
import java.util.List;
4142
import java.util.function.IntFunction;
@@ -47,13 +48,19 @@ public abstract class QueryList {
4748
protected final SearchExecutionContext searchExecutionContext;
4849
protected final MappedFieldType field;
4950
protected final Block block;
50-
protected final boolean onlySingleValues;
51+
@Nullable
52+
protected final OnlySingleValueParams onlySingleValueParams;
5153

52-
protected QueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, boolean onlySingleValues) {
54+
protected QueryList(
55+
MappedFieldType field,
56+
SearchExecutionContext searchExecutionContext,
57+
Block block,
58+
OnlySingleValueParams onlySingleValueParams
59+
) {
5360
this.searchExecutionContext = searchExecutionContext;
5461
this.field = field;
5562
this.block = block;
56-
this.onlySingleValues = onlySingleValues;
63+
this.onlySingleValueParams = onlySingleValueParams;
5764
}
5865

5966
/**
@@ -66,19 +73,27 @@ int getPositionCount() {
6673
/**
6774
* Returns a copy of this query list that only returns queries for single-valued positions.
6875
* That is, it returns `null` queries for either multivalued or null positions.
76+
* <p>
77+
* Whenever a multi-value position is encountered, whether in the input block or in the queried index, a warning is emitted.
78+
* </p>
6979
*/
70-
public abstract QueryList onlySingleValues();
80+
public abstract QueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage);
7181

7282
final Query getQuery(int position) {
7383
final int valueCount = block.getValueCount(position);
74-
if (onlySingleValues && valueCount != 1) {
84+
if (onlySingleValueParams != null && valueCount != 1) {
85+
if (valueCount > 1) {
86+
onlySingleValueParams.warnings.registerException(
87+
new IllegalArgumentException(onlySingleValueParams.multiValueWarningMessage)
88+
);
89+
}
7590
return null;
7691
}
7792
final int firstValueIndex = block.getFirstValueIndex(position);
7893

7994
Query query = doGetQuery(position, firstValueIndex, valueCount);
8095

81-
if (onlySingleValues) {
96+
if (onlySingleValueParams != null) {
8297
query = wrapSingleValueQuery(query);
8398
}
8499

@@ -92,22 +107,24 @@ final Query getQuery(int position) {
92107
abstract Query doGetQuery(int position, int firstValueIndex, int valueCount);
93108

94109
private Query wrapSingleValueQuery(Query query) {
110+
assert onlySingleValueParams != null : "Requested to wrap single value query without single value params";
111+
95112
SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery(
96113
searchExecutionContext.getForField(field, MappedFieldType.FielddataOperation.SEARCH),
97114
// Not emitting warnings for multivalued fields not matching
98-
Warnings.NOOP_WARNINGS
115+
onlySingleValueParams.warnings,
116+
onlySingleValueParams.multiValueWarningMessage
99117
);
100118

101-
Query rewrite = singleValueQuery;
119+
Query rewrite;
102120
try {
103121
rewrite = singleValueQuery.rewrite(searchExecutionContext.searcher());
104122
if (rewrite instanceof MatchAllDocsQuery) {
105123
// nothing to filter
106124
return query;
107125
}
108126
} catch (IOException e) {
109-
// ignore
110-
// TODO: Should we do something with the exception?
127+
throw new UncheckedIOException("Error while rewriting SingleValueQuery", e);
111128
}
112129

113130
BooleanQuery.Builder builder = new BooleanQuery.Builder();
@@ -152,7 +169,7 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC
152169
case COMPOSITE -> throw new IllegalArgumentException("can't read values from [composite] block");
153170
case UNKNOWN -> throw new IllegalArgumentException("can't read values from [" + block + "]");
154171
};
155-
return new TermQueryList(field, searchExecutionContext, block, false, blockToJavaObject);
172+
return new TermQueryList(field, searchExecutionContext, block, null, blockToJavaObject);
156173
}
157174

158175
/**
@@ -162,7 +179,7 @@ public static QueryList rawTermQueryList(MappedFieldType field, SearchExecutionC
162179
public static QueryList ipTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, BytesRefBlock block) {
163180
BytesRef scratch = new BytesRef();
164181
byte[] ipBytes = new byte[InetAddressPoint.BYTES];
165-
return new TermQueryList(field, searchExecutionContext, block, false, offset -> {
182+
return new TermQueryList(field, searchExecutionContext, block, null, offset -> {
166183
final var bytes = block.getBytesRef(offset, scratch);
167184
if (ipBytes.length != bytes.length) {
168185
// Lucene only support 16-byte IP addresses, even IPv4 is encoded in 16 bytes
@@ -182,7 +199,7 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution
182199
field,
183200
searchExecutionContext,
184201
block,
185-
false,
202+
null,
186203
field instanceof RangeFieldMapper.RangeFieldType rangeFieldType
187204
? offset -> rangeFieldType.dateTimeFormatter().formatMillis(block.getLong(offset))
188205
: block::getLong
@@ -193,7 +210,7 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution
193210
* Returns a list of geo_shape queries for the given field and the input block.
194211
*/
195212
public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block) {
196-
return new GeoShapeQueryList(field, searchExecutionContext, block, false);
213+
return new GeoShapeQueryList(field, searchExecutionContext, block, null);
197214
}
198215

199216
private static class TermQueryList extends QueryList {
@@ -203,16 +220,22 @@ private TermQueryList(
203220
MappedFieldType field,
204221
SearchExecutionContext searchExecutionContext,
205222
Block block,
206-
boolean onlySingleValues,
223+
OnlySingleValueParams onlySingleValueParams,
207224
IntFunction<Object> blockValueReader
208225
) {
209-
super(field, searchExecutionContext, block, onlySingleValues);
226+
super(field, searchExecutionContext, block, onlySingleValueParams);
210227
this.blockValueReader = blockValueReader;
211228
}
212229

213230
@Override
214-
public TermQueryList onlySingleValues() {
215-
return new TermQueryList(field, searchExecutionContext, block, true, blockValueReader);
231+
public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage) {
232+
return new TermQueryList(
233+
field,
234+
searchExecutionContext,
235+
block,
236+
new OnlySingleValueParams(warnings, multiValueWarningMessage),
237+
blockValueReader
238+
);
216239
}
217240

218241
@Override
@@ -241,17 +264,22 @@ private GeoShapeQueryList(
241264
MappedFieldType field,
242265
SearchExecutionContext searchExecutionContext,
243266
Block block,
244-
boolean onlySingleValues
267+
OnlySingleValueParams onlySingleValueParams
245268
) {
246-
super(field, searchExecutionContext, block, onlySingleValues);
269+
super(field, searchExecutionContext, block, onlySingleValueParams);
247270

248271
this.blockValueReader = blockToGeometry(block);
249272
this.shapeQuery = shapeQuery();
250273
}
251274

252275
@Override
253-
public GeoShapeQueryList onlySingleValues() {
254-
return new GeoShapeQueryList(field, searchExecutionContext, block, true);
276+
public GeoShapeQueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage) {
277+
return new GeoShapeQueryList(
278+
field,
279+
searchExecutionContext,
280+
block,
281+
new OnlySingleValueParams(warnings, multiValueWarningMessage)
282+
);
255283
}
256284

257285
@Override
@@ -295,4 +323,6 @@ private IntFunction<Query> shapeQuery() {
295323
throw new IllegalArgumentException("Unsupported field type for geo_match ENRICH: " + field.typeName());
296324
}
297325
}
326+
327+
protected record OnlySingleValueParams(Warnings warnings, String multiValueWarningMessage) {}
298328
}

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,14 @@ public final class SingleValueMatchQuery extends Query {
4646
* This avoids reporting warnings when queries are not matching multi-values
4747
*/
4848
private static final int MULTI_VALUE_MATCH_COST = 1000;
49-
private static final IllegalArgumentException MULTI_VALUE_EXCEPTION = new IllegalArgumentException(
50-
"single-value function encountered multi-value"
51-
);
5249
private final IndexFieldData<?> fieldData;
5350
private final Warnings warnings;
51+
private final String multiValueExceptionMessage;
5452

55-
public SingleValueMatchQuery(IndexFieldData<?> fieldData, Warnings warnings) {
53+
public SingleValueMatchQuery(IndexFieldData<?> fieldData, Warnings warnings, String multiValueExceptionMessage) {
5654
this.fieldData = fieldData;
5755
this.warnings = warnings;
56+
this.multiValueExceptionMessage = multiValueExceptionMessage;
5857
}
5958

6059
@Override
@@ -123,7 +122,7 @@ private ScorerSupplier scorerSupplier(
123122
return false;
124123
}
125124
if (sortedNumerics.docValueCount() != 1) {
126-
warnings.registerException(MULTI_VALUE_EXCEPTION);
125+
registerMultiValueException();
127126
return false;
128127
}
129128
return true;
@@ -158,7 +157,7 @@ private ScorerSupplier scorerSupplier(
158157
return false;
159158
}
160159
if (sortedSetDocValues.docValueCount() != 1) {
161-
warnings.registerException(MULTI_VALUE_EXCEPTION);
160+
registerMultiValueException();
162161
return false;
163162
}
164163
return true;
@@ -187,7 +186,7 @@ private ScorerSupplier scorerSupplier(
187186
return false;
188187
}
189188
if (sortedBinaryDocValues.docValueCount() != 1) {
190-
warnings.registerException(MULTI_VALUE_EXCEPTION);
189+
registerMultiValueException();
191190
return false;
192191
}
193192
return true;
@@ -267,6 +266,10 @@ public long cost() {
267266
}
268267
}
269268

269+
private void registerMultiValueException() {
270+
warnings.registerException(IllegalArgumentException.class, multiValueExceptionMessage);
271+
}
272+
270273
private static class PredicateScorerSupplier extends ScorerSupplier {
271274
private final float score;
272275
private final ScoreMode scoreMode;

0 commit comments

Comments
 (0)