Skip to content

Commit 99c3aec

Browse files
quux00albertzaharovits
quux00
authored and
albertzaharovits
committed
Search coordinator uses event.ingested in cluster state to do rewrites (#110352)
Min/max range for the event.ingested timestamp field (part of Elastic Common Schema) was added to IndexMetadata in cluster state for searchable snapshots in #106252. This commit modifies the search coordinator to rewrite searches to MatchNone if the query searches a range of event.ingested that, from the min/max range in cluster state, is known to not overlap. This is the same behavior we currently have for the @timestamp field.
1 parent 5eca6b8 commit 99c3aec

File tree

12 files changed

+1034
-173
lines changed

12 files changed

+1034
-173
lines changed

‎docs/changelog/110352.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 110352
2+
summary: Search coordinator uses `event.ingested` in cluster state to do rewrites
3+
area: Search
4+
type: enhancement
5+
issues: []

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public void testGetTimestampFieldTypeForTsdbDataStream() throws IOException {
6161
DocWriteResponse indexResponse = indexDoc();
6262

6363
var indicesService = getInstanceFromNode(IndicesService.class);
64-
var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
64+
var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
6565
assertThat(result, notNullValue());
6666
}
6767

@@ -70,7 +70,7 @@ public void testGetTimestampFieldTypeForDataStream() throws IOException {
7070
DocWriteResponse indexResponse = indexDoc();
7171

7272
var indicesService = getInstanceFromNode(IndicesService.class);
73-
var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex());
73+
var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex());
7474
assertThat(result, nullValue());
7575
}
7676

‎server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java

Lines changed: 95 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99
package org.elasticsearch.index.query;
1010

1111
import org.elasticsearch.client.internal.Client;
12+
import org.elasticsearch.cluster.metadata.DataStream;
13+
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.common.Strings;
1215
import org.elasticsearch.core.Nullable;
13-
import org.elasticsearch.index.mapper.DateFieldMapper;
1416
import org.elasticsearch.index.mapper.MappedFieldType;
1517
import org.elasticsearch.index.mapper.MappingLookup;
1618
import org.elasticsearch.index.shard.IndexLongFieldRange;
19+
import org.elasticsearch.indices.DateFieldRangeInfo;
1720
import org.elasticsearch.xcontent.XContentParserConfiguration;
1821

1922
import java.util.Collections;
@@ -23,19 +26,24 @@
2326
* Context object used to rewrite {@link QueryBuilder} instances into simplified version in the coordinator.
2427
* Instances of this object rely on information stored in the {@code IndexMetadata} for certain indices.
2528
* Right now this context object is able to rewrite range queries that include a known timestamp field
26-
* (i.e. the timestamp field for DataStreams) into a MatchNoneQueryBuilder and skip the shards that
27-
* don't hold queried data. See IndexMetadata#getTimestampRange() for more details
29+
* (i.e. the timestamp field for DataStreams or the 'event.ingested' field in ECS) into a MatchNoneQueryBuilder
30+
* and skip the shards that don't hold queried data. See IndexMetadata for more details.
2831
*/
2932
public class CoordinatorRewriteContext extends QueryRewriteContext {
30-
private final IndexLongFieldRange indexLongFieldRange;
31-
private final DateFieldMapper.DateFieldType timestampFieldType;
33+
private final DateFieldRangeInfo dateFieldRangeInfo;
3234

35+
/**
36+
* Context for coordinator search rewrites based on time ranges for the @timestamp field and/or 'event.ingested' field
37+
* @param parserConfig
38+
* @param client
39+
* @param nowInMillis
40+
* @param dateFieldRangeInfo range and field type info for @timestamp and 'event.ingested'
41+
*/
3342
public CoordinatorRewriteContext(
3443
XContentParserConfiguration parserConfig,
3544
Client client,
3645
LongSupplier nowInMillis,
37-
IndexLongFieldRange indexLongFieldRange,
38-
DateFieldMapper.DateFieldType timestampFieldType
46+
DateFieldRangeInfo dateFieldRangeInfo
3947
) {
4048
super(
4149
parserConfig,
@@ -53,29 +61,98 @@ public CoordinatorRewriteContext(
5361
null,
5462
null
5563
);
56-
this.indexLongFieldRange = indexLongFieldRange;
57-
this.timestampFieldType = timestampFieldType;
64+
this.dateFieldRangeInfo = dateFieldRangeInfo;
5865
}
5966

60-
long getMinTimestamp() {
61-
return indexLongFieldRange.getMin();
67+
/**
68+
* Get min timestamp for either '@timestamp' or 'event.ingested' fields. Any other field
69+
* passed in will cause an {@link IllegalArgumentException} to be thrown, as these are the only
70+
* two fields supported for coordinator rewrites (based on time range).
71+
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
72+
* @return min timestamp for the field from IndexMetadata in cluster state.
73+
*/
74+
long getMinTimestamp(String fieldName) {
75+
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
76+
return dateFieldRangeInfo.getTimestampRange().getMin();
77+
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
78+
return dateFieldRangeInfo.getEventIngestedRange().getMin();
79+
} else {
80+
throw new IllegalArgumentException(
81+
Strings.format(
82+
"Only [%s] or [%s] fields are supported for min timestamp coordinator rewrites, but got: [%s]",
83+
DataStream.TIMESTAMP_FIELD_NAME,
84+
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
85+
fieldName
86+
)
87+
);
88+
}
6289
}
6390

64-
long getMaxTimestamp() {
65-
return indexLongFieldRange.getMax();
91+
/**
92+
* Get max timestamp for either '@timestamp' or 'event.ingested' fields. Any other field
93+
* passed in will cause an {@link IllegalArgumentException} to be thrown, as these are the only
94+
* two fields supported for coordinator rewrites (based on time range).
95+
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
96+
* @return max timestamp for the field from IndexMetadata in cluster state.
97+
*/
98+
long getMaxTimestamp(String fieldName) {
99+
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
100+
return dateFieldRangeInfo.getTimestampRange().getMax();
101+
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
102+
return dateFieldRangeInfo.getEventIngestedRange().getMax();
103+
} else {
104+
throw new IllegalArgumentException(
105+
Strings.format(
106+
"Only [%s] or [%s] fields are supported for max timestamp coordinator rewrites, but got: [%s]",
107+
DataStream.TIMESTAMP_FIELD_NAME,
108+
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
109+
fieldName
110+
)
111+
);
112+
}
66113
}
67114

68-
boolean hasTimestampData() {
69-
return indexLongFieldRange.isComplete() && indexLongFieldRange != IndexLongFieldRange.EMPTY;
115+
/**
116+
* Determine whether either '@timestamp' or 'event.ingested' fields has useful timestamp ranges
117+
* stored in cluster state for this context.
118+
* Any other fieldname will cause an {@link IllegalArgumentException} to be thrown, as these are the only
119+
* two fields supported for coordinator rewrites (based on time range).
120+
* @param fieldName Must be DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME
121+
* @return min timestamp for the field from IndexMetadata in cluster state.
122+
*/
123+
boolean hasTimestampData(String fieldName) {
124+
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
125+
return dateFieldRangeInfo.getTimestampRange().isComplete()
126+
&& dateFieldRangeInfo.getTimestampRange() != IndexLongFieldRange.EMPTY;
127+
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
128+
return dateFieldRangeInfo.getEventIngestedRange().isComplete()
129+
&& dateFieldRangeInfo.getEventIngestedRange() != IndexLongFieldRange.EMPTY;
130+
} else {
131+
throw new IllegalArgumentException(
132+
Strings.format(
133+
"Only [%s] or [%s] fields are supported for min/max timestamp coordinator rewrites, but got: [%s]",
134+
DataStream.TIMESTAMP_FIELD_NAME,
135+
IndexMetadata.EVENT_INGESTED_FIELD_NAME,
136+
fieldName
137+
)
138+
);
139+
}
70140
}
71141

142+
/**
143+
* @param fieldName Get MappedFieldType for either '@timestamp' or 'event.ingested' fields.
144+
* @return min timestamp for the field from IndexMetadata in cluster state or null if fieldName was not
145+
* DataStream.TIMESTAMP_FIELD_NAME or IndexMetadata.EVENT_INGESTED_FIELD_NAME.
146+
*/
72147
@Nullable
73148
public MappedFieldType getFieldType(String fieldName) {
74-
if (fieldName.equals(timestampFieldType.name()) == false) {
149+
if (DataStream.TIMESTAMP_FIELD_NAME.equals(fieldName)) {
150+
return dateFieldRangeInfo.getTimestampFieldType();
151+
} else if (IndexMetadata.EVENT_INGESTED_FIELD_NAME.equals(fieldName)) {
152+
return dateFieldRangeInfo.getEventIngestedFieldType();
153+
} else {
75154
return null;
76155
}
77-
78-
return timestampFieldType;
79156
}
80157

81158
@Override

‎server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.index.Index;
1515
import org.elasticsearch.index.mapper.DateFieldMapper;
1616
import org.elasticsearch.index.shard.IndexLongFieldRange;
17+
import org.elasticsearch.indices.DateFieldRangeInfo;
1718
import org.elasticsearch.xcontent.XContentParserConfiguration;
1819

1920
import java.util.function.Function;
@@ -25,14 +26,14 @@ public class CoordinatorRewriteContextProvider {
2526
private final Client client;
2627
private final LongSupplier nowInMillis;
2728
private final Supplier<ClusterState> clusterStateSupplier;
28-
private final Function<Index, DateFieldMapper.DateFieldType> mappingSupplier;
29+
private final Function<Index, DateFieldRangeInfo> mappingSupplier;
2930

3031
public CoordinatorRewriteContextProvider(
3132
XContentParserConfiguration parserConfig,
3233
Client client,
3334
LongSupplier nowInMillis,
3435
Supplier<ClusterState> clusterStateSupplier,
35-
Function<Index, DateFieldMapper.DateFieldType> mappingSupplier
36+
Function<Index, DateFieldRangeInfo> mappingSupplier
3637
) {
3738
this.parserConfig = parserConfig;
3839
this.client = client;
@@ -49,18 +50,33 @@ public CoordinatorRewriteContext getCoordinatorRewriteContext(Index index) {
4950
if (indexMetadata == null) {
5051
return null;
5152
}
52-
DateFieldMapper.DateFieldType dateFieldType = mappingSupplier.apply(index);
53-
if (dateFieldType == null) {
53+
54+
DateFieldRangeInfo dateFieldRangeInfo = mappingSupplier.apply(index);
55+
if (dateFieldRangeInfo == null) {
5456
return null;
5557
}
58+
59+
DateFieldMapper.DateFieldType timestampFieldType = dateFieldRangeInfo.getTimestampFieldType();
60+
DateFieldMapper.DateFieldType eventIngestedFieldType = dateFieldRangeInfo.getEventIngestedFieldType();
5661
IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange();
62+
IndexLongFieldRange eventIngestedRange = indexMetadata.getEventIngestedRange();
63+
5764
if (timestampRange.containsAllShardRanges() == false) {
58-
timestampRange = indexMetadata.getTimeSeriesTimestampRange(dateFieldType);
59-
if (timestampRange == null) {
65+
// if @timestamp range is not present or not ready in cluster state, fallback to using time series range (if present)
66+
timestampRange = indexMetadata.getTimeSeriesTimestampRange(timestampFieldType);
67+
// if timestampRange in the time series is null AND the eventIngestedRange is not ready for use, return null (no coord rewrite)
68+
if (timestampRange == null && eventIngestedRange.containsAllShardRanges() == false) {
6069
return null;
6170
}
6271
}
6372

64-
return new CoordinatorRewriteContext(parserConfig, client, nowInMillis, timestampRange, dateFieldType);
73+
// the DateFieldRangeInfo from the mappingSupplier only has field types, but not ranges
74+
// so create a new object with ranges pulled from cluster state
75+
return new CoordinatorRewriteContext(
76+
parserConfig,
77+
client,
78+
nowInMillis,
79+
new DateFieldRangeInfo(timestampFieldType, timestampRange, eventIngestedFieldType, eventIngestedRange)
80+
);
6581
}
6682
}

‎server/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -436,11 +436,11 @@ public String getWriteableName() {
436436
protected MappedFieldType.Relation getRelation(final CoordinatorRewriteContext coordinatorRewriteContext) {
437437
final MappedFieldType fieldType = coordinatorRewriteContext.getFieldType(fieldName);
438438
if (fieldType instanceof final DateFieldMapper.DateFieldType dateFieldType) {
439-
if (coordinatorRewriteContext.hasTimestampData() == false) {
439+
if (coordinatorRewriteContext.hasTimestampData(fieldName) == false) {
440440
return MappedFieldType.Relation.DISJOINT;
441441
}
442-
long minTimestamp = coordinatorRewriteContext.getMinTimestamp();
443-
long maxTimestamp = coordinatorRewriteContext.getMaxTimestamp();
442+
long minTimestamp = coordinatorRewriteContext.getMinTimestamp(fieldName);
443+
long maxTimestamp = coordinatorRewriteContext.getMaxTimestamp(fieldName);
444444
DateMathParser dateMathParser = getForceDateParser();
445445
return dateFieldType.isFieldWithinQuery(
446446
minTimestamp,

‎server/src/main/java/org/elasticsearch/indices/DateFieldRangeInfo.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.indices;
10+
11+
import org.elasticsearch.index.mapper.DateFieldMapper;
12+
import org.elasticsearch.index.shard.IndexLongFieldRange;
13+
14+
/**
15+
* Data holder of timestamp fields held in cluster state IndexMetadata.
16+
*/
17+
public final class DateFieldRangeInfo {
18+
19+
private final DateFieldMapper.DateFieldType timestampFieldType;
20+
private final IndexLongFieldRange timestampRange;
21+
private final DateFieldMapper.DateFieldType eventIngestedFieldType;
22+
private final IndexLongFieldRange eventIngestedRange;
23+
24+
public DateFieldRangeInfo(
25+
DateFieldMapper.DateFieldType timestampFieldType,
26+
IndexLongFieldRange timestampRange,
27+
DateFieldMapper.DateFieldType eventIngestedFieldType,
28+
IndexLongFieldRange eventIngestedRange
29+
) {
30+
this.timestampFieldType = timestampFieldType;
31+
this.timestampRange = timestampRange;
32+
this.eventIngestedFieldType = eventIngestedFieldType;
33+
this.eventIngestedRange = eventIngestedRange;
34+
}
35+
36+
public DateFieldMapper.DateFieldType getTimestampFieldType() {
37+
return timestampFieldType;
38+
}
39+
40+
public IndexLongFieldRange getTimestampRange() {
41+
return timestampRange;
42+
}
43+
44+
public DateFieldMapper.DateFieldType getEventIngestedFieldType() {
45+
return eventIngestedFieldType;
46+
}
47+
48+
public IndexLongFieldRange getEventIngestedRange() {
49+
return eventIngestedRange;
50+
}
51+
}

‎server/src/main/java/org/elasticsearch/indices/IndicesService.java

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,6 @@
9898
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
9999
import org.elasticsearch.index.flush.FlushStats;
100100
import org.elasticsearch.index.get.GetStats;
101-
import org.elasticsearch.index.mapper.DateFieldMapper;
102101
import org.elasticsearch.index.mapper.IdFieldMapper;
103102
import org.elasticsearch.index.mapper.MapperMetrics;
104103
import org.elasticsearch.index.mapper.MapperRegistry;
@@ -1764,7 +1763,13 @@ public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {
17641763
}
17651764

17661765
public CoordinatorRewriteContextProvider getCoordinatorRewriteContextProvider(LongSupplier nowInMillis) {
1767-
return new CoordinatorRewriteContextProvider(parserConfig, client, nowInMillis, clusterService::state, this::getTimestampFieldType);
1766+
return new CoordinatorRewriteContextProvider(
1767+
parserConfig,
1768+
client,
1769+
nowInMillis,
1770+
clusterService::state,
1771+
this::getTimestampFieldTypeInfo
1772+
);
17681773
}
17691774

17701775
/**
@@ -1854,14 +1859,16 @@ public boolean allPendingDanglingIndicesWritten() {
18541859
}
18551860

18561861
/**
1857-
* @return the field type of the {@code @timestamp} field of the given index, or {@code null} if:
1862+
* @return DateFieldRangeInfo holding the field types of the {@code @timestamp} and {@code event.ingested} fields of the index.
1863+
* or {@code null} if:
18581864
* - the index is not found,
18591865
* - the field is not found, or
1860-
* - the field is not a timestamp field.
1866+
* - the mapping is not known yet, or
1867+
* - the index does not have a useful timestamp field.
18611868
*/
18621869
@Nullable
1863-
public DateFieldMapper.DateFieldType getTimestampFieldType(Index index) {
1864-
return timestampFieldMapperService.getTimestampFieldType(index);
1870+
public DateFieldRangeInfo getTimestampFieldTypeInfo(Index index) {
1871+
return timestampFieldMapperService.getTimestampFieldTypeMap(index);
18651872
}
18661873

18671874
public IndexScopedSettings getIndexScopedSettings() {

0 commit comments

Comments
 (0)