Skip to content

Commit f15018c

Browse files
leontyevdvelasticsearchmachine
leontyevdv
and
elasticsearchmachine
authored
ES|QL: Make _tsid available in metadata (#135204)
* ES|QL: Make _tsid available in metadata Add _tsid into the list of available attributes in metadata. Closes #133205 --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent 5420780 commit f15018c

File tree

23 files changed

+382
-35
lines changed

23 files changed

+382
-35
lines changed

‎docs/changelog/135204.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 135204
2+
summary: Make `_tsid` available in metadata
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 133205

‎server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,15 @@ public Query termQuery(Object value, SearchExecutionContext context) {
134134
throw new IllegalArgumentException("[" + NAME + "] is not searchable");
135135
}
136136

137+
@Override
138+
public Object valueForDisplay(Object value) {
139+
if (value == null) {
140+
return null;
141+
}
142+
BytesRef binaryValue = (BytesRef) value;
143+
return TimeSeriesIdFieldMapper.encodeTsid(binaryValue);
144+
}
145+
137146
@Override
138147
public BlockLoader blockLoader(BlockLoaderContext blContext) {
139148
return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());

‎server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -768,4 +768,23 @@ public void testParseWithDynamicMappingInvalidRoutingHash() {
768768
});
769769
assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Illegal base64 character 20"));
770770
}
771+
772+
public void testValueForDisplay() throws Exception {
773+
DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> {
774+
b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject();
775+
b.startObject("b").field("type", "long").field("time_series_dimension", true).endObject();
776+
}));
777+
778+
ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100));
779+
BytesRef tsidBytes = doc.rootDoc().getBinaryValue("_tsid");
780+
assertThat(tsidBytes, not(nullValue()));
781+
782+
TimeSeriesIdFieldMapper.TimeSeriesIdFieldType fieldType = TimeSeriesIdFieldMapper.FIELD_TYPE;
783+
Object displayValue = fieldType.valueForDisplay(tsidBytes);
784+
Object encodedValue = TimeSeriesIdFieldMapper.encodeTsid(tsidBytes);
785+
786+
assertThat(displayValue, equalTo(encodedValue));
787+
assertThat(displayValue.getClass(), is(String.class));
788+
assertThat(fieldType.valueForDisplay(null), nullValue());
789+
}
771790
}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ public class MetadataAttribute extends TypedAttribute {
4545
Map.entry(IgnoredFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
4646
Map.entry(SourceFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.SOURCE, false)),
4747
Map.entry(IndexModeFieldMapper.NAME, new MetadataAttributeConfiguration(DataType.KEYWORD, true)),
48-
Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false))
48+
Map.entry(SCORE, new MetadataAttributeConfiguration(DataType.DOUBLE, false)),
49+
Map.entry(TSID_FIELD, new MetadataAttributeConfiguration(DataType.TSID_DATA_TYPE, false))
4950
);
5051

5152
private record MetadataAttributeConfiguration(DataType dataType, boolean searchable) {}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,7 @@ public enum DataType implements Writeable {
414414
}
415415

416416
private static final Collection<DataType> TYPES = Arrays.stream(values())
417-
.filter(d -> d != DOC_DATA_TYPE && d != TSID_DATA_TYPE)
417+
.filter(d -> d != DOC_DATA_TYPE)
418418
.sorted(Comparator.comparing(DataType::typeName))
419419
.toList();
420420

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.geometry.utils.Geohash;
4646
import org.elasticsearch.h3.H3;
4747
import org.elasticsearch.index.IndexMode;
48+
import org.elasticsearch.index.mapper.RoutingPathFields;
4849
import org.elasticsearch.license.XPackLicenseState;
4950
import org.elasticsearch.search.SearchService;
5051
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
@@ -901,8 +902,9 @@ public static Literal randomLiteral(DataType type) {
901902
throw new UncheckedIOException(e);
902903
}
903904
}
905+
case TSID_DATA_TYPE -> randomTsId().toBytesRef();
904906
case DENSE_VECTOR -> Arrays.asList(randomArray(10, 10, i -> new Float[10], ESTestCase::randomFloat));
905-
case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException(
907+
case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException(
906908
"can't make random values for [" + type.typeName() + "]"
907909
);
908910
}, type);
@@ -918,6 +920,22 @@ static Version randomVersion() {
918920
};
919921
}
920922

923+
static BytesReference randomTsId() {
924+
RoutingPathFields routingPathFields = new RoutingPathFields(null);
925+
926+
int numDimensions = randomIntBetween(1, 4);
927+
for (int i = 0; i < numDimensions; i++) {
928+
String fieldName = "dim" + i;
929+
if (randomBoolean()) {
930+
routingPathFields.addString(fieldName, randomAlphaOfLength(randomIntBetween(3, 10)));
931+
} else {
932+
routingPathFields.addLong(fieldName, randomLongBetween(1, 1000));
933+
}
934+
}
935+
936+
return routingPathFields.buildHash();
937+
}
938+
921939
public static WildcardLike wildcardLike(Expression left, String exp) {
922940
return new WildcardLike(EMPTY, left, new WildcardPattern(exp), false);
923941
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,3 +525,37 @@ true | two | 2024-05-10T00:18:00.000Z
525525
false | two | 2024-05-10T00:20:00.000Z
526526
false | two | 2024-05-10T00:22:00.000Z
527527
;
528+
529+
tsidMetadataAttributeCount
530+
required_capability: ts_command_v0
531+
required_capability: metadata_tsid_field
532+
533+
TS k8s METADATA _tsid
534+
| STATS cnt = count_distinct(_tsid)
535+
;
536+
537+
cnt:long
538+
9
539+
;
540+
541+
tsidMetadataAttributeAggregation
542+
required_capability: ts_command_v0
543+
required_capability: metadata_tsid_field
544+
545+
TS k8s METADATA _tsid
546+
| STATS cnt = count_distinct(_tsid) BY cluster, pod
547+
| SORT cluster
548+
;
549+
ignoreOrder:true
550+
551+
cnt:long | cluster:keyword | pod:keyword
552+
1 | staging | one
553+
1 | staging | two
554+
1 | staging | three
555+
1 | qa | one
556+
1 | qa | two
557+
1 | qa | three
558+
1 | prod | one
559+
1 | prod | two
560+
1 | prod | three
561+
;

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClusterTimeSeriesIT.java‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.action;
9+
10+
import org.elasticsearch.common.Randomness;
11+
import org.elasticsearch.common.Strings;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.common.unit.ByteSizeValue;
14+
15+
import java.util.ArrayList;
16+
import java.util.HashMap;
17+
import java.util.List;
18+
import java.util.Map;
19+
20+
import static org.elasticsearch.index.mapper.DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER;
21+
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
22+
import static org.hamcrest.Matchers.equalTo;
23+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
24+
import static org.hamcrest.Matchers.hasSize;
25+
26+
// @TestLogging(value = "org.elasticsearch.xpack.esql.session:DEBUG", reason = "to better understand planning")
27+
public class CrossClusterTimeSeriesIT extends AbstractCrossClusterTestCase {
28+
29+
private static final String INDEX_NAME = "hosts";
30+
31+
record Doc(String host, String cluster, long timestamp, int requestCount, double cpu, ByteSizeValue memory) {}
32+
33+
public void testTsIdMetadataInResponse() {
34+
populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME);
35+
populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME);
36+
37+
try (EsqlQueryResponse resp = runQuery("TS hosts, cluster-a:hosts METADATA _tsid", Boolean.TRUE)) {
38+
assertNotNull(
39+
resp.columns().stream().map(ColumnInfoImpl::name).filter(name -> name.equalsIgnoreCase("_tsid")).findFirst().orElse(null)
40+
);
41+
42+
assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2);
43+
}
44+
}
45+
46+
public void testTsIdMetadataInResponseWithFailure() {
47+
populateTimeSeriesIndex(LOCAL_CLUSTER, INDEX_NAME);
48+
populateTimeSeriesIndex(REMOTE_CLUSTER_1, INDEX_NAME);
49+
50+
try (
51+
EsqlQueryResponse resp = runQuery(
52+
"TS hosts, cluster-a:hosts METADATA _tsid | WHERE host IS NOT NULL | STATS cnt = count_distinct(_tsid)",
53+
Boolean.TRUE
54+
)
55+
) {
56+
List<List<Object>> values = getValuesList(resp);
57+
assertThat(values, hasSize(1));
58+
assertNotNull(values.getFirst().getFirst());
59+
assertCCSExecutionInfoDetails(resp.getExecutionInfo(), 2);
60+
}
61+
}
62+
63+
private void populateTimeSeriesIndex(String clusterAlias, String indexName) {
64+
int numShards = randomIntBetween(1, 5);
65+
String clusterTag = Strings.isEmpty(clusterAlias) ? "local" : clusterAlias;
66+
Settings settings = Settings.builder()
67+
.put("mode", "time_series")
68+
.putList("routing_path", List.of("host", "cluster"))
69+
.put("index.number_of_shards", numShards)
70+
.build();
71+
72+
client(clusterAlias).admin()
73+
.indices()
74+
.prepareCreate(indexName)
75+
.setSettings(settings)
76+
.setMapping(
77+
"@timestamp",
78+
"type=date",
79+
"host",
80+
"type=keyword,time_series_dimension=true",
81+
"cluster",
82+
"type=keyword,time_series_dimension=true",
83+
"cpu",
84+
"type=double,time_series_metric=gauge",
85+
"memory",
86+
"type=long,time_series_metric=gauge",
87+
"request_count",
88+
"type=integer,time_series_metric=counter",
89+
"cluster_tag",
90+
"type=keyword"
91+
)
92+
.get();
93+
94+
final List<Doc> docs = getRandomDocs();
95+
96+
for (Doc doc : docs) {
97+
client().prepareIndex(indexName)
98+
.setSource(
99+
"@timestamp",
100+
doc.timestamp,
101+
"host",
102+
doc.host,
103+
"cluster",
104+
doc.cluster,
105+
"cpu",
106+
doc.cpu,
107+
"memory",
108+
doc.memory.getBytes(),
109+
"request_count",
110+
doc.requestCount,
111+
"cluster_tag",
112+
clusterTag
113+
)
114+
.get();
115+
}
116+
client().admin().indices().prepareRefresh(indexName).get();
117+
}
118+
119+
private List<Doc> getRandomDocs() {
120+
final List<Doc> docs = new ArrayList<>();
121+
122+
Map<String, String> hostToClusters = new HashMap<>();
123+
for (int i = 0; i < 5; i++) {
124+
hostToClusters.put("p" + i, randomFrom("qa", "prod"));
125+
}
126+
long timestamp = DEFAULT_DATE_TIME_FORMATTER.parseMillis("2024-04-15T00:00:00Z");
127+
128+
Map<String, Integer> requestCounts = new HashMap<>();
129+
int numDocs = between(20, 100);
130+
for (int i = 0; i < numDocs; i++) {
131+
List<String> hosts = randomSubsetOf(between(1, hostToClusters.size()), hostToClusters.keySet());
132+
timestamp += between(1, 10) * 1000L;
133+
for (String host : hosts) {
134+
var requestCount = requestCounts.compute(host, (k, curr) -> {
135+
if (curr == null || randomInt(100) <= 20) {
136+
return randomIntBetween(0, 10);
137+
} else {
138+
return curr + randomIntBetween(1, 10);
139+
}
140+
});
141+
int cpu = randomIntBetween(0, 100);
142+
ByteSizeValue memory = ByteSizeValue.ofBytes(randomIntBetween(1024, 1024 * 1024));
143+
docs.add(new Doc(host, hostToClusters.get(host), timestamp, requestCount, cpu, memory));
144+
}
145+
}
146+
147+
Randomness.shuffle(docs);
148+
149+
return docs;
150+
}
151+
152+
private static void assertCCSExecutionInfoDetails(EsqlExecutionInfo executionInfo, int expectedNumClusters) {
153+
assertNotNull(executionInfo);
154+
assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L));
155+
assertTrue(executionInfo.isCrossClusterSearch());
156+
assertThat(executionInfo.getClusters().size(), equalTo(expectedNumClusters));
157+
158+
List<EsqlExecutionInfo.Cluster> clusters = executionInfo.clusterAliases().stream().map(executionInfo::getCluster).toList();
159+
160+
for (EsqlExecutionInfo.Cluster cluster : clusters) {
161+
assertThat(cluster.getTook().millis(), greaterThanOrEqualTo(0L));
162+
assertThat(cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL));
163+
assertThat(cluster.getSkippedShards(), equalTo(0));
164+
assertThat(cluster.getFailedShards(), equalTo(0));
165+
}
166+
}
167+
168+
}

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -686,4 +686,21 @@ public void testNullMetricsAreSkipped() {
686686
assertEquals("Did not filter nulls on counter type", 50, resp.documentsFound());
687687
}
688688
}
689+
690+
public void testTSIDMetadataAttribute() {
691+
List<ColumnInfoImpl> columns = List.of(
692+
new ColumnInfoImpl("_tsid", DataType.TSID_DATA_TYPE, null),
693+
new ColumnInfoImpl("cluster", DataType.KEYWORD, null)
694+
);
695+
696+
try (EsqlQueryResponse resp = run(" TS hosts METADATA _tsid | KEEP _tsid, cluster | LIMIT 1")) {
697+
assertThat(resp.columns(), equalTo(columns));
698+
699+
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
700+
assertThat(values, hasSize(1));
701+
assertThat(values.getFirst().get(0), Matchers.notNullValue());
702+
assertThat(values.getFirst().get(1), Matchers.notNullValue());
703+
}
704+
}
705+
689706
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1558,9 +1558,12 @@ public enum Cap {
15581558

15591559
INLINE_STATS_FIX_OPTIMIZED_AS_LOCAL_RELATION(INLINESTATS_V11.enabled),
15601560

1561-
DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS
1561+
DENSE_VECTOR_AGG_METRIC_DOUBLE_IF_FNS,
15621562

1563-
;
1563+
/**
1564+
* Support for requesting the "_tsid" metadata field.
1565+
*/
1566+
METADATA_TSID_FIELD;
15641567

15651568
private final boolean enabled;
15661569

0 commit comments

Comments
 (0)