Skip to content

Commit a7a79f7

Browse files
not-napoleonnot-napoleon
authored
ESQL - transport version change to support TSDB metadata (#129649)
Relates to #128621 This is a first step in making the ES|QL query planner aware of TSDB Dimensions and Metric field metadata. This is purposefully small to only touch the serialization change we'll need for this. The plan is get the TSDB metadata field type out of Field Caps and to store this information on EsField. This PR adds a place to store such a field, and adds it to the serialization for EsField and its sub-classes. As of this PR, we don't do anything with this data. That's intentional, to minimize the footprint of the transport version change. Further PRs in this project will load and act on this data. I've added some constructors here to minimize the number of files I'm touching in this PR. I hope that as we begin loading this data (as opposed to just defaulting it right now) we can get rid of some of these default value constructors.
1 parent 136442d commit a7a79f7

File tree

10 files changed

+185
-28
lines changed

10 files changed

+185
-28
lines changed

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,7 @@ static TransportVersion def(int id) {
327327
public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN = def(9_111_0_00);
328328
public static final TransportVersion MAPPINGS_IN_DATA_STREAMS = def(9_112_0_00);
329329

330+
public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_113_0_00);
330331
/*
331332
* STOP! READ THIS FIRST! No, really,
332333
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,22 +21,35 @@
2121
public class DateEsField extends EsField {
2222

2323
public static DateEsField dateEsField(String name, Map<String, EsField> properties, boolean hasDocValues) {
24-
return new DateEsField(name, DataType.DATETIME, properties, hasDocValues);
24+
return new DateEsField(name, DataType.DATETIME, properties, hasDocValues, TimeSeriesFieldType.UNKNOWN);
2525
}
2626

27-
private DateEsField(String name, DataType dataType, Map<String, EsField> properties, boolean hasDocValues) {
28-
super(name, dataType, properties, hasDocValues);
27+
private DateEsField(
28+
String name,
29+
DataType dataType,
30+
Map<String, EsField> properties,
31+
boolean hasDocValues,
32+
TimeSeriesFieldType timeSeriesFieldType
33+
) {
34+
super(name, dataType, properties, hasDocValues, timeSeriesFieldType);
2935
}
3036

3137
protected DateEsField(StreamInput in) throws IOException {
32-
this(readCachedStringWithVersionCheck(in), DataType.DATETIME, in.readImmutableMap(EsField::readFrom), in.readBoolean());
38+
this(
39+
readCachedStringWithVersionCheck(in),
40+
DataType.DATETIME,
41+
in.readImmutableMap(EsField::readFrom),
42+
in.readBoolean(),
43+
readTimeSeriesFieldType(in)
44+
);
3345
}
3446

3547
@Override
3648
public void writeContent(StreamOutput out) throws IOException {
3749
writeCachedStringWithVersionCheck(out, getName());
3850
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
3951
out.writeBoolean(isAggregatable());
52+
writeTimeSeriesFieldType(out);
4053
}
4154

4255
public String getWriteableName() {

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,40 @@
2626
*/
2727
public class EsField implements Writeable {
2828

29-
private static Map<String, Writeable.Reader<? extends EsField>> readers = Map.ofEntries(
29+
/**
30+
* Fields in a TSDB can be either dimensions or metrics. This enum provides a way to store, serialize, and operate on those field
31+
* roles within the ESQL query processing pipeline.
32+
*/
33+
public enum TimeSeriesFieldType implements Writeable {
34+
UNKNOWN(0),
35+
NONE(1),
36+
METRIC(2),
37+
DIMENSION(3);
38+
39+
private final int id;
40+
41+
TimeSeriesFieldType(int id) {
42+
this.id = id;
43+
}
44+
45+
@Override
46+
public void writeTo(StreamOutput out) throws IOException {
47+
out.writeVInt(id);
48+
}
49+
50+
public static TimeSeriesFieldType readFromStream(StreamInput in) throws IOException {
51+
int id = in.readVInt();
52+
return switch (id) {
53+
case 0 -> UNKNOWN;
54+
case 1 -> NONE;
55+
case 2 -> METRIC;
56+
case 3 -> DIMENSION;
57+
default -> throw new IOException("Unexpected value for TimeSeriesFieldType: " + id);
58+
};
59+
}
60+
}
61+
62+
private static Map<String, Reader<? extends EsField>> readers = Map.ofEntries(
3063
Map.entry("DateEsField", DateEsField::new),
3164
Map.entry("EsField", EsField::new),
3265
Map.entry("InvalidMappedField", InvalidMappedField::new),
@@ -37,7 +70,7 @@ public class EsField implements Writeable {
3770
Map.entry("UnsupportedEsField", UnsupportedEsField::new)
3871
);
3972

40-
public static Writeable.Reader<? extends EsField> getReader(String name) {
73+
public static Reader<? extends EsField> getReader(String name) {
4174
Reader<? extends EsField> result = readers.get(name);
4275
if (result == null) {
4376
throw new IllegalArgumentException("Invalid EsField type [" + name + "]");
@@ -50,17 +83,41 @@ public static Writeable.Reader<? extends EsField> getReader(String name) {
5083
private final Map<String, EsField> properties;
5184
private final String name;
5285
private final boolean isAlias;
86+
// Because the subclasses all reimplement serialization, this needs to be writeable from subclass constructors
87+
private final TimeSeriesFieldType timeSeriesFieldType;
5388

5489
public EsField(String name, DataType esDataType, Map<String, EsField> properties, boolean aggregatable) {
55-
this(name, esDataType, properties, aggregatable, false);
90+
this(name, esDataType, properties, aggregatable, false, TimeSeriesFieldType.UNKNOWN);
91+
}
92+
93+
public EsField(
94+
String name,
95+
DataType esDataType,
96+
Map<String, EsField> properties,
97+
boolean aggregatable,
98+
TimeSeriesFieldType timeSeriesFieldType
99+
) {
100+
this(name, esDataType, properties, aggregatable, false, timeSeriesFieldType);
56101
}
57102

58103
public EsField(String name, DataType esDataType, Map<String, EsField> properties, boolean aggregatable, boolean isAlias) {
104+
this(name, esDataType, properties, aggregatable, isAlias, TimeSeriesFieldType.UNKNOWN);
105+
}
106+
107+
public EsField(
108+
String name,
109+
DataType esDataType,
110+
Map<String, EsField> properties,
111+
boolean aggregatable,
112+
boolean isAlias,
113+
TimeSeriesFieldType timeSeriesFieldType
114+
) {
59115
this.name = name;
60116
this.esDataType = esDataType;
61117
this.aggregatable = aggregatable;
62118
this.properties = properties;
63119
this.isAlias = isAlias;
120+
this.timeSeriesFieldType = timeSeriesFieldType;
64121
}
65122

66123
public EsField(StreamInput in) throws IOException {
@@ -69,6 +126,7 @@ public EsField(StreamInput in) throws IOException {
69126
this.properties = in.readImmutableMap(EsField::readFrom);
70127
this.aggregatable = in.readBoolean();
71128
this.isAlias = in.readBoolean();
129+
this.timeSeriesFieldType = readTimeSeriesFieldType(in);
72130
}
73131

74132
private DataType readDataType(StreamInput in) throws IOException {
@@ -107,6 +165,21 @@ public void writeContent(StreamOutput out) throws IOException {
107165
out.writeMap(properties, (o, x) -> x.writeTo(out));
108166
out.writeBoolean(aggregatable);
109167
out.writeBoolean(isAlias);
168+
writeTimeSeriesFieldType(out);
169+
}
170+
171+
protected void writeTimeSeriesFieldType(StreamOutput out) throws IOException {
172+
if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE)) {
173+
this.timeSeriesFieldType.writeTo(out);
174+
}
175+
}
176+
177+
protected static TimeSeriesFieldType readTimeSeriesFieldType(StreamInput in) throws IOException {
178+
if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE)) {
179+
return TimeSeriesFieldType.readFromStream(in);
180+
} else {
181+
return TimeSeriesFieldType.UNKNOWN;
182+
}
110183
}
111184

112185
/**

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public class InvalidMappedField extends EsField {
3434
private final Map<String, Set<String>> typesToIndices;
3535

3636
public InvalidMappedField(String name, String errorMessage, Map<String, EsField> properties) {
37-
this(name, errorMessage, properties, Map.of());
37+
this(name, errorMessage, properties, Map.of(), TimeSeriesFieldType.UNKNOWN);
3838
}
3939

4040
public InvalidMappedField(String name, String errorMessage) {
@@ -45,17 +45,29 @@ public InvalidMappedField(String name, String errorMessage) {
4545
* Constructor supporting union types, used in ES|QL.
4646
*/
4747
public InvalidMappedField(String name, Map<String, Set<String>> typesToIndices) {
48-
this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices);
48+
this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices, TimeSeriesFieldType.UNKNOWN);
4949
}
5050

51-
private InvalidMappedField(String name, String errorMessage, Map<String, EsField> properties, Map<String, Set<String>> typesToIndices) {
52-
super(name, DataType.UNSUPPORTED, properties, false);
51+
private InvalidMappedField(
52+
String name,
53+
String errorMessage,
54+
Map<String, EsField> properties,
55+
Map<String, Set<String>> typesToIndices,
56+
TimeSeriesFieldType type
57+
) {
58+
super(name, DataType.UNSUPPORTED, properties, false, type);
5359
this.errorMessage = errorMessage;
5460
this.typesToIndices = typesToIndices;
5561
}
5662

5763
protected InvalidMappedField(StreamInput in) throws IOException {
58-
this(readCachedStringWithVersionCheck(in), in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom));
64+
this(
65+
readCachedStringWithVersionCheck(in),
66+
in.readString(),
67+
in.readImmutableMap(StreamInput::readString, EsField::readFrom),
68+
Map.of(),
69+
readTimeSeriesFieldType(in)
70+
);
5971
}
6072

6173
public Set<DataType> types() {
@@ -67,6 +79,7 @@ public void writeContent(StreamOutput out) throws IOException {
6779
writeCachedStringWithVersionCheck(out, getName());
6880
out.writeString(errorMessage);
6981
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
82+
writeTimeSeriesFieldType(out);
7083
}
7184

7285
public String getWriteableName() {

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public KeywordEsField(
4242
boolean normalized,
4343
boolean isAlias
4444
) {
45-
this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias);
45+
this(name, KEYWORD, properties, hasDocValues, precision, normalized, isAlias, TimeSeriesFieldType.UNKNOWN);
4646
}
4747

4848
protected KeywordEsField(
@@ -52,9 +52,10 @@ protected KeywordEsField(
5252
boolean hasDocValues,
5353
int precision,
5454
boolean normalized,
55-
boolean isAlias
55+
boolean isAlias,
56+
TimeSeriesFieldType timeSeriesFieldType
5657
) {
57-
super(name, esDataType, properties, hasDocValues, isAlias);
58+
super(name, esDataType, properties, hasDocValues, isAlias, timeSeriesFieldType);
5859
this.precision = precision;
5960
this.normalized = normalized;
6061
}
@@ -67,7 +68,8 @@ public KeywordEsField(StreamInput in) throws IOException {
6768
in.readBoolean(),
6869
in.readInt(),
6970
in.readBoolean(),
70-
in.readBoolean()
71+
in.readBoolean(),
72+
readTimeSeriesFieldType(in)
7173
);
7274
}
7375

@@ -79,6 +81,7 @@ public void writeContent(StreamOutput out) throws IOException {
7981
out.writeInt(precision);
8082
out.writeBoolean(normalized);
8183
out.writeBoolean(isAlias());
84+
writeTimeSeriesFieldType(out);
8285
}
8386

8487
public String getWriteableName() {

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,24 @@ public MultiTypeEsField(String name, DataType dataType, boolean aggregatable, Ma
3838
this.indexToConversionExpressions = indexToConversionExpressions;
3939
}
4040

41+
public MultiTypeEsField(
42+
String name,
43+
DataType dataType,
44+
boolean aggregatable,
45+
Map<String, Expression> indexToConversionExpressions,
46+
TimeSeriesFieldType timeSeriesFieldType
47+
) {
48+
super(name, dataType, Map.of(), aggregatable, timeSeriesFieldType);
49+
this.indexToConversionExpressions = indexToConversionExpressions;
50+
}
51+
4152
protected MultiTypeEsField(StreamInput in) throws IOException {
4253
this(
4354
readCachedStringWithVersionCheck(in),
4455
DataType.readFrom(in),
4556
in.readBoolean(),
46-
in.readImmutableMap(i -> i.readNamedWriteable(Expression.class))
57+
in.readImmutableMap(i -> i.readNamedWriteable(Expression.class)),
58+
readTimeSeriesFieldType(in)
4759
);
4860
}
4961

@@ -53,6 +65,7 @@ public void writeContent(StreamOutput out) throws IOException {
5365
getDataType().writeTo(out);
5466
out.writeBoolean(isAggregatable());
5567
out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v));
68+
writeTimeSeriesFieldType(out);
5669
}
5770

5871
public String getWriteableName() {

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,27 @@ public TextEsField(String name, Map<String, EsField> properties, boolean hasDocV
3030
}
3131

3232
public TextEsField(String name, Map<String, EsField> properties, boolean hasDocValues, boolean isAlias) {
33-
super(name, TEXT, properties, hasDocValues, isAlias);
33+
super(name, TEXT, properties, hasDocValues, isAlias, TimeSeriesFieldType.UNKNOWN);
34+
}
35+
36+
public TextEsField(
37+
String name,
38+
Map<String, EsField> properties,
39+
boolean hasDocValues,
40+
boolean isAlias,
41+
TimeSeriesFieldType timeSeriesFieldType
42+
) {
43+
super(name, TEXT, properties, hasDocValues, isAlias, timeSeriesFieldType);
3444
}
3545

3646
protected TextEsField(StreamInput in) throws IOException {
37-
this(readCachedStringWithVersionCheck(in), in.readImmutableMap(EsField::readFrom), in.readBoolean(), in.readBoolean());
47+
this(
48+
readCachedStringWithVersionCheck(in),
49+
in.readImmutableMap(EsField::readFrom),
50+
in.readBoolean(),
51+
in.readBoolean(),
52+
readTimeSeriesFieldType(in)
53+
);
3854
}
3955

4056
@Override
@@ -43,6 +59,7 @@ public void writeContent(StreamOutput out) throws IOException {
4359
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
4460
out.writeBoolean(isAggregatable());
4561
out.writeBoolean(isAlias());
62+
writeTimeSeriesFieldType(out);
4663
}
4764

4865
public String getWriteableName() {

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,29 @@ public UnsupportedEsField(String name, List<String> originalTypes) {
3535
}
3636

3737
public UnsupportedEsField(String name, List<String> originalTypes, String inherited, Map<String, EsField> properties) {
38-
super(name, DataType.UNSUPPORTED, properties, false);
38+
this(name, originalTypes, inherited, properties, TimeSeriesFieldType.UNKNOWN);
39+
}
40+
41+
public UnsupportedEsField(
42+
String name,
43+
List<String> originalTypes,
44+
String inherited,
45+
Map<String, EsField> properties,
46+
TimeSeriesFieldType timeSeriesFieldType
47+
) {
48+
super(name, DataType.UNSUPPORTED, properties, false, timeSeriesFieldType);
3949
this.originalTypes = originalTypes;
4050
this.inherited = inherited;
4151
}
4252

4353
public UnsupportedEsField(StreamInput in) throws IOException {
44-
this(readCachedStringWithVersionCheck(in), readOriginalTypes(in), in.readOptionalString(), in.readImmutableMap(EsField::readFrom));
54+
this(
55+
readCachedStringWithVersionCheck(in),
56+
readOriginalTypes(in),
57+
in.readOptionalString(),
58+
in.readImmutableMap(EsField::readFrom),
59+
readTimeSeriesFieldType(in)
60+
);
4561
}
4662

4763
private static List<String> readOriginalTypes(StreamInput in) throws IOException {
@@ -64,6 +80,7 @@ public void writeContent(StreamOutput out) throws IOException {
6480
}
6581
out.writeOptionalString(getInherited());
6682
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
83+
writeTimeSeriesFieldType(out);
6784
}
6885

6986
public String getWriteableName() {

0 commit comments

Comments
 (0)