Skip to content

Commit 5cca15f

Browse files
nik9000nik9000
authored
ESQL: Drop old serialization (#134727)
Drops a bunch of serialization against versions of Elasticsearch prior to 8.18. `main` will never have to talk to such old versions again and it simplifies future changes.
1 parent 6d6a6ec commit 5cca15f

File tree

67 files changed

+229
-773
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+229
-773
lines changed

‎test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2727,16 +2727,14 @@ protected static MapMatcher getProfileMatcher() {
27272727
.entry("plans", instanceOf(List.class));
27282728
}
27292729

2730-
protected static MapMatcher getResultMatcher(boolean includeMetadata, boolean includePartial, boolean includeDocumentsFound) {
2730+
protected static MapMatcher getResultMatcher(boolean includePartial, boolean includeDocumentsFound) {
27312731
MapMatcher mapMatcher = matchesMap();
27322732
if (includeDocumentsFound) {
27332733
// Older versions may not return documents_found and values_loaded.
27342734
mapMatcher = mapMatcher.entry("documents_found", greaterThanOrEqualTo(0));
27352735
mapMatcher = mapMatcher.entry("values_loaded", greaterThanOrEqualTo(0));
27362736
}
2737-
if (includeMetadata) {
2738-
mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
2739-
}
2737+
mapMatcher = mapMatcher.entry("took", greaterThanOrEqualTo(0));
27402738
// Older version may not have is_partial
27412739
if (includePartial) {
27422740
mapMatcher = mapMatcher.entry("is_partial", false);
@@ -2748,7 +2746,7 @@ protected static MapMatcher getResultMatcher(boolean includeMetadata, boolean in
27482746
* Create empty result matcher from result, taking into account all metadata items.
27492747
*/
27502748
protected static MapMatcher getResultMatcher(Map<String, Object> result) {
2751-
return getResultMatcher(result.containsKey("took"), result.containsKey("is_partial"), result.containsKey("documents_found"));
2749+
return getResultMatcher(result.containsKey("is_partial"), result.containsKey("documents_found"));
27522750
}
27532751

27542752
/**

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
import java.util.Objects;
2323

2424
import static org.elasticsearch.TransportVersions.ESQL_FIELD_ATTRIBUTE_DROP_TYPE;
25-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
26-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
2725

2826
/**
2927
* Attribute for an ES field.
@@ -105,7 +103,7 @@ private static FieldAttribute innerReadFrom(StreamInput in) throws IOException {
105103
Source source = Source.readFrom((StreamInput & PlanStreamInput) in);
106104
String parentName = ((PlanStreamInput) in).readOptionalCachedString();
107105
String qualifier = readQualifier((PlanStreamInput) in, in.getTransportVersion());
108-
String name = readCachedStringWithVersionCheck(in);
106+
String name = ((PlanStreamInput) in).readCachedString();
109107
if (in.getTransportVersion().before(ESQL_FIELD_ATTRIBUTE_DROP_TYPE)) {
110108
DataType.readFrom(in);
111109
}
@@ -125,7 +123,7 @@ public void writeTo(StreamOutput out) throws IOException {
125123
Source.EMPTY.writeTo(out);
126124
((PlanStreamOutput) out).writeOptionalCachedString(parentName);
127125
checkAndSerializeQualifier((PlanStreamOutput) out, out.getTransportVersion());
128-
writeCachedStringWithVersionCheck(out, name());
126+
((PlanStreamOutput) out).writeCachedString(name());
129127
if (out.getTransportVersion().before(ESQL_FIELD_ATTRIBUTE_DROP_TYPE)) {
130128
dataType().writeTo(out);
131129
}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java‎

Lines changed: 2 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import org.apache.lucene.util.Accountable;
1010
import org.apache.lucene.util.BytesRef;
1111
import org.apache.lucene.util.RamUsageEstimator;
12-
import org.elasticsearch.TransportVersions;
1312
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1413
import org.elasticsearch.common.io.stream.StreamInput;
1514
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -24,10 +23,8 @@
2423
import java.io.IOException;
2524
import java.time.Duration;
2625
import java.util.Collection;
27-
import java.util.List;
2826
import java.util.Objects;
2927

30-
import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
3128
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
3229
import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
3330
import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
@@ -82,13 +79,13 @@ private static Literal readFrom(StreamInput in) throws IOException {
8279
Source source = Source.readFrom((StreamInput & PlanStreamInput) in);
8380
Object value = in.readGenericValue();
8481
DataType dataType = DataType.readFrom(in);
85-
return new Literal(source, mapToLiteralValue(in, dataType, value), dataType);
82+
return new Literal(source, value, dataType);
8683
}
8784

8885
@Override
8986
public void writeTo(StreamOutput out) throws IOException {
9087
Source.EMPTY.writeTo(out);
91-
out.writeGenericValue(mapFromLiteralValue(out, dataType, value));
88+
out.writeGenericValue(value);
9289
dataType.writeTo(out);
9390
}
9491

@@ -232,44 +229,6 @@ public static Literal fromLong(Source source, Long literal) {
232229
return new Literal(source, literal, LONG);
233230
}
234231

235-
/**
236-
* Not all literal values are currently supported in StreamInput/StreamOutput as generic values.
237-
* This mapper allows for addition of new and interesting values without (yet) adding to StreamInput/Output.
238-
* This makes the most sense during the pre-GA version of ESQL. When we get near GA we might want to push this down.
239-
* <p>
240-
* For the spatial point type support we need to care about the fact that 8.12.0 uses encoded longs for serializing
241-
* while 8.13 uses WKB.
242-
*/
243-
private static Object mapFromLiteralValue(StreamOutput out, DataType dataType, Object value) {
244-
if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) {
245-
// In 8.12.0 we serialized point literals as encoded longs, but now use WKB
246-
if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) {
247-
if (value instanceof List<?> list) {
248-
return list.stream().map(v -> mapFromLiteralValue(out, dataType, v)).toList();
249-
}
250-
return wkbAsLong(dataType, (BytesRef) value);
251-
}
252-
}
253-
return value;
254-
}
255-
256-
/**
257-
* Not all literal values are currently supported in StreamInput/StreamOutput as generic values.
258-
* This mapper allows for addition of new and interesting values without (yet) changing StreamInput/Output.
259-
*/
260-
private static Object mapToLiteralValue(StreamInput in, DataType dataType, Object value) {
261-
if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) {
262-
// In 8.12.0 we serialized point literals as encoded longs, but now use WKB
263-
if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) {
264-
if (value instanceof List<?> list) {
265-
return list.stream().map(v -> mapToLiteralValue(in, dataType, v)).toList();
266-
}
267-
return longAsWKB(dataType, (Long) value);
268-
}
269-
}
270-
return value;
271-
}
272-
273232
private static BytesRef longAsWKB(DataType dataType, long encoded) {
274233
return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded);
275234
}

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import org.elasticsearch.index.mapper.SourceFieldMapper;
1515
import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper;
1616
import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
17+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
18+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1719

1820
import java.io.IOException;
1921
import java.math.BigInteger;
@@ -30,8 +32,6 @@
3032
import java.util.function.Function;
3133

3234
import static java.util.stream.Collectors.toMap;
33-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
34-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
3535

3636
/**
3737
* This enum represents data types the ES|QL query processing layer is able to
@@ -718,12 +718,11 @@ public DataType counter() {
718718
}
719719

720720
public void writeTo(StreamOutput out) throws IOException {
721-
writeCachedStringWithVersionCheck(out, typeName);
721+
((PlanStreamOutput) out).writeCachedString(typeName);
722722
}
723723

724724
public static DataType readFrom(StreamInput in) throws IOException {
725-
// TODO: Use our normal enum serialization pattern
726-
return readFrom(readCachedStringWithVersionCheck(in));
725+
return readFrom(((PlanStreamInput) in).readCachedString());
727726
}
728727

729728
/**

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@
88

99
import org.elasticsearch.common.io.stream.StreamInput;
1010
import org.elasticsearch.common.io.stream.StreamOutput;
11+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
12+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1113

1214
import java.io.IOException;
1315
import java.util.Map;
1416

15-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
16-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
17-
1817
/**
1918
* Information about a field in an ES index with the {@code date} type
2019
*/
@@ -36,7 +35,7 @@ private DateEsField(
3635

3736
protected DateEsField(StreamInput in) throws IOException {
3837
this(
39-
readCachedStringWithVersionCheck(in),
38+
((PlanStreamInput) in).readCachedString(),
4039
DataType.DATETIME,
4140
in.readImmutableMap(EsField::readFrom),
4241
in.readBoolean(),
@@ -46,7 +45,7 @@ protected DateEsField(StreamInput in) throws IOException {
4645

4746
@Override
4847
public void writeContent(StreamOutput out) throws IOException {
49-
writeCachedStringWithVersionCheck(out, getName());
48+
((PlanStreamOutput) out).writeCachedString(getName());
5049
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
5150
out.writeBoolean(isAggregatable());
5251
writeTimeSeriesFieldType(out);

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
package org.elasticsearch.xpack.esql.core.type;
88

99
import org.elasticsearch.TransportVersion;
10-
import org.elasticsearch.TransportVersions;
1110
import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities;
1211
import org.elasticsearch.common.io.stream.StreamInput;
1312
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -20,9 +19,6 @@
2019
import java.util.Map;
2120
import java.util.Objects;
2221

23-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
24-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
25-
2622
/**
2723
* Information about a field in an ES index.
2824
*/
@@ -157,30 +153,14 @@ public EsField(
157153
}
158154

159155
public EsField(StreamInput in) throws IOException {
160-
this.name = readCachedStringWithVersionCheck(in);
161-
this.esDataType = readDataType(in);
156+
this.name = ((PlanStreamInput) in).readCachedString();
157+
this.esDataType = DataType.readFrom(in);
162158
this.properties = in.readImmutableMap(EsField::readFrom);
163159
this.aggregatable = in.readBoolean();
164160
this.isAlias = in.readBoolean();
165161
this.timeSeriesFieldType = readTimeSeriesFieldType(in);
166162
}
167163

168-
private DataType readDataType(StreamInput in) throws IOException {
169-
String name = readCachedStringWithVersionCheck(in);
170-
if (in.getTransportVersion().before(TransportVersions.V_8_16_0) && name.equalsIgnoreCase("NESTED")) {
171-
/*
172-
* The "nested" data type existed in older versions of ESQL but was
173-
* entirely used to filter mappings away. Those versions will still
174-
* sometimes send it inside EsField when hitting `nested` fields in
175-
* indices. But the rest of ESQL will never see that type. Thus, we
176-
* translate it here. We translate to UNSUPPORTED because that seems
177-
* to work. We've already performed any required filtering.
178-
*/
179-
return DataType.UNSUPPORTED;
180-
}
181-
return DataType.readFrom(name);
182-
}
183-
184164
public static <A extends EsField> A readFrom(StreamInput in) throws IOException {
185165
return ((PlanStreamInput) in).readEsFieldWithCache();
186166
}
@@ -196,7 +176,7 @@ public void writeTo(StreamOutput out) throws IOException {
196176
* This needs to be overridden by subclasses for specific serialization
197177
*/
198178
public void writeContent(StreamOutput out) throws IOException {
199-
writeCachedStringWithVersionCheck(out, name);
179+
((PlanStreamOutput) out).writeCachedString(name);
200180
esDataType.writeTo(out);
201181
out.writeMap(properties, (o, x) -> x.writeTo(out));
202182
out.writeBoolean(aggregatable);

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
import org.elasticsearch.common.io.stream.StreamInput;
1111
import org.elasticsearch.common.io.stream.StreamOutput;
1212
import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
13+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
14+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1315

1416
import java.io.IOException;
1517
import java.util.Map;
@@ -18,9 +20,6 @@
1820
import java.util.TreeMap;
1921
import java.util.stream.Collectors;
2022

21-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
22-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
23-
2423
/**
2524
* Representation of field mapped differently across indices.
2625
* Used during mapping discovery only.
@@ -62,7 +61,7 @@ private InvalidMappedField(
6261

6362
protected InvalidMappedField(StreamInput in) throws IOException {
6463
this(
65-
readCachedStringWithVersionCheck(in),
64+
((PlanStreamInput) in).readCachedString(),
6665
in.readString(),
6766
in.readImmutableMap(StreamInput::readString, EsField::readFrom),
6867
Map.of(),
@@ -76,7 +75,7 @@ public Set<DataType> types() {
7675

7776
@Override
7877
public void writeContent(StreamOutput out) throws IOException {
79-
writeCachedStringWithVersionCheck(out, getName());
78+
((PlanStreamOutput) out).writeCachedString(getName());
8079
out.writeString(errorMessage);
8180
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
8281
writeTimeSeriesFieldType(out);

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@
88

99
import org.elasticsearch.common.io.stream.StreamInput;
1010
import org.elasticsearch.common.io.stream.StreamOutput;
11+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
12+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1113

1214
import java.io.IOException;
1315
import java.util.Map;
1416
import java.util.Objects;
1517

1618
import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
17-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
18-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
1919

2020
/**
2121
* Information about a field in an ES index with the {@code keyword} type.
@@ -54,7 +54,7 @@ protected KeywordEsField(
5454

5555
public KeywordEsField(StreamInput in) throws IOException {
5656
this(
57-
readCachedStringWithVersionCheck(in),
57+
((PlanStreamInput) in).readCachedString(),
5858
KEYWORD,
5959
in.readImmutableMap(EsField::readFrom),
6060
in.readBoolean(),
@@ -67,7 +67,7 @@ public KeywordEsField(StreamInput in) throws IOException {
6767

6868
@Override
6969
public void writeContent(StreamOutput out) throws IOException {
70-
writeCachedStringWithVersionCheck(out, getName());
70+
((PlanStreamOutput) out).writeCachedString(getName());
7171
out.writeMap(getProperties(), (o, x) -> x.writeTo(out));
7272
out.writeBoolean(isAggregatable());
7373
out.writeInt(precision);

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,15 @@
1010
import org.elasticsearch.common.io.stream.StreamInput;
1111
import org.elasticsearch.common.io.stream.StreamOutput;
1212
import org.elasticsearch.xpack.esql.core.expression.Expression;
13+
import org.elasticsearch.xpack.esql.core.util.PlanStreamInput;
14+
import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput;
1315

1416
import java.io.IOException;
1517
import java.util.HashMap;
1618
import java.util.Map;
1719
import java.util.Objects;
1820
import java.util.Set;
1921

20-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamInput.readCachedStringWithVersionCheck;
21-
import static org.elasticsearch.xpack.esql.core.util.PlanStreamOutput.writeCachedStringWithVersionCheck;
22-
2322
/**
2423
* During IndexResolution it could occur that the same field is mapped to different types in different indices.
2524
* The class MultiTypeEfField.UnresolvedField holds that information and allows for later resolution of the field
@@ -46,7 +45,7 @@ public MultiTypeEsField(
4645

4746
protected MultiTypeEsField(StreamInput in) throws IOException {
4847
this(
49-
readCachedStringWithVersionCheck(in),
48+
((PlanStreamInput) in).readCachedString(),
5049
DataType.readFrom(in),
5150
in.readBoolean(),
5251
in.readImmutableMap(i -> i.readNamedWriteable(Expression.class)),
@@ -56,7 +55,7 @@ protected MultiTypeEsField(StreamInput in) throws IOException {
5655

5756
@Override
5857
public void writeContent(StreamOutput out) throws IOException {
59-
writeCachedStringWithVersionCheck(out, getName());
58+
((PlanStreamOutput) out).writeCachedString(getName());
6059
getDataType().writeTo(out);
6160
out.writeBoolean(isAggregatable());
6261
out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v));

0 commit comments

Comments
 (0)