Skip to content

Commit 2d1c02d

Browse files
committed
Commented out float features, everything pass either way expect a single bug arpung the exostence of COUNTER_FLOAT.
All test pass with the comments. Without them ther obe which fail is in EsqlQueryRerponseTest.
1 parent f91cc68 commit 2d1c02d

File tree

13 files changed

+114
-14
lines changed

13 files changed

+114
-14
lines changed

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77
package org.elasticsearch.xpack.esql.core.type;
88

9+
import org.apache.logging.log4j.core.util.Booleans;
910
import org.apache.lucene.util.BytesRef;
1011
import org.elasticsearch.common.io.stream.StreamInput;
1112
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -140,6 +141,7 @@
140141
* </ul>
141142
*/
142143
public enum DataType {
144+
143145
/**
144146
* Fields of this type are unsupported by any functions and are always
145147
* rendered as {@code null} in the response.
@@ -182,7 +184,15 @@ public enum DataType {
182184
* aggregation, and casting to their parent numeric type.
183185
*/
184186
COUNTER_DOUBLE(builder().esType("counter_double").estimatedSize(Double.BYTES).docValues().counter()),
185-
187+
/**
188+
* 32-bit floating point numbers labeled as metric counters in time-series indices.
189+
* Although stored internally as numeric fields, they represent cumulative
190+
* metrics and must not be treated as regular numeric fields. Therefore,
191+
* we define them differently and separately from their parent numeric field.
192+
* These fields are strictly for use in retrieval from indices, rate
193+
* aggregation, and casting to their parent numeric type.
194+
*/
195+
//COUNTER_FLOAT(builder().esType("counter_float").estimatedSize(Float.BYTES).docValues().counter()),
186196
/**
187197
* 64-bit signed numbers loaded as a java {@code long}.
188198
*/
@@ -217,7 +227,7 @@ public enum DataType {
217227
* Values of this type never escape type resolution and functions,
218228
* operators, and results should never encounter one.
219229
*/
220-
FLOAT(builder().esType("float").estimatedSize(Float.BYTES).rationalNumber().docValues().widenSmallNumeric(DOUBLE)),
230+
FLOAT(builder().esType("float").estimatedSize(Float.BYTES).rationalNumber().docValues()/*.counter(COUNTER_FLOAT)*/.widenSmallNumeric(DOUBLE)),
221231
/**
222232
* 16-bit floating point numbers widened on load to {@link #DOUBLE}.
223233
* Values of this type never escape type resolution and functions,
@@ -559,7 +569,7 @@ public static boolean isRepresentable(DataType t) {
559569
}
560570

561571
public static boolean isCounter(DataType t) {
562-
return t == COUNTER_DOUBLE || t == COUNTER_INTEGER || t == COUNTER_LONG;
572+
return Set.of(COUNTER_DOUBLE, COUNTER_INTEGER, COUNTER_LONG).contains(t);
563573
}
564574

565575
public static boolean isSpatialPoint(DataType t) {
@@ -654,7 +664,10 @@ public boolean isCounter() {
654664
* widen it into, otherwise this returns {@code this}.
655665
*/
656666
public DataType widenSmallNumeric() {
657-
return widenSmallNumeric == null ? this : widenSmallNumeric;
667+
return switch (this) {
668+
case FLOAT -> NATIVE_FLOATS_FEATURE_FLAG ? FLOAT : DOUBLE;
669+
default -> widenSmallNumeric == null ? this : widenSmallNumeric;
670+
};
658671
}
659672

660673
/**
@@ -843,4 +856,5 @@ Builder counter(DataType counter) {
843856
return this;
844857
}
845858
}
859+
public static final boolean NATIVE_FLOATS_FEATURE_FLAG = Booleans.parseBoolean(System.getProperty("esql_native_floats_enabled"), false);
846860
}

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/EvaluatorImplementer.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT;
4545
import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR;
4646
import static org.elasticsearch.compute.gen.Types.EXPRESSION_EVALUATOR_FACTORY;
47+
import static org.elasticsearch.compute.gen.Types.FLOAT_BLOCK;
4748
import static org.elasticsearch.compute.gen.Types.INT_BLOCK;
4849
import static org.elasticsearch.compute.gen.Types.LONG_BLOCK;
4950
import static org.elasticsearch.compute.gen.Types.PAGE;
@@ -1091,6 +1092,8 @@ static boolean isBlockType(TypeName type) {
10911092
return type.equals(INT_BLOCK)
10921093
|| type.equals(LONG_BLOCK)
10931094
|| type.equals(DOUBLE_BLOCK)
1095+
// TODO NATIVE FLOATS
1096+
// || type.equals(FLOAT_BLOCK)
10941097
|| type.equals(BOOLEAN_BLOCK)
10951098
|| type.equals(BYTES_REF_BLOCK);
10961099
}

x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR_BUILDER;
4040
import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR_FIXED_BUILDER;
4141
import static org.elasticsearch.compute.gen.Types.FLOAT_BLOCK_BUILDER;
42+
import static org.elasticsearch.compute.gen.Types.FLOAT_VECTOR;
4243
import static org.elasticsearch.compute.gen.Types.FLOAT_VECTOR_BUILDER;
4344
import static org.elasticsearch.compute.gen.Types.FLOAT_VECTOR_FIXED_BUILDER;
4445
import static org.elasticsearch.compute.gen.Types.INT_BLOCK;
@@ -226,7 +227,7 @@ static String appendMethod(TypeName t) {
226227
if (t.equals(TypeName.DOUBLE) || t.equals(DOUBLE_BLOCK) || t.equals(DOUBLE_VECTOR)) {
227228
return "appendDouble";
228229
}
229-
if (t.equals(TypeName.FLOAT) || t.equals(FLOAT_BLOCK_BUILDER)) {
230+
if (t.equals(TypeName.FLOAT) || t.equals(FLOAT_BLOCK_BUILDER) /* TODO NATIVE FLOATS || t.equals(FLOAT_VECTOR) */ ) {
230231
return "appendFloat";
231232
}
232233
throw new IllegalArgumentException("unknown append method for [" + t + "]");
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
apply plugin: 'elasticsearch.internal-java-rest-test'
2+
apply plugin: 'elasticsearch.internal-yaml-rest-test'
3+
// Necessary to use tests in Serverless
4+
apply plugin: 'elasticsearch.internal-test-artifact'
5+
apply plugin: org.elasticsearch.gradle.internal.precommit.CheckstylePrecommitPlugin
6+
apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenApisPrecommitPlugin
7+
apply plugin: org.elasticsearch.gradle.internal.precommit.ForbiddenPatternsPrecommitPlugin
8+
apply plugin: org.elasticsearch.gradle.internal.precommit.FilePermissionsPrecommitPlugin
9+
apply plugin: org.elasticsearch.gradle.internal.precommit.LoggerUsagePrecommitPlugin
10+
apply plugin: org.elasticsearch.gradle.internal.precommit.TestingConventionsPrecommitPlugin
11+
12+
dependencies {
13+
javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
14+
javaRestTestImplementation project(xpackModule('esql:qa:server'))
15+
javaRestTestImplementation project(xpackModule('esql:tools'))
16+
javaRestTestImplementation project(xpackModule('esql'))
17+
yamlRestTestImplementation project(xpackModule('esql:qa:server'))
18+
19+
javaRestTestImplementation('org.apache.arrow:arrow-vector:18.3.0')
20+
javaRestTestImplementation('org.apache.arrow:arrow-format:18.3.0')
21+
javaRestTestImplementation('org.apache.arrow:arrow-memory-core:18.3.0')
22+
javaRestTestImplementation('org.checkerframework:checker-qual:3.42.0')
23+
javaRestTestImplementation('com.google.flatbuffers:flatbuffers-java:23.5.26')
24+
javaRestTestImplementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
25+
javaRestTestImplementation("com.fasterxml.jackson.core:jackson-core:${versions.jackson}")
26+
javaRestTestImplementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson}")
27+
javaRestTestImplementation("org.slf4j:slf4j-api:${versions.slf4j}")
28+
javaRestTestImplementation("org.slf4j:slf4j-nop:${versions.slf4j}")
29+
javaRestTestImplementation('org.apache.arrow:arrow-memory-unsafe:18.3.0')
30+
31+
clusterPlugins project(':plugins:mapper-size')
32+
clusterPlugins project(':plugins:mapper-murmur3')
33+
clusterPlugins project(':x-pack:plugin:inference:qa:test-service-plugin')
34+
}
35+
36+
restResources {
37+
restApi {
38+
include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster', 'capabilities', 'index'
39+
}
40+
restTests {
41+
includeXpack 'esql'
42+
}
43+
}
44+
45+
tasks.named('javaRestTest') {
46+
usesDefaultDistribution("to be triaged")
47+
maxParallelForks = 1
48+
jvmArgs('--add-opens=java.base/java.nio=ALL-UNNAMED', 'esql_native_floats_data_type=true')
49+
}
50+
51+
tasks.named('yamlRestTest') {
52+
usesDefaultDistribution("to be triaged")
53+
maxParallelForks = 1
54+
}

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -827,7 +827,7 @@ public static Literal randomLiteral(DataType type) {
827827
case DATETIME -> randomMillisUpToYear9999();
828828
case DATE_NANOS -> randomLongBetween(0, Long.MAX_VALUE);
829829
case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble();
830-
case FLOAT -> randomFloat();
830+
case FLOAT /* TODO native floats ,COUNTER_FLOAT */ -> randomFloat();
831831
case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));
832832
case KEYWORD -> new BytesRef(randomAlphaOfLength(5));
833833
case IP -> new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())));

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,14 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
8484
return builder.value(((DoubleBlock) block).getDouble(valueIndex));
8585
}
8686
};
87+
/* TODO NATIVE FLOATS
88+
case FLOAT, COUNTER_FLOAT -> new PositionToXContent(block) {
89+
@Override
90+
protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
91+
throws IOException {
92+
return builder.value(((FloatBlock) block).getFloat(valueIndex));
93+
}
94+
}; */
8795
case UNSIGNED_LONG -> new PositionToXContent(block) {
8896
@Override
8997
protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
@@ -190,8 +198,10 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
190198
return builder.value(((FloatBlock) block).getFloat(valueIndex));
191199
}
192200
};
193-
case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT,
194-
PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]");
201+
case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT,
202+
/* TODO native floats COUNTER_FLOAT, */ HALF_FLOAT, SCALED_FLOAT, PARTIAL_AGG -> throw new IllegalArgumentException(
203+
"can't convert values of type [" + columnInfo.type() + "]"
204+
);
195205
};
196206
}
197207
}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
117117
case LONG, COUNTER_LONG -> ((LongBlock) block).getLong(offset);
118118
case INTEGER, COUNTER_INTEGER -> ((IntBlock) block).getInt(offset);
119119
case DOUBLE, COUNTER_DOUBLE -> ((DoubleBlock) block).getDouble(offset);
120+
// TODO NATIVE FLOATS
121+
// case FLOAT, COUNTER_FLOAT -> ((FloatBlock) block).getFloat(offset);
120122
case KEYWORD, TEXT -> ((BytesRefBlock) block).getBytesRef(offset, scratch).utf8ToString();
121123
case IP -> {
122124
BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch);
@@ -149,8 +151,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
149151
}
150152
}
151153
case DENSE_VECTOR -> ((FloatBlock) block).getFloat(offset);
152-
case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL,
153-
PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
154+
case SHORT, BYTE, FLOAT, /* TODO native floats COUNTER_FLOAT, */ HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION,
155+
DOC_DATA_TYPE, TSID_DATA_TYPE, NULL, PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType);
154156
};
155157
}
156158
}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ public class Match extends FullTextFunction implements OptionalArgument, PostAna
9696
DATETIME,
9797
DATE_NANOS,
9898
DOUBLE,
99+
// TODO NATIVE FLOATS
100+
// FLOAT,
99101
INTEGER,
100102
IP,
101103
LONG,
@@ -108,6 +110,8 @@ public class Match extends FullTextFunction implements OptionalArgument, PostAna
108110
DATETIME,
109111
DATE_NANOS,
110112
DOUBLE,
113+
// TODO NATIVE FLOATS
114+
// FLOAT,
111115
INTEGER,
112116
IP,
113117
LONG,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,8 +209,9 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
209209
case KEYWORD, TEXT, CARTESIAN_POINT, CARTESIAN_SHAPE, GEO_POINT, GEO_SHAPE, IP, VERSION -> CoalesceBytesRefEvaluator
210210
.toEvaluator(toEvaluator, children());
211211
case NULL -> EvalOperator.CONSTANT_NULL_FACTORY;
212-
case UNSUPPORTED, SHORT, BYTE, DATE_PERIOD, OBJECT, DOC_DATA_TYPE, SOURCE, TIME_DURATION, FLOAT, HALF_FLOAT, TSID_DATA_TYPE,
213-
SCALED_FLOAT, PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> throw new UnsupportedOperationException(
212+
case UNSUPPORTED, SHORT, BYTE, DATE_PERIOD, OBJECT, DOC_DATA_TYPE, SOURCE, TIME_DURATION, FLOAT,
213+
HALF_FLOAT, /* TODO native floats COUNTER_FLOAT, */
214+
TSID_DATA_TYPE, SCALED_FLOAT, PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> throw new UnsupportedOperationException(
214215
dataType() + " can’t be coalesced"
215216
);
216217
};

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -490,8 +490,9 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte
490490
case VERSION -> TopNEncoder.VERSION;
491491
case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_NANOS, DATE_PERIOD, TIME_DURATION,
492492
OBJECT, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE;
493-
case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, SOURCE,
494-
AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> TopNEncoder.DEFAULT_UNSORTABLE;
493+
case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER,
494+
COUNTER_DOUBLE, /* TODO native floats COUNTER_FLOAT, */
495+
SOURCE, AGGREGATE_METRIC_DOUBLE, DENSE_VECTOR -> TopNEncoder.DEFAULT_UNSORTABLE;
495496
// unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point
496497
case PARTIAL_AGG, UNSUPPORTED -> TopNEncoder.UNSUPPORTED;
497498
};

0 commit comments

Comments
 (0)