Skip to content

Commit 87e82d1

Browse files
JonasKunzJonasKunz
authored
ESQL: Add exponential histogram CSV Tests (elastic#137619)
1 parent 1d9ab12 commit 87e82d1

File tree

12 files changed

+463
-15
lines changed

12 files changed

+463
-15
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
2626
import org.elasticsearch.xpack.esql.CsvTestsDataLoader;
2727
import org.elasticsearch.xpack.esql.SpecReader;
28+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
2829
import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
30+
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
2931
import org.junit.AfterClass;
3032
import org.junit.ClassRule;
3133
import org.junit.rules.RuleChain;
@@ -422,4 +424,17 @@ protected boolean supportsTook() throws IOException {
422424
// We don't read took properly in multi-cluster tests.
423425
return false;
424426
}
427+
428+
@Override
429+
protected boolean supportsExponentialHistograms() {
430+
try {
431+
return RestEsqlTestCase.hasCapabilities(client(), List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM.capabilityName()))
432+
&& RestEsqlTestCase.hasCapabilities(
433+
remoteClusterClient(),
434+
List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM.capabilityName())
435+
);
436+
} catch (IOException e) {
437+
throw new RuntimeException(e);
438+
}
439+
}
425440
}

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
import org.elasticsearch.xcontent.XContentBuilder;
1717
import org.elasticsearch.xcontent.json.JsonXContent;
1818
import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
19+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
1920
import org.elasticsearch.xpack.esql.planner.PlannerSettings;
2021
import org.elasticsearch.xpack.esql.plugin.ComputeService;
2122
import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
23+
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
2224
import org.junit.Before;
2325
import org.junit.ClassRule;
2426

2527
import java.io.IOException;
28+
import java.util.List;
2629

2730
@ThreadLeakFilters(filters = TestClustersThreadFilter.class)
2831
public class EsqlSpecIT extends EsqlSpecTestCase {
@@ -51,6 +54,11 @@ protected boolean supportsSourceFieldMapping() {
5154
return cluster.getNumNodes() == 1;
5255
}
5356

57+
@Override
58+
protected boolean supportsExponentialHistograms() {
59+
return RestEsqlTestCase.hasCapabilities(client(), List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM.capabilityName()));
60+
}
61+
5462
@Before
5563
public void configureChunks() throws IOException {
5664
assumeTrue("test clusters were broken", testClustersOk);

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import org.elasticsearch.client.ResponseException;
1717
import org.elasticsearch.client.RestClient;
1818
import org.elasticsearch.common.xcontent.XContentHelper;
19+
import org.elasticsearch.core.Types;
20+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
1921
import org.elasticsearch.features.NodeFeature;
2022
import org.elasticsearch.geometry.Geometry;
2123
import org.elasticsearch.geometry.Point;
@@ -26,11 +28,14 @@
2628
import org.elasticsearch.test.MapMatcher;
2729
import org.elasticsearch.test.rest.ESRestTestCase;
2830
import org.elasticsearch.test.rest.TestFeatureService;
31+
import org.elasticsearch.xcontent.XContentParser;
32+
import org.elasticsearch.xcontent.XContentParserConfiguration;
2933
import org.elasticsearch.xcontent.XContentType;
3034
import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
3135
import org.elasticsearch.xpack.esql.CsvTestUtils;
3236
import org.elasticsearch.xpack.esql.EsqlTestUtils;
3337
import org.elasticsearch.xpack.esql.SpecReader;
38+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
3439
import org.elasticsearch.xpack.esql.plugin.EsqlFeatures;
3540
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode;
3641
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.RequestObjectBuilder;
@@ -173,7 +178,14 @@ public void setup() {
173178
if (supportsInferenceTestService()) {
174179
createInferenceEndpoints(adminClient());
175180
}
176-
loadDataSetIntoEs(client(), supportsIndexModeLookup(), supportsSourceFieldMapping(), supportsInferenceTestService());
181+
loadDataSetIntoEs(
182+
client(),
183+
supportsIndexModeLookup(),
184+
supportsSourceFieldMapping(),
185+
supportsInferenceTestService(),
186+
false,
187+
supportsExponentialHistograms()
188+
);
177189
return null;
178190
});
179191
}
@@ -274,6 +286,10 @@ protected boolean supportsSourceFieldMapping() throws IOException {
274286
return true;
275287
}
276288

289+
protected boolean supportsExponentialHistograms() {
290+
return RestEsqlTestCase.hasCapabilities(client(), List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM.capabilityName()));
291+
}
292+
277293
protected void doTest() throws Throwable {
278294
doTest(testCase.query);
279295
}
@@ -390,6 +406,18 @@ private Object valueMapper(CsvTestUtils.Type type, Object value) {
390406
value = s.replaceAll("\\\\n", "\n");
391407
}
392408
}
409+
if (type == CsvTestUtils.Type.EXPONENTIAL_HISTOGRAM) {
410+
if (value instanceof Map<?, ?> map) {
411+
return ExponentialHistogramXContent.parseForTesting(Types.<Map<String, Object>>forciblyCast(map));
412+
}
413+
if (value instanceof String json) {
414+
try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, json)) {
415+
return ExponentialHistogramXContent.parseForTesting(parser);
416+
} catch (IOException e) {
417+
throw new RuntimeException(e);
418+
}
419+
}
420+
}
393421
return value.toString();
394422
}
395423

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ private static List<String> originalTypes(Map<String, ?> x) {
263263
}
264264

265265
private List<String> availableIndices() throws IOException {
266-
return availableDatasetsForEs(true, supportsSourceFieldMapping(), false, requiresTimeSeries()).stream()
266+
return availableDatasetsForEs(true, supportsSourceFieldMapping(), false, requiresTimeSeries(), false).stream()
267267
.filter(x -> x.requiresInferenceEndpoint() == false)
268268
.map(x -> x.indexName())
269269
.toList();

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.common.time.DateFormatter;
1313
import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder;
1414
import org.elasticsearch.compute.data.Page;
15+
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
1516
import org.elasticsearch.geometry.utils.Geohash;
1617
import org.elasticsearch.h3.H3;
1718
import org.elasticsearch.logging.Logger;
@@ -46,6 +47,7 @@
4647
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
4748
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
4849
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.aggregateMetricDoubleLiteralToString;
50+
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.exponentialHistogramToString;
4951
import static org.hamcrest.MatcherAssert.assertThat;
5052
import static org.hamcrest.Matchers.instanceOf;
5153
import static org.junit.Assert.assertEquals;
@@ -431,6 +433,11 @@ private static Object convertExpectedValue(Type expectedType, Object expectedVal
431433
AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral.class,
432434
x -> aggregateMetricDoubleLiteralToString((AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral) x)
433435
);
436+
case EXPONENTIAL_HISTOGRAM -> rebuildExpected(
437+
expectedValue,
438+
ExponentialHistogram.class,
439+
x -> exponentialHistogramToString((ExponentialHistogram) x)
440+
);
434441
default -> expectedValue;
435442
};
436443
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,21 @@
2323
import org.elasticsearch.compute.data.ElementType;
2424
import org.elasticsearch.compute.data.Page;
2525
import org.elasticsearch.core.Booleans;
26+
import org.elasticsearch.core.Nullable;
2627
import org.elasticsearch.core.Releasable;
2728
import org.elasticsearch.core.Releasables;
2829
import org.elasticsearch.core.Strings;
2930
import org.elasticsearch.core.Tuple;
31+
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
32+
import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
3033
import org.elasticsearch.geometry.utils.Geohash;
3134
import org.elasticsearch.h3.H3;
3235
import org.elasticsearch.logging.Logger;
3336
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
3437
import org.elasticsearch.test.VersionUtils;
38+
import org.elasticsearch.xcontent.XContentParser;
39+
import org.elasticsearch.xcontent.XContentParserConfiguration;
40+
import org.elasticsearch.xcontent.json.JsonXContent;
3541
import org.elasticsearch.xpack.esql.action.ResponseValueUtils;
3642
import org.elasticsearch.xpack.esql.core.type.DataType;
3743
import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -57,7 +63,6 @@
5763
import java.util.regex.Pattern;
5864
import java.util.stream.Stream;
5965

60-
import static org.elasticsearch.common.Strings.delimitedListToStringArray;
6166
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
6267
import static org.elasticsearch.xpack.esql.EsqlTestUtils.reader;
6368
import static org.elasticsearch.xpack.esql.SpecReader.shouldSkipLine;
@@ -143,13 +148,14 @@ void append(String stringValue) {
143148
return;
144149
}
145150
stringValue = mvStrings[0].replace(ESCAPED_COMMA_SEQUENCE, ",");
146-
} else if (stringValue.contains(",") && type != Type.AGGREGATE_METRIC_DOUBLE) {// multi-value field
151+
} else if (stringValue.matches(".*" + COMMA_ESCAPING_REGEX + ".*") && type != Type.AGGREGATE_METRIC_DOUBLE) {// multi-value
152+
// field
147153
builderWrapper().builder().beginPositionEntry();
148154

149-
String[] arrayOfValues = delimitedListToStringArray(stringValue, ",");
155+
String[] arrayOfValues = stringValue.split(COMMA_ESCAPING_REGEX, -1);
150156
List<Object> convertedValues = new ArrayList<>(arrayOfValues.length);
151157
for (String value : arrayOfValues) {
152-
convertedValues.add(type.convert(value));
158+
convertedValues.add(type.convert(value.replace(ESCAPED_COMMA_SEQUENCE, ",")));
153159
}
154160
Stream<Object> convertedValuesStream = convertedValues.stream();
155161
if (type.sortMultiValues()) {
@@ -161,7 +167,7 @@ void append(String stringValue) {
161167
return;
162168
}
163169

164-
var converted = stringValue.length() == 0 ? null : type.convert(stringValue);
170+
var converted = stringValue.length() == 0 ? null : type.convert(stringValue.replace(ESCAPED_COMMA_SEQUENCE, ","));
165171
builderWrapper().append().accept(converted);
166172
}
167173

@@ -498,6 +504,7 @@ public enum Type {
498504
AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral.class
499505
),
500506
DENSE_VECTOR(Float::parseFloat, Float.class, false),
507+
EXPONENTIAL_HISTOGRAM(CsvTestUtils::parseExponentialHistogram, ExponentialHistogram.class),
501508
UNSUPPORTED(Type::convertUnsupported, Void.class);
502509

503510
private static Void convertUnsupported(String s) {
@@ -593,7 +600,7 @@ public static Type asType(ElementType elementType, Type actualType) {
593600
case DOC -> throw new IllegalArgumentException("can't assert on doc blocks");
594601
case COMPOSITE -> throw new IllegalArgumentException("can't assert on composite blocks");
595602
case AGGREGATE_METRIC_DOUBLE -> AGGREGATE_METRIC_DOUBLE;
596-
case EXPONENTIAL_HISTOGRAM -> throw new IllegalArgumentException("exponential histogram blocks not supported yet");
603+
case EXPONENTIAL_HISTOGRAM -> EXPONENTIAL_HISTOGRAM;
597604
case UNKNOWN -> throw new IllegalArgumentException("Unknown block types cannot be handled");
598605
};
599606
}
@@ -699,4 +706,15 @@ private static double scaledFloat(String value, String factor) {
699706
double scalingFactor = Double.parseDouble(factor);
700707
return new BigDecimal(value).multiply(BigDecimal.valueOf(scalingFactor)).longValue() / scalingFactor;
701708
}
709+
710+
private static ExponentialHistogram parseExponentialHistogram(@Nullable String json) {
711+
if (json == null) {
712+
return null;
713+
}
714+
try (XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, json)) {
715+
return ExponentialHistogramXContent.parseForTesting(parser);
716+
} catch (IOException e) {
717+
throw new IllegalArgumentException(e);
718+
}
719+
}
702720
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ public class CsvTestsDataLoader {
172172
private static final TestDataset DENSE_VECTOR = new TestDataset("dense_vector");
173173
private static final TestDataset COLORS = new TestDataset("colors");
174174
private static final TestDataset COLORS_CMYK_LOOKUP = new TestDataset("colors_cmyk").withSetting("lookup-settings.json");
175+
private static final TestDataset EXP_HISTO_SAMPLE = new TestDataset("exp_histo_sample");
175176

176177
public static final Map<String, TestDataset> CSV_DATASET_MAP = Map.ofEntries(
177178
Map.entry(EMPLOYEES.indexName, EMPLOYEES),
@@ -239,7 +240,8 @@ public class CsvTestsDataLoader {
239240
Map.entry(COLORS.indexName, COLORS),
240241
Map.entry(COLORS_CMYK_LOOKUP.indexName, COLORS_CMYK_LOOKUP),
241242
Map.entry(MULTI_COLUMN_JOINABLE.indexName, MULTI_COLUMN_JOINABLE),
242-
Map.entry(MULTI_COLUMN_JOINABLE_LOOKUP.indexName, MULTI_COLUMN_JOINABLE_LOOKUP)
243+
Map.entry(MULTI_COLUMN_JOINABLE_LOOKUP.indexName, MULTI_COLUMN_JOINABLE_LOOKUP),
244+
Map.entry(EXP_HISTO_SAMPLE.indexName, EXP_HISTO_SAMPLE)
243245
);
244246

245247
private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");
@@ -331,7 +333,7 @@ public static void main(String[] args) throws IOException {
331333
}
332334

333335
try (RestClient client = builder.build()) {
334-
loadDataSetIntoEs(client, true, true, false, false, (restClient, indexName, indexMapping, indexSettings) -> {
336+
loadDataSetIntoEs(client, true, true, false, false, true, (restClient, indexName, indexMapping, indexSettings) -> {
335337
// don't use ESRestTestCase methods here or, if you do, test running the main method before making the change
336338
StringBuilder jsonBody = new StringBuilder("{");
337339
if (indexSettings != null && indexSettings.isEmpty() == false) {
@@ -354,15 +356,17 @@ public static Set<TestDataset> availableDatasetsForEs(
354356
boolean supportsIndexModeLookup,
355357
boolean supportsSourceFieldMapping,
356358
boolean inferenceEnabled,
357-
boolean requiresTimeSeries
359+
boolean requiresTimeSeries,
360+
boolean exponentialHistogramFieldSupported
358361
) throws IOException {
359362
Set<TestDataset> testDataSets = new HashSet<>();
360363

361364
for (TestDataset dataset : CSV_DATASET_MAP.values()) {
362365
if ((inferenceEnabled || dataset.requiresInferenceEndpoint == false)
363366
&& (supportsIndexModeLookup || isLookupDataset(dataset) == false)
364367
&& (supportsSourceFieldMapping || isSourceMappingDataset(dataset) == false)
365-
&& (requiresTimeSeries == false || isTimeSeries(dataset))) {
368+
&& (requiresTimeSeries == false || isTimeSeries(dataset))
369+
&& (exponentialHistogramFieldSupported || containsExponentialHistogramFields(dataset) == false)) {
366370
testDataSets.add(dataset);
367371
}
368372
}
@@ -386,6 +390,27 @@ private static boolean isSourceMappingDataset(TestDataset dataset) throws IOExce
386390
return mappingNode.get("_source") != null;
387391
}
388392

393+
private static boolean containsExponentialHistogramFields(TestDataset dataset) throws IOException {
394+
if (dataset.mappingFileName() == null) {
395+
return false;
396+
}
397+
String mappingJsonText = readTextFile(getResource("/" + dataset.mappingFileName()));
398+
JsonNode mappingNode = new ObjectMapper().readTree(mappingJsonText);
399+
JsonNode properties = mappingNode.get("properties");
400+
if (properties != null) {
401+
for (var fieldWithValue : properties.properties()) {
402+
JsonNode fieldProperties = fieldWithValue.getValue();
403+
if (fieldProperties != null) {
404+
JsonNode typeNode = fieldProperties.get("type");
405+
if (typeNode != null && typeNode.asText().equals("exponential_histogram")) {
406+
return true;
407+
}
408+
}
409+
}
410+
}
411+
return false;
412+
}
413+
389414
private static boolean isTimeSeries(TestDataset dataset) throws IOException {
390415
Settings settings = dataset.readSettingsFile();
391416
String mode = settings.get("index.mode");
@@ -398,22 +423,24 @@ public static void loadDataSetIntoEs(
398423
boolean supportsSourceFieldMapping,
399424
boolean inferenceEnabled
400425
) throws IOException {
401-
loadDataSetIntoEs(client, supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, false);
426+
loadDataSetIntoEs(client, supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, false, false);
402427
}
403428

404429
public static void loadDataSetIntoEs(
405430
RestClient client,
406431
boolean supportsIndexModeLookup,
407432
boolean supportsSourceFieldMapping,
408433
boolean inferenceEnabled,
409-
boolean timeSeriesOnly
434+
boolean timeSeriesOnly,
435+
boolean exponentialHistogramFieldSupported
410436
) throws IOException {
411437
loadDataSetIntoEs(
412438
client,
413439
supportsIndexModeLookup,
414440
supportsSourceFieldMapping,
415441
inferenceEnabled,
416442
timeSeriesOnly,
443+
exponentialHistogramFieldSupported,
417444
(restClient, indexName, indexMapping, indexSettings) -> {
418445
ESRestTestCase.createIndex(restClient, indexName, indexSettings, indexMapping, null);
419446
}
@@ -426,13 +453,20 @@ private static void loadDataSetIntoEs(
426453
boolean supportsSourceFieldMapping,
427454
boolean inferenceEnabled,
428455
boolean timeSeriesOnly,
456+
boolean exponentialHistogramFieldSupported,
429457
IndexCreator indexCreator
430458
) throws IOException {
431459
Logger logger = LogManager.getLogger(CsvTestsDataLoader.class);
432460

433461
Set<String> loadedDatasets = new HashSet<>();
434462
logger.info("Loading test datasets");
435-
for (var dataset : availableDatasetsForEs(supportsIndexModeLookup, supportsSourceFieldMapping, inferenceEnabled, timeSeriesOnly)) {
463+
for (var dataset : availableDatasetsForEs(
464+
supportsIndexModeLookup,
465+
supportsSourceFieldMapping,
466+
inferenceEnabled,
467+
timeSeriesOnly,
468+
exponentialHistogramFieldSupported
469+
)) {
436470
load(client, dataset, logger, indexCreator);
437471
loadedDatasets.add(dataset.indexName);
438472
}

0 commit comments

Comments
 (0)