Skip to content

Commit 4c4dee6

Browse files
mridula-s109mridula-s109
authored
Merge branch 'main' into pinned-retriever
2 parents fae7a05 + 0700b24 commit 4c4dee6

File tree

31 files changed

+3008
-45
lines changed

31 files changed

+3008
-45
lines changed

‎docs/changelog/127139.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127139
2+
summary: Add `suggested_cast`
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎qa/smoke-test-http/src/internalClusterTest/java/org/elasticsearch/http/SearchRestCancellationIT.java‎

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
import org.apache.http.nio.entity.NByteArrayEntity;
1313
import org.apache.logging.log4j.LogManager;
1414
import org.apache.lucene.util.SetOnce;
15-
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
16-
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
1715
import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
1816
import org.elasticsearch.action.bulk.BulkRequestBuilder;
1917
import org.elasticsearch.action.search.MultiSearchRequest;
@@ -46,7 +44,6 @@
4644
import java.util.ArrayList;
4745
import java.util.Collection;
4846
import java.util.Collections;
49-
import java.util.HashMap;
5047
import java.util.List;
5148
import java.util.Map;
5249
import java.util.concurrent.CancellationException;
@@ -94,7 +91,7 @@ public void testAutomaticCancellationMultiSearchDuringQueryPhase() throws Except
9491
}
9592

9693
void verifyCancellationDuringQueryPhase(String searchAction, Request searchRequest) throws Exception {
97-
Map<String, String> nodeIdToName = readNodesInfo();
94+
Map<String, String> nodeIdToName = nodeIdsToNames();
9895

9996
List<ScriptedBlockPlugin> plugins = initBlockFactory();
10097
indexTestData();
@@ -137,7 +134,7 @@ public void testAutomaticCancellationMultiSearchDuringFetchPhase() throws Except
137134
}
138135

139136
void verifyCancellationDuringFetchPhase(String searchAction, Request searchRequest) throws Exception {
140-
Map<String, String> nodeIdToName = readNodesInfo();
137+
Map<String, String> nodeIdToName = nodeIdsToNames();
141138

142139
List<ScriptedBlockPlugin> plugins = initBlockFactory();
143140
indexTestData();
@@ -153,16 +150,6 @@ void verifyCancellationDuringFetchPhase(String searchAction, Request searchReque
153150
expectThrows(CancellationException.class, future::actionGet);
154151
}
155152

156-
private static Map<String, String> readNodesInfo() {
157-
Map<String, String> nodeIdToName = new HashMap<>();
158-
NodesInfoResponse nodesInfoResponse = clusterAdmin().prepareNodesInfo().get();
159-
assertFalse(nodesInfoResponse.hasFailures());
160-
for (NodeInfo node : nodesInfoResponse.getNodes()) {
161-
nodeIdToName.put(node.getNode().getId(), node.getNode().getName());
162-
}
163-
return nodeIdToName;
164-
}
165-
166153
private static void ensureSearchTaskIsCancelled(String transportAction, Function<String, String> nodeIdToName) throws Exception {
167154
SetOnce<TaskInfo> searchTask = new SetOnce<>();
168155
ListTasksResponse listTasksResponse = clusterAdmin().prepareListTasks().get();

‎test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@
193193
import java.util.Arrays;
194194
import java.util.Collection;
195195
import java.util.Collections;
196+
import java.util.HashMap;
196197
import java.util.HashSet;
197198
import java.util.IdentityHashMap;
198199
import java.util.List;
@@ -1207,6 +1208,19 @@ public static String getNodeId(String nodeName) {
12071208
return internalCluster().getInstance(ClusterService.class, nodeName).localNode().getId();
12081209
}
12091210

1211+
/**
1212+
* @return A map of the cluster node Ids to their node names.
1213+
*/
1214+
public static Map<String, String> nodeIdsToNames() {
1215+
var names = internalCluster().getNodeNames();
1216+
Map<String, String> nodeIdsToNames = new HashMap<>();
1217+
for (var name : names) {
1218+
nodeIdsToNames.put(getNodeId(name), name);
1219+
}
1220+
return nodeIdsToNames;
1221+
1222+
}
1223+
12101224
/**
12111225
* Waits until at least a give number of document is visible for searchers
12121226
*

‎test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2314,6 +2314,11 @@ public int numMasterNodes() {
23142314
return filterNodes(nodes, NodeAndClient::isMasterEligible).size();
23152315
}
23162316

2317+
public Set<String> masterEligibleNodeNames() {
2318+
var masterEligibleNodes = filterNodes(nodes, NodeAndClient::isMasterEligible);
2319+
return masterEligibleNodes.stream().map(nodeAndClient -> nodeAndClient.name).collect(Collectors.toSet());
2320+
}
2321+
23172322
public void setDisruptionScheme(ServiceDisruptionScheme scheme) {
23182323
assert activeDisruptionScheme == null
23192324
: "there is already and active disruption [" + activeDisruptionScheme + "]. call clearDisruptionScheme first";

‎x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -708,6 +708,29 @@ public boolean isDate() {
708708
};
709709
}
710710

711+
public static DataType suggestedCast(Set<DataType> originalTypes) {
712+
if (originalTypes.isEmpty() || originalTypes.contains(UNSUPPORTED)) {
713+
return null;
714+
}
715+
if (originalTypes.contains(DATE_NANOS) && originalTypes.contains(DATETIME) && originalTypes.size() == 2) {
716+
return DATE_NANOS;
717+
}
718+
if (originalTypes.contains(AGGREGATE_METRIC_DOUBLE)) {
719+
boolean allNumeric = true;
720+
for (DataType type : originalTypes) {
721+
if (type.isNumeric() == false && type != AGGREGATE_METRIC_DOUBLE) {
722+
allNumeric = false;
723+
break;
724+
}
725+
}
726+
if (allNumeric) {
727+
return AGGREGATE_METRIC_DOUBLE;
728+
}
729+
}
730+
731+
return KEYWORD;
732+
}
733+
711734
/**
712735
* Named parameters with default values. It's just easier to do this with
713736
* a builder in java....

‎x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.xcontent.XContentBuilder;
2828
import org.elasticsearch.xcontent.XContentType;
2929
import org.elasticsearch.xcontent.json.JsonXContent;
30+
import org.elasticsearch.xpack.esql.core.type.DataType;
3031
import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase;
3132
import org.elasticsearch.xpack.esql.tools.ProfileParser;
3233
import org.hamcrest.Matchers;
@@ -40,11 +41,14 @@
4041
import java.nio.charset.StandardCharsets;
4142
import java.util.ArrayList;
4243
import java.util.Arrays;
44+
import java.util.Comparator;
4345
import java.util.HashSet;
4446
import java.util.List;
4547
import java.util.Locale;
4648
import java.util.Map;
4749
import java.util.Set;
50+
import java.util.stream.Collectors;
51+
import java.util.stream.Stream;
4852

4953
import static org.elasticsearch.test.ListMatcher.matchesList;
5054
import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -648,6 +652,120 @@ public void testForceSleepsProfile() throws IOException {
648652
}
649653
}
650654

655+
public void testSuggestedCast() throws IOException {
656+
// TODO: Figure out how best to make sure we don't leave out new types
657+
Map<DataType, String> typesAndValues = Map.ofEntries(
658+
Map.entry(DataType.BOOLEAN, "\"true\""),
659+
Map.entry(DataType.LONG, "-1234567890234567"),
660+
Map.entry(DataType.INTEGER, "123"),
661+
Map.entry(DataType.UNSIGNED_LONG, "1234567890234567"),
662+
Map.entry(DataType.DOUBLE, "12.4"),
663+
Map.entry(DataType.KEYWORD, "\"keyword\""),
664+
Map.entry(DataType.TEXT, "\"some text\""),
665+
Map.entry(DataType.DATE_NANOS, "\"2015-01-01T12:10:30.123456789Z\""),
666+
Map.entry(DataType.DATETIME, "\"2015-01-01T12:10:30Z\""),
667+
Map.entry(DataType.IP, "\"192.168.30.1\""),
668+
Map.entry(DataType.VERSION, "\"8.19.0\""),
669+
Map.entry(DataType.GEO_POINT, "[-71.34, 41.12]"),
670+
Map.entry(DataType.GEO_SHAPE, """
671+
{
672+
"type": "Point",
673+
"coordinates": [-77.03653, 38.897676]
674+
}
675+
"""),
676+
Map.entry(DataType.AGGREGATE_METRIC_DOUBLE, """
677+
{
678+
"max": 14983.1
679+
}
680+
""")
681+
);
682+
Set<DataType> shouldBeSupported = Stream.of(DataType.values()).filter(DataType::isRepresentable).collect(Collectors.toSet());
683+
shouldBeSupported.remove(DataType.CARTESIAN_POINT);
684+
shouldBeSupported.remove(DataType.CARTESIAN_SHAPE);
685+
shouldBeSupported.remove(DataType.NULL);
686+
shouldBeSupported.remove(DataType.DOC_DATA_TYPE);
687+
shouldBeSupported.remove(DataType.TSID_DATA_TYPE);
688+
for (DataType type : shouldBeSupported) {
689+
assertTrue(typesAndValues.containsKey(type));
690+
}
691+
assertThat(typesAndValues.size(), equalTo(shouldBeSupported.size()));
692+
693+
for (DataType type : typesAndValues.keySet()) {
694+
String additionalProperties = "";
695+
if (type == DataType.AGGREGATE_METRIC_DOUBLE) {
696+
additionalProperties += """
697+
,
698+
"metrics": ["max"],
699+
"default_metric": "max"
700+
""";
701+
}
702+
createIndex("index-" + type.esType(), null, """
703+
"properties": {
704+
"my_field": {
705+
"type": "%s" %s
706+
}
707+
}
708+
""".formatted(type.esType(), additionalProperties));
709+
Request doc = new Request("PUT", "index-" + type.esType() + "/_doc/1");
710+
doc.setJsonEntity("{\"my_field\": " + typesAndValues.get(type) + "}");
711+
client().performRequest(doc);
712+
}
713+
714+
List<DataType> listOfTypes = new ArrayList<>(typesAndValues.keySet());
715+
listOfTypes.sort(Comparator.comparing(DataType::typeName));
716+
717+
for (int i = 0; i < listOfTypes.size(); i++) {
718+
for (int j = i + 1; j < listOfTypes.size(); j++) {
719+
String query = """
720+
{
721+
"query": "FROM index-%s,index-%s | LIMIT 100 | KEEP my_field"
722+
}
723+
""".formatted(listOfTypes.get(i).esType(), listOfTypes.get(j).esType());
724+
Request request = new Request("POST", "/_query");
725+
request.setJsonEntity(query);
726+
Response resp = client().performRequest(request);
727+
Map<String, Object> results = entityAsMap(resp);
728+
List<?> columns = (List<?>) results.get("columns");
729+
DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j)));
730+
assertThat(
731+
columns,
732+
equalTo(
733+
List.of(
734+
Map.ofEntries(
735+
Map.entry("name", "my_field"),
736+
Map.entry("type", "unsupported"),
737+
Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())),
738+
Map.entry("suggested_cast", suggestedCast.typeName())
739+
)
740+
)
741+
)
742+
);
743+
744+
String castedQuery = """
745+
{
746+
"query": "FROM index-%s,index-%s | LIMIT 100 | EVAL my_field = my_field::%s"
747+
}
748+
""".formatted(
749+
listOfTypes.get(i).esType(),
750+
listOfTypes.get(j).esType(),
751+
suggestedCast == DataType.KEYWORD ? "STRING" : suggestedCast.nameUpper()
752+
);
753+
Request castedRequest = new Request("POST", "/_query");
754+
castedRequest.setJsonEntity(castedQuery);
755+
Response castedResponse = client().performRequest(castedRequest);
756+
Map<String, Object> castedResults = entityAsMap(castedResponse);
757+
List<?> castedColumns = (List<?>) castedResults.get("columns");
758+
assertThat(
759+
castedColumns,
760+
equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", suggestedCast.typeName()))))
761+
);
762+
}
763+
}
764+
for (DataType type : typesAndValues.keySet()) {
765+
deleteIndex("index-" + type.esType());
766+
}
767+
}
768+
651769
static MapMatcher commonProfile() {
652770
return matchesMap() //
653771
.entry("description", any(String.class))

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.xcontent.XContentType;
3333
import org.elasticsearch.xcontent.json.JsonXContent;
3434
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
35+
import org.elasticsearch.xpack.esql.core.type.DataType;
3536
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
3637
import org.hamcrest.Matcher;
3738
import org.junit.Before;
@@ -45,8 +46,10 @@
4546
import java.util.List;
4647
import java.util.Locale;
4748
import java.util.Map;
49+
import java.util.Objects;
4850
import java.util.TreeMap;
4951
import java.util.function.Function;
52+
import java.util.stream.Collectors;
5053

5154
import static org.elasticsearch.test.ListMatcher.matchesList;
5255
import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -690,7 +693,7 @@ public void testByteFieldWithIntSubfieldTooBig() throws IOException {
690693
* </pre>.
691694
*/
692695
public void testIncompatibleTypes() throws IOException {
693-
assumeOriginalTypesReported();
696+
assumeSuggestedCastReported();
694697
keywordTest().createIndex("test1", "f");
695698
index("test1", """
696699
{"f": "f1"}""");
@@ -764,7 +767,7 @@ public void testDistinctInEachIndex() throws IOException {
764767
* </pre>.
765768
*/
766769
public void testMergeKeywordAndObject() throws IOException {
767-
assumeOriginalTypesReported();
770+
assumeSuggestedCastReported();
768771
keywordTest().createIndex("test1", "file");
769772
index("test1", """
770773
{"file": "f1"}""");
@@ -959,7 +962,7 @@ public void testIntegerDocValuesConflict() throws IOException {
959962
* In an ideal world we'd promote the {@code integer} to an {@code long} and just go.
960963
*/
961964
public void testLongIntegerConflict() throws IOException {
962-
assumeOriginalTypesReported();
965+
assumeSuggestedCastReported();
963966
longTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no");
964967
index("test1", """
965968
{"emp_no": 1}""");
@@ -1002,7 +1005,7 @@ public void testLongIntegerConflict() throws IOException {
10021005
* In an ideal world we'd promote the {@code short} to an {@code integer} and just go.
10031006
*/
10041007
public void testIntegerShortConflict() throws IOException {
1005-
assumeOriginalTypesReported();
1008+
assumeSuggestedCastReported();
10061009
intTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no");
10071010
index("test1", """
10081011
{"emp_no": 1}""");
@@ -1051,7 +1054,7 @@ public void testIntegerShortConflict() throws IOException {
10511054
* </pre>.
10521055
*/
10531056
public void testTypeConflictInObject() throws IOException {
1054-
assumeOriginalTypesReported();
1057+
assumeSuggestedCastReported();
10551058
createIndex("test1", empNoInObject("integer"));
10561059
index("test1", """
10571060
{"foo": {"emp_no": 1}}""");
@@ -1379,6 +1382,12 @@ private void assumeOriginalTypesReported() throws IOException {
13791382
assumeTrue("This test makes sense for versions that report original types", requiredClusterCapability);
13801383
}
13811384

1385+
private void assumeSuggestedCastReported() throws IOException {
1386+
var capsName = EsqlCapabilities.Cap.SUGGESTED_CAST.name().toLowerCase(Locale.ROOT);
1387+
boolean requiredClusterCapability = clusterHasCapability("POST", "/_query", List.of(), List.of(capsName)).orElse(false);
1388+
assumeTrue("This test makes sense for versions that report suggested casts", requiredClusterCapability);
1389+
}
1390+
13821391
private CheckedConsumer<XContentBuilder, IOException> empNoInObject(String empNoType) {
13831392
return index -> {
13841393
index.startObject("properties");
@@ -1715,7 +1724,19 @@ private static Map<String, Object> columnInfo(String name, String type) {
17151724
}
17161725

17171726
private static Map<String, Object> unsupportedColumnInfo(String name, String... originalTypes) {
1718-
return Map.of("name", name, "type", "unsupported", "original_types", List.of(originalTypes));
1727+
DataType suggested = DataType.suggestedCast(
1728+
List.of(originalTypes).stream().map(DataType::fromTypeName).filter(Objects::nonNull).collect(Collectors.toSet())
1729+
);
1730+
if (suggested == null) {
1731+
return Map.of("name", name, "type", "unsupported", "original_types", List.of(originalTypes));
1732+
} else {
1733+
return Map.ofEntries(
1734+
Map.entry("name", name),
1735+
Map.entry("type", "unsupported"),
1736+
Map.entry("original_types", List.of(originalTypes)),
1737+
Map.entry("suggested_cast", suggested.typeName())
1738+
);
1739+
}
17191740
}
17201741

17211742
private static void index(String name, String... docs) throws IOException {

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/convert.csv-spec‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -481,3 +481,23 @@ x:integer | agg_metric:aggregate_metric_double
481481
[5032, 11111, 40814] | {"min":5032.0,"max":40814.0,"sum":56957.0,"value_count":3}
482482
//end::toAggregateMetricDoubleMv-result[]
483483
;
484+
485+
convertToAggregateMetricDoubleCastingOperatorFromDouble
486+
required_capability: suggested_cast
487+
ROW x = 29384.1256
488+
| EVAL agg_metric = x::aggregate_metric_double
489+
;
490+
491+
x:double | agg_metric:aggregate_metric_double
492+
29384.1256 | {"min":29384.1256,"max":29384.1256,"sum":29384.1256,"value_count":1}
493+
;
494+
495+
convertToAggregateMetricDoubleCastingOperatorFromInt
496+
required_capability: suggested_cast
497+
ROW x = 55555
498+
| EVAL agg_metric = x::aggregate_metric_double
499+
;
500+
501+
x:integer | agg_metric:aggregate_metric_double
502+
55555 | {"min":55555,"max":55555,"sum":55555,"value_count":1}
503+
;

0 commit comments

Comments
 (0)