Skip to content

Commit 8dca0e2

Browse files
carlosdelestcarlosdelest
committed
Add casting support for match operator
1 parent a845b5a commit 8dca0e2

File tree

8 files changed

+976
-843
lines changed

8 files changed

+976
-843
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
import static org.elasticsearch.test.ESTestCase.randomLong;
119119
import static org.elasticsearch.test.ESTestCase.randomLongBetween;
120120
import static org.elasticsearch.test.ESTestCase.randomMillisUpToYear9999;
121+
import static org.elasticsearch.test.ESTestCase.randomNonNegativeLong;
121122
import static org.elasticsearch.test.ESTestCase.randomShort;
122123
import static org.elasticsearch.test.ESTestCase.randomZone;
123124
import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
@@ -723,10 +724,10 @@ public static Literal randomLiteral(DataType type) {
723724
case BYTE -> randomByte();
724725
case SHORT -> randomShort();
725726
case INTEGER, COUNTER_INTEGER -> randomInt();
726-
case UNSIGNED_LONG, LONG, COUNTER_LONG -> randomLong();
727+
case LONG, COUNTER_LONG -> randomLong();
728+
case UNSIGNED_LONG, DATE_NANOS -> randomNonNegativeLong();
727729
case DATE_PERIOD -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32));
728730
case DATETIME -> randomMillisUpToYear9999();
729-
case DATE_NANOS -> randomLong();
730731
case DOUBLE, SCALED_FLOAT, COUNTER_DOUBLE -> randomDouble();
731732
case FLOAT -> randomFloat();
732733
case HALF_FLOAT -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat()));

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-all-types.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
"date": {
1818
"type": "date"
1919
},
20+
"date_nanos": {
21+
"type": "date_nanos"
22+
},
2023
"double": {
2124
"type": "double"
2225
},

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ regexBooleanExpression
7878
;
7979

8080
matchBooleanExpression
81-
: fieldExp=qualifiedName COLON queryString=constant
81+
: fieldExp=qualifiedName COLON matchQuery=constant (CAST_OP dataType)?
8282
;
8383

8484
valueExpression

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java‎

Lines changed: 849 additions & 819 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -627,13 +627,16 @@ public String visitIdentifierOrParameter(EsqlBaseParser.IdentifierOrParameterCon
627627

628628
@Override
629629
public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) {
630-
Source source = source(ctx);
631-
DataType dataType = typedParsing(this, ctx.dataType(), DataType.class);
630+
return castToType(source(ctx), ctx.primaryExpression(), ctx.dataType());
631+
}
632+
633+
private Expression castToType(Source source, ParseTree parseTree, EsqlBaseParser.DataTypeContext dataTypeCtx) {
634+
DataType dataType = typedParsing(this, dataTypeCtx, DataType.class);
632635
var converterToFactory = EsqlDataTypeConverter.converterFunctionFactory(dataType);
633636
if (converterToFactory == null) {
634637
throw new ParsingException(source, "Unsupported conversion to type [{}]", dataType);
635638
}
636-
Expression expr = expression(ctx.primaryExpression());
639+
Expression expr = expression(parseTree);
637640
return converterToFactory.apply(source, expr);
638641
}
639642

@@ -923,6 +926,13 @@ String unresolvedAttributeNameInParam(ParserRuleContext ctx, Expression param) {
923926

924927
@Override
925928
public Expression visitMatchBooleanExpression(EsqlBaseParser.MatchBooleanExpressionContext ctx) {
926-
return new Match(source(ctx), expression(ctx.fieldExp), expression(ctx.queryString));
929+
final Expression matchQueryExpression;
930+
if (ctx.dataType() != null) {
931+
matchQueryExpression = castToType(source(ctx), ctx.matchQuery, ctx.dataType());
932+
} else {
933+
matchQueryExpression = expression(ctx.matchQuery);
934+
}
935+
936+
return new Match(source(ctx), expression(ctx.fieldExp), matchQueryExpression);
927937
}
928938
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java‎

Lines changed: 82 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1111

1212
import org.apache.lucene.search.IndexSearcher;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.common.network.NetworkAddress;
1315
import org.elasticsearch.common.settings.Settings;
1416
import org.elasticsearch.core.Tuple;
1517
import org.elasticsearch.index.IndexMode;
@@ -41,7 +43,6 @@
4143
import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
4244
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
4345
import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
44-
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchTests;
4546
import org.elasticsearch.xpack.esql.index.EsIndex;
4647
import org.elasticsearch.xpack.esql.index.IndexResolution;
4748
import org.elasticsearch.xpack.esql.parser.ParsingException;
@@ -67,12 +68,17 @@
6768
import org.elasticsearch.xpack.esql.stats.SearchContextStats;
6869
import org.elasticsearch.xpack.esql.stats.SearchStats;
6970
import org.elasticsearch.xpack.kql.query.KqlQueryBuilder;
71+
import org.elasticsearch.xpack.versionfield.Version;
7072
import org.junit.Before;
7173

7274
import java.io.IOException;
75+
import java.math.BigInteger;
76+
import java.net.InetAddress;
77+
import java.net.UnknownHostException;
7378
import java.util.List;
7479
import java.util.Locale;
7580
import java.util.Map;
81+
import java.util.function.BiFunction;
7682

7783
import static java.util.Arrays.asList;
7884
import static org.elasticsearch.compute.aggregation.AggregatorMode.FINAL;
@@ -1273,33 +1279,95 @@ public void testMissingFieldsDoNotGetExtracted() {
12731279
* estimatedRowSize[324]
12741280
*/
12751281
public void testSingleMatchFilterPushdown() {
1276-
// Check for every possible query data type
1277-
for (DataType queryDataType : Match.DATA_TYPES) {
1278-
var query = MatchTests.randomQuery(queryDataType);
1279-
var queryText = query;
1280-
if (query instanceof String) {
1281-
queryText = "\"" + query + "\"";
1282+
var analyzer = makeAnalyzer("mapping-all-types.json");
1283+
1284+
BiFunction<Object, DataType, String> queryValueProvider = (value, dataType) -> {
1285+
if (dataType == DataType.TEXT || dataType == DataType.KEYWORD) {
1286+
return queryValueAsString(value, dataType);
12821287
}
1283-
var esqlQuery = """
1284-
from test
1285-
| where first_name:""" + queryText;
1286-
try {
1287-
var plan = plannerOptimizer.plan(esqlQuery);
1288+
return queryValueAsCasting(value, dataType);
1289+
};
12881290

1291+
// Check for every possible query data type
1292+
for (DataType fieldDataType : Match.FIELD_DATA_TYPES) {
1293+
var queryValue = randomQueryValue(fieldDataType);
1294+
1295+
String fieldName = fieldDataType == DataType.DATETIME ? "date" : fieldDataType.name().toLowerCase(Locale.ROOT);
1296+
var esqlQuery = String.format(
1297+
Locale.ROOT,
1298+
"from test | where %s:%s",
1299+
fieldName,
1300+
queryValueAsCasting(queryValue, fieldDataType)
1301+
);
1302+
1303+
try {
1304+
var plan = plannerOptimizer.plan(esqlQuery, IS_SV_STATS, analyzer);
12891305
var limit = as(plan, LimitExec.class);
12901306
var exchange = as(limit.child(), ExchangeExec.class);
12911307
var project = as(exchange.child(), ProjectExec.class);
12921308
var fieldExtract = as(project.child(), FieldExtractExec.class);
12931309
var actualLuceneQuery = as(fieldExtract.child(), EsQueryExec.class).query();
12941310

1295-
var expectedLuceneQuery = new MatchQueryBuilder("first_name", query);
1296-
assertThat("Unexpected match query for data type " + queryDataType, actualLuceneQuery, equalTo(expectedLuceneQuery));
1311+
var expectedLuceneQuery = new MatchQueryBuilder(fieldName, queryValue);
1312+
assertThat("Unexpected match query for data type " + fieldDataType, actualLuceneQuery, equalTo(expectedLuceneQuery));
12971313
} catch (ParsingException e) {
12981314
fail("Error parsing ESQL query: " + esqlQuery + "\n" + e.getMessage());
12991315
}
13001316
}
13011317
}
13021318

1319+
private static Object randomQueryValue(DataType dataType) {
1320+
if (Match.FIELD_DATA_TYPES.contains(dataType) == false) {
1321+
throw new IllegalArgumentException("Unsupported type in tests: " + dataType);
1322+
}
1323+
Object value = EsqlTestUtils.randomLiteral(dataType).value();
1324+
if (value instanceof BytesRef bytesRef) {
1325+
switch (dataType) {
1326+
case TEXT, KEYWORD -> value = bytesRef.utf8ToString();
1327+
case VERSION -> value = new Version(bytesRef).toString();
1328+
case IP -> {
1329+
try {
1330+
value = NetworkAddress.format(InetAddress.getByAddress(bytesRef.bytes));
1331+
} catch (UnknownHostException e) {
1332+
throw new IllegalArgumentException(e);
1333+
}
1334+
}
1335+
default -> throw new IllegalArgumentException("Unexpected type: " + dataType + " has BytesRef as value");
1336+
}
1337+
} else if (dataType == DataType.UNSIGNED_LONG) {
1338+
value = BigInteger.valueOf((Long) value);
1339+
}
1340+
1341+
return value;
1342+
}
1343+
1344+
private static String queryValueAsCasting(Object value, DataType dataType) {
1345+
if (Match.FIELD_DATA_TYPES.contains(dataType) == false) {
1346+
throw new IllegalArgumentException("Unsupported type in tests: " + dataType);
1347+
}
1348+
if (value instanceof String) {
1349+
value = "\"" + value + "\"";
1350+
}
1351+
return switch (dataType) {
1352+
case VERSION -> value + "::VERSION";
1353+
case IP -> value + "::IP";
1354+
case DATETIME -> value + "::DATETIME";
1355+
case DATE_NANOS -> value + "::DATE_NANOS";
1356+
case INTEGER -> value + "::INTEGER";
1357+
case LONG -> value + "::LONG";
1358+
case BOOLEAN -> String.valueOf(value).toLowerCase(Locale.ROOT);
1359+
case UNSIGNED_LONG -> "\"" + value + "\"::UNSIGNED_LONG";
1360+
default -> value.toString();
1361+
};
1362+
}
1363+
1364+
private static String queryValueAsString(Object value, DataType dataType) {
1365+
if (Match.FIELD_DATA_TYPES.contains(dataType) == false) {
1366+
throw new IllegalArgumentException("Unsupported type in tests: " + dataType);
1367+
}
1368+
return "\"" + String.valueOf(value) + "\"";
1369+
}
1370+
13031371
/**
13041372
* Expects
13051373
* EvalExec[[CONCAT([65 6d 70 5f 6e 6f 3a 20][KEYWORD],TOSTRING(emp_no{f}#12),[2c 20 6e 61 6d 65 3a 20][KEYWORD],first_nam

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
3030
import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
3131
import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
32+
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIP;
3233
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
3334
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
3435
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
@@ -2314,7 +2315,6 @@ public void testInvalidMatchOperator() {
23142315
"from test | WHERE field:CONCAT(\"hello\", \"world\")",
23152316
"line 1:25: mismatched input 'CONCAT' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, "
23162317
);
2317-
expectError("from test | WHERE field:123::STRING", "line 1:28: mismatched input '::' expecting {<EOF>, '|', 'and', 'or'}");
23182318
expectError(
23192319
"from test | WHERE field:(true OR false)",
23202320
"line 1:25: extraneous input '(' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, "
@@ -2323,7 +2323,7 @@ public void testInvalidMatchOperator() {
23232323
"from test | WHERE field:another_field_or_value",
23242324
"line 1:25: mismatched input 'another_field_or_value' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, "
23252325
);
2326-
expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+' expecting {<EOF>, '|', 'and', 'or'}");
2326+
expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+'");
23272327
expectError(
23282328
"from test | WHERE \"field\":\"value\"",
23292329
"line 1:26: mismatched input ':' expecting {<EOF>, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}"
@@ -2333,4 +2333,25 @@ public void testInvalidMatchOperator() {
23332333
"line 1:37: mismatched input ':' expecting {<EOF>, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}"
23342334
);
23352335
}
2336+
2337+
public void testMatchFunctionCasting() {
2338+
var plan = statement("FROM test | WHERE match(field, \"value\"::IP)");
2339+
var filter = as(plan, Filter.class);
2340+
var function = (UnresolvedFunction) filter.condition();
2341+
assertTrue(function.children().get(0) instanceof UnresolvedAttribute);
2342+
var toIp = (ToIP) function.children().get(1);
2343+
var literal = (Literal) toIp.field();
2344+
assertThat(literal.value(), equalTo("value"));
2345+
}
2346+
2347+
public void testMatchOperatorCasting() {
2348+
var plan = statement("FROM test | WHERE field:\"value\"::IP)");
2349+
var filter = as(plan, Filter.class);
2350+
var match = (Match) filter.condition();
2351+
var matchField = (UnresolvedAttribute) match.field();
2352+
assertThat(matchField.name(), equalTo("field"));
2353+
var toIp = (ToIP) match.query();
2354+
var literal = (Literal) toIp.field();
2355+
assertThat(literal.value(), equalTo("value"));
2356+
}
23362357
}

0 commit comments

Comments
 (0)