diff --git a/docs/changelog/134317.yaml b/docs/changelog/134317.yaml new file mode 100644 index 0000000000000..83b16bcb38b30 --- /dev/null +++ b/docs/changelog/134317.yaml @@ -0,0 +1,5 @@ +pr: 134317 +summary: ES|QL - Allow multivalued query parameters +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 528f9ac2f57ea..b7ab4dd8fbd1f 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -22,6 +22,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -456,41 +457,51 @@ public static DataType fromEs(String name) { } public static DataType fromJava(Object value) { - if (value == null) { - return NULL; - } - if (value instanceof Integer) { - return INTEGER; - } - if (value instanceof Long) { - return LONG; - } - if (value instanceof BigInteger) { - return UNSIGNED_LONG; - } - if (value instanceof Boolean) { - return BOOLEAN; - } - if (value instanceof Double) { - return DOUBLE; - } - if (value instanceof Float) { - return FLOAT; - } - if (value instanceof Byte) { - return BYTE; - } - if (value instanceof Short) { - return SHORT; - } - if (value instanceof ZonedDateTime) { - return DATETIME; - } - if (value instanceof String || value instanceof Character || value instanceof BytesRef) { - return KEYWORD; + switch (value) { + case null -> { + return NULL; + } + case Integer i -> { + return INTEGER; + } + case Long l -> { + return LONG; + } + case BigInteger bigInteger -> { + return UNSIGNED_LONG; + } + case Boolean b -> { + return BOOLEAN; + } + case Double v -> { + return DOUBLE; + } + case Float v -> { + return FLOAT; + } + case Byte b -> { + return BYTE; + } + case Short i -> { + return SHORT; + } + case ZonedDateTime zonedDateTime -> { + return DATETIME; + } + case List list -> { + if (list.isEmpty()) { + return null; + } + return fromJava(list.getFirst()); + } + default -> { + if (value instanceof String || value instanceof Character || value instanceof BytesRef) { + return KEYWORD; + } + return null; + } } - return null; } public static boolean isUnsupported(DataType from) { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 61cc5c80cbc00..8c3c243de3e02 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -687,14 +687,125 @@ public void testErrorMessageForInvalidIntervalParams() throws IOException { ); } - public void testErrorMessageForArrayValuesInParams() throws IOException { + public void testArrayValuesAllowedInValueParams() throws IOException { + Map responseMap = runEsql( + RequestObjectBuilder.jsonBuilder() + .query("row a = ?a | eval b = ?b, c = ?c, d = ?d, e = ?e") + .params( + "[{\"a\" : [\"a1\", \"a2\"]}, {\"b\" : [1, 2]}, {\"c\": [true, false]}, {\"d\": [1.1, 2.2]}," + + " {\"e\": [1674835275193, 1674835275193]}]" + ) + ); + System.out.println(responseMap); + + ListMatcher values = matchesList().item( + matchesList().item(matchesList().item("a1").item("a2")) + .item(matchesList().item(1).item(2)) + .item(matchesList().item(true).item(false)) + .item(matchesList().item(1.1).item(2.2)) + .item(matchesList().item(1674835275193L).item(1674835275193L)) + ); + } + + public void testArrayValuesNullsNotAllowedInValueParams() throws IOException { + ResponseException re = expectThrows( + ResponseException.class, + () -> runEsqlSync( + RequestObjectBuilder.jsonBuilder() + .query("row a = ?a | eval b = ?b, c = ?c, d = ?d, e = ?e, f = ?f") + .params( + "[{\"a\" : [null, \"a2\"]}, {\"b\" : [null, 2]}, {\"c\": [null, false]}, {\"d\": [null, 2.2]}," + + " {\"e\": [null, 1674835275193]}, {\"f\": [null, null]}]" + ) + ) + ); + + String error = EntityUtils.toString(re.getResponse().getEntity()).replaceAll("\\\\\n\s+\\\\", ""); + assertThat( + error, + containsString( + "[1:79] Parameter [a] contains a null entry: [null, a2]. " + "Null values are not allowed in multivalued params;" + ) + ); + assertThat( + error, + containsString( + "[1:101] Parameter [b] contains a null entry: [null, 2]. " + "Null values are not allowed in multivalued params;" + ) + ); + assertThat( + error, + containsString( + "[1:120] Parameter [c] contains a null entry: [null, false]. " + "Null values are not allowed in multivalued params;" + ) + ); + assertThat( + error, + containsString( + "[1:142] Parameter [d] contains a null entry: [null, 2.2]. " + "Null values are not allowed in multivalued params;" + ) + ); + assertThat( + error, + containsString( + "[1:162] Parameter [e] contains a null entry: [null, 1674835275193]. " + + "Null values are not allowed in multivalued params;" + ) + ); + assertThat( + error, + containsString( + "[1:192] Parameter [f] contains a null entry: [null, null]. " + "Null values are not allowed in multivalued params" + ) + ); + } + + public void testArrayValuesAllowedInUnnamedParams() throws IOException { + Map responseMap = runEsql( + RequestObjectBuilder.jsonBuilder() + .query("row a = ? | eval b = ?, c = ?, d = ?, e = ?") + .params("[[\"a1\", \"a2\"], [1, 2], [true, false], [1.1, 2.2], [1674835275193, 1674835275193]]") + ); + System.out.println(responseMap); + + ListMatcher values = matchesList().item( + matchesList().item(matchesList().item("a1").item("a2")) + .item(matchesList().item(1).item(2)) + .item(matchesList().item(true).item(false)) + .item(matchesList().item(1.1).item(2.2)) + .item(matchesList().item(1674835275193L).item(1674835275193L)) + ); + + assertResultMap( + responseMap, + matchesList().item(matchesMap().entry("name", "a").entry("type", "keyword")) + .item(matchesMap().entry("name", "b").entry("type", "integer")) + .item(matchesMap().entry("name", "c").entry("type", "boolean")) + .item(matchesMap().entry("name", "d").entry("type", "double")) + .item(matchesMap().entry("name", "e").entry("type", "long")), + values + ); + } + + public void testErrorMessageForArrayValuesInNonValueParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(RequestObjectBuilder.jsonBuilder().query("row a = 1 | eval x = ?").params("[{\"n1\": [5, 6, 7]}]")) + () -> runEsql( + RequestObjectBuilder.jsonBuilder().query("row a = 1 | eval ?n1 = 5").params("[{\"n1\" : {\"identifier\" : [\"integer\"]}}]") + ) + ); + assertThat( + EntityUtils.toString(re.getResponse().getEntity()), + containsString("\"Failed to parse params: [1:47] n1={identifier=[integer]} parameter is multivalued") + ); + + re = expectThrows( + ResponseException.class, + () -> runEsql(RequestObjectBuilder.jsonBuilder().query("row a = 1 | keep ?n1").params("[{\"n1\" : {\"pattern\" : [\"a*\"]}}]")) ); assertThat( EntityUtils.toString(re.getResponse().getEntity()), - containsString("Failed to parse params: [1:45] n1=[5, 6, 7] is not supported as a parameter") + containsString("\"Failed to parse params: [1:43] n1={pattern=[a*]} parameter is multivalued") ); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java index 01da9f253b695..c9efb5c3ae409 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KnnFunctionIT.java @@ -22,6 +22,11 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.parser.ParserUtils; +import org.elasticsearch.xpack.esql.parser.QueryParam; +import org.elasticsearch.xpack.esql.parser.QueryParams; import org.junit.Before; import java.io.IOException; @@ -130,6 +135,33 @@ public void testKnnOptions() { } } + public void testDenseVectorQueryParams() { + float[] queryVector = new float[numDims]; + Arrays.fill(queryVector, 0); + EsqlQueryRequest queryRequest = new EsqlQueryRequest(); + QueryParams queryParams = new QueryParams( + List.of(new QueryParam("queryVector", Arrays.asList(queryVector), DataType.INTEGER, ParserUtils.ParamClassification.VALUE)) + ); + + queryRequest.params(queryParams); + + var query = String.format(Locale.ROOT, """ + FROM test METADATA _score + | WHERE knn(vector, %s) OR id > 100 + | KEEP id, _score, vector + | SORT _score DESC + | LIMIT 5 + """, Arrays.toString(queryVector)); + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("id", "_score", "vector")); + assertColumnTypes(resp.columns(), List.of("integer", "double", "dense_vector")); + + List> valuesList = EsqlTestUtils.getValuesList(resp); + assertEquals(5, valuesList.size()); + } + } + public void testKnnNonPushedDown() { float[] queryVector = new float[numDims]; Arrays.fill(queryVector, 0.0f); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 60e8d6ffbd0c0..e472c81a1ca90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1477,7 +1477,12 @@ public enum Cap { /** * Support present_over_time aggregation that gets evaluated per time-series */ - PRESENT_OVER_TIME(Build.current().isSnapshot()); + PRESENT_OVER_TIME(Build.current().isSnapshot()), + + /** + * Multivalued query parameters + */ + QUERY_PARAMS_MULTI_VALUES(); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java index da5c6c761ae93..df3ba240c2d6d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java @@ -167,7 +167,6 @@ private static QueryParams parseParams(XContentParser p) throws IOException { } for (Map.Entry entry : param.fields.entrySet()) { ParserUtils.ParamClassification classification = null; - paramValue = null; String paramName = entry.getKey(); checkParamNameValidity(paramName, errors, loc); @@ -177,17 +176,15 @@ private static QueryParams parseParams(XContentParser p) throws IOException { classification = getParamClassification(keyName.toString(), errors, loc); if (classification != null) { paramValue = value.get(keyName); - checkParamValueValidity(classification, paramValue, loc, errors); + checkParamValueValidity(entry, classification, paramValue, loc, errors); } } - } else {// parameter specifies as a value only + } else {// parameter specifies a single or multi value paramValue = entry.getValue(); classification = VALUE; + checkParamValueValidity(entry, classification, paramValue, loc, errors); } type = DataType.fromJava(paramValue); - if (type == null) { - errors.add(new XContentParseException(loc, entry + " is not supported as a parameter")); - } currentParam = new QueryParam( paramName, paramValue, @@ -197,32 +194,30 @@ private static QueryParams parseParams(XContentParser p) throws IOException { namedParams.add(currentParam); } } else { - paramValue = null; - if (token == XContentParser.Token.VALUE_STRING) { - paramValue = p.text(); - type = DataType.KEYWORD; - } else if (token == XContentParser.Token.VALUE_NUMBER) { - XContentParser.NumberType numberType = p.numberType(); - if (numberType == XContentParser.NumberType.INT) { - paramValue = p.intValue(); - type = DataType.INTEGER; - } else if (numberType == XContentParser.NumberType.LONG) { - paramValue = p.longValue(); - type = DataType.LONG; - } else if (numberType == XContentParser.NumberType.DOUBLE) { - paramValue = p.doubleValue(); - type = DataType.DOUBLE; + if (token == XContentParser.Token.START_ARRAY) { + DataType arrayType = DataType.NULL; + List paramValues = new ArrayList<>(); + boolean nullValueFound = false, mixedTypesFound = false; + while ((p.nextToken()) != XContentParser.Token.END_ARRAY) { + ParamValueAndType valueAndDataType = parseSingleParamValue(p, errors); + DataType currentType = valueAndDataType.type; + nullValueFound = nullValueFound | (currentType == DataType.NULL); + mixedTypesFound = mixedTypesFound | (arrayType != DataType.NULL && arrayType != currentType); + if (currentType != DataType.NULL) { + arrayType = currentType; + } + paramValues.add(valueAndDataType.value); + } + if (nullValueFound) { + addNullEntryError(errors, loc, null, paramValues); + } else if (mixedTypesFound) { + addMixedTypesError(errors, loc, null, paramValues); } - } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - paramValue = p.booleanValue(); - type = DataType.BOOLEAN; - } else if (token == XContentParser.Token.VALUE_NULL) { - type = DataType.NULL; + unNamedParams.add(new QueryParam(null, paramValues, arrayType, VALUE)); } else { - errors.add(new XContentParseException(loc, token + " is not supported as a parameter")); + ParamValueAndType valueAndDataType = parseSingleParamValue(p, errors); + unNamedParams.add(new QueryParam(null, valueAndDataType.value, valueAndDataType.type, VALUE)); } - currentParam = new QueryParam(null, paramValue, type, VALUE); - unNamedParams.add(currentParam); } } } @@ -246,6 +241,75 @@ private static QueryParams parseParams(XContentParser p) throws IOException { return new QueryParams(namedParams.isEmpty() ? unNamedParams : namedParams); } + private static void addMixedTypesError( + List errors, + XContentLocation loc, + String paramName, + List paramValues + ) { + errors.add( + new XContentParseException( + loc, + "Parameter " + + (paramName == null ? "" : "[" + paramName + "] ") + + "contains mixed data types: " + + paramValues + + ". Mixed data types are not allowed in multivalued params." + ) + ); + } + + private static void addNullEntryError( + List errors, + XContentLocation loc, + String paramName, + List paramValues + ) { + errors.add( + new XContentParseException( + loc, + "Parameter " + + (paramName == null ? "" : "[" + paramName + "] ") + + "contains a null entry: " + + paramValues + + ". Null values are not allowed in multivalued params" + ) + ); + } + + private record ParamValueAndType(Object value, DataType type) {} + + private static ParamValueAndType parseSingleParamValue(XContentParser p, List errors) throws IOException { + Object paramValue = null; + DataType type = DataType.NULL; + XContentParser.Token token = p.currentToken(); + if (token == XContentParser.Token.VALUE_STRING) { + paramValue = p.text(); + type = DataType.KEYWORD; + } else if (token == XContentParser.Token.VALUE_NUMBER) { + XContentParser.NumberType numberType = p.numberType(); + if (numberType == XContentParser.NumberType.INT) { + paramValue = p.intValue(); + type = DataType.INTEGER; + } else if (numberType == XContentParser.NumberType.LONG) { + paramValue = p.longValue(); + type = DataType.LONG; + } else if (numberType == XContentParser.NumberType.DOUBLE) { + paramValue = p.doubleValue(); + type = DataType.DOUBLE; + } + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + paramValue = p.booleanValue(); + type = DataType.BOOLEAN; + } else if (token == XContentParser.Token.VALUE_NULL) { + type = DataType.NULL; + } else { + XContentLocation loc = p.getTokenLocation(); + errors.add(new XContentParseException(loc, token + " is not supported as a parameter")); + } + return new ParamValueAndType(paramValue, type); + } + private static void checkParamNameValidity(String name, List errors, XContentLocation loc) { if (isValidParamName(name) == false) { errors.add( @@ -317,11 +381,44 @@ private static ParserUtils.ParamClassification getParamClassification( } private static void checkParamValueValidity( + Map.Entry entry, ParserUtils.ParamClassification classification, Object value, XContentLocation loc, List errors ) { + if (value instanceof List valueList) { + if (classification != VALUE) { + errors.add( + new XContentParseException( + loc, + entry + " parameter is multivalued, only " + VALUE.name() + " parameters can be multivalued" + ) + ); + return; + } + // Multivalued field + DataType arrayType = null; + for (Object currentValue : valueList) { + checkParamValueValidity(entry, classification, currentValue, loc, errors); + DataType currentType = DataType.fromJava(currentValue); + if (currentType == DataType.NULL) { + addNullEntryError(errors, loc, entry.getKey(), valueList); + break; + } else if (arrayType != null && arrayType != currentType) { + addMixedTypesError(errors, loc, entry.getKey(), valueList); + break; + } + arrayType = currentType; + } + return; + } + + DataType type = DataType.fromJava(value); + if (type == null) { + errors.add(new XContentParseException(loc, entry + " is not supported as a parameter")); + } + // If a param is an "identifier" or a "pattern", validate it is a string. // If a param is a "pattern", validate it contains *. if (classification == IDENTIFIER || classification == PATTERN) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 451be4db743da..e726b48888a73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -1047,8 +1047,12 @@ private Expression visitParam(EsqlBaseParser.ParameterContext ctx, QueryParam pa return new UnresolvedAttribute(source, value.toString()); } } - if ((type == KEYWORD || type == TEXT) && value instanceof String) { - value = BytesRefs.toBytesRef(value); + if ((type == KEYWORD || type == TEXT)) { + if (value instanceof String) { + value = BytesRefs.toBytesRef(value); + } else if (value instanceof List list) { + value = list.stream().map(v -> v instanceof String ? BytesRefs.toBytesRef(v) : v).toList(); + } } return new Literal(source, value, type); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java index 00ac4837a1fe7..b77ab2d903672 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java @@ -35,6 +35,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.parser.ParserUtils; import org.elasticsearch.xpack.esql.parser.ParsingException; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; @@ -104,7 +106,10 @@ public void testNamedParams() throws IOException { ,"params":[ {"n1" : "8.15.0"}, { "n2" : 0.05}, {"n3" : -799810013}, {"n4" : "127.0.0.1"}, {"n5" : "esql"}, {"n_6" : null}, {"n7_" : false}, {"_n1" : "8.15.0"}, { "__n2" : 0.05}, {"__3" : -799810013}, - {"__4n" : "127.0.0.1"}, {"_n5" : "esql"}, {"_n6" : null}, {"_n7" : false}] }"""; + {"__4n" : "127.0.0.1"}, {"_n5" : "esql"}, {"_n6" : null}, {"_n7" : false}, + {"_n8": ["8.15.0", "8.19.0"]}, {"_n9": ["x", "y"]}, {"_n10": [true, false]}, {"_n11": [1.0, 1.1, 1.2]}, + {"_n12": [-799810013, 0, 799810013]} + ] }"""; List params = List.of( paramAsConstant("n1", "8.15.0"), @@ -120,7 +125,54 @@ public void testNamedParams() throws IOException { paramAsConstant("__4n", "127.0.0.1"), paramAsConstant("_n5", "esql"), paramAsConstant("_n6", null), - paramAsConstant("_n7", false) + paramAsConstant("_n7", false), + new QueryParam("_n8", List.of("8.15.0", "8.19.0"), KEYWORD, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n9", List.of("x", "y"), KEYWORD, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n10", List.of(true, false), BOOLEAN, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n11", List.of(1.0, 1.1, 1.2), DOUBLE, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n12", List.of(-799810013, 0, 799810013), DataType.INTEGER, ParserUtils.ParamClassification.VALUE) + // TODO add mixed null values, or check all elements, and separate into a new method + ); + String json = String.format(Locale.ROOT, """ + { + "query": "%s", + "columnar": %s, + "locale": "%s", + "filter": %s + %s""", query, columnar, locale.toLanguageTag(), filter, paramsString); + + EsqlQueryRequest request = parseEsqlQueryRequestSync(json); + + assertEquals(query, request.query()); + assertEquals(columnar, request.columnar()); + assertEquals(locale.toLanguageTag(), request.locale().toLanguageTag()); + assertEquals(locale, request.locale()); + assertEquals(filter, request.filter()); + assertEquals(params.size(), request.params().size()); + + for (int i = 0; i < request.params().size(); i++) { + assertEquals(params.get(i), request.params().get(i + 1)); + } + } + + public void testNamedMultivaluedParams() throws IOException { + String query = randomAlphaOfLengthBetween(1, 100); + boolean columnar = randomBoolean(); + Locale locale = randomLocale(random()); + QueryBuilder filter = randomQueryBuilder(); + + String paramsString = """ + ,"params":[ + {"_n1": ["8.15.0", "8.19.0"]}, {"_n2": ["x", "y"]}, {"_n3": [true, false]}, {"_n4": [1.0, 1.1, 1.2]}, + {"_n5": [-799810013, 0, 799810013]} + ] }"""; + + List params = List.of( + new QueryParam("_n1", List.of("8.15.0", "8.19.0"), KEYWORD, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n2", List.of("x", "y"), KEYWORD, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n3", List.of(true, false), BOOLEAN, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n4", List.of(1.0, 1.1, 1.2), DOUBLE, ParserUtils.ParamClassification.VALUE), + new QueryParam("_n5", List.of(-799810013, 0, 799810013), DataType.INTEGER, ParserUtils.ParamClassification.VALUE) ); String json = String.format(Locale.ROOT, """ { @@ -256,6 +308,106 @@ public void testInvalidParams() throws IOException { ); } + public void testInvalidMultivaluedNamedParams() throws IOException { + String query = randomAlphaOfLengthBetween(1, 100); + boolean columnar = randomBoolean(); + Locale locale = randomLocale(random()); + QueryBuilder filter = randomQueryBuilder(); + + // invalid named parameter for multivalued constants + String paramsString = """ + "params":[ + {"_n1": [null, "8.15.0"]}, {"_n2": [null, null, "x"]}, {"_n3": [null, true, false]}, + {"_n4": [null, 1.0, null]}, {"_n5": [null, -799810013, null, 799810013]}, + {"n6" : [{"value" : {"a5" : "v5"}}]}, {"n7" : [{"identifier" : ["x", "y"]}]}, {"n8" : [{"pattern" : ["x*", "y*"]}]} + ]"""; + String json1 = String.format(Locale.ROOT, """ + { + %s, + "query": "%s", + "columnar": %s, + "locale": "%s", + "filter": %s + }""", paramsString, query, columnar, locale.toLanguageTag(), filter); + + Exception e1 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json1)); + assertThat( + e1.getCause().getMessage(), + containsString( + "[3:2] Parameter [_n1] contains a null entry: [null, 8.15.0]. Null values are not allowed in multivalued params;" + ) + ); + assertThat( + e1.getCause().getMessage(), + containsString( + "[3:29] Parameter [_n2] contains a null entry: [null, null, x]. Null values are not allowed in multivalued params;" + ) + ); + assertThat( + e1.getCause().getMessage(), + containsString( + "[3:57] Parameter [_n3] contains a null entry: [null, true, false]. Null values are not allowed in multivalued params;" + ) + ); + assertThat( + e1.getCause().getMessage(), + containsString( + "[4:2] Parameter [_n4] contains a null entry: [null, 1.0, null]. Null values are not allowed in multivalued params;" + ) + ); + assertThat( + e1.getCause().getMessage(), + containsString( + "[4:30] Parameter [_n5] contains a null entry: [null, -799810013, null, 799810013]. " + + "Null values are not allowed in multivalued params;" + ) + ); + assertThat(e1.getCause().getMessage(), containsString("[5:2] n6=[{value={a5=v5}}] is not supported as a parameter")); + assertThat(e1.getCause().getMessage(), containsString("[5:40] n7=[{identifier=[x, y]}] is not supported as a parameter")); + assertThat(e1.getCause().getMessage(), containsString("[5:80] n8=[{pattern=[x*, y*]}] is not supported as a parameter")); + } + + public void testInvalidMultivaluedUnnamedParams() throws IOException { + String query = randomAlphaOfLengthBetween(1, 100); + boolean columnar = randomBoolean(); + Locale locale = randomLocale(random()); + QueryBuilder filter = randomQueryBuilder(); + + // invalid named parameter for multivalued constants + String paramsString = """ + "params":[ + [null, "8.15.0"], [null, null, "x"], [null, true, false], [null, 1.0, null], [null, -799810013, null, 799810013] + ]"""; + String json1 = String.format(Locale.ROOT, """ + { + %s, + "query": "%s", + "columnar": %s, + "locale": "%s", + "filter": %s + }""", paramsString, query, columnar, locale.toLanguageTag(), filter); + + Exception e1 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json1)); + assertThat( + e1.getCause().getMessage(), + containsString("[3:2] Parameter contains a null entry: [null, 8.15.0]. Null values are not allowed in multivalued params;") + ); + assertThat( + e1.getCause().getMessage(), + containsString("[3:20] Parameter contains a null entry: [null, null, x]. Null values are not allowed in multivalued params;") + ); + assertThat( + e1.getCause().getMessage(), + containsString( + "[3:39] Parameter contains a null entry: [null, true, false]. Null values are not allowed in multivalued params;" + ) + ); + assertThat( + e1.getCause().getMessage(), + containsString("[3:60] Parameter contains a null entry: [null, 1.0, null]. Null values are not allowed in multivalued params;") + ); + } + public void testInvalidParamsForIdentifiersPatterns() throws IOException { String query = randomAlphaOfLengthBetween(1, 100); boolean columnar = randomBoolean(); @@ -280,40 +432,88 @@ public void testInvalidParamsForIdentifiersPatterns() throws IOException { }""", paramsString1, query, columnar, locale.toLanguageTag(), filter); Exception e1 = expectThrows(XContentParseException.class, () -> parseEsqlQueryRequestSync(json1)); + String message = e1.getCause().getMessage(); assertThat( - e1.getCause().getMessage(), + message, + containsString("[2:15] [v] is not a valid param attribute, a valid attribute is any of VALUE, IDENTIFIER, PATTERN; ") + ); + assertThat( + message, + containsString( + "[2:38] [n2] has multiple param attributes [identifier, pattern], " + + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param;" + ) + ); + assertThat( + message, + containsString( + "[2:38] [v2] is not a valid value for PATTERN parameter, " + + "a valid value for PATTERN parameter is a string and contains *;" + ) + ); + assertThat( + message, + containsString( + "[3:1] [n3] has multiple param attributes [identifier, pattern], " + + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param;" + ) + ); + assertThat( + message, + containsString( + "[3:1] [v3] is not a valid value for PATTERN parameter, " + + "a valid value for PATTERN parameter is a string and contains *;" + ) + ); + assertThat( + message, + containsString( + "[3:51] [n4] has multiple param attributes [pattern, value], " + + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param;" + ) + ); + assertThat( + message, + containsString( + "[3:51] [v4.1] is not a valid value for PATTERN parameter, " + + "a valid value for PATTERN parameter is a string and contains *;" + ) + ); + assertThat(message, containsString("[4:1] n5={value={a5=v5}} is not supported as a parameter;")); + assertThat(message, containsString("[4:36] [{a6.1=v6.1, a6.2=v6.2}] is not a valid value for IDENTIFIER parameter,")); + assertThat(message, containsString("a valid value for IDENTIFIER parameter is a string;")); + assertThat(message, containsString("[4:36] n6={identifier={a6.1=v6.1, a6.2=v6.2}} is not supported as a parameter;")); + assertThat( + message, + containsString("[4:98] [n7] has no valid param attribute, only one of VALUE, IDENTIFIER, PATTERN can be defined in a param;") + ); + assertThat( + message, + containsString("[5:34] n9={identifier=[x, y]} parameter is multivalued, only VALUE parameters can be multivalued;") + ); + assertThat( + message, + containsString("[5:72] n10={pattern=[x*, y*]} parameter is multivalued, only VALUE parameters can be multivalued;") + ); + assertThat(message, containsString("a valid value for PATTERN parameter is a string and contains *;")); + assertThat( + message, + containsString("[6:1] [1] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string;") + ); + assertThat(message, containsString("[6:31] [true] is not a valid value for PATTERN parameter,")); + assertThat(message, containsString("a valid value for PATTERN parameter is a string and contains *;")); + assertThat( + message, + containsString( + "[6:61] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string;" + ) + ); + assertThat(message, containsString("[6:94] [v14] is not a valid value for PATTERN parameter,")); + assertThat(message, containsString("a valid value for PATTERN parameter is a string and contains *;")); + assertThat( + message, containsString( - "[2:15] [v] is not a valid param attribute, a valid attribute is any of VALUE, IDENTIFIER, PATTERN; " - + "[2:38] [n2] has multiple param attributes [identifier, pattern], " - + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; " - + "[2:38] [v2] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[3:1] [n3] has multiple param attributes [identifier, pattern], " - + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; " - + "[3:1] [v3] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[3:51] [n4] has multiple param attributes [pattern, value], " - + "only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; " - + "[3:51] [v4.1] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[4:1] n5={value={a5=v5}} is not supported as a parameter; " - + "[4:36] [{a6.1=v6.1, a6.2=v6.2}] is not a valid value for IDENTIFIER parameter, " - + "a valid value for IDENTIFIER parameter is a string; " - + "[4:36] n6={identifier={a6.1=v6.1, a6.2=v6.2}} is not supported as a parameter; " - + "[4:98] [n7] has no valid param attribute, only one of VALUE, IDENTIFIER, PATTERN can be defined in a param; " - + "[5:1] n8={value=[x, y]} is not supported as a parameter; " - + "[5:34] [[x, y]] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " - + "[5:34] n9={identifier=[x, y]} is not supported as a parameter; " - + "[5:72] [[x*, y*]] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[5:72] n10={pattern=[x*, y*]} is not supported as a parameter; " - + "[6:1] [1] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " - + "[6:31] [true] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[6:61] [null] is not a valid value for IDENTIFIER parameter, a valid value for IDENTIFIER parameter is a string; " - + "[6:94] [v14] is not a valid value for PATTERN parameter, " - + "a valid value for PATTERN parameter is a string and contains *; " - + "[7:1] Cannot parse more than one key:value pair as parameter, found [{n16:{identifier=v16}}, {n15:{pattern=v15*}}]" + "[7:1] Cannot parse more than one key:value pair as parameter, found [{n16:{identifier=v16}}, {n15:{pattern=v15*}}" ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index d1dbdb9466614..417eb0f1a7834 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -2407,6 +2407,25 @@ private static void checkDenseVectorEvalCastingKnn(String fieldName) { assertThat(queryVector.name(), is("query")); } + public void testDenseVectorImplicitCastingKnnQueryParams() { + checkDenseVectorCastingKnnQueryParams("float_vector"); + checkDenseVectorCastingKnnQueryParams("byte_vector"); + checkDenseVectorCastingKnnQueryParams("bit_vector"); + } + + private void checkDenseVectorCastingKnnQueryParams(String fieldName) { + var plan = analyze(String.format(Locale.ROOT, """ + from test | where knn(%s, ?query_vector) + """, fieldName), "mapping-dense_vector.json", new QueryParams(List.of(paramAsConstant("query_vector", List.of(0, 1, 2))))); + + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var knn = as(filter.condition(), Knn.class); + var queryVector = as(knn.query(), ToDenseVector.class); + var literal = as(queryVector.field(), Literal.class); + assertThat(literal.value(), equalTo(List.of(0, 1, 2))); + } + public void testDenseVectorImplicitCastingSimilarityFunctions() { assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml index b9d20d4cd40cf..e38b661b2baa8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml @@ -412,6 +412,38 @@ FROM EVAL SORT LIMIT with documents_found: - match: {values.1: ["1",2.0,null,true,123,1674835275193]} - match: {values.2: ["1",2.0,null,true,123,1674835275193]} +--- +"Test Unnamed, Multivalued Input Params": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: POST + path: /_query + parameters: [ ] + capabilities: [ query_params_multi_values ] + reason: "multivalued parameters" + - do: + esql.query: + body: + query: 'from test | eval x = ?, y = ?, z = ?, t = ?, u = ? | keep x, y, z, t, u | limit 3' + params: [["1", "2"], [2.0, 3.0], [true, false], [123, 456], [1674835275193, 1674835275193]] + + - length: {columns: 5} + - match: {columns.0.name: "x"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "y"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "z"} + - match: {columns.2.type: "boolean"} + - match: {columns.3.name: "t"} + - match: {columns.3.type: "integer"} + - match: {columns.4.name: "u"} + - match: {columns.4.type: "long"} + - length: {values: 3} + - match: {values.0: [["1", "2"], [2.0, 3.0], [true, false], [123, 456], [1674835275193, 1674835275193]]} + - match: {values.1: [["1", "2"], [2.0, 3.0], [true, false], [123, 456], [1674835275193, 1674835275193]]} + - match: {values.2: [["1", "2"], [2.0, 3.0], [true, false], [123, 456], [1674835275193, 1674835275193]]} + --- "Test Unnamed Input Params Also For Limit And Sample": - requires: @@ -486,6 +518,39 @@ FROM EVAL SORT LIMIT with documents_found: - match: {values.1: ["1",2.0,null,true,123,1674835275193]} - match: {values.2: ["1",2.0,null,true,123,1674835275193]} +--- +"Test Named, Multivalued Input Params": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: POST + path: /_query + parameters: [ ] + capabilities: [ parameter_for_limit, parameter_for_sample, query_params_multi_values ] + reason: "named or positional parameters, multivalued parameters" + + - do: + esql.query: + body: + query: 'from test | eval x = ?n1, y = ?n2, z = ?n3, t = ?n4, u = ?n5 | keep x, y, z, t, u | limit ?l' + params: [{"n1" : ["1", "2"]}, {"n2" : [2.0, 3.0]}, {"n3" : [true, false]}, {"n4" : [123, 456]}, {"n5": [1674835275193, -1674835275193]}, {"l": 3}] + + - length: {columns: 5} + - match: {columns.0.name: "x"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "y"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "z"} + - match: {columns.2.type: "boolean"} + - match: {columns.3.name: "t"} + - match: {columns.3.type: "integer"} + - match: {columns.4.name: "u"} + - match: {columns.4.type: "long"} + - length: {values: 3} + - match: {values.0: [["1", "2"], [2.0, 3.0], [true, false] ,[123, 456], [1674835275193, -1674835275193]]} + - match: {values.1: [["1", "2"], [2.0, 3.0], [true, false] ,[123, 456], [1674835275193, -1674835275193]]} + - match: {values.2: [["1", "2"], [2.0, 3.0], [true, false] ,[123, 456], [1674835275193, -1674835275193]]} + --- "Test Interval in Input Params": - requires: