Skip to content

Commit d821c47

Browse files
fang-xing-esqlelasticmachine
fang-xing-esql
and
elasticmachine
authored
[8.x] [ES|QL] Named parameter for field names and field name patterns (#112905) (#114529)
* [ES|QL] Named parameter for field names and field name patterns (#112905) * named parameters for field name and pattern (cherry picked from commit 4122d89) * update for backward compatibility on older jvm --------- Co-authored-by: Elastic Machine <[email protected]>
1 parent 24e246a commit d821c47

File tree

28 files changed

+3115
-1794
lines changed

28 files changed

+3115
-1794
lines changed

‎docs/changelog/112905.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 112905
2+
summary: "[ES|QL] Named parameter for field names and field name patterns"
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java‎

Lines changed: 129 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import java.util.regex.Pattern;
5656

5757
import static java.util.Collections.emptySet;
58+
import static java.util.Map.entry;
5859
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
5960
import static org.elasticsearch.test.ListMatcher.matchesList;
6061
import static org.elasticsearch.test.MapMatcher.assertMap;
@@ -648,6 +649,134 @@ public void testErrorMessageForInvalidIntervalParams() throws IOException {
648649
);
649650
}
650651

652+
public void testErrorMessageForArrayValuesInParams() throws IOException {
653+
ResponseException re = expectThrows(
654+
ResponseException.class,
655+
() -> runEsql(RequestObjectBuilder.jsonBuilder().query("row a = 1 | eval x = ?").params("[{\"n1\": [5, 6, 7]}]"))
656+
);
657+
assertThat(
658+
EntityUtils.toString(re.getResponse().getEntity()),
659+
containsString("Failed to parse params: [1:45] n1=[5, 6, 7] is not supported as a parameter")
660+
);
661+
}
662+
663+
public void testNamedParamsForIdentifierAndIdentifierPatterns() throws IOException {
664+
bulkLoadTestData(10);
665+
// positive
666+
var query = requestObjectBuilder().query(
667+
format(
668+
null,
669+
"from {} | eval x1 = ?n1 | where ?n2 == x1 | stats xx2 = ?fn1(?n3) by ?n4 | keep ?n4, ?n5 | sort ?n4",
670+
testIndexName()
671+
)
672+
)
673+
.params(
674+
"[{\"n1\" : {\"value\" : \"integer\" , \"kind\" : \"identifier\"}},"
675+
+ "{\"n2\" : {\"value\" : \"short\" , \"kind\" : \"identifier\"}}, "
676+
+ "{\"n3\" : {\"value\" : \"double\" , \"kind\" : \"identifier\"}},"
677+
+ "{\"n4\" : {\"value\" : \"boolean\" , \"kind\" : \"identifier\"}}, "
678+
+ "{\"n5\" : {\"value\" : \"xx*\" , \"kind\" : \"pattern\"}}, "
679+
+ "{\"fn1\" : {\"value\" : \"max\" , \"kind\" : \"identifier\"}}]"
680+
);
681+
Map<String, Object> result = runEsql(query);
682+
Map<String, String> colA = Map.of("name", "boolean", "type", "boolean");
683+
Map<String, String> colB = Map.of("name", "xx2", "type", "double");
684+
assertEquals(List.of(colA, colB), result.get("columns"));
685+
assertEquals(List.of(List.of(false, 9.1), List.of(true, 8.1)), result.get("values"));
686+
687+
// missing params
688+
ResponseException re = expectThrows(
689+
ResponseException.class,
690+
() -> runEsqlSync(
691+
requestObjectBuilder().query(
692+
format(
693+
null,
694+
"from {} | eval x1 = ?n1 | where ?n2 == x1 | stats xx2 = max(?n3) by ?n4 | keep ?n4, ?n5 | sort ?n4",
695+
testIndexName()
696+
)
697+
).params("[]")
698+
)
699+
);
700+
String error = re.getMessage();
701+
assertThat(error, containsString("ParsingException"));
702+
assertThat(error, containsString("Unknown query parameter [n1]"));
703+
704+
// param inside backquote is not recognized as a param
705+
Map<String, Integer> commandsWithLineNumber = Map.ofEntries(
706+
entry("eval x1 = `?n1`", 33),
707+
entry("where `?n1` == 1", 29),
708+
entry("stats x = max(n2) by `?n1`", 44),
709+
entry("stats x = max(`?n1`) by n2", 37),
710+
entry("keep `?n1`", 28),
711+
entry("sort `?n1`", 28)
712+
);
713+
for (Map.Entry<String, Integer> command : commandsWithLineNumber.entrySet()) {
714+
re = expectThrows(
715+
ResponseException.class,
716+
() -> runEsqlSync(
717+
requestObjectBuilder().query(format(null, "from {} | {}", testIndexName(), command.getKey()))
718+
.params(
719+
"[{\"n1\" : {\"value\" : \"integer\" , \"kind\" : \"identifier\"}},"
720+
+ "{\"n2\" : {\"value\" : \"short\" , \"kind\" : \"identifier\"}}]"
721+
)
722+
)
723+
);
724+
error = re.getMessage();
725+
assertThat(error, containsString("VerificationException"));
726+
assertThat(error, containsString("line 1:" + command.getValue() + ": Unknown column [?n1]"));
727+
}
728+
729+
commandsWithLineNumber = Map.ofEntries(
730+
entry("rename ?n1 as ?n2", 30),
731+
entry("enrich idx2 ON ?n1 WITH ?n2 = ?n3", 38),
732+
entry("keep ?n1", 28),
733+
entry("drop ?n1", 28)
734+
);
735+
for (Map.Entry<String, Integer> command : commandsWithLineNumber.entrySet()) {
736+
re = expectThrows(
737+
ResponseException.class,
738+
() -> runEsqlSync(
739+
requestObjectBuilder().query(format(null, "from {} | {}", testIndexName(), command.getKey()))
740+
.params(
741+
"[{\"n1\" : {\"value\" : \"`n1`\" , \"kind\" : \"identifier\"}},"
742+
+ "{\"n2\" : {\"value\" : \"`n2`\" , \"kind\" : \"identifier\"}}, "
743+
+ "{\"n3\" : {\"value\" : \"`n3`\" , \"kind\" : \"identifier\"}}]"
744+
)
745+
)
746+
);
747+
error = re.getMessage();
748+
assertThat(error, containsString("VerificationException"));
749+
assertThat(error, containsString("line 1:" + command.getValue() + ": Unknown column [`n1`]"));
750+
}
751+
752+
// param cannot be used as a command name
753+
Map<String, String> paramsAsCommandNames = Map.ofEntries(
754+
entry("eval", "x = 1"),
755+
entry("where", "x == 1"),
756+
entry("stats", "x = count(*)"),
757+
entry("keep", "x"),
758+
entry("drop", "x"),
759+
entry("rename", "x as y"),
760+
entry("sort", "x"),
761+
entry("dissect", "x \"%{foo}\""),
762+
entry("grok", "x \"%{WORD:foo}\""),
763+
entry("enrich", "idx2 ON x"),
764+
entry("mvExpand", "x")
765+
);
766+
for (Map.Entry<String, String> command : paramsAsCommandNames.entrySet()) {
767+
re = expectThrows(
768+
ResponseException.class,
769+
() -> runEsqlSync(
770+
requestObjectBuilder().query(format(null, "from {} | ?cmd {}", testIndexName(), command.getValue()))
771+
.params("[{\"cmd\" : {\"value\" : \"" + command.getKey() + "\", \"kind\" : \"identifier\"}}]")
772+
)
773+
);
774+
error = re.getMessage();
775+
assertThat(error, containsString("ParsingException"));
776+
assertThat(error, containsString("line 1:23: mismatched input '?cmd' expecting {'dissect', 'drop'"));
777+
}
778+
}
779+
651780
public void testErrorMessageForLiteralDateMathOverflow() throws IOException {
652781
List<String> dateMathOverflowExpressions = List.of(
653782
"2147483647 day + 1 day",
@@ -687,14 +816,6 @@ private void assertExceptionForDateMath(String dateMathString, String errorSubst
687816
assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400));
688817
}
689818

690-
public void testErrorMessageForArrayValuesInParams() throws IOException {
691-
ResponseException re = expectThrows(
692-
ResponseException.class,
693-
() -> runEsql(requestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"n1\": [5, 6, 7]}]"))
694-
);
695-
assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("n1=[5, 6, 7] is not supported as a parameter"));
696-
}
697-
698819
public void testComplexFieldNames() throws IOException {
699820
bulkLoadTestData(1);
700821
// catch verification exception, field names not found

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
5656
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
5757
import org.elasticsearch.xpack.esql.index.EsIndex;
58+
import org.elasticsearch.xpack.esql.parser.QueryParam;
5859
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
5960
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
6061
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
@@ -119,8 +120,12 @@
119120
import static org.elasticsearch.test.MapMatcher.assertMap;
120121
import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
121122
import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
123+
import static org.elasticsearch.xpack.esql.core.type.DataType.NULL;
122124
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN;
123125
import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO;
126+
import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.IDENTIFIER;
127+
import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.PATTERN;
128+
import static org.elasticsearch.xpack.esql.parser.ParserUtils.ParamClassification.VALUE;
124129
import static org.hamcrest.Matchers.instanceOf;
125130
import static org.junit.Assert.assertTrue;
126131

@@ -681,4 +686,16 @@ public static WildcardLike wildcardLike(Expression left, String exp) {
681686
public static RLike rlike(Expression left, String exp) {
682687
return new RLike(EMPTY, left, new RLikePattern(exp));
683688
}
689+
690+
public static QueryParam paramAsConstant(String name, Object value) {
691+
return new QueryParam(name, value, DataType.fromJava(value), VALUE);
692+
}
693+
694+
public static QueryParam paramAsIdentifier(String name, Object value) {
695+
return new QueryParam(name, value, NULL, IDENTIFIER);
696+
}
697+
698+
public static QueryParam paramAsPattern(String name, Object value) {
699+
return new QueryParam(name, value, NULL, PATTERN);
700+
}
684701
}

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,8 @@ mode PROJECT_MODE;
306306
PROJECT_PIPE : PIPE -> type(PIPE), popMode;
307307
PROJECT_DOT: DOT -> type(DOT);
308308
PROJECT_COMMA : COMMA -> type(COMMA);
309+
PROJECT_PARAM : PARAM -> type(PARAM);
310+
PROJECT_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
309311
310312
fragment UNQUOTED_ID_BODY_WITH_PATTERN
311313
: (LETTER | DIGIT | UNDERSCORE | ASTERISK)
@@ -339,6 +341,8 @@ RENAME_PIPE : PIPE -> type(PIPE), popMode;
339341
RENAME_ASSIGN : ASSIGN -> type(ASSIGN);
340342
RENAME_COMMA : COMMA -> type(COMMA);
341343
RENAME_DOT: DOT -> type(DOT);
344+
RENAME_PARAM : PARAM -> type(PARAM);
345+
RENAME_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
342346
343347
AS : 'as';
344348
@@ -410,6 +414,9 @@ ENRICH_FIELD_QUOTED_IDENTIFIER
410414
: QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)
411415
;
412416

417+
ENRICH_FIELD_PARAM : PARAM -> type(PARAM);
418+
ENRICH_FIELD_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
419+
413420
ENRICH_FIELD_LINE_COMMENT
414421
: LINE_COMMENT -> channel(HIDDEN)
415422
;
@@ -425,6 +432,8 @@ ENRICH_FIELD_WS
425432
mode MVEXPAND_MODE;
426433
MVEXPAND_PIPE : PIPE -> type(PIPE), popMode;
427434
MVEXPAND_DOT: DOT -> type(DOT);
435+
MVEXPAND_PARAM : PARAM -> type(PARAM);
436+
MVEXPAND_NAMED_OR_POSITIONAL_PARAM : NAMED_OR_POSITIONAL_PARAM -> type(NAMED_OR_POSITIONAL_PARAM);
428437

429438
MVEXPAND_QUOTED_IDENTIFIER
430439
: QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER)

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ primaryExpression
101101
;
102102

103103
functionExpression
104-
: identifier LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP
104+
: identifierOrParameter LP (ASTERISK | (booleanExpression (COMMA booleanExpression)*))? RP
105105
;
106106

107107
dataType
@@ -165,7 +165,7 @@ statsCommand
165165
;
166166

167167
qualifiedName
168-
: identifier (DOT identifier)*
168+
: identifierOrParameter (DOT identifierOrParameter)*
169169
;
170170

171171
qualifiedNamePattern
@@ -183,6 +183,7 @@ identifier
183183

184184
identifierPattern
185185
: ID_PATTERN
186+
| parameter
186187
;
187188

188189
constant
@@ -191,18 +192,23 @@ constant
191192
| decimalValue #decimalLiteral
192193
| integerValue #integerLiteral
193194
| booleanValue #booleanLiteral
194-
| params #inputParams
195+
| parameter #inputParameter
195196
| string #stringLiteral
196197
| OPENING_BRACKET numericValue (COMMA numericValue)* CLOSING_BRACKET #numericArrayLiteral
197198
| OPENING_BRACKET booleanValue (COMMA booleanValue)* CLOSING_BRACKET #booleanArrayLiteral
198199
| OPENING_BRACKET string (COMMA string)* CLOSING_BRACKET #stringArrayLiteral
199200
;
200201

201-
params
202+
parameter
202203
: PARAM #inputParam
203204
| NAMED_OR_POSITIONAL_PARAM #inputNamedOrPositionalParam
204205
;
205206

207+
identifierOrParameter
208+
: identifier
209+
| parameter
210+
;
211+
206212
limitCommand
207213
: LIMIT INTEGER_LITERAL
208214
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,12 @@ public enum Cap {
350350
/**
351351
* Compute year differences in full calendar years.
352352
*/
353-
DATE_DIFF_YEAR_CALENDARIAL;
353+
DATE_DIFF_YEAR_CALENDARIAL,
354+
355+
/**
356+
* Support named parameters for field names.
357+
*/
358+
NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES;
354359

355360
private final boolean snapshotOnly;
356361
private final FeatureFlag featureFlag;

0 commit comments

Comments
 (0)