diff --git a/docs/changelog/124958.yaml b/docs/changelog/124958.yaml new file mode 100644 index 0000000000000..be7f646b7dcae --- /dev/null +++ b/docs/changelog/124958.yaml @@ -0,0 +1,6 @@ +pr: 124958 +summary: Catch parsing exception +area: ES|QL +type: bug +issues: + - 119025 \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index 620a25e0170ea..a8ee18d8b2777 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -14,25 +14,79 @@ import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.VocabularyImpl; import org.antlr.v4.runtime.atn.PredictionMode; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import java.util.BitSet; +import java.util.EmptyStackException; +import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.elasticsearch.xpack.esql.core.util.StringUtils.isInteger; +import static org.elasticsearch.xpack.esql.parser.ParserUtils.nameOrPosition; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; public class EsqlParser { private static final Logger log = LogManager.getLogger(EsqlParser.class); + /** + * Maximum number of characters in an ESQL query. Antlr may parse the entire + * query into tokens to make the choices, buffering the world. There's a lot we + * can do in the grammar to prevent that, but let's be paranoid and assume we'll + * fail at preventing antlr from slurping in the world. Instead, let's make sure + * that the world just isn't that big. + */ + public static final int MAX_LENGTH = 1_000_000; + + private static void replaceSymbolWithLiteral(Map symbolReplacements, String[] literalNames, String[] symbolicNames) { + for (int i = 0, replacements = symbolReplacements.size(); i < symbolicNames.length && replacements > 0; i++) { + String symName = symbolicNames[i]; + if (symName != null) { + String replacement = symbolReplacements.get(symName); + if (replacement != null && literalNames[i] == null) { + // literals are single quoted + literalNames[i] = "'" + replacement + "'"; + replacements--; + } + } + } + } + + /** + * Add the literal name to a number of tokens that due to ANTLR internals/ATN + * have their symbolic name returns instead during error reporting. + * When reporting token errors, ANTLR uses the Vocabulary class to get the displayName + * (if set), otherwise falls back to the literal one and eventually uses the symbol name. + * Since the Vocabulary is static and not pluggable, this code modifies the underlying + * arrays by setting the literal string manually based on the token index. + * This is needed since some symbols, especially around setting up the mode, end up losing + * their literal representation. + * NB: this code is highly dependent on the ANTLR internals and thus will likely break + * during upgrades. + * NB: Can't use this for replacing DEV_ since the Vocabular is static while DEV_ replacement occurs per runtime configuration + */ + static { + Map symbolReplacements = Map.of("LP", "(", "OPENING_BRACKET", "["); + + // the vocabularies have the same content however are different instances + // for extra reliability, perform the replacement for each map + VocabularyImpl parserVocab = (VocabularyImpl) EsqlBaseParser.VOCABULARY; + replaceSymbolWithLiteral(symbolReplacements, parserVocab.getLiteralNames(), parserVocab.getSymbolicNames()); + + VocabularyImpl lexerVocab = (VocabularyImpl) EsqlBaseLexer.VOCABULARY; + replaceSymbolWithLiteral(symbolReplacements, lexerVocab.getLiteralNames(), lexerVocab.getSymbolicNames()); + } + private EsqlConfig config = new EsqlConfig(); public EsqlConfig config() { @@ -43,25 +97,34 @@ public void setEsqlConfig(EsqlConfig config) { this.config = config; } + // testing utility public LogicalPlan createStatement(String query) { return createStatement(query, new QueryParams()); } + // testing utility public LogicalPlan createStatement(String query, QueryParams params) { + return createStatement(query, params, new PlanTelemetry(new EsqlFunctionRegistry())); + } + + public LogicalPlan createStatement(String query, QueryParams params, PlanTelemetry metrics) { if (log.isDebugEnabled()) { log.debug("Parsing as statement: {}", query); } - return invokeParser(query, params, EsqlBaseParser::singleStatement, AstBuilder::plan); + return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan); } private T invokeParser( String query, QueryParams params, + PlanTelemetry metrics, Function parseFunction, BiFunction result ) { + if (query.length() > MAX_LENGTH) { + throw new ParsingException("ESQL statement is too large [{} characters > {}]", query.length(), MAX_LENGTH); + } try { - // new CaseChangingCharStream() EsqlBaseLexer lexer = new EsqlBaseLexer(CharStreams.fromString(query)); lexer.removeErrorListeners(); @@ -88,9 +151,12 @@ private T invokeParser( log.trace("Parse tree: {}", tree.toStringTree()); } - return result.apply(new AstBuilder(params), tree); + return result.apply(new AstBuilder(new ExpressionBuilder.ParsingContext(params, metrics)), tree); } catch (StackOverflowError e) { throw new ParsingException("ESQL statement is too large, causing stack overflow when generating the parsing tree: [{}]", query); + // likely thrown by an invalid popMode (such as extra closing parenthesis) + } catch (EmptyStackException ese) { + throw new ParsingException("Invalid query [{}]", query); } } @@ -121,11 +187,14 @@ public void syntaxError( String message, RecognitionException e ) { - if (recognizer instanceof EsqlBaseParser parser && parser.isDevVersion() == false) { - Matcher m = REPLACE_DEV.matcher(message); - message = m.replaceAll(StringUtils.EMPTY); - } + if (recognizer instanceof EsqlBaseParser parser) { + Matcher m; + if (parser.isDevVersion() == false) { + m = REPLACE_DEV.matcher(message); + message = m.replaceAll(StringUtils.EMPTY); + } + } throw new ParsingException(message, e, line, charPositionInLine); } }; @@ -152,7 +221,7 @@ private static class ParametrizedTokenSource extends DelegatingTokenSource { @Override public Token nextToken() { Token token = delegate.nextToken(); - if (token.getType() == EsqlBaseLexer.PARAM) { + if (token.getType() == EsqlBaseLexer.PARAM || token.getType() == EsqlBaseLexer.DOUBLE_PARAMS) { checkAnonymousParam(token); if (param > params.size()) { throw new ParsingException(source(token), "Not enough actual parameters {}", params.size()); @@ -161,8 +230,9 @@ public Token nextToken() { param++; } - if (token.getType() == EsqlBaseLexer.NAMED_OR_POSITIONAL_PARAM) { - if (isInteger(token.getText().substring(1))) { + String nameOrPosition = nameOrPosition(token); + if (nameOrPosition.isBlank() == false) { + if (isInteger(nameOrPosition)) { checkPositionalParam(token); } else { checkNamedParam(token); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 11ffe587155e2..9d2e7be770a18 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.parser; import org.elasticsearch.Build; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -15,22 +16,25 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; -import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; +import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; +import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod; @@ -39,14 +43,16 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.plan.TableIdentifier; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Dedup; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Explain; import org.elasticsearch.xpack.esql.plan.logical.Filter; +import org.elasticsearch.xpack.esql.plan.logical.Fork; import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Keep; @@ -58,10 +64,14 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; +import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -72,6 +82,12 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsIdentifier; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsPattern; import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.Features.CROSS_CLUSTER; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.Features.WILDCARD_PATTERN; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.randomIndexPattern; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.randomIndexPatterns; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.unquoteIndexPattern; +import static org.elasticsearch.xpack.esql.IdentifierGenerator.without; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -296,18 +312,18 @@ public void testStatsWithoutGroups() { ); } - public void testStatsWithoutAggs() throws Exception { + public void testStatsWithoutAggs() { assertEquals( new Aggregate(EMPTY, PROCESSING_CMD_INPUT, Aggregate.AggregateType.STANDARD, List.of(attribute("a")), List.of(attribute("a"))), processingCommand("stats by a") ); } - public void testStatsWithoutAggsOrGroup() throws Exception { + public void testStatsWithoutAggsOrGroup() { expectError("from text | stats", "At least one aggregation or grouping expression required in [stats]"); } - public void testAggsWithGroupKeyAsAgg() throws Exception { + public void testAggsWithGroupKeyAsAgg() { var queries = new String[] { """ row a = 1, b = 2 | stats a by a @@ -328,7 +344,7 @@ public void testAggsWithGroupKeyAsAgg() throws Exception { } } - public void testStatsWithGroupKeyAndAggFilter() throws Exception { + public void testStatsWithGroupKeyAndAggFilter() { var a = attribute("a"); var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); @@ -338,7 +354,7 @@ public void testStatsWithGroupKeyAndAggFilter() throws Exception { ); } - public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception { + public void testStatsWithGroupKeyAndMixedAggAndFilter() { var a = attribute("a"); var min = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var max = new UnresolvedFunction(EMPTY, "max", DEFAULT, List.of(a)); @@ -373,7 +389,7 @@ public void testStatsWithGroupKeyAndMixedAggAndFilter() throws Exception { ); } - public void testStatsWithoutGroupKeyMixedAggAndFilter() throws Exception { + public void testStatsWithoutGroupKeyMixedAggAndFilter() { var a = attribute("a"); var f = new UnresolvedFunction(EMPTY, "min", DEFAULT, List.of(a)); var filter = new Alias(EMPTY, "min(a) where a > 1", new FilteredExpression(EMPTY, f, new GreaterThan(EMPTY, a, integer(1)))); @@ -386,8 +402,11 @@ public void testStatsWithoutGroupKeyMixedAggAndFilter() throws Exception { public void testInlineStatsWithGroups() { var query = "inlinestats b = min(a) by c, d.e"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> processingCommand(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'inlinestats' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'inlinestats' expecting {"), + () -> processingCommand(query) + ); return; } assertEquals( @@ -412,8 +431,11 @@ public void testInlineStatsWithGroups() { public void testInlineStatsWithoutGroups() { var query = "inlinestats min(a), c = 1"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> processingCommand(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'inlinestats' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'inlinestats' expecting {"), + () -> processingCommand(query) + ); return; } assertEquals( @@ -477,7 +499,8 @@ public void testStringAsIndexPattern() { clusterAndIndexAsIndexPattern(command, "cluster:index"); clusterAndIndexAsIndexPattern(command, "cluster:.index"); clusterAndIndexAsIndexPattern(command, "cluster*:index*"); - clusterAndIndexAsIndexPattern(command, "cluster*:*"); + clusterAndIndexAsIndexPattern(command, "cluster*:*");// this is not a valid pattern, * should be inside <> + clusterAndIndexAsIndexPattern(command, "cluster*:"); clusterAndIndexAsIndexPattern(command, "cluster*:*"); clusterAndIndexAsIndexPattern(command, "*:index*"); clusterAndIndexAsIndexPattern(command, "*:*"); @@ -525,10 +548,10 @@ private void clusterAndIndexAsLookupIndexPattern(String clusterAndIndex) { public void testInvalidCharacterInIndexPattern() { Map commands = new HashMap<>(); - commands.put("FROM {}", "line 1:7: "); + commands.put("FROM {}", "line 1:6: "); if (Build.current().isSnapshot()) { - commands.put("METRICS {}", "line 1:10: "); - commands.put("ROW x = 1 | LOOKUP_🐔 {} ON j", "line 1:23: "); + commands.put("METRICS {}", "line 1:9: "); + commands.put("ROW x = 1 | LOOKUP_🐔 {} ON j", "line 1:22: "); } String lineNumber; for (String command : commands.keySet()) { @@ -572,10 +595,13 @@ public void testInvalidCharacterInIndexPattern() { continue; } - lineNumber = command.contains("FROM") ? "line 1:21: " : "line 1:24: "; + lineNumber = command.contains("FROM") ? "line 1:20: " : "line 1:23: "; expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"- , -\"", commands.get(command), "", "must not be empty"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern,-\"", commands.get(command), "", "must not be empty"); + clustersAndIndices(command, "indexpattern", "*-"); clustersAndIndices(command, "indexpattern", "-indexpattern"); } @@ -585,7 +611,7 @@ public void testInvalidCharacterInIndexPattern() { if (command.contains("LOOKUP_🐔")) { continue; } - lineNumber = command.contains("FROM") ? "line 1:10: " : "line 1:13: "; + lineNumber = command.contains("FROM") ? "line 1:9: " : "line 1:12: "; clustersAndIndices(command, "*", "-index#pattern"); clustersAndIndices(command, "index*", "-index#pattern"); clustersAndIndices(command, "*", "-<--logstash-{now/M{yyyy.MM}}>"); @@ -620,11 +646,8 @@ public void testInvalidQuotingAsFromIndexPattern() { expectError("FROM \"foo\"bar\"", ": token recognition error at: '\"'"); expectError("FROM \"foo\"\"bar\"", ": extraneous input '\"bar\"' expecting "); - expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}"); - expectError( - "FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", - ": mismatched input '\"bar\"' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}" - ); + expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', 'metadata'}"); + expectError("FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", ": mismatched input '\"bar\"' expecting {, '|', ',', 'metadata'}"); } public void testInvalidQuotingAsMetricsIndexPattern() { @@ -845,16 +868,17 @@ public void testSuggestAvailableSourceCommandsOnParsingError() { Tuple.tuple("a/*hi*/", "a"), Tuple.tuple("explain [ frm a ]", "frm") )) { - ParsingException pe = expectThrows(ParsingException.class, () -> statement(queryWithUnexpectedCmd.v1())); - assertThat( - pe.getMessage(), + expectThrows( + ParsingException.class, allOf( containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"), containsString("'explain'"), containsString("'from'"), containsString("'row'") - ) + ), + () -> statement(queryWithUnexpectedCmd.v1()) ); + } } @@ -869,15 +893,15 @@ public void testSuggestAvailableProcessingCommandsOnParsingError() { Tuple.tuple("from a | a/*hi*/", "a"), Tuple.tuple("explain [ from a | evl b = c ]", "evl") )) { - ParsingException pe = expectThrows(ParsingException.class, () -> statement(queryWithUnexpectedCmd.v1())); - assertThat( - pe.getMessage(), + expectThrows( + ParsingException.class, allOf( containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"), containsString("'eval'"), containsString("'stats'"), containsString("'where'") - ) + ), + () -> statement(queryWithUnexpectedCmd.v1()) ); } } @@ -885,18 +909,25 @@ public void testSuggestAvailableProcessingCommandsOnParsingError() { public void testDeprecatedIsNullFunction() { expectError( "from test | eval x = is_null(f)", - "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + "line 1:22: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" ); expectError( "row x = is_null(f)", - "line 1:10: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + "line 1:9: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" ); if (Build.current().isSnapshot()) { expectError( "from test | eval x = ?fn1(f)", List.of(paramAsIdentifier("fn1", "IS_NULL")), - "line 1:23: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + "line 1:22: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" + ); + } + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError( + "from test | eval x = ??fn1(f)", + List.of(paramAsConstant("fn1", "IS_NULL")), + "line 1:22: is_null function is not supported anymore, please use 'is null'/'is not null' predicates instead" ); } } @@ -904,30 +935,27 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting "); expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'"); - expectError( - "explain [from foo] metadata _index", - "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}" - ); + expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}"); } public void testMetadataFieldMultipleDeclarations() { - expectError("from test metadata _index, _version, _index", "1:39: metadata field [_index] already declared [@1:20]"); + expectError("from test metadata _index, _version, _index", "1:38: metadata field [_index] already declared [@1:20]"); } public void testMetadataFieldUnsupportedPrimitiveType() { - expectError("from test metadata _tier", "line 1:21: unsupported metadata field [_tier]"); + expectError("from test metadata _tier", "line 1:20: unsupported metadata field [_tier]"); } public void testMetadataFieldUnsupportedCustomType() { - expectError("from test metadata _feature", "line 1:21: unsupported metadata field [_feature]"); + expectError("from test metadata _feature", "line 1:20: unsupported metadata field [_feature]"); } public void testMetadataFieldNotFoundNonExistent() { - expectError("from test metadata _doesnot_compute", "line 1:21: unsupported metadata field [_doesnot_compute]"); + expectError("from test metadata _doesnot_compute", "line 1:20: unsupported metadata field [_doesnot_compute]"); } public void testMetadataFieldNotFoundNormalField() { - expectError("from test metadata emp_no", "line 1:21: unsupported metadata field [emp_no]"); + expectError("from test metadata emp_no", "line 1:20: unsupported metadata field [emp_no]"); } public void testDissectPattern() { @@ -971,10 +999,10 @@ public void testGrokPattern() { assertEquals("%{WORD:foo}", grok.parser().pattern()); assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); - ParsingException pe = expectThrows(ParsingException.class, () -> statement("row a = \"foo bar\" | grok a \"%{_invalid_:x}\"")); - assertThat( - pe.getMessage(), - containsString("Invalid pattern [%{_invalid_:x}] for grok: Unable to find pattern [_invalid_] in Grok's pattern dictionary") + expectThrows( + ParsingException.class, + containsString("Invalid pattern [%{_invalid_:x}] for grok: Unable to find pattern [_invalid_] in Grok's pattern dictionary"), + () -> statement("row a = \"foo bar\" | grok a \"%{_invalid_:x}\"") ); cmd = processingCommand("grok a \"%{WORD:foo} %{WORD:foo}\""); @@ -985,13 +1013,13 @@ public void testGrokPattern() { expectError( "row a = \"foo bar\" | GROK a \"%{NUMBER:foo} %{WORD:foo}\"", - "line 1:22: Invalid GROK pattern [%{NUMBER:foo} %{WORD:foo}]:" + "line 1:21: Invalid GROK pattern [%{NUMBER:foo} %{WORD:foo}]:" + " the attribute [foo] is defined multiple times with different types" ); expectError( "row a = \"foo\" | GROK a \"(?P.+)\"", - "line 1:18: Invalid grok pattern [(?P.+)]: [undefined group option]" + "line 1:17: Invalid grok pattern [(?P.+)]: [undefined group option]" ); } @@ -1015,7 +1043,7 @@ public void testLikeRLike() { expectError( "from a | where foo like \"(?i)(^|[^a-zA-Z0-9_-])nmap($|\\\\.)\"", - "line 1:17: Invalid pattern for LIKE [(?i)(^|[^a-zA-Z0-9_-])nmap($|\\.)]: " + "line 1:16: Invalid pattern for LIKE [(?i)(^|[^a-zA-Z0-9_-])nmap($|\\.)]: " + "[Invalid sequence - escape character is not followed by special wildcard char]" ); } @@ -1089,7 +1117,7 @@ public void testEnrich() { ); expectError( "from a | enrich typo:countries on foo", - "line 1:18: Unrecognized value [typo], ENRICH policy qualifier needs to be one of [_ANY, _COORDINATOR, _REMOTE]" + "line 1:17: Unrecognized value [typo], ENRICH policy qualifier needs to be one of [_ANY, _COORDINATOR, _REMOTE]" ); } @@ -1113,8 +1141,7 @@ public void testKeepStarMvExpand() { public void testUsageOfProject() { String query = "from test | project foo, bar"; - ParsingException e = expectThrows(ParsingException.class, "Expected syntax error for " + query, () -> statement(query)); - assertThat(e.getMessage(), containsString("mismatched input 'project' expecting")); + expectThrows(ParsingException.class, containsString("mismatched input 'project' expecting"), () -> statement(query)); } public void testInputParams() { @@ -1141,55 +1168,60 @@ public void testInputParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2")); + assertThat(alias.child().fold(FoldContext.small()), is("2")); field = row.fields().get(2); assertThat(field.name(), is("a")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2 days")); + assertThat(alias.child().fold(FoldContext.small()), is("2 days")); field = row.fields().get(3); assertThat(field.name(), is("b")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("4 hours")); + assertThat(alias.child().fold(FoldContext.small()), is("4 hours")); field = row.fields().get(4); assertThat(field.name(), is("c")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("1.2.3")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("1.2.3")); field = row.fields().get(5); assertThat(field.name(), is("d")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("127.0.0.1")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("127.0.0.1")); field = row.fields().get(6); assertThat(field.name(), is("e")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(9)); + assertThat(alias.child().fold(FoldContext.small()), is(9)); field = row.fields().get(7); assertThat(field.name(), is("f")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(11)); + assertThat(alias.child().fold(FoldContext.small()), is(11)); } public void testMissingInputParams() { expectError("row x = ?, y = ?", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError("from test | eval x = ??, y = ??", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + expectError("from test | eval x = ??, y = ?", List.of(paramAsConstant(null, 1)), "Not enough actual parameters 1"); + } } public void testNamedParams() { @@ -1202,13 +1234,13 @@ public void testNamedParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidNamedParams() { @@ -1228,15 +1260,23 @@ public void testInvalidNamedParams() { expectError("from test | where x < ?#1", List.of(paramAsConstant("#1", 5)), "token recognition error at: '#'"); - expectError( - "from test | where x < ??", - List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), - "extraneous input '?' expecting " - ); - expectError("from test | where x < ?Å", List.of(paramAsConstant("Å", 5)), "line 1:24: token recognition error at: 'Å'"); expectError("from test | eval x = ?Å", List.of(paramAsConstant("Å", 5)), "line 1:23: token recognition error at: 'Å'"); + + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError( + "from test | where x < ???", + List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), + "extraneous input '?' expecting " + ); + } else { + expectError( + "from test | where x < ??", + List.of(paramAsConstant("n_1", 5), paramAsConstant("n_2", 5)), + "extraneous input '?' expecting " + ); + } } public void testPositionalParams() { @@ -1249,13 +1289,13 @@ public void testPositionalParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidPositionalParams() { @@ -1274,8 +1314,8 @@ public void testInvalidPositionalParams() { expectError( "from test | where x < ?0 and y < ?2", List.of(paramAsConstant(null, 5)), - "line 1:24: No parameter is defined for position 0, did you mean position 1?; " - + "line 1:35: No parameter is defined for position 2, did you mean position 1?" + "line 1:23: No parameter is defined for position 0, did you mean position 1?; " + + "line 1:34: No parameter is defined for position 2, did you mean position 1?" ); expectError( @@ -1538,33 +1578,39 @@ public void testParamInAggFunction() { } public void testParamMixed() { - expectError( - "from test | where x < ? | eval y = ?n2 + ?n3 | limit ?n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and anonymous" - ); - - expectError( - "from test | where x < ? | eval y = ?_n2 + ?n3 | limit ?_4", - List.of(paramAsConstant("n1", 5), paramAsConstant("_n2", -1), paramAsConstant("n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and anonymous" - ); - - expectError( - "from test | where x < ?1 | eval y = ?n2 + ?_n3 | limit ?n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("_n3", 100), paramAsConstant("n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of named and positional" + Map, String> mixedParams = new HashMap<>( + Map.ofEntries( + Map.entry(List.of("?", "?n2", "?n3"), "named and anonymous"), + Map.entry(List.of("?", "?_n2", "?n3"), "named and anonymous"), + Map.entry(List.of("?1", "?n2", "?_n3"), "named and positional"), + Map.entry(List.of("?", "?2", "?n3"), "positional and anonymous") + ) ); - expectError( - "from test | where x < ? | eval y = ?2 + ?n3 | limit ?_n4", - List.of(paramAsConstant("n1", 5), paramAsConstant("n2", -1), paramAsConstant("n3", 100), paramAsConstant("_n4", 10)), - "Inconsistent parameter declaration, " - + "use one of positional, named or anonymous params but not a combination of positional and anonymous" - ); + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + mixedParams.put(List.of("??", "??n2", "??n3"), "named and anonymous"); + mixedParams.put(List.of("?", "??_n2", "?n3"), "named and anonymous"); + mixedParams.put(List.of("??1", "?n2", "?_n3"), "named and positional"); + mixedParams.put(List.of("?", "??2", "?n3"), "positional and anonymous"); + } + for (Map.Entry, String> mixedParam : mixedParams.entrySet()) { + List params = mixedParam.getKey(); + String errorMessage = mixedParam.getValue(); + String query = LoggerMessageFormat.format( + null, + "from test | where x < {} | eval y = {}() + {}", + params.get(0), + params.get(1), + params.get(2) + ); + expectError( + query, + List.of(paramAsConstant("n1", "f1"), paramAsConstant("n2", "fn2"), paramAsConstant("n3", "f3")), + "Inconsistent parameter declaration, " + + "use one of positional, named or anonymous params but not a combination of " + + errorMessage + ); + } } public void testIntervalParam() { @@ -1584,10 +1630,7 @@ public void testIntervalParam() { } public void testParamForIdentifier() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); + // TODO will be replaced by testDoubleParamsForIdentifier after providing an identifier with a single parameter marker is deprecated // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand // eval, where assertEquals( @@ -1699,8 +1742,7 @@ public void testParamForIdentifier() { List.of(new Order(EMPTY, attribute("f.11..f.12.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) ), attribute("f.*.13.f.14*"), - attribute("f.*.13.f.14*"), - null + attribute("f.*.13.f.14*") ), statement( """ @@ -1846,10 +1888,6 @@ public void testParamForIdentifier() { } public void testParamForIdentifierPattern() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // name patterns can appear in keep and drop // all patterns LogicalPlan plan = statement( @@ -1939,10 +1977,6 @@ public void testParamForIdentifierPattern() { } public void testParamInInvalidPosition() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // param for pattern is not supported in eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand // where/stats/sort/dissect/grok are covered in RestEsqlTestCase List invalidParamPositions = List.of("eval ?f1 = 1", "stats x = ?f1(*)", "mv_expand ?f1", "rename ?f1 as ?f2"); @@ -1994,10 +2028,6 @@ public void testParamInInvalidPosition() { } public void testMissingParam() { - assumeTrue( - "named parameters for identifiers and patterns require snapshot build", - EsqlCapabilities.Cap.NAMED_PARAMETER_FOR_FIELD_AND_FUNCTION_NAMES_SIMPLIFIED_SYNTAX.isEnabled() - ); // cover all processing commands eval/where/stats/sort/rename/dissect/grok/enrich/mvexpand/keep/drop String error = "Unknown query parameter [f1], did you mean [f4]?"; String errorMvExpandFunctionNameCommandOption = "Query parameter [?f1] is null or undefined, cannot be used as an identifier"; @@ -2022,6 +2052,9 @@ public void testMissingParam() { missingParamGroupB.contains(missingParam) ? errorMvExpandFunctionNameCommandOption : error ); } + if (EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled()) { + expectError("from test | " + missingParam.replace("?", "??"), List.of(paramAsConstant("f4", "f1*")), error); + } } } @@ -2050,63 +2083,65 @@ public void testQuotedName() { private void assertStringAsIndexPattern(String string, String statement) { if (Build.current().isSnapshot() == false && statement.contains("METRIC")) { - var e = expectThrows(ParsingException.class, () -> statement(statement)); - assertThat(e.getMessage(), containsString("mismatched input 'METRICS' expecting {")); + expectThrows(ParsingException.class, containsString("mismatched input 'METRICS' expecting {"), () -> statement(statement)); return; } LogicalPlan from = statement(statement); assertThat(from, instanceOf(UnresolvedRelation.class)); UnresolvedRelation table = (UnresolvedRelation) from; - assertThat(table.table().index(), is(string)); + assertThat(table.indexPattern().indexPattern(), is(string)); } private void assertStringAsLookupIndexPattern(String string, String statement) { if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> statement(statement)); - assertThat(e.getMessage(), containsString("line 1:14: LOOKUP_🐔 is in preview and only available in SNAPSHOT build")); + expectThrows( + ParsingException.class, + containsString("line 1:14: LOOKUP_🐔 is in preview and only available in SNAPSHOT build"), + () -> statement(statement) + ); return; } var plan = statement(statement); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo(string)); + assertThat(tableName.fold(FoldContext.small()), equalTo(string)); } - public void testIdPatternUnquoted() throws Exception { + public void testIdPatternUnquoted() { var string = "regularString"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternQuoted() throws Exception { + public void testIdPatternQuoted() { var string = "`escaped string`"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternQuotedWithDoubleBackticks() throws Exception { + public void testIdPatternQuotedWithDoubleBackticks() { var string = "`escaped``string`"; assertThat(breakIntoFragments(string), contains(string)); } - public void testIdPatternUnquotedAndQuoted() throws Exception { + public void testIdPatternUnquotedAndQuoted() { var string = "this`is`a`mix`of`ids`"; assertThat(breakIntoFragments(string), contains("this", "`is`", "a", "`mix`", "of", "`ids`")); } - public void testIdPatternQuotedTraling() throws Exception { + public void testIdPatternQuotedTraling() { var string = "`foo`*"; assertThat(breakIntoFragments(string), contains("`foo`", "*")); } - public void testIdPatternWithDoubleQuotedStrings() throws Exception { + public void testIdPatternWithDoubleQuotedStrings() { var string = "`this``is`a`quoted `` string``with`backticks"; assertThat(breakIntoFragments(string), contains("`this``is`", "a", "`quoted `` string``with`", "backticks")); } - public void testSpaceNotAllowedInIdPattern() throws Exception { + public void testSpaceNotAllowedInIdPattern() { expectError("ROW a = 1| RENAME a AS this is `not okay`", "mismatched input 'is' expecting {, '|', ',', '.'}"); } - public void testSpaceNotAllowedInIdPatternKeep() throws Exception { + public void testSpaceNotAllowedInIdPatternKeep() { expectError("ROW a = 1, b = 1| KEEP a b", "extraneous input 'b'"); } @@ -2120,31 +2155,34 @@ public void testEnrichOnMatchField() { } public void testInlineConvertWithNonexistentType() { - expectError("ROW 1::doesnotexist", "line 1:9: Unknown data type named [doesnotexist]"); - expectError("ROW \"1\"::doesnotexist", "line 1:11: Unknown data type named [doesnotexist]"); - expectError("ROW false::doesnotexist", "line 1:13: Unknown data type named [doesnotexist]"); - expectError("ROW abs(1)::doesnotexist", "line 1:14: Unknown data type named [doesnotexist]"); - expectError("ROW (1+2)::doesnotexist", "line 1:13: Unknown data type named [doesnotexist]"); + expectError("ROW 1::doesnotexist", "line 1:8: Unknown data type named [doesnotexist]"); + expectError("ROW \"1\"::doesnotexist", "line 1:10: Unknown data type named [doesnotexist]"); + expectError("ROW false::doesnotexist", "line 1:12: Unknown data type named [doesnotexist]"); + expectError("ROW abs(1)::doesnotexist", "line 1:13: Unknown data type named [doesnotexist]"); + expectError("ROW (1+2)::doesnotexist", "line 1:12: Unknown data type named [doesnotexist]"); } public void testLookup() { String query = "ROW a = 1 | LOOKUP_🐔 t ON j"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> statement(query)); - assertThat(e.getMessage(), containsString("line 1:13: mismatched input 'LOOKUP_🐔' expecting {")); + expectThrows( + ParsingException.class, + containsString("line 1:13: mismatched input 'LOOKUP_🐔' expecting {"), + () -> statement(query) + ); return; } var plan = statement(query); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo("t")); + assertThat(tableName.fold(FoldContext.small()), equalTo("t")); assertThat(lookup.matchFields(), hasSize(1)); var matchField = as(lookup.matchFields().get(0), UnresolvedAttribute.class); assertThat(matchField.name(), equalTo("j")); } public void testInlineConvertUnsupportedType() { - expectError("ROW 3::BYTE", "line 1:6: Unsupported conversion to type [BYTE]"); + expectError("ROW 3::BYTE", "line 1:5: Unsupported conversion to type [BYTE]"); } public void testMetricsWithoutStats() { @@ -2287,21 +2325,17 @@ public void testInvalidAlias() { expectError("from test | eval A = coalesce(\"Å\", Å)", "line 1:36: token recognition error at: 'Å'"); } + public void testInvalidRemoteClusterPattern() { + expectError("from \"rem:ote\":index", "cluster string [rem:ote] must not contain ':'"); + } + private LogicalPlan unresolvedRelation(String index) { - return new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, null, index), false, List.of(), IndexMode.STANDARD, null, "FROM"); + return new UnresolvedRelation(EMPTY, new IndexPattern(EMPTY, index), false, List.of(), IndexMode.STANDARD, null, "FROM"); } private LogicalPlan unresolvedTSRelation(String index) { List metadata = List.of(new MetadataAttribute(EMPTY, MetadataAttribute.TSID_FIELD, DataType.KEYWORD, false)); - return new UnresolvedRelation( - EMPTY, - new TableIdentifier(EMPTY, null, index), - false, - metadata, - IndexMode.TIME_SERIES, - null, - "FROM TS" - ); + return new UnresolvedRelation(EMPTY, new IndexPattern(EMPTY, index), false, metadata, IndexMode.TIME_SERIES, null, "FROM TS"); } public void testMetricWithGroupKeyAsAgg() { @@ -2315,10 +2349,10 @@ public void testMetricWithGroupKeyAsAgg() { public void testMatchOperatorConstantQueryString() { var plan = statement("FROM test | WHERE field:\"value\""); var filter = as(plan, Filter.class); - var match = (Match) filter.condition(); + var match = (MatchOperator) filter.condition(); var matchField = (UnresolvedAttribute) match.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(match.query().fold(), equalTo("value")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } public void testInvalidMatchOperator() { @@ -2327,7 +2361,6 @@ public void testInvalidMatchOperator() { "from test | WHERE field:CONCAT(\"hello\", \"world\")", "line 1:25: mismatched input 'CONCAT' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " ); - expectError("from test | WHERE field:123::STRING", "line 1:28: mismatched input '::' expecting {, '|', 'and', 'or'}"); expectError( "from test | WHERE field:(true OR false)", "line 1:25: extraneous input '(' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " @@ -2336,7 +2369,7 @@ public void testInvalidMatchOperator() { "from test | WHERE field:another_field_or_value", "line 1:25: mismatched input 'another_field_or_value' expecting {QUOTED_STRING, INTEGER_LITERAL, DECIMAL_LITERAL, " ); - expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+' expecting {, '|', 'and', 'or'}"); + expectError("from test | WHERE field:2+3", "line 1:26: mismatched input '+'"); expectError( "from test | WHERE \"field\":\"value\"", "line 1:26: mismatched input ':' expecting {, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}" @@ -2346,4 +2379,1569 @@ public void testInvalidMatchOperator() { "line 1:37: mismatched input ':' expecting {, '|', 'and', '::', 'or', '+', '-', '*', '/', '%'}" ); } + + public void testMatchFunctionFieldCasting() { + var plan = statement("FROM test | WHERE match(field::int, \"value\")"); + var filter = as(plan, Filter.class); + var function = (UnresolvedFunction) filter.condition(); + var toInteger = (ToInteger) function.children().get(0); + var matchField = (UnresolvedAttribute) toInteger.field(); + assertThat(matchField.name(), equalTo("field")); + assertThat(function.children().get(1).fold(FoldContext.small()), equalTo("value")); + } + + public void testMatchOperatorFieldCasting() { + var plan = statement("FROM test | WHERE field::int : \"value\""); + var filter = as(plan, Filter.class); + var match = (MatchOperator) filter.condition(); + var toInteger = (ToInteger) match.field(); + var matchField = (UnresolvedAttribute) toInteger.field(); + assertThat(matchField.name(), equalTo("field")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); + } + + public void testFailingMetadataWithSquareBrackets() { + expectError( + "FROM test [METADATA _index] | STATS count(*)", + "line 1:11: mismatched input '[' expecting {, '|', ',', 'metadata'}" + ); + } + + public void testNamedFunctionArgumentInMap() { + // functions can be scalar, grouping and aggregation + // functions can be in eval/where/stats/sort/dissect/grok commands, commands in snapshot are not covered + // positive + // In eval and where clause as function arguments + LinkedHashMap expectedMap1 = new LinkedHashMap<>(4); + expectedMap1.put("option1", "string"); + expectedMap1.put("option2", 1); + expectedMap1.put("option3", List.of(2.0, 3.0, 4.0)); + expectedMap1.put("option4", List.of(true, false)); + LinkedHashMap expectedMap2 = new LinkedHashMap<>(4); + expectedMap2.put("option1", List.of("string1", "string2")); + expectedMap2.put("option2", List.of(1, 2, 3)); + expectedMap2.put("option3", 2.0); + expectedMap2.put("option4", true); + LinkedHashMap expectedMap3 = new LinkedHashMap<>(4); + expectedMap3.put("option1", "string"); + expectedMap3.put("option2", 2.0); + expectedMap3.put("option3", List.of(1, 2, 3)); + expectedMap3.put("option4", List.of(true, false)); + + assertEquals( + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of( + new Alias( + EMPTY, + "x", + function( + "fn1", + List.of(attribute("f1"), new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap1)) + ) + ) + ) + ), + new Equals( + EMPTY, + attribute("y"), + function("fn2", List.of(new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap2))) + ) + ), + statement(""" + from test + | eval x = fn1(f1, "testString", {"option1":"string","option2":1,"option3":[2.0,3.0,4.0],"option4":[true,false]}) + | where y == fn2("testString", {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"option4":true}) + """) + ); + + // In stats, by and sort as function arguments + assertEquals( + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of( + new Alias( + EMPTY, + "fn2(f3, {\"option1\":[\"string1\",\"string2\"],\"option2\":[1,2,3],\"option3\":2.0,\"option4\":true})", + function("fn2", List.of(attribute("f3"), mapExpression(expectedMap2))) + ) + ), + List.of( + new Alias(EMPTY, "x", function("fn1", List.of(attribute("f1"), attribute("f2"), mapExpression(expectedMap1)))), + attribute("fn2(f3, {\"option1\":[\"string1\",\"string2\"],\"option2\":[1,2,3],\"option3\":2.0,\"option4\":true})") + ) + ), + List.of( + new Order( + EMPTY, + function("fn3", List.of(attribute("f4"), mapExpression(expectedMap3))), + Order.OrderDirection.ASC, + Order.NullsPosition.LAST + ) + ) + ), + statement(""" + from test + | stats x = fn1(f1, f2, {"option1":"string","option2":1,"option3":[2.0,3.0,4.0],"option4":[true,false]}) + by fn2(f3, {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"option4":true}) + | sort fn3(f4, {"option1":"string","option2":2.0,"option3":[1,2,3],"option4":[true,false]}) + """) + ); + + // In dissect and grok as function arguments + LogicalPlan plan = statement(""" + from test + | dissect fn1(f1, f2, {"option1":"string", "option2":1,"option3":[2.0,3.0,4.0],"option4":[true,false]}) "%{bar}" + | grok fn2(f3, {"option1":["string1","string2"],"option2":[1,2,3],"option3":2.0,"option4":true}) "%{WORD:foo}" + """); + Grok grok = as(plan, Grok.class); + assertEquals(function("fn2", List.of(attribute("f3"), mapExpression(expectedMap2))), grok.input()); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(function("fn1", List.of(attribute("f1"), attribute("f2"), mapExpression(expectedMap1))), dissect.input()); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + UnresolvedRelation ur = as(dissect.child(), UnresolvedRelation.class); + assertEquals(ur, relation("test")); + } + + public void testNamedFunctionArgumentInMapWithNamedParameters() { + // map entry values provided in named parameter, arrays are not supported by named parameters yet + LinkedHashMap expectedMap1 = new LinkedHashMap<>(4); + expectedMap1.put("option1", "string"); + expectedMap1.put("option2", 1); + expectedMap1.put("option3", List.of(2.0, 3.0, 4.0)); + expectedMap1.put("option4", List.of(true, false)); + LinkedHashMap expectedMap2 = new LinkedHashMap<>(4); + expectedMap2.put("option1", List.of("string1", "string2")); + expectedMap2.put("option2", List.of(1, 2, 3)); + expectedMap2.put("option3", 2.0); + expectedMap2.put("option4", true); + LinkedHashMap expectedMap3 = new LinkedHashMap<>(4); + expectedMap3.put("option1", "string"); + expectedMap3.put("option2", 2.0); + expectedMap3.put("option3", List.of(1, 2, 3)); + expectedMap3.put("option4", List.of(true, false)); + assertEquals( + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of( + new Alias( + EMPTY, + "x", + function( + "fn1", + List.of(attribute("f1"), new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap1)) + ) + ) + ) + ), + new Equals( + EMPTY, + attribute("y"), + function("fn2", List.of(new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap2))) + ) + ), + statement( + """ + from test + | eval x = ?fn1(?n1, ?n2, {"option1":?n3,"option2":?n4,"option3":[2.0,3.0,4.0],"option4":[true,false]}) + | where y == ?fn2(?n2, {"option1":["string1","string2"],"option2":[1,2,3],"option3":?n5,"option4":?n6}) + """, + new QueryParams( + List.of( + paramAsIdentifier("fn1", "fn1"), + paramAsIdentifier("fn2", "fn2"), + paramAsIdentifier("n1", "f1"), + paramAsConstant("n2", "testString"), + paramAsConstant("n3", "string"), + paramAsConstant("n4", 1), + paramAsConstant("n5", 2.0), + paramAsConstant("n6", true) + ) + ) + ) + ); + + assertEquals( + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of( + new Alias( + EMPTY, + "?fn2(?n7, {\"option1\":[\"string1\",\"string2\"],\"option2\":[1,2,3],\"option3\":?n5,\"option4\":?n6})", + function("fn2", List.of(attribute("f3"), mapExpression(expectedMap2))) + ) + ), + List.of( + new Alias(EMPTY, "x", function("fn1", List.of(attribute("f1"), attribute("f2"), mapExpression(expectedMap1)))), + attribute("?fn2(?n7, {\"option1\":[\"string1\",\"string2\"],\"option2\":[1,2,3],\"option3\":?n5,\"option4\":?n6})") + ) + ), + List.of( + new Order( + EMPTY, + function("fn3", List.of(attribute("f4"), mapExpression(expectedMap3))), + Order.OrderDirection.ASC, + Order.NullsPosition.LAST + ) + ) + ), + statement( + """ + from test + | stats x = ?fn1(?n1, ?n2, {"option1":?n3,"option2":?n4,"option3":[2.0,3.0,4.0],"option4":[true,false]}) + by ?fn2(?n7, {"option1":["string1","string2"],"option2":[1,2,3],"option3":?n5,"option4":?n6}) + | sort ?fn3(?n8, {"option1":?n3,"option2":?n5,"option3":[1,2,3],"option4":[true,false]}) + """, + new QueryParams( + List.of( + paramAsIdentifier("fn1", "fn1"), + paramAsIdentifier("fn2", "fn2"), + paramAsIdentifier("fn3", "fn3"), + paramAsIdentifier("n1", "f1"), + paramAsIdentifier("n2", "f2"), + paramAsConstant("n3", "string"), + paramAsConstant("n4", 1), + paramAsConstant("n5", 2.0), + paramAsConstant("n6", true), + paramAsIdentifier("n7", "f3"), + paramAsIdentifier("n8", "f4") + ) + ) + ) + ); + + LogicalPlan plan = statement( + """ + from test + | dissect ?fn1(?n1, ?n2, {"option1":?n3,"option2":?n4,"option3":[2.0,3.0,4.0],"option4":[true,false]}) "%{bar}" + | grok ?fn2(?n7, {"option1":["string1","string2"],"option2":[1,2,3],"option3":?n5,"option4":?n6}) "%{WORD:foo}" + """, + new QueryParams( + List.of( + paramAsIdentifier("fn1", "fn1"), + paramAsIdentifier("fn2", "fn2"), + paramAsIdentifier("n1", "f1"), + paramAsIdentifier("n2", "f2"), + paramAsConstant("n3", "string"), + paramAsConstant("n4", 1), + paramAsConstant("n5", 2.0), + paramAsConstant("n6", true), + paramAsIdentifier("n7", "f3") + ) + ) + ); + Grok grok = as(plan, Grok.class); + assertEquals(function("fn2", List.of(attribute("f3"), mapExpression(expectedMap2))), grok.input()); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(function("fn1", List.of(attribute("f1"), attribute("f2"), mapExpression(expectedMap1))), dissect.input()); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + UnresolvedRelation ur = as(dissect.child(), UnresolvedRelation.class); + assertEquals(ur, relation("test")); + } + + public void testNamedFunctionArgumentWithCaseSensitiveKeys() { + LinkedHashMap expectedMap1 = new LinkedHashMap<>(3); + expectedMap1.put("option", "string"); + expectedMap1.put("Option", 1); + expectedMap1.put("oPtion", List.of(2.0, 3.0, 4.0)); + LinkedHashMap expectedMap2 = new LinkedHashMap<>(3); + expectedMap2.put("option", List.of("string1", "string2")); + expectedMap2.put("Option", List.of(1, 2, 3)); + expectedMap2.put("oPtion", 2.0); + + assertEquals( + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of( + new Alias( + EMPTY, + "x", + function( + "fn1", + List.of(attribute("f1"), new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap1)) + ) + ) + ) + ), + new Equals( + EMPTY, + attribute("y"), + function("fn2", List.of(new Literal(EMPTY, "testString", KEYWORD), mapExpression(expectedMap2))) + ) + ), + statement(""" + from test + | eval x = fn1(f1, "testString", {"option":"string","Option":1,"oPtion":[2.0,3.0,4.0]}) + | where y == fn2("testString", {"option":["string1","string2"],"Option":[1,2,3],"oPtion":2.0}) + """) + ); + } + + public void testMultipleNamedFunctionArgumentsNotAllowed() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "41"), + Map.entry("where {}", "38"), + Map.entry("stats {}", "38"), + Map.entry("stats agg() by {}", "47"), + Map.entry("sort {}", "37"), + Map.entry("dissect {} \"%{bar}\"", "40"), + Map.entry("grok {} \"%{WORD:foo}\"", "37") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + String errorMessage = cmd.startsWith("dissect") || cmd.startsWith("grok") + ? "mismatched input ',' expecting ')'" + : "no viable alternative at input 'fn(f1,"; + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"option\":1}, {\"option\":2})"), + LoggerMessageFormat.format(null, "line 1:{}: {}", error, errorMessage) + ); + } + } + + public void testNamedFunctionArgumentNotInMap() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "38"), + Map.entry("where {}", "35"), + Map.entry("stats {}", "35"), + Map.entry("stats agg() by {}", "44"), + Map.entry("sort {}", "34"), + Map.entry("dissect {} \"%{bar}\"", "37"), + Map.entry("grok {} \"%{WORD:foo}\"", "34") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + String errorMessage = cmd.startsWith("dissect") || cmd.startsWith("grok") + ? "extraneous input ':' expecting {',', ')'}" + : "no viable alternative at input 'fn(f1, \"option1\":'"; + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, \"option1\":\"string\")"), + LoggerMessageFormat.format(null, "line 1:{}: {}", error, errorMessage) + ); + } + } + + public void testNamedFunctionArgumentNotConstant() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", new String[] { "31", "35" }), + Map.entry("where {}", new String[] { "28", "32" }), + Map.entry("stats {}", new String[] { "28", "32" }), + Map.entry("stats agg() by {}", new String[] { "37", "41" }), + Map.entry("sort {}", new String[] { "27", "31" }), + Map.entry("dissect {} \"%{bar}\"", new String[] { "30", "34" }), + Map.entry("grok {} \"%{WORD:foo}\"", new String[] { "27", "31" }) + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error1 = command.getValue()[0]; + String error2 = command.getValue()[1]; + String errorMessage1 = cmd.startsWith("dissect") || cmd.startsWith("grok") + ? "mismatched input '1' expecting QUOTED_STRING" + : "no viable alternative at input 'fn(f1, { 1'"; + String errorMessage2 = cmd.startsWith("dissect") || cmd.startsWith("grok") + ? "mismatched input 'string' expecting {QUOTED_STRING" + : "no viable alternative at input 'fn(f1, {"; + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, { 1:\"string\" })"), + LoggerMessageFormat.format(null, "line 1:{}: {}", error1, errorMessage1) + ); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, { \"1\":string })"), + LoggerMessageFormat.format(null, "line 1:{}: {}", error2, errorMessage2) + ); + } + } + + public void testNamedFunctionArgumentEmptyMap() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "30"), + Map.entry("where {}", "27"), + Map.entry("stats {}", "27"), + Map.entry("stats agg() by {}", "36"), + Map.entry("sort {}", "26"), + Map.entry("dissect {} \"%{bar}\"", "29"), + Map.entry("grok {} \"%{WORD:foo}\"", "26") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + String errorMessage = cmd.startsWith("dissect") || cmd.startsWith("grok") + ? "mismatched input '}' expecting QUOTED_STRING" + : "no viable alternative at input 'fn(f1, {}'"; + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {}})"), + LoggerMessageFormat.format(null, "line 1:{}: {}", error, errorMessage) + ); + } + } + + public void testNamedFunctionArgumentMapWithNULL() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "29"), + Map.entry("where {}", "26"), + Map.entry("stats {}", "26"), + Map.entry("stats agg() by {}", "35"), + Map.entry("sort {}", "25"), + Map.entry("dissect {} \"%{bar}\"", "28"), + Map.entry("grok {} \"%{WORD:foo}\"", "25") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"option\":null})"), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Invalid named function argument [\"option\":null], NULL is not supported" + ) + ); + } + } + + public void testNamedFunctionArgumentMapWithEmptyKey() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "29"), + Map.entry("where {}", "26"), + Map.entry("stats {}", "26"), + Map.entry("stats agg() by {}", "35"), + Map.entry("sort {}", "25"), + Map.entry("dissect {} \"%{bar}\"", "28"), + Map.entry("grok {} \"%{WORD:foo}\"", "25") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"\":1})"), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Invalid named function argument [\"\":1], empty key is not supported" + ) + ); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\" \":1})"), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Invalid named function argument [\" \":1], empty key is not supported" + ) + ); + } + } + + public void testNamedFunctionArgumentMapWithDuplicatedKey() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "29"), + Map.entry("where {}", "26"), + Map.entry("stats {}", "26"), + Map.entry("stats agg() by {}", "35"), + Map.entry("sort {}", "25"), + Map.entry("dissect {} \"%{bar}\"", "28"), + Map.entry("grok {} \"%{WORD:foo}\"", "25") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"dup\":1,\"dup\":2})"), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Duplicated function arguments with the same name [dup] is not supported" + ) + ); + } + } + + public void testNamedFunctionArgumentInInvalidPositions() { + // negative, named arguments are not supported outside of a functionExpression where booleanExpression or indexPattern is supported + String map = "{\"option1\":\"string\", \"option2\":1}"; + + Map commands = Map.ofEntries( + Map.entry("from {}", "line 1:7: mismatched input '\"option1\"' expecting {, '|', ',', 'metadata'}"), + Map.entry("row x = {}", "line 1:9: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL"), + Map.entry("eval x = {}", "line 1:22: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL"), + Map.entry("where x > {}", "line 1:23: no viable alternative at input 'x > {'"), + Map.entry("stats agg() by {}", "line 1:28: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL"), + Map.entry("sort {}", "line 1:18: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL"), + Map.entry("keep {}", "line 1:18: token recognition error at: '{'"), + Map.entry("drop {}", "line 1:18: token recognition error at: '{'"), + Map.entry("rename a as {}", "line 1:25: token recognition error at: '{'"), + Map.entry("mv_expand {}", "line 1:23: token recognition error at: '{'"), + Map.entry("limit {}", "line 1:19: mismatched input '{' expecting INTEGER_LITERAL"), + Map.entry("enrich idx2 on f1 with f2 = {}", "line 1:41: token recognition error at: '{'"), + Map.entry("dissect {} \"%{bar}\"", "line 1:21: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL"), + Map.entry("grok {} \"%{WORD:foo}\"", "line 1:18: extraneous input '{' expecting {QUOTED_STRING, INTEGER_LITERAL") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String errorMessage = command.getValue(); + String from = cmd.startsWith("row") || cmd.startsWith("from") ? "" : "from test | "; + expectError(LoggerMessageFormat.format(null, from + cmd, map), errorMessage); + } + } + + public void testNamedFunctionArgumentWithUnsupportedNamedParameterTypes() { + Map commands = Map.ofEntries( + Map.entry("eval x = {}", "29"), + Map.entry("where {}", "26"), + Map.entry("stats {}", "26"), + Map.entry("stats agg() by {}", "35"), + Map.entry("sort {}", "25"), + Map.entry("dissect {} \"%{bar}\"", "28"), + Map.entry("grok {} \"%{WORD:foo}\"", "25") + ); + + for (Map.Entry command : commands.entrySet()) { + String cmd = command.getKey(); + String error = command.getValue(); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"option1\":?n1})"), + List.of(paramAsIdentifier("n1", "v1")), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Invalid named function argument [\"option1\":?n1], only constant value is supported" + ) + ); + expectError( + LoggerMessageFormat.format(null, "from test | " + cmd, "fn(f1, {\"option1\":?n1})"), + List.of(paramAsPattern("n1", "v1")), + LoggerMessageFormat.format( + null, + "line 1:{}: {}", + error, + "Invalid named function argument [\"option1\":?n1], only constant value is supported" + ) + ); + } + } + + public void testValidFromPattern() { + var basePattern = randomIndexPatterns(); + + var plan = statement("FROM " + basePattern); + + assertThat(as(plan, UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(basePattern))); + } + + public void testValidJoinPattern() { + assumeTrue("LOOKUP JOIN requires corresponding capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + + var basePattern = randomIndexPatterns(without(CROSS_CLUSTER)); + var joinPattern = randomIndexPattern(without(WILDCARD_PATTERN), without(CROSS_CLUSTER)); + var onField = randomIdentifier(); + + var plan = statement("FROM " + basePattern + " | LOOKUP JOIN " + joinPattern + " ON " + onField); + + var join = as(plan, LookupJoin.class); + assertThat(as(join.left(), UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(basePattern))); + assertThat(as(join.right(), UnresolvedRelation.class).indexPattern().indexPattern(), equalTo(unquoteIndexPattern(joinPattern))); + + var joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertThat(joinType.columns(), hasSize(1)); + assertThat(as(joinType.columns().getFirst(), UnresolvedAttribute.class).name(), equalTo(onField)); + assertThat(joinType.coreJoin().joinName(), equalTo("LEFT OUTER")); + } + + public void testInvalidJoinPatterns() { + assumeTrue("LOOKUP JOIN requires corresponding capability", EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()); + + { + // wildcard + var joinPattern = randomIndexPattern(WILDCARD_PATTERN, without(CROSS_CLUSTER)); + expectError( + "FROM " + randomIndexPatterns() + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], * is not allowed in LOOKUP JOIN" + ); + } + { + // remote cluster on the right + var fromPatterns = randomIndexPatterns(without(CROSS_CLUSTER)); + var joinPattern = randomIndexPattern(CROSS_CLUSTER, without(WILDCARD_PATTERN)); + expectError( + "FROM " + fromPatterns + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(joinPattern) + "], remote clusters are not supported in LOOKUP JOIN" + ); + } + { + // remote cluster on the left + var fromPatterns = randomIndexPatterns(CROSS_CLUSTER); + var joinPattern = randomIndexPattern(without(CROSS_CLUSTER), without(WILDCARD_PATTERN)); + expectError( + "FROM " + fromPatterns + " | LOOKUP JOIN " + joinPattern + " ON " + randomIdentifier(), + "invalid index pattern [" + unquoteIndexPattern(fromPatterns) + "], remote clusters are not supported in LOOKUP JOIN" + ); + } + } + + public void testInvalidInsistAsterisk() { + assumeTrue("requires snapshot build", Build.current().isSnapshot()); + expectError("FROM text | EVAL x = 4 | INSIST_🐔 *", "INSIST doesn't support wildcards, found [*]"); + expectError("FROM text | EVAL x = 4 | INSIST_🐔 foo*", "INSIST doesn't support wildcards, found [foo*]"); + } + + public void testValidFork() { + assumeTrue("FORK requires corresponding capability", EsqlCapabilities.Cap.FORK.isEnabled()); + + var plan = statement(""" + FROM foo* + | FORK ( WHERE a:"baz" | LIMIT 11 ) + ( WHERE b:"bar" | SORT b ) + ( WHERE c:"bat" ) + ( SORT c ) + ( LIMIT 5 ) + """); + var fork = as(plan, Fork.class); + var subPlans = fork.subPlans(); + + // first subplan + var eval = as(subPlans.get(0), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork1")))); + var limit = as(eval.child(), Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertThat(((Literal) limit.limit()).value(), equalTo(11)); + var filter = as(limit.child(), Filter.class); + var match = (MatchOperator) filter.condition(); + var matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("a")); + assertThat(match.query().fold(FoldContext.small()), equalTo("baz")); + + // second subplan + eval = as(subPlans.get(1), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork2")))); + var orderBy = as(eval.child(), OrderBy.class); + assertThat(orderBy.order().size(), equalTo(1)); + Order order = orderBy.order().get(0); + assertThat(order.child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) order.child()).name(), equalTo("b")); + filter = as(orderBy.child(), Filter.class); + match = (MatchOperator) filter.condition(); + matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("b")); + assertThat(match.query().fold(FoldContext.small()), equalTo("bar")); + + // third subplan + eval = as(subPlans.get(2), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork3")))); + filter = as(eval.child(), Filter.class); + match = (MatchOperator) filter.condition(); + matchField = (UnresolvedAttribute) match.field(); + assertThat(matchField.name(), equalTo("c")); + assertThat(match.query().fold(FoldContext.small()), equalTo("bat")); + + // fourth subplan + eval = as(subPlans.get(3), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork4")))); + orderBy = as(eval.child(), OrderBy.class); + assertThat(orderBy.order().size(), equalTo(1)); + order = orderBy.order().get(0); + assertThat(order.child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) order.child()).name(), equalTo("c")); + + // fifth subplan + eval = as(subPlans.get(4), Eval.class); + assertThat(as(eval.fields().get(0), Alias.class), equalTo(alias("_fork", literalString("fork5")))); + limit = as(eval.child(), Limit.class); + assertThat(limit.limit(), instanceOf(Literal.class)); + assertThat(((Literal) limit.limit()).value(), equalTo(5)); + } + + public void testInvalidFork() { + assumeTrue("FORK requires corresponding capability", EsqlCapabilities.Cap.FORK.isEnabled()); + + expectError("FROM foo* | FORK (WHERE a:\"baz\")", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (LIMIT 10)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (SORT a)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | FORK (WHERE x>1 | LIMIT 5)", "line 1:13: Fork requires at least two branches"); + expectError("FROM foo* | WHERE x>1 | FORK (WHERE a:\"baz\")", "Fork requires at least two branches"); + + expectError("FROM foo* | FORK (LIMIT 10) (EVAL x = 1)", "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}"); + expectError("FROM foo* | FORK (EVAL x = 1) (LIMIT 10)", "line 1:19: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}"); + expectError( + "FROM foo* | FORK (WHERE x>1 |EVAL x = 1) (WHERE x>1)", + "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK (WHERE x>1 |EVAL x = 1) (WHERE x>1)", + "line 1:30: mismatched input 'EVAL' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK (WHERE x>1 |STATS count(x) by y) (WHERE x>1)", + "line 1:30: mismatched input 'STATS' expecting {'limit', 'sort', 'where'}" + ); + expectError( + "FROM foo* | FORK ( FORK (WHERE x>1) (WHERE y>1)) (WHERE z>1)", + "line 1:20: mismatched input 'FORK' expecting {'limit', 'sort', 'where'}" + ); + expectError("FROM foo* | FORK ( x+1 ) ( WHERE y>2 )", "line 1:20: mismatched input 'x+1' expecting {'limit', 'sort', 'where'}"); + expectError("FROM foo* | FORK ( LIMIT 10 ) ( y+2 )", "line 1:33: mismatched input 'y+2' expecting {'limit', 'sort', 'where'}"); + } + + public void testFieldNamesAsCommands() throws Exception { + String[] keywords = new String[] { + "dissect", + "drop", + "enrich", + "eval", + "explain", + "from", + "grok", + "keep", + "limit", + "mv_expand", + "rename", + "sort", + "stats" }; + for (String keyword : keywords) { + var plan = statement("FROM test | STATS avg(" + keyword + ")"); + var aggregate = as(plan, Aggregate.class); + } + } + + // [ and ( are used to trigger a double mode causing their symbol name (instead of text) to be used in error reporting + // this test checks that their are properly replaced in the error message + public void testPreserveParanthesis() { + // test for ( + expectError("row a = 1 not in", "line 1:17: mismatched input '' expecting '('"); + expectError("row a = 1 | where a not in", "line 1:27: mismatched input '' expecting '('"); + expectError("row a = 1 | where a not in (1", "line 1:30: mismatched input '' expecting {',', ')'}"); + expectError("row a = 1 | where a not in [1", "line 1:28: missing '(' at '['"); + expectError("row a = 1 | where a not in 123", "line 1:28: missing '(' at '123'"); + // test for [ + expectError("explain", "line 1:8: mismatched input '' expecting '['"); + expectError("explain ]", "line 1:9: token recognition error at: ']'"); + expectError("explain [row x = 1", "line 1:19: missing ']' at ''"); + } + + static Alias alias(String name, Expression value) { + return new Alias(EMPTY, name, value); + } + + public void testValidRrf() { + assumeTrue("RRF requires corresponding capability", EsqlCapabilities.Cap.RRF.isEnabled()); + + LogicalPlan plan = statement(""" + FROM foo* METADATA _id, _index, _score + | FORK ( WHERE a:"baz" ) + ( WHERE b:"bar" ) + | RRF + """); + + var orderBy = as(plan, OrderBy.class); + assertThat(orderBy.order().size(), equalTo(3)); + + assertThat(orderBy.order().get(0).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(0).child()).name(), equalTo("_score")); + assertThat(orderBy.order().get(1).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(1).child()).name(), equalTo("_id")); + assertThat(orderBy.order().get(2).child(), instanceOf(UnresolvedAttribute.class)); + assertThat(((UnresolvedAttribute) orderBy.order().get(2).child()).name(), equalTo("_index")); + + var dedup = as(orderBy.child(), Dedup.class); + assertThat(dedup.groupings().size(), equalTo(2)); + assertThat(dedup.groupings().get(0), instanceOf(UnresolvedAttribute.class)); + assertThat(dedup.groupings().get(0).name(), equalTo("_id")); + assertThat(dedup.groupings().get(1), instanceOf(UnresolvedAttribute.class)); + assertThat(dedup.groupings().get(1).name(), equalTo("_index")); + assertThat(dedup.aggregates().size(), equalTo(1)); + assertThat(dedup.aggregates().get(0), instanceOf(Alias.class)); + + var rrfScoreEval = as(dedup.child(), RrfScoreEval.class); + assertThat(rrfScoreEval.scoreAttribute(), instanceOf(UnresolvedAttribute.class)); + assertThat(rrfScoreEval.scoreAttribute().name(), equalTo("_score")); + assertThat(rrfScoreEval.forkAttribute(), instanceOf(UnresolvedAttribute.class)); + assertThat(rrfScoreEval.forkAttribute().name(), equalTo("_fork")); + + assertThat(rrfScoreEval.child(), instanceOf(Fork.class)); + } + + public void testDoubleParamsForIdentifier() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // There are three variations of double parameters - named, positional or anonymous, e.g. ??n, ??1 or ??, covered. + // Each query is executed three times with the three variations. + + // field names can appear in eval/where/stats/sort/keep/drop/rename/dissect/grok/enrich/mvexpand + // eval, where + List> doubleParams = new ArrayList<>(3); + List namedDoubleParams = List.of("??f0", "??fn1", "??f1", "??f2", "??f3"); + List positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5"); + List anonymousDoubleParams = List.of("??", "??", "??", "??", "??"); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | eval {} = {}({}) + | where {} == {} + | limit 1""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval(EMPTY, relation("test"), List.of(new Alias(EMPTY, "x", function("toString", List.of(attribute("f1.")))))), + new Equals(EMPTY, attribute("f.2"), attribute("f3")) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("f1", "f1."), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3") + ) + ) + ) + ); + } + + namedDoubleParams = List.of("??f0", "??fn1", "??f1", "??f2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | eval {} = {}({}.{}) + | where {}.{} == {}.{} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7) + ); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of(new Alias(EMPTY, "x", function("toString", List.of(attribute("f1..f.2"))))) + ), + new Equals(EMPTY, attribute("f3.*.f.4."), attribute("f.5.*.f.*.6")) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("f1", "f1."), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3.*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.*.6") + ) + ) + ) + ); + } + + // stats, sort, mv_expand + namedDoubleParams = List.of("??fn2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | stats y = {}({}) by {} + | sort {} + | mv_expand {}""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.4.")), + List.of(new Alias(EMPTY, "y", function("count", List.of(attribute("f3.*")))), attribute("f.4.")) + ), + List.of(new Order(EMPTY, attribute("f.5.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.6*"), + attribute("f.6*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("f3", "f3.*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + + namedDoubleParams = List.of("??fn2", "??f7", "??f8", "??f9", "??f10", "??f11", "??f12", "??f13", "??f14"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8", "??9"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | stats y = {}({}.{}) by {}.{} + | sort {}.{} + | mv_expand {}.{}""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8) + ); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.9.f10.*")), + List.of(new Alias(EMPTY, "y", function("count", List.of(attribute("f.7*.f8.")))), attribute("f.9.f10.*")) + ), + List.of(new Order(EMPTY, attribute("f.11..f.12.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.*.13.f.14*"), + attribute("f.*.13.f.14*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("f7", "f.7*"), + paramAsConstant("f8", "f8."), + paramAsConstant("f9", "f.9"), + paramAsConstant("f10", "f10.*"), + paramAsConstant("f11", "f.11."), + paramAsConstant("f12", "f.12.*"), + paramAsConstant("f13", "f.*.13"), + paramAsConstant("f14", "f.14*") + ) + ) + ) + ); + } + + // keep, drop, rename, grok, dissect, lookup join + namedDoubleParams = List.of("??f1", "??f2", "??f3", "??f4", "??f5", "??f6", "??f7", "??f8", "??f9"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6", "??7", "??8", "??9"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | keep {}, {} + | drop {}, {} + | dissect {} "%{bar}" + | grok {} "%{WORD:foo}" + | rename {} as {} + | lookup join idx on {} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8) + ); + LogicalPlan plan = statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3."), + paramAsConstant("f4", "f4.*"), + paramAsConstant("f5", "f.5*"), + paramAsConstant("f6", "f.6."), + paramAsConstant("f7", "f7*."), + paramAsConstant("f8", "f.8"), + paramAsConstant("f9", "f9") + ) + ) + ); + Limit limit = as(plan, Limit.class); + LookupJoin join = as(limit.child(), LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f9"))); + Rename rename = as(join.left(), Rename.class); + assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f.8", attribute("f7*.")))); + Grok grok = as(rename.child(), Grok.class); + assertEquals(grok.input(), attribute("f.6.")); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(dissect.input(), attribute("f.5*")); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + Drop drop = as(dissect.child(), Drop.class); + List removals = drop.removals(); + assertEquals(removals, List.of(attribute("f3."), attribute("f4.*"))); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections(), List.of(attribute("f.1.*"), attribute("f.2"))); + } + + namedDoubleParams = List.of( + "??f1", + "??f2", + "??f3", + "??f4", + "??f5", + "??f6", + "??f7", + "??f8", + "??f9", + "??f10", + "??f11", + "??f12", + "??f13", + "??f14" + ); + positionalDoubleParams = List.of( + "??1", + "??2", + "??3", + "??4", + "??5", + "??6", + "??7", + "??8", + "??9", + "??10", + "??11", + "??12", + "??13", + "??14" + ); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + """ + from test + | keep {}.{} + | drop {}.{} + | dissect {}.{} "%{bar}" + | grok {}.{} "%{WORD:foo}" + | rename {}.{} as {}.{} + | lookup join idx on {}.{} + | limit 1""", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5), + params.get(6), + params.get(7), + params.get(8), + params.get(9), + params.get(10), + params.get(11), + params.get(12), + params.get(13) + ); + LogicalPlan plan = statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f3."), + paramAsConstant("f4", "f4.*"), + paramAsConstant("f5", "f.5*"), + paramAsConstant("f6", "f.6."), + paramAsConstant("f7", "f7*."), + paramAsConstant("f8", "f.8"), + paramAsConstant("f9", "f.9*"), + paramAsConstant("f10", "f.10."), + paramAsConstant("f11", "f11*."), + paramAsConstant("f12", "f.12"), + paramAsConstant("f13", "f13"), + paramAsConstant("f14", "f14") + ) + ) + ); + Limit limit = as(plan, Limit.class); + LookupJoin join = as(limit.child(), LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f13.f14"))); + Rename rename = as(join.left(), Rename.class); + assertEquals(rename.renamings(), List.of(new Alias(EMPTY, "f11*..f.12", attribute("f.9*.f.10.")))); + Grok grok = as(rename.child(), Grok.class); + assertEquals(grok.input(), attribute("f7*..f.8")); + assertEquals("%{WORD:foo}", grok.parser().pattern()); + assertEquals(List.of(referenceAttribute("foo", KEYWORD)), grok.extractedFields()); + Dissect dissect = as(grok.child(), Dissect.class); + assertEquals(dissect.input(), attribute("f.5*.f.6.")); + assertEquals("%{bar}", dissect.parser().pattern()); + assertEquals("", dissect.parser().appendSeparator()); + assertEquals(List.of(referenceAttribute("bar", KEYWORD)), dissect.extractedFields()); + Drop drop = as(dissect.child(), Drop.class); + List removals = drop.removals(); + assertEquals(removals, List.of(attribute("f3..f4.*"))); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections(), List.of(attribute("f.1.*.f.2"))); + } + + // enrich, lookup join + namedDoubleParams = List.of("??f1", "??f2", "??f3"); + positionalDoubleParams = List.of("??1", "??2", "??3"); + anonymousDoubleParams = List.of("??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + "from idx1 | ENRICH idx2 ON {} WITH {} = {}", + params.get(0), + params.get(1), + params.get(2) + ); + assertEquals( + new Enrich( + EMPTY, + relation("idx1"), + null, + new Literal(EMPTY, "idx2", KEYWORD), + attribute("f.1.*"), + null, + Map.of(), + List.of(new Alias(EMPTY, "f.2", attribute("f.3*"))) + ), + statement( + query, + new QueryParams(List.of(paramAsConstant("f1", "f.1.*"), paramAsConstant("f2", "f.2"), paramAsConstant("f3", "f.3*"))) + ) + ); + } + + namedDoubleParams = List.of("??f1", "??f2", "??f3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "??2", "??3", "??4", "??5", "??6"); + anonymousDoubleParams = List.of("??", "??", "??", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format( + null, + "from idx1 | ENRICH idx2 ON {}.{} WITH {}.{} = {}.{}", + params.get(0), + params.get(1), + params.get(2), + params.get(3), + params.get(4), + params.get(5) + ); + assertEquals( + new Enrich( + EMPTY, + relation("idx1"), + null, + new Literal(EMPTY, "idx2", KEYWORD), + attribute("f.1.*.f.2"), + null, + Map.of(), + List.of(new Alias(EMPTY, "f.3*.f.4.*", attribute("f.5.f.6*"))) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f1", "f.1.*"), + paramAsConstant("f2", "f.2"), + paramAsConstant("f3", "f.3*"), + paramAsConstant("f4", "f.4.*"), + paramAsConstant("f5", "f.5"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + } + + public void testMixedSingleDoubleParams() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // This is a subset of testDoubleParamsForIdentifier, with single and double parameter markers mixed in the queries + // Single parameter markers represent a constant value or pattern + // double parameter markers represent identifiers - field or function names + + // mixed constant and identifier, eval/where + List> doubleParams = new ArrayList<>(3); + List namedDoubleParams = List.of("??f0", "??fn1", "?v1", "??f2", "?v3"); + List positionalDoubleParams = List.of("??1", "??2", "?3", "??4", "?5"); + List anonymousDoubleParams = List.of("??", "??", "?", "??", "?"); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | eval {} = {}({}) + | where {} == {} + | limit 1""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new Limit( + EMPTY, + new Literal(EMPTY, 1, INTEGER), + new Filter( + EMPTY, + new Eval( + EMPTY, + relation("test"), + List.of(new Alias(EMPTY, "x", function("toString", List.of(new Literal(EMPTY, "constant_value", KEYWORD))))) + ), + new Equals(EMPTY, attribute("f.2"), new Literal(EMPTY, 100, INTEGER)) + ) + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("f0", "x"), + paramAsConstant("fn1", "toString"), + paramAsConstant("v1", "constant_value"), + paramAsConstant("f2", "f.2"), + paramAsConstant("v3", 100) + ) + ) + ) + ); + } + + // mixed constant and identifier, stats/sort/mv_expand + namedDoubleParams = List.of("??fn2", "?v3", "??f4", "??f5", "??f6"); + positionalDoubleParams = List.of("??1", "?2", "??3", "??4", "??5"); + anonymousDoubleParams = List.of("??", "?", "??", "??", "??"); + doubleParams.clear(); + doubleParams.add(namedDoubleParams); + doubleParams.add(positionalDoubleParams); + doubleParams.add(anonymousDoubleParams); + for (List params : doubleParams) { + String query = LoggerMessageFormat.format(null, """ + from test + | stats y = {}({}) by {} + | sort {} + | mv_expand {}""", params.get(0), params.get(1), params.get(2), params.get(3), params.get(4)); + assertEquals( + new MvExpand( + EMPTY, + new OrderBy( + EMPTY, + new Aggregate( + EMPTY, + relation("test"), + Aggregate.AggregateType.STANDARD, + List.of(attribute("f.4.")), + List.of(new Alias(EMPTY, "y", function("count", List.of(new Literal(EMPTY, "*", KEYWORD)))), attribute("f.4.")) + ), + List.of(new Order(EMPTY, attribute("f.5.*"), Order.OrderDirection.ASC, Order.NullsPosition.LAST)) + ), + attribute("f.6*"), + attribute("f.6*") + ), + statement( + query, + new QueryParams( + List.of( + paramAsConstant("fn2", "count"), + paramAsConstant("v3", "*"), + paramAsConstant("f4", "f.4."), + paramAsConstant("f5", "f.5.*"), + paramAsConstant("f6", "f.6*") + ) + ) + ) + ); + } + + // mixed field name and field name pattern + LogicalPlan plan = statement( + "from test | keep ??f1, ?f2 | drop ?f3, ??f4 | lookup join idx on ??f5", + new QueryParams( + List.of( + paramAsConstant("f1", "f*1."), + paramAsPattern("f2", "f.2*"), + paramAsPattern("f3", "f3.*"), + paramAsConstant("f4", "f.4.*"), + paramAsConstant("f5", "f5") + ) + ) + ); + + LookupJoin join = as(plan, LookupJoin.class); + UnresolvedRelation ur = as(join.right(), UnresolvedRelation.class); + assertEquals(ur.indexPattern().indexPattern(), "idx"); + JoinTypes.UsingJoinType joinType = as(join.config().type(), JoinTypes.UsingJoinType.class); + assertEquals(joinType.coreJoin().joinName(), "LEFT OUTER"); + assertEquals(joinType.columns(), List.of(attribute("f5"))); + Drop drop = as(join.left(), Drop.class); + List removals = drop.removals(); + assertEquals(removals.size(), 2); + UnresolvedNamePattern up = as(removals.get(0), UnresolvedNamePattern.class); + assertEquals(up.name(), "f3.*"); + assertEquals(up.pattern(), "f3.*"); + UnresolvedAttribute ua = as(removals.get(1), UnresolvedAttribute.class); + assertEquals(ua.name(), "f.4.*"); + Keep keep = as(drop.child(), Keep.class); + assertEquals(keep.projections().size(), 2); + ua = as(keep.projections().get(0), UnresolvedAttribute.class); + assertEquals(ua.name(), "f*1."); + up = as(keep.projections().get(1), UnresolvedNamePattern.class); + assertEquals(up.name(), "f.2*"); + assertEquals(up.pattern(), "f.2*"); + ur = as(keep.child(), UnresolvedRelation.class); + assertEquals(ur, relation("test")); + + // test random single and double params + // commands in group1 take both constants(?) and identifiers(??) + List commandWithRandomSingleOrDoubleParamsGroup1 = List.of( + "eval x = {}f1, y = {}f2, z = {}f3", + "eval x = fn({}f1), y = {}f2 + {}f3", + "where {}f1 == \"a\" and {}f2 > 1 and {}f3 in (1, 2)", + "stats x = fn({}f1) by {}f2, {}f3", + "sort {}f1, {}f2, {}f3", + "dissect {}f1 \"%{bar}\"", + "grok {}f1 \"%{WORD:foo}\"" + ); + for (String command : commandWithRandomSingleOrDoubleParamsGroup1) { + String param1 = randomBoolean() ? "?" : "??"; + String param2 = randomBoolean() ? "?" : "??"; + String param3 = randomBoolean() ? "?" : "??"; + plan = statement( + LoggerMessageFormat.format(null, "from test | " + command, param1, param2, param3), + new QueryParams(List.of(paramAsConstant("f1", "f1"), paramAsConstant("f2", "f2"), paramAsConstant("f3", "f3"))) + ); + assertNotNull(plan); + } + // commands in group2 only take identifiers(??) + List commandWithRandomSingleOrDoubleParamsGroup2 = List.of( + "eval x = {}f1(), y = {}f2(), z = {}f3()", + "where {}f1 : \"b\" and {}f2() > 0 and {}f3()", + "stats x = {}f1(), {}f2(), {}f3()", + "rename {}f1 as {}f2, {}f3 as x", + "enrich idx2 ON {}f1 WITH {}f2 = {}f3", + "keep {}f1, {}f2, {}f3", + "drop {}f1, {}f2, {}f3", + "mv_expand {}f1 | mv_expand {}f2 | mv_expand {}f3", + "lookup join idx1 on {}f1 | lookup join idx2 on {}f2 | lookup join idx3 on {}f3" + ); + + for (String command : commandWithRandomSingleOrDoubleParamsGroup2) { + String param1 = randomBoolean() ? "?" : "??"; + String param2 = randomBoolean() ? "?" : "??"; + String param3 = randomBoolean() ? "?" : "??"; + if (param1.equals("?") || param2.equals("?") || param3.equals("?")) { + expectError( + LoggerMessageFormat.format(null, "from test | " + command, param1, param2, param3), + List.of(paramAsConstant("f1", "f1"), paramAsConstant("f2", "f2"), paramAsConstant("f3", "f3")), + command.contains("join") + ? "JOIN ON clause only supports fields at the moment" + : "declared as a constant, cannot be used as an identifier" + ); + } + } + } + + public void testInvalidDoubleParamsNames() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + expectError( + "from test | where x < ??n1 | eval y = ??n2", + List.of(paramAsConstant("n1", "f1"), paramAsConstant("n3", "f2")), + "line 1:39: Unknown query parameter [n2], did you mean any of [n3, n1]?" + ); + + expectError("from test | where x < ??@1", List.of(paramAsConstant("@1", "f1")), "line 1:25: extraneous input '@1' expecting "); + + expectError("from test | where x < ??#1", List.of(paramAsConstant("#1", "f1")), "line 1:25: token recognition error at: '#'"); + + expectError("from test | where x < ??Å", List.of(paramAsConstant("Å", "f1")), "line 1:25: token recognition error at: 'Å'"); + + expectError("from test | eval x = ??Å", List.of(paramAsConstant("Å", "f1")), "line 1:24: token recognition error at: 'Å'"); + } + + public void testInvalidDoubleParamsPositions() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + expectError( + "from test | where x < ??0", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 0, did you mean position 1" + ); + + expectError( + "from test | where x < ??2", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 2, did you mean position 1" + ); + + expectError( + "from test | where x < ??0 and y < ??2", + List.of(paramAsConstant(null, "f1")), + "line 1:23: No parameter is defined for position 0, did you mean position 1?; " + + "line 1:35: No parameter is defined for position 2, did you mean position 1?" + ); + + expectError( + "from test | where x < ??0", + List.of(paramAsConstant(null, "f1"), paramAsConstant(null, "f2")), + "line 1:23: No parameter is defined for position 0, did you mean any position between 1 and 2?" + ); + } + + public void testInvalidDoubleParamsType() { + assumeTrue( + "double parameters markers for identifiers requires snapshot build", + EsqlCapabilities.Cap.DOUBLE_PARAMETER_MARKERS_FOR_IDENTIFIERS.isEnabled() + ); + // double parameter markers cannot be declared as identifier patterns + String error = "Query parameter [??f1][f1] declared as a pattern, cannot be used as an identifier"; + List commandWithDoubleParams = List.of( + "eval x = ??f1", + "eval x = ??f1(f1)", + "where ??f1 == \"a\"", + "stats x = count(??f1)", + "sort ??f1", + "rename ??f1 as ??f2", + "dissect ??f1 \"%{bar}\"", + "grok ??f1 \"%{WORD:foo}\"", + "enrich idx2 ON ??f1 WITH ??f2 = ??f3", + "keep ??f1", + "drop ??f1", + "mv_expand ??f1", + "lookup join idx on ??f1" + ); + for (String command : commandWithDoubleParams) { + expectError( + "from test | " + command, + List.of(paramAsPattern("f1", "f1*"), paramAsPattern("f2", "f2*"), paramAsPattern("f3", "f3*")), + error + ); + } + } + + public void testUnclosedParenthesis() { + String[] queries = { "row a = )", "row ]", "from source | eval x = [1,2,3]]" }; + for (String q : queries) { + expectError(q, "Invalid query"); + } + } }