Skip to content

Commit d3f4c1b

Browse files
SwiddisSwiddis
committed
Merge remote-tracking branch 'upstream/main' into feature/pre-fetch-batches-in-enumeration
2 parents a5a0fbf + c203bda commit d3f4c1b

File tree

144 files changed

+11233
-759
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

144 files changed

+11233
-759
lines changed

‎async-query-core/src/main/java/org/opensearch/sql/spark/rest/model/LangType.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
/** Language type accepted in async query apis. */
99
public enum LangType {
1010
SQL("sql"),
11-
PPL("ppl");
11+
PPL("ppl"),
12+
PROMQL("promql");
1213
private final String text;
1314

1415
LangType(String text) {

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2330,13 +2330,15 @@ private void buildParseRelNode(Parse node, CalcitePlanContext context) {
23302330
};
23312331
if (ParseMethod.PATTERNS.equals(parseMethod)) {
23322332
rexNodeList = ArrayUtils.add(rexNodeList, context.relBuilder.literal("<*>"));
2333+
} else {
2334+
rexNodeList = ArrayUtils.add(rexNodeList, context.relBuilder.literal(parseMethod.getName()));
23332335
}
23342336
List<RexNode> newFields = new ArrayList<>();
23352337
for (String groupCandidate : groupCandidates) {
23362338
RexNode innerRex =
23372339
PPLFuncImpTable.INSTANCE.resolve(
23382340
context.rexBuilder, ParseUtils.BUILTIN_FUNCTION_MAP.get(parseMethod), rexNodeList);
2339-
if (ParseMethod.GROK.equals(parseMethod)) {
2341+
if (!ParseMethod.PATTERNS.equals(parseMethod)) {
23402342
newFields.add(
23412343
PPLFuncImpTable.INSTANCE.resolve(
23422344
context.rexBuilder,

‎core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ private PPLOperandTypes() {}
9292
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.INTEGER_INTEGER);
9393
public static final UDFOperandMetadata STRING_STRING =
9494
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.CHARACTER_CHARACTER);
95+
public static final UDFOperandMetadata STRING_STRING_STRING =
96+
UDFOperandMetadata.wrap(
97+
OperandTypes.family(
98+
SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER));
9599
public static final UDFOperandMetadata NUMERIC_NUMERIC =
96100
UDFOperandMetadata.wrap((FamilyOperandTypeChecker) OperandTypes.NUMERIC_NUMERIC);
97101
public static final UDFOperandMetadata STRING_INTEGER =

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,8 +323,8 @@ public enum BuiltinFunctionName {
323323
INTERNAL_PATTERN_PARSER(FunctionName.of("pattern_parser")),
324324
INTERNAL_PATTERN(FunctionName.of("pattern")),
325325
INTERNAL_UNCOLLECT_PATTERNS(FunctionName.of("uncollect_patterns")),
326-
INTERNAL_REGEXP_EXTRACT(FunctionName.of("regexp_extract"), true),
327326
INTERNAL_GROK(FunctionName.of("grok"), true),
327+
INTERNAL_PARSE(FunctionName.of("parse"), true),
328328
INTERNAL_REGEXP_REPLACE_3(FunctionName.of("regexp_replace_3"), true),
329329
INTERNAL_REGEXP_REPLACE_PG_4(FunctionName.of("regexp_replace_pg_4"), true),
330330
INTERNAL_REGEXP_REPLACE_5(FunctionName.of("regexp_replace_5"), true),

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl;
5858
import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl;
5959
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
60-
import org.opensearch.sql.expression.function.udf.GrokFunction;
60+
import org.opensearch.sql.expression.function.udf.ParseFunction;
6161
import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction;
6262
import org.opensearch.sql.expression.function.udf.RexExtractFunction;
6363
import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction;
@@ -369,7 +369,10 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
369369
PPLOperandTypes.NONE)
370370
.toUDF("UTC_TIMESTAMP");
371371
public static final SqlOperator WEEK = new WeekFunction().toUDF("WEEK");
372-
public static final SqlOperator GROK = new GrokFunction().toUDF("GROK");
372+
public static final SqlOperator GROK = new ParseFunction().toUDF("GROK");
373+
// TODO: Figure out if there is other option to perform multiple group match in Calcite
374+
// For now, keep V2's regexExpression logic to avoid breaking change
375+
public static final SqlOperator PARSE = new ParseFunction().toUDF("PARSE");
373376
public static final SqlOperator PATTERN_PARSER =
374377
new PatternParserFunctionImpl().toUDF("PATTERN_PARSER");
375378

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@
8080
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL;
8181
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK;
8282
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM;
83+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE;
8384
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN;
8485
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN_PARSER;
85-
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_EXTRACT;
8686
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_3;
8787
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_5;
8888
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_PG_4;
@@ -712,7 +712,6 @@ void populate() {
712712
registerOperator(LOG2, SqlLibraryOperators.LOG2);
713713
registerOperator(MD5, SqlLibraryOperators.MD5);
714714
registerOperator(SHA1, SqlLibraryOperators.SHA1);
715-
registerOperator(INTERNAL_REGEXP_EXTRACT, SqlLibraryOperators.REGEXP_EXTRACT);
716715
registerOperator(INTERNAL_REGEXP_REPLACE_3, SqlLibraryOperators.REGEXP_REPLACE_3);
717716
registerOperator(INTERNAL_REGEXP_REPLACE_PG_4, SqlLibraryOperators.REGEXP_REPLACE_PG_4);
718717
registerOperator(INTERNAL_REGEXP_REPLACE_5, SqlLibraryOperators.REGEXP_REPLACE_5);
@@ -735,6 +734,7 @@ void populate() {
735734
registerOperator(SHA2, PPLBuiltinOperators.SHA2);
736735
registerOperator(CIDRMATCH, PPLBuiltinOperators.CIDRMATCH);
737736
registerOperator(INTERNAL_GROK, PPLBuiltinOperators.GROK);
737+
registerOperator(INTERNAL_PARSE, PPLBuiltinOperators.PARSE);
738738
registerOperator(MATCH, PPLBuiltinOperators.MATCH);
739739
registerOperator(MATCH_PHRASE, PPLBuiltinOperators.MATCH_PHRASE);
740740
registerOperator(MATCH_BOOL_PREFIX, PPLBuiltinOperators.MATCH_BOOL_PREFIX);

‎core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.google.common.collect.ImmutableMap;
99
import java.lang.reflect.Method;
1010
import java.util.Arrays;
11+
import java.util.Collections;
1112
import java.util.Comparator;
1213
import java.util.HashMap;
1314
import java.util.List;
@@ -18,7 +19,6 @@
1819
import org.apache.calcite.adapter.enumerable.RexImpTable;
1920
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
2021
import org.apache.calcite.linq4j.function.Parameter;
21-
import org.apache.calcite.linq4j.function.Strict;
2222
import org.apache.calcite.linq4j.tree.Expression;
2323
import org.apache.calcite.linq4j.tree.Types;
2424
import org.apache.calcite.rel.type.RelDataType;
@@ -37,8 +37,11 @@
3737
import org.opensearch.sql.common.patterns.PatternUtils.ParseResult;
3838

3939
public class PatternParserFunctionImpl extends ImplementorUDF {
40+
private static final Map<String, Object> EMPTY_RESULT =
41+
ImmutableMap.of(PatternUtils.PATTERN, "", PatternUtils.TOKENS, Collections.emptyMap());
42+
4043
protected PatternParserFunctionImpl() {
41-
super(new PatternParserImplementor(), NullPolicy.ARG0);
44+
super(new PatternParserImplementor(), NullPolicy.NONE);
4245
}
4346

4447
@Override
@@ -90,11 +93,10 @@ private Method getMethod(Class<?> paramType, String methodName) {
9093
* A simple and general label pattern algorithm given aggregated patterns, which is adopted from
9194
* Drain algorithm(see https://ieeexplore.ieee.org/document/8029742).
9295
*/
93-
@Strict
9496
public static Object evalAgg(
9597
@Parameter(name = "field") String field, @Parameter(name = "aggObject") Object aggObject) {
9698
if (Strings.isBlank(field)) {
97-
return ImmutableMap.of();
99+
return EMPTY_RESULT;
98100
}
99101
List<Map<String, Object>> aggResult = (List<Map<String, Object>>) aggObject;
100102
List<String> preprocessedTokens =
@@ -125,11 +127,10 @@ public static Object evalAgg(
125127
}
126128
}
127129

128-
@Strict
129130
public static Object evalField(
130131
@Parameter(name = "pattern") String pattern, @Parameter(name = "field") String field) {
131132
if (Strings.isBlank(field)) {
132-
return ImmutableMap.of();
133+
return EMPTY_RESULT;
133134
}
134135

135136
Map<String, List<String>> tokensMap = new HashMap<>();
@@ -143,12 +144,11 @@ public static Object evalField(
143144
tokensMap);
144145
}
145146

146-
@Strict
147147
public static Object evalSamples(
148148
@Parameter(name = "pattern") String pattern,
149149
@Parameter(name = "sample_logs") List<String> sampleLogs) {
150150
if (sampleLogs.isEmpty()) {
151-
return ImmutableMap.of();
151+
return EMPTY_RESULT;
152152
}
153153
Map<String, List<String>> tokensMap = new HashMap<>();
154154
ParseResult parseResult = PatternUtils.parsePattern(pattern, PatternUtils.WILDCARD_PATTERN);

‎core/src/main/java/org/opensearch/sql/expression/function/udf/GrokFunction.java‎ renamed to ‎core/src/main/java/org/opensearch/sql/expression/function/udf/ParseFunction.java‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import java.util.Collections;
1111
import java.util.List;
12+
import java.util.Locale;
1213
import java.util.Map;
1314
import java.util.stream.Collectors;
1415
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
@@ -22,18 +23,20 @@
2223
import org.apache.calcite.sql.type.SqlTypeName;
2324
import org.apache.calcite.sql.type.SqlTypeUtil;
2425
import org.apache.commons.lang3.tuple.Pair;
26+
import org.opensearch.sql.ast.expression.ParseMethod;
2527
import org.opensearch.sql.calcite.utils.PPLOperandTypes;
2628
import org.opensearch.sql.data.model.ExprValue;
2729
import org.opensearch.sql.expression.DSL;
2830
import org.opensearch.sql.expression.LiteralExpression;
2931
import org.opensearch.sql.expression.function.ImplementorUDF;
3032
import org.opensearch.sql.expression.function.UDFOperandMetadata;
31-
import org.opensearch.sql.expression.parse.GrokExpression;
33+
import org.opensearch.sql.expression.parse.ParseExpression;
34+
import org.opensearch.sql.utils.ParseUtils;
3235

33-
public final class GrokFunction extends ImplementorUDF {
36+
public final class ParseFunction extends ImplementorUDF {
3437

35-
public GrokFunction() {
36-
super(new GrokImplementor(), NullPolicy.ANY);
38+
public ParseFunction() {
39+
super(new ParseImplementor(), NullPolicy.NONE);
3740
}
3841

3942
@Override
@@ -48,29 +51,33 @@ public SqlReturnTypeInference getReturnTypeInference() {
4851

4952
@Override
5053
public UDFOperandMetadata getOperandMetadata() {
51-
return PPLOperandTypes.STRING_STRING;
54+
return PPLOperandTypes.STRING_STRING_STRING;
5255
}
5356

54-
public static class GrokImplementor implements NotNullImplementor {
57+
public static class ParseImplementor implements NotNullImplementor {
58+
5559
@Override
5660
public Expression implement(
5761
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
58-
return Expressions.call(GrokFunction.GrokImplementor.class, "grok", translatedOperands);
62+
return Expressions.call(ParseFunction.ParseImplementor.class, "parse", translatedOperands);
5963
}
6064

61-
public static Map<String, String> grok(String input, String regex) {
65+
public static Map<String, String> parse(String input, String regex, String parseMethod) {
66+
ParseMethod method = ParseMethod.valueOf(parseMethod.toUpperCase(Locale.ROOT));
67+
List<String> namedFields =
68+
ParseUtils.getNamedGroupCandidates(method, regex, Collections.emptyMap());
6269
if (input == null) {
63-
return Collections.EMPTY_MAP;
70+
return namedFields.stream().collect(Collectors.toMap(element -> element, element -> ""));
6471
}
6572
LiteralExpression inputExpr = DSL.literal(input);
6673
LiteralExpression regexExpr = DSL.literal(regex);
67-
List<String> namedFields = GrokExpression.getNamedGroupCandidates(regex);
6874
return namedFields.stream()
6975
.map(
7076
namedField -> {
71-
GrokExpression grokExpression =
72-
new GrokExpression(inputExpr, regexExpr, DSL.literal(namedField));
73-
ExprValue parsedValue = grokExpression.parseValue(inputExpr.valueOf());
77+
ParseExpression parseExpr =
78+
ParseUtils.createParseExpression(
79+
method, inputExpr, regexExpr, DSL.literal(namedField));
80+
ExprValue parsedValue = parseExpr.parseValue(inputExpr.valueOf());
7481
return Pair.of(namedField, parsedValue.stringValue());
7582
})
7683
.collect(Collectors.toMap(Pair::getKey, Pair::getValue));

‎core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,5 @@ public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
7171
return visitor.visitParse(this, context);
7272
}
7373

74-
abstract ExprValue parseValue(ExprValue value) throws ExpressionEvaluationException;
74+
public abstract ExprValue parseValue(ExprValue value) throws ExpressionEvaluationException;
7575
}

‎core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ public RegexExpression(Expression sourceField, Expression pattern, Expression id
3636
}
3737

3838
@Override
39-
ExprValue parseValue(ExprValue value) throws ExpressionEvaluationException {
39+
public ExprValue parseValue(ExprValue value) throws ExpressionEvaluationException {
4040
String rawString = value.stringValue();
4141

4242
String extracted = RegexCommonUtils.extractNamedGroup(rawString, regexPattern, identifierStr);

0 commit comments

Comments
 (0)