Skip to content

Commit 29b1155

Browse files
fang-xing-esqlfang-xing-esql
committed
remove grok and dissect, add enrich
1 parent 02e2260 commit 29b1155

File tree

4 files changed

+88
-40
lines changed

4 files changed

+88
-40
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1982,3 +1982,45 @@ c:long | mo:date_nanos
19821982
22 | 1986-01-01T00:00:00.000Z
19831983
22 | 1985-01-01T00:00:00.000Z
19841984
;
1985+
1986+
1987+
MultiTypedEnrichOnNumericField
1988+
required_capability: enrich_load
1989+
required_capability: implicit_casting_union_typed_numeric_and_date
1990+
1991+
from employees, employees_incompatible
1992+
| enrich languages_policy on languages
1993+
| keep emp_no, language_name
1994+
| sort emp_no
1995+
| limit 6
1996+
;
1997+
1998+
emp_no:long |language_name:keyword
1999+
10001 | French
2000+
10001 | French
2001+
10002 | null
2002+
10002 | null
2003+
10003 | German
2004+
10003 | German
2005+
;
2006+
2007+
MultiTypedEnrichOnNumericFieldWithEval
2008+
required_capability: enrich_load
2009+
required_capability: implicit_casting_union_typed_numeric_and_date
2010+
2011+
from employees, employees_incompatible
2012+
| eval languages = languages + 1
2013+
| enrich languages_policy on languages
2014+
| keep emp_no, language_name
2015+
| sort emp_no
2016+
| limit 6
2017+
;
2018+
2019+
emp_no:long |language_name:keyword
2020+
10001 | Spanish
2021+
10001 | Spanish
2022+
10002 | null
2023+
10002 | null
2024+
10003 | null
2025+
10003 | null
2026+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,11 @@
7676
import org.elasticsearch.xpack.esql.plan.IndexPattern;
7777
import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
7878
import org.elasticsearch.xpack.esql.plan.logical.Dedup;
79-
import org.elasticsearch.xpack.esql.plan.logical.Dissect;
8079
import org.elasticsearch.xpack.esql.plan.logical.Drop;
8180
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
8281
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
8382
import org.elasticsearch.xpack.esql.plan.logical.Eval;
8483
import org.elasticsearch.xpack.esql.plan.logical.Fork;
85-
import org.elasticsearch.xpack.esql.plan.logical.Grok;
8684
import org.elasticsearch.xpack.esql.plan.logical.Insist;
8785
import org.elasticsearch.xpack.esql.plan.logical.Keep;
8886
import org.elasticsearch.xpack.esql.plan.logical.Limit;
@@ -91,7 +89,6 @@
9189
import org.elasticsearch.xpack.esql.plan.logical.MvExpand;
9290
import org.elasticsearch.xpack.esql.plan.logical.OrderBy;
9391
import org.elasticsearch.xpack.esql.plan.logical.Project;
94-
import org.elasticsearch.xpack.esql.plan.logical.RegexExtract;
9592
import org.elasticsearch.xpack.esql.plan.logical.Rename;
9693
import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval;
9794
import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation;
@@ -1065,7 +1062,10 @@ private LogicalPlan resolveEnrich(Enrich enrich, List<Attribute> childrenOutput)
10651062
final DataType dataType = resolved.dataType();
10661063
String matchType = enrich.policy().getType();
10671064
DataType[] allowed = allowedEnrichTypes(matchType);
1068-
if (Arrays.asList(allowed).contains(dataType) == false) {
1065+
if (Arrays.asList(allowed).contains(dataType) == false && multiTypedField(resolved) == null) { // leave multi-typed
1066+
// fields to
1067+
// ImplicitCasting to
1068+
// deal with
10691069
String suffix = "only ["
10701070
+ Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", "))
10711071
+ "] allowed for type ["
@@ -1269,10 +1269,10 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) {
12691269
if (p instanceof OrderBy
12701270
|| p instanceof EsqlProject
12711271
|| p instanceof Aggregate
1272-
|| p instanceof RegexExtract
12731272
|| p instanceof MvExpand
12741273
|| p instanceof Eval
1275-
|| p instanceof LookupJoin) {
1274+
|| p instanceof LookupJoin
1275+
|| p instanceof Enrich) {
12761276
return castInvalidMappedFieldInLogicalPlan(p, false);
12771277
}
12781278
return p;
@@ -1317,10 +1317,10 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan,
13171317
case EsqlProject project -> project.projections();
13181318
case OrderBy ob -> ob.order();
13191319
case Aggregate agg -> agg.groupings();
1320-
case RegexExtract re -> List.of(re.input());
13211320
case MvExpand me -> List.of(me.target());
1322-
case Eval e -> e.fields();
1321+
case Eval eval -> eval.fields();
13231322
case LookupJoin lj -> lj.config().leftFields();
1323+
case Enrich enrich -> List.of(enrich.matchField());
13241324
// The other types of plans are unexpected
13251325
default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan);
13261326
};
@@ -1355,32 +1355,13 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan,
13551355
}
13561356
return agg.with(evalForInvalidMappedField(agg.source(), agg.child(), aliases), newProjections, newAggs);
13571357
}
1358-
case Dissect d -> {
1359-
return new Dissect(
1360-
plan.source(),
1361-
evalForInvalidMappedField(d.source(), d.child(), aliases),
1362-
newProjections.get(0),
1363-
d.parser(),
1364-
d.extractedFields()
1365-
);
1366-
}
1367-
case Grok g -> {
1368-
return new Grok(
1369-
plan.source(),
1370-
evalForInvalidMappedField(g.source(), g.child(), aliases),
1371-
newProjections.get(0),
1372-
g.parser(),
1373-
g.extractedFields()
1374-
);
1375-
}
13761358
case MvExpand mve -> {
13771359
NamedExpression newTarget = Expressions.attribute(newProjections.get(0));
13781360
return new MvExpand(
1379-
plan.source(),
1361+
mve.source(),
13801362
evalForInvalidMappedField(mve.source(), mve.child(), aliases),
13811363
newTarget,
1382-
new ReferenceAttribute(newTarget.source(), newTarget.name(), newTarget.dataType(), newTarget.nullable(),
1383-
mve.expanded().id(), false)
1364+
newTarget.toAttribute()
13841365
);
13851366
}
13861367
case Eval ev -> {
@@ -1400,6 +1381,20 @@ private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan,
14001381
newJoinConfig
14011382
);
14021383
}
1384+
case Enrich enrich -> {
1385+
NamedExpression newMatchField = Expressions.attribute(newProjections.get(0));
1386+
return new Enrich(
1387+
enrich.source(),
1388+
evalForInvalidMappedField(enrich.source(), enrich.child(), aliases),
1389+
enrich.mode(),
1390+
enrich.policyName(),
1391+
// Let resolveEnrich check whether the data type is supported, e.g. Enrich does not support data_nanos
1392+
new UnresolvedAttribute(newMatchField.source(), newMatchField.name()),
1393+
enrich.policy(),
1394+
enrich.concreteIndices(),
1395+
enrich.enrichFields()
1396+
);
1397+
}
14031398
// The other types of plans are unexpected
14041399
default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan);
14051400
}
@@ -1462,13 +1457,11 @@ private static Tuple<List<Alias>, List<Expression>> castInvalidMappedFields(
14621457
Expression e = Alias.unwrap(exp);
14631458
e = e instanceof Order o ? o.child() : e;
14641459
String alias = exp instanceof Alias a ? a.name() : null;
1465-
if (e.resolved()
1466-
&& e.dataType() == UNSUPPORTED
1467-
&& e instanceof FieldAttribute fa
1468-
&& fa.field() instanceof InvalidMappedField imf) {
1460+
InvalidMappedField multiTypedField = multiTypedField(e);
1461+
if (multiTypedField != null) {
14691462
// This is an invalid mapped field, find a common data type and cast to it.
14701463
DataType targetType = null;
1471-
for (DataType type : imf.types()) {
1464+
for (DataType type : multiTypedField.types()) {
14721465
if (targetType == null) { // Initialize the target type to the first type.
14731466
targetType = type;
14741467
} else {
@@ -1480,9 +1473,9 @@ private static Tuple<List<Alias>, List<Expression>> castInvalidMappedFields(
14801473
}
14811474
}
14821475
if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) {
1483-
alias = alias != null ? alias : fa.name();
1484-
Source source = fa.source();
1485-
Expression newChild = castInvalidMappedField(targetType, fa);
1476+
alias = alias != null ? alias : multiTypedField.getName();
1477+
Source source = e.source();
1478+
Expression newChild = castInvalidMappedField(targetType, e);
14861479
Alias newAlias = new Alias(source, alias, newChild);
14871480
newAliases.add(newAlias);
14881481
if (createNewChildPlan) {
@@ -1525,7 +1518,7 @@ private static Eval evalForInvalidMappedField(Source source, LogicalPlan childPl
15251518
/**
15261519
* Do implicit casting for data, date_nanos and numeric types only
15271520
*/
1528-
private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) {
1521+
private static Expression castInvalidMappedField(DataType targetType, Expression fa) {
15291522
Source source = fa.source();
15301523
return switch (targetType) {
15311524
case INTEGER -> new ToInteger(source, fa);
@@ -1956,4 +1949,15 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) {
19561949
return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput);
19571950
}
19581951
}
1952+
1953+
private static InvalidMappedField multiTypedField(Expression e) {
1954+
Expression exp = Alias.unwrap(e);
1955+
if (exp.resolved()
1956+
&& exp.dataType() == UNSUPPORTED
1957+
&& exp instanceof FieldAttribute fa
1958+
&& fa.field() instanceof InvalidMappedField imf) {
1959+
return imf;
1960+
}
1961+
return null;
1962+
}
19591963
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
3333
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
3434
import org.elasticsearch.xpack.esql.core.tree.Source;
35+
import org.elasticsearch.xpack.esql.core.type.DataType;
3536
import org.elasticsearch.xpack.esql.index.EsIndex;
3637
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
3738
import org.elasticsearch.xpack.esql.plan.GeneratingPlan;
@@ -208,6 +209,7 @@ public boolean expressionsResolved() {
208209
return policyName.resolved()
209210
&& matchField instanceof EmptyAttribute == false // matchField not defined in the query, needs to be resolved from the policy
210211
&& matchField.resolved()
212+
&& matchField.dataType() != DataType.UNSUPPORTED
211213
&& Resolvables.resolved(enrichFields());
212214
}
213215

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,8 @@ public void testUnsupportedAndMultiTypedFields() {
133133
error("from test* | enrich client_cidr on unsupported", analyzer)
134134
);
135135
assertEquals(
136-
"1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];"
137-
+ " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]",
136+
"1:36: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types: "
137+
+ "[ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]",
138138
error("from test* | enrich client_cidr on multi_typed", analyzer)
139139
);
140140

0 commit comments

Comments
 (0)