Skip to content

Commit bb06a63

Browse files
julian-elasticjulian-elastic
committed
Integrate with latest from main, add UTs
1 parent 1a43e72 commit bb06a63

File tree

6 files changed

+676
-588
lines changed

6 files changed

+676
-588
lines changed

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java‎

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.junit.rules.TestRule;
3030

3131
import java.io.IOException;
32-
import java.util.ArrayList;
3332
import java.util.List;
3433
import java.util.Map;
3534
import java.util.Set;
@@ -205,25 +204,24 @@ private Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder reques
205204
}
206205
}
207206

208-
private <C, V> void assertResultMapForLike(
207+
private <C, V> void assertResultMapWithCapabilities(
209208
boolean includeCCSMetadata,
210209
Map<String, Object> result,
211210
C columns,
212211
V values,
213212
boolean remoteOnly,
214-
boolean requireLikeListCapability
213+
List<String> fullResultCapabilities
215214
) throws IOException {
216-
List<String> requiredCapabilities = new ArrayList<>(List.of("like_on_index_fields"));
217-
if (requireLikeListCapability) {
218-
requiredCapabilities.add("like_list_on_index_fields");
219-
}
220215
// the feature is completely supported if both local and remote clusters support it
221-
boolean isSupported = capabilitiesSupportedNewAndOld(requiredCapabilities);
222-
216+
// otherwise we expect a partial result, and will not check the data
217+
boolean isSupported = capabilitiesSupportedNewAndOld(fullResultCapabilities);
223218
if (isSupported) {
224219
assertResultMap(includeCCSMetadata, result, columns, values, remoteOnly);
225220
} else {
226-
logger.info("--> skipping data check for like index test, cluster does not support like index feature");
221+
logger.info(
222+
"--> skipping data check for a test, cluster does not support all of [{}] capabilities",
223+
String.join(",", fullResultCapabilities)
224+
);
227225
// just verify that we did not get a partial result
228226
var clusters = result.get("_clusters");
229227
var reason = "unexpected partial results" + (clusters != null ? ": _clusters=" + clusters : "");
@@ -526,7 +524,7 @@ public void testLikeIndex() throws Exception {
526524
""", includeCCSMetadata);
527525
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
528526
var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
529-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
527+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
530528
}
531529

532530
public void testLikeIndexLegacySettingNoResults() throws Exception {
@@ -548,7 +546,7 @@ public void testLikeIndexLegacySettingNoResults() throws Exception {
548546
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
549547
// we expect empty result, since the setting is false
550548
var values = List.of();
551-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
549+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
552550
}
553551
}
554552

@@ -572,7 +570,7 @@ public void testLikeIndexLegacySettingResults() throws Exception {
572570
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
573571
// we expect results, since the setting is false, but there is : in the LIKE query
574572
var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
575-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
573+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
576574
}
577575
}
578576

@@ -586,7 +584,7 @@ public void testNotLikeIndex() throws Exception {
586584
""", includeCCSMetadata);
587585
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
588586
var values = List.of(List.of(localDocs.size(), localIndex));
589-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
587+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
590588
}
591589

592590
public void testLikeListIndex() throws Exception {
@@ -601,7 +599,14 @@ public void testLikeListIndex() throws Exception {
601599
""", includeCCSMetadata);
602600
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
603601
var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
604-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
602+
assertResultMapWithCapabilities(
603+
includeCCSMetadata,
604+
result,
605+
columns,
606+
values,
607+
false,
608+
List.of("like_on_index_fields", "like_list_on_index_fields")
609+
);
605610
}
606611

607612
public void testNotLikeListIndex() throws Exception {
@@ -615,7 +620,14 @@ public void testNotLikeListIndex() throws Exception {
615620
""", includeCCSMetadata);
616621
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
617622
var values = List.of(List.of(localDocs.size(), localIndex));
618-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
623+
assertResultMapWithCapabilities(
624+
includeCCSMetadata,
625+
result,
626+
columns,
627+
values,
628+
false,
629+
List.of("like_on_index_fields", "like_list_on_index_fields")
630+
);
619631
}
620632

621633
public void testNotLikeListKeyword() throws Exception {
@@ -633,7 +645,14 @@ public void testNotLikeListKeyword() throws Exception {
633645
List.of((int) remoteDocs.stream().filter(filter).count(), REMOTE_CLUSTER_NAME + ":" + remoteIndex),
634646
List.of((int) localDocs.stream().filter(filter).count(), localIndex)
635647
);
636-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
648+
assertResultMapWithCapabilities(
649+
includeCCSMetadata,
650+
result,
651+
columns,
652+
values,
653+
false,
654+
List.of("like_on_index_fields", "like_list_on_index_fields")
655+
);
637656
}
638657

639658
public void testRLikeIndex() throws Exception {
@@ -646,7 +665,7 @@ public void testRLikeIndex() throws Exception {
646665
""", includeCCSMetadata);
647666
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
648667
var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
649-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
668+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
650669
}
651670

652671
public void testNotRLikeIndex() throws Exception {
@@ -659,7 +678,37 @@ public void testNotRLikeIndex() throws Exception {
659678
""", includeCCSMetadata);
660679
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
661680
var values = List.of(List.of(localDocs.size(), localIndex));
662-
assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
681+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
682+
}
683+
684+
public void testRLikeListIndex() throws Exception {
685+
assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("rlike_with_list_of_patterns")));
686+
boolean includeCCSMetadata = includeCCSMetadata();
687+
Map<String, Object> result = run("""
688+
FROM test-local-index,*:test-remote-index METADATA _index
689+
| WHERE _index RLIKE (".*remote.*", ".*not-exist.*")
690+
| STATS c = COUNT(*) BY _index
691+
| SORT _index ASC
692+
""", includeCCSMetadata);
693+
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
694+
var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
695+
// we depend on the code in like_on_index_fields to serialize an ExpressionQueryBuilder
696+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
697+
}
698+
699+
public void testNotRLikeListIndex() throws Exception {
700+
assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("rlike_with_list_of_patterns")));
701+
boolean includeCCSMetadata = includeCCSMetadata();
702+
Map<String, Object> result = run("""
703+
FROM test-local-index,*:test-remote-index METADATA _index
704+
| WHERE _index NOT RLIKE (".*remote.*", ".*not-exist.*")
705+
| STATS c = COUNT(*) BY _index
706+
| SORT _index ASC
707+
""", includeCCSMetadata);
708+
var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
709+
var values = List.of(List.of(localDocs.size(), localIndex));
710+
// we depend on the code in like_on_index_fields to serialize an ExpressionQueryBuilder
711+
assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
663712
}
664713

665714
private RestClient remoteClusterClient() throws IOException {

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1237,7 +1237,7 @@ public enum Cap {
12371237
/**
12381238
* Support for the RLIKE operator with a list of regexes.
12391239
*/
1240-
RLIKE_WITH_LIST_OF_PATTERNS;
1240+
RLIKE_WITH_LIST_OF_PATTERNS,
12411241

12421242
/**
12431243
* FUSE command

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLikeList.java‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,29 @@
77

88
package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
99

10+
import org.apache.lucene.search.MultiTermQuery;
11+
import org.apache.lucene.util.automaton.Automaton;
12+
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1013
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1114
import org.elasticsearch.common.io.stream.StreamInput;
1215
import org.elasticsearch.common.io.stream.StreamOutput;
16+
import org.elasticsearch.index.mapper.MappedFieldType;
17+
import org.elasticsearch.index.query.SearchExecutionContext;
1318
import org.elasticsearch.xpack.esql.core.expression.Expression;
1419
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
1520
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
1621
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePatternList;
17-
import org.elasticsearch.xpack.esql.core.querydsl.query.EsqlAutomatonQuery;
1822
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
1923
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
2024
import org.elasticsearch.xpack.esql.core.tree.Source;
2125
import org.elasticsearch.xpack.esql.expression.function.Param;
26+
import org.elasticsearch.xpack.esql.io.stream.ExpressionQuery;
2227
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
2328
import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
2429
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
2530

2631
import java.io.IOException;
32+
import java.util.function.Supplier;
2733
import java.util.stream.Collectors;
2834

2935
public class RLikeList extends RegexMatch<RLikePatternList> {
@@ -33,6 +39,30 @@ public class RLikeList extends RegexMatch<RLikePatternList> {
3339
RLikeList::new
3440
);
3541

42+
Supplier<Automaton> automatonSupplier = new Supplier<>() {
43+
Automaton cached;
44+
45+
@Override
46+
public Automaton get() {
47+
if (cached == null) {
48+
cached = pattern().createAutomaton(caseInsensitive());
49+
}
50+
return cached;
51+
}
52+
};
53+
54+
Supplier<CharacterRunAutomaton> characterRunAutomatonSupplier = new Supplier<>() {
55+
CharacterRunAutomaton cached;
56+
57+
@Override
58+
public CharacterRunAutomaton get() {
59+
if (cached == null) {
60+
cached = new CharacterRunAutomaton(automatonSupplier.get());
61+
}
62+
return cached;
63+
}
64+
};
65+
3666
/**
3767
* The documentation for this function is in RLike, and shown to the users as `RLIKE` in the docs.
3868
*/
@@ -97,15 +127,30 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand
97127
}
98128

99129
private Query translateField(String targetFieldName) {
100-
return new EsqlAutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription());
130+
return new ExpressionQuery(source(), targetFieldName, this);
131+
}
132+
133+
@Override
134+
public org.apache.lucene.search.Query asLuceneQuery(
135+
MappedFieldType fieldType,
136+
MultiTermQuery.RewriteMethod constantScoreRewrite,
137+
SearchExecutionContext context
138+
) {
139+
return fieldType.automatonQuery(
140+
automatonSupplier,
141+
characterRunAutomatonSupplier,
142+
constantScoreRewrite,
143+
context,
144+
getLuceneQueryDescription()
145+
);
101146
}
102147

103148
@Override
104149
protected NodeInfo<? extends Expression> info() {
105150
return NodeInfo.create(this, RLikeList::new, field(), pattern(), caseInsensitive());
106151
}
107152

108-
private String getAutomatonDescription() {
153+
private String getLuceneQueryDescription() {
109154
// we use the information used to create the automaton to describe the query here
110155
String patternDesc = pattern().patternList().stream().map(RLikePattern::pattern).collect(Collectors.joining("\", \""));
111156
return "RLIKE(\"" + patternDesc + "\"), caseInsensitive=" + caseInsensitive();

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,6 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand
146146

147147
private boolean supportsPushdown(TransportVersion version) {
148148
return version == null || version.onOrAfter(TransportVersions.ESQL_FIXED_INDEX_LIKE);
149-
return new EsqlAutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription());
150149
}
151150

152151
@Override

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp‎

Lines changed: 1 addition & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)