Skip to content

Commit e698178

Browse files
iveraseiverase
authored
[8.19] Improve execution of terms queries over wildcard fields (#129051)
1 parent ef46998 commit e698178

File tree

4 files changed

+223
-0
lines changed

4 files changed

+223
-0
lines changed

‎docs/changelog/128986.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 128986
2+
summary: Improve execution of terms queries over wildcard fields
3+
area: Search
4+
type: bug
5+
issues:
6+
- 128201

‎x-pack/plugin/wildcard/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
22

33
apply plugin: 'elasticsearch.internal-es-plugin'
44
apply plugin: 'elasticsearch.internal-yaml-rest-test'
5+
apply plugin: 'elasticsearch.internal-cluster-test'
56

67
esplugin {
78
name = 'wildcard'

‎x-pack/plugin/wildcard/src/internalClusterTest/java/org/elasticsearch/xpack/wildcard/search/WildcardSearchIT.java‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.wildcard.search;
9+
10+
import org.elasticsearch.action.bulk.BulkRequestBuilder;
11+
import org.elasticsearch.action.index.IndexRequest;
12+
import org.elasticsearch.index.query.TermQueryBuilder;
13+
import org.elasticsearch.index.query.TermsQueryBuilder;
14+
import org.elasticsearch.plugins.Plugin;
15+
import org.elasticsearch.test.ESIntegTestCase;
16+
import org.elasticsearch.xcontent.XContentBuilder;
17+
import org.elasticsearch.xcontent.XContentFactory;
18+
import org.elasticsearch.xcontent.XContentType;
19+
import org.elasticsearch.xpack.wildcard.Wildcard;
20+
import org.hamcrest.Matchers;
21+
import org.junit.Before;
22+
23+
import java.io.IOException;
24+
import java.util.ArrayList;
25+
import java.util.Arrays;
26+
import java.util.Collection;
27+
import java.util.List;
28+
29+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
30+
31+
public class WildcardSearchIT extends ESIntegTestCase {
32+
33+
private List<String> terms = null;
34+
35+
@Override
36+
protected Collection<Class<? extends Plugin>> nodePlugins() {
37+
return List.of(Wildcard.class);
38+
}
39+
40+
@Before
41+
public void setup() throws IOException {
42+
terms = new ArrayList<>();
43+
XContentBuilder xcb = XContentFactory.jsonBuilder()
44+
.startObject()
45+
.startObject("properties")
46+
.startObject("wildcard")
47+
.field("type", "wildcard")
48+
.endObject()
49+
.startObject("keyword")
50+
.field("type", "keyword")
51+
.endObject()
52+
.endObject()
53+
.endObject();
54+
indicesAdmin().prepareCreate("test").setMapping(xcb).get();
55+
final int numDocs = randomIntBetween(100, 1000);
56+
final BulkRequestBuilder builder = client().prepareBulk();
57+
for (int i = 0; i < numDocs; i++) {
58+
if (rarely()) {
59+
indexMultiValue(builder);
60+
} else {
61+
indexSingleValue(builder);
62+
}
63+
}
64+
assertFalse(builder.get().hasFailures());
65+
indicesAdmin().prepareRefresh("test").get();
66+
}
67+
68+
private void indexSingleValue(BulkRequestBuilder builder) {
69+
String term = randomIndexString();
70+
builder.add(
71+
new IndexRequest("test").source("{\"wildcard\" : \"" + term + "\", \"keyword\" : \"" + term + "\"}", XContentType.JSON)
72+
);
73+
terms.add(term);
74+
}
75+
76+
private void indexMultiValue(BulkRequestBuilder builder) {
77+
int docSize = randomIntBetween(1, 10);
78+
String[] docTerms = new String[docSize];
79+
for (int i = 0; i < docSize; i++) {
80+
String term = randomIndexString();
81+
terms.add(term);
82+
docTerms[i] = "\"" + term + "\"";
83+
}
84+
builder.add(
85+
new IndexRequest("test").source(
86+
"{\"wildcard\" : " + Arrays.toString(docTerms) + ", \"keyword\" : " + Arrays.toString(docTerms) + "}",
87+
XContentType.JSON
88+
)
89+
);
90+
}
91+
92+
public void testTermQueryDuel() {
93+
for (int i = 0; i < 50; i++) {
94+
String term = randomQueryString(terms);
95+
TermQueryBuilder termQueryBuilder1 = new TermQueryBuilder("wildcard", term);
96+
TermQueryBuilder termQueryBuilder2 = new TermQueryBuilder("keyword", term);
97+
assertResponse(
98+
client().prepareSearch("test").setQuery(termQueryBuilder1),
99+
response -> assertResponse(
100+
client().prepareSearch("test").setQuery(termQueryBuilder2),
101+
response2 -> assertThat(
102+
response.getHits().getTotalHits().value,
103+
Matchers.equalTo(response2.getHits().getTotalHits().value)
104+
)
105+
)
106+
);
107+
}
108+
}
109+
110+
public void testTermsQueryDuel() {
111+
for (int i = 0; i < 10; i++) {
112+
String[] terms = new String[randomIntBetween(2, 8192)];
113+
for (int j = 0; j < terms.length; j++) {
114+
terms[j] = randomQueryString(this.terms);
115+
}
116+
TermsQueryBuilder termsQueryBuilder1 = new TermsQueryBuilder("wildcard", terms);
117+
TermsQueryBuilder termsQueryBuilder2 = new TermsQueryBuilder("keyword", terms);
118+
assertResponse(
119+
client().prepareSearch("test").setQuery(termsQueryBuilder1),
120+
response -> assertResponse(
121+
client().prepareSearch("test").setQuery(termsQueryBuilder2),
122+
response2 -> assertThat(
123+
response.getHits().getTotalHits().value,
124+
Matchers.equalTo(response2.getHits().getTotalHits().value)
125+
)
126+
)
127+
);
128+
}
129+
}
130+
131+
private static String randomIndexString() {
132+
String string = randomAlphaOfLength(randomIntBetween(0, 30));
133+
if (rarely()) {
134+
return string + "*";
135+
} else if (rarely()) {
136+
return "*" + string;
137+
} else if (rarely()) {
138+
return "*" + string + "*";
139+
} else {
140+
return string;
141+
}
142+
}
143+
144+
private static String randomQueryString(List<String> terms) {
145+
if (rarely()) {
146+
return terms.get(randomIntBetween(0, terms.size() - 1));
147+
} else if (randomBoolean()) {
148+
return randomAlphaOfLength(randomIntBetween(0, 30));
149+
} else {
150+
return randomAlphaOfLength(1) + "*";
151+
}
152+
}
153+
}

‎x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.lucene.search.BooleanClause;
2626
import org.apache.lucene.search.BooleanClause.Occur;
2727
import org.apache.lucene.search.BooleanQuery;
28+
import org.apache.lucene.search.ConstantScoreQuery;
2829
import org.apache.lucene.search.FieldExistsQuery;
2930
import org.apache.lucene.search.FuzzyQuery;
3031
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -33,6 +34,7 @@
3334
import org.apache.lucene.search.MultiTermQuery.RewriteMethod;
3435
import org.apache.lucene.search.PrefixQuery;
3536
import org.apache.lucene.search.Query;
37+
import org.apache.lucene.search.TermInSetQuery;
3638
import org.apache.lucene.search.TermQuery;
3739
import org.apache.lucene.search.TermRangeQuery;
3840
import org.apache.lucene.search.WildcardQuery;
@@ -81,11 +83,14 @@
8183
import java.nio.charset.StandardCharsets;
8284
import java.time.ZoneId;
8385
import java.util.ArrayList;
86+
import java.util.Collection;
8487
import java.util.HashSet;
88+
import java.util.Iterator;
8589
import java.util.LinkedHashSet;
8690
import java.util.List;
8791
import java.util.Map;
8892
import java.util.Set;
93+
import java.util.TreeSet;
8994

9095
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
9196

@@ -96,6 +101,7 @@ public class WildcardFieldMapper extends FieldMapper {
96101

97102
public static final String CONTENT_TYPE = "wildcard";
98103
public static short MAX_CLAUSES_IN_APPROXIMATION_QUERY = 10;
104+
private static final int WILDCARD_TERMS_EXPANSION_LIMIT = 16;
99105
public static final int NGRAM_SIZE = 3;
100106

101107
static final NamedAnalyzer WILDCARD_ANALYZER_7_10 = new NamedAnalyzer("_wildcard_7_10", AnalyzerScope.GLOBAL, new Analyzer() {
@@ -835,6 +841,63 @@ public Query termQuery(Object value, SearchExecutionContext context) {
835841
}
836842
}
837843

844+
@Override
845+
public Query termsQuery(Collection<?> values, @Nullable SearchExecutionContext context) {
846+
final BytesRef[] terms = buildTerms(values);
847+
final Query aproxQuery;
848+
if (terms.length < WILDCARD_TERMS_EXPANSION_LIMIT) {
849+
// If there are few terms, we can approximate each term using a BooleanQuery.
850+
final BooleanQuery.Builder builder = new BooleanQuery.Builder();
851+
for (BytesRef term : terms) {
852+
final BooleanQuery.Builder rewritten = new BooleanQuery.Builder();
853+
final Integer numClauses = getApproxWildCardQuery(escapeWildcardSyntax(term.utf8ToString()), rewritten);
854+
if (numClauses != null && numClauses > 0) {
855+
builder.add(rewritten.build(), Occur.SHOULD);
856+
}
857+
}
858+
aproxQuery = builder.build();
859+
} else {
860+
// If there are too many terms, we cannot rewrite approximate into a BooleanQuery as it will use too much memory.
861+
// Instead, we generate a TermInSetQuery. In order to match the necessary documents we need to add at least one token
862+
// per term, ideally we should add the token that makes the term most different from the others.
863+
final Set<String> tokens = new LinkedHashSet<>();
864+
final Set<BytesRef> tokenList = new TreeSet<>();
865+
for (BytesRef term : terms) {
866+
// Break search term into tokens
867+
final boolean matchAll = breakIntoTokens(escapeWildcardSyntax(term.utf8ToString()), tokens);
868+
assert matchAll == false;
869+
if (tokens.isEmpty() == false) {
870+
// If there are tokens, we take the middle one to represent the term
871+
// which is probably the most different one.
872+
tokenList.add(getMiddleToken(tokens));
873+
}
874+
tokens.clear();
875+
}
876+
aproxQuery = new TermInSetQuery(name(), tokenList);
877+
}
878+
return BinaryDvConfirmedQuery.fromTerms(new ConstantScoreQuery(aproxQuery), name(), terms);
879+
}
880+
881+
private static BytesRef getMiddleToken(Set<String> tokens) {
882+
int mid = (tokens.size() + 1) / 2;
883+
Iterator<String> iterator = tokens.iterator();
884+
for (int i = 0; i < mid - 1; i++) {
885+
iterator.next();
886+
}
887+
assert iterator.hasNext();
888+
return BytesRefs.toBytesRef(iterator.next());
889+
}
890+
891+
private static BytesRef[] buildTerms(Collection<?> values) {
892+
final Set<?> dedupe = new HashSet<>(values);
893+
final BytesRef[] terms = new BytesRef[dedupe.size()];
894+
final Iterator<?> iterator = dedupe.iterator();
895+
for (int i = 0; i < dedupe.size(); i++) {
896+
terms[i] = BytesRefs.toBytesRef(iterator.next());
897+
}
898+
return terms;
899+
}
900+
838901
private static String escapeWildcardSyntax(String term) {
839902
StringBuilder result = new StringBuilder();
840903
for (int i = 0; i < term.length();) {

0 commit comments

Comments
 (0)