Skip to content

Commit 2faea54

Browse files
svilen-mihaylov-elasticsvilen-mihaylov-elastic
committed
Add ability to set "max_analyzed_offet" implicitly to "index.highlight
.max_analyzed_offset", by setting it excplicitly to "-1".
1 parent e087f3d commit 2faea54

File tree

8 files changed

+74
-19
lines changed

8 files changed

+74
-19
lines changed

‎plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
1818
import org.elasticsearch.index.query.SearchExecutionContext;
1919
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
20+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
2021
import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
2122
import org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter;
2223
import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
@@ -52,7 +53,7 @@ protected List<Object> loadFieldValues(
5253
}
5354

5455
@Override
55-
protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
56+
protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
5657
return new AnnotatedHighlighterAnalyzer(super.wrapAnalyzer(analyzer, maxAnalyzedOffset));
5758
}
5859

‎plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
4040
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper;
4141
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
42+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
4243
import org.elasticsearch.lucene.search.uhighlight.Snippet;
4344
import org.elasticsearch.search.fetch.subphase.highlight.LimitTokenOffsetAnalyzer;
4445
import org.elasticsearch.test.ESTestCase;
@@ -85,7 +86,7 @@ private void assertHighlightOneDoc(
8586
int noMatchSize,
8687
String[] expectedPassages,
8788
int maxAnalyzedOffset,
88-
Integer queryMaxAnalyzedOffset
89+
Integer queryMaxAnalyzedOffsetIn
8990
) throws Exception {
9091

9192
try (Directory dir = newDirectory()) {
@@ -116,8 +117,9 @@ private void assertHighlightOneDoc(
116117
for (int i = 0; i < markedUpInputs.length; i++) {
117118
annotations[i] = AnnotatedText.parse(markedUpInputs[i]);
118119
}
120+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffsetIn, maxAnalyzedOffset);
119121
if (queryMaxAnalyzedOffset != null) {
120-
wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset);
122+
wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset.getNotNull());
121123
}
122124
AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer);
123125
hiliteAnalyzer.setAnnotations(annotations);
@@ -311,6 +313,19 @@ public void testExceedMaxAnalyzedOffset() throws Exception {
311313
e.getMessage()
312314
);
313315

316+
// Same as before, but force using index maxOffset (20) as queryMaxOffset by passing -1.
317+
assertHighlightOneDoc(
318+
"text",
319+
new String[] { "[Long Text exceeds](Long+Text+exceeds) MAX analyzed offset)" },
320+
query,
321+
Locale.ROOT,
322+
breakIterator,
323+
0,
324+
new String[] { "Long Text [exceeds](_hit_term=exceeds) MAX analyzed offset)" },
325+
20,
326+
-1
327+
);
328+
314329
assertHighlightOneDoc(
315330
"text",
316331
new String[] { "[Long Text Exceeds](Long+Text+Exceeds) MAX analyzed offset [Long Text Exceeds](Long+Text+Exceeds)" },

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
3434
private final Locale breakIteratorLocale;
3535
private final int noMatchSize;
3636
private String fieldValue;
37-
private final Integer queryMaxAnalyzedOffset;
37+
private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
3838

3939
CustomFieldHighlighter(
4040
String field,
@@ -47,7 +47,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
4747
PassageFormatter passageFormatter,
4848
Comparator<Passage> passageSortComparator,
4949
int noMatchSize,
50-
Integer queryMaxAnalyzedOffset
50+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset
5151
) {
5252
super(
5353
field,
@@ -113,7 +113,7 @@ protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) {
113113
@Override
114114
protected Passage[] highlightOffsetsEnums(OffsetsEnum off) throws IOException {
115115
if (queryMaxAnalyzedOffset != null) {
116-
off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset);
116+
off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset.getNotNull());
117117
}
118118
return super.highlightOffsetsEnums(off);
119119
}

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public final class CustomUnifiedHighlighter extends UnifiedHighlighter {
6666
private final int noMatchSize;
6767
private final CustomFieldHighlighter fieldHighlighter;
6868
private final int maxAnalyzedOffset;
69-
private final Integer queryMaxAnalyzedOffset;
69+
private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
7070

7171
/**
7272
* Creates a new instance of {@link CustomUnifiedHighlighter}
@@ -94,7 +94,7 @@ public CustomUnifiedHighlighter(
9494
int noMatchSize,
9595
int maxPassages,
9696
int maxAnalyzedOffset,
97-
Integer queryMaxAnalyzedOffset,
97+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset,
9898
boolean requireFieldMatch,
9999
boolean weightMatchesEnabled
100100
) {
@@ -125,9 +125,9 @@ public Snippet[] highlightField(LeafReader reader, int docId, CheckedSupplier<St
125125
return null;
126126
}
127127
int fieldValueLength = fieldValue.length();
128-
if (((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset)
128+
if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
129129
&& (getOffsetSource(field) == OffsetSource.ANALYSIS)
130-
&& (fieldValueLength > maxAnalyzedOffset))) {
130+
&& (fieldValueLength > maxAnalyzedOffset)) {
131131
throw new IllegalArgumentException(
132132
"The length ["
133133
+ fieldValueLength

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene.search.uhighlight;
11+
12+
public class QueryMaxAnalyzedOffset {
13+
private final int queryMaxAnalyzedOffset;
14+
15+
private QueryMaxAnalyzedOffset(final int queryMaxAnalyzedOffset) {
16+
// If we have a negative value, grab value for the actual maximum from the index.
17+
this.queryMaxAnalyzedOffset = queryMaxAnalyzedOffset;
18+
}
19+
20+
public static QueryMaxAnalyzedOffset create(final Integer queryMaxAnalyzedOffset, final int indexMaxAnalyzedOffset) {
21+
if (queryMaxAnalyzedOffset == null) {
22+
return null;
23+
}
24+
return new QueryMaxAnalyzedOffset(queryMaxAnalyzedOffset < 0 ? indexMaxAnalyzedOffset : queryMaxAnalyzedOffset);
25+
}
26+
27+
public int getNotNull() {
28+
return queryMaxAnalyzedOffset;
29+
}
30+
}

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.lucene.search.uhighlight.BoundedBreakIteratorScanner;
3232
import org.elasticsearch.lucene.search.uhighlight.CustomPassageFormatter;
3333
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
34+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
3435
import org.elasticsearch.lucene.search.uhighlight.Snippet;
3536
import org.elasticsearch.search.fetch.FetchContext;
3637
import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -121,7 +122,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
121122
int maxAnalyzedOffset = indexSettings.getHighlightMaxAnalyzedOffset();
122123
boolean weightMatchesEnabled = indexSettings.isWeightMatchesEnabled();
123124
int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments();
124-
Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
125+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(
126+
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
127+
maxAnalyzedOffset
128+
);
125129
Analyzer analyzer = wrapAnalyzer(
126130
fieldContext.context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
127131
queryMaxAnalyzedOffset
@@ -171,7 +175,7 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
171175
fieldContext.field.fieldOptions().noMatchSize(),
172176
highlighterNumberOfFragments,
173177
maxAnalyzedOffset,
174-
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
178+
queryMaxAnalyzedOffset,
175179
fieldContext.field.fieldOptions().requireFieldMatch(),
176180
weightMatchesEnabled
177181
);
@@ -186,9 +190,9 @@ protected PassageFormatter getPassageFormatter(SearchHighlightContext.Field fiel
186190
);
187191
}
188192

189-
protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
193+
protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
190194
if (maxAnalyzedOffset != null) {
191-
analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
195+
analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
192196
}
193197
return analyzer;
194198
}

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.common.text.Text;
2828
import org.elasticsearch.index.IndexSettings;
2929
import org.elasticsearch.index.mapper.MappedFieldType;
30+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
3031
import org.elasticsearch.search.fetch.FetchContext;
3132
import org.elasticsearch.search.fetch.FetchSubPhase;
3233

@@ -107,7 +108,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
107108
ArrayList<OrderedTextFragment> fragsList = new ArrayList<>();
108109
List<Object> textsToHighlight;
109110
final int maxAnalyzedOffset = context.getSearchExecutionContext().getIndexSettings().getHighlightMaxAnalyzedOffset();
110-
Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
111+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = QueryMaxAnalyzedOffset.create(
112+
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
113+
maxAnalyzedOffset
114+
);
111115
Analyzer analyzer = wrapAnalyzer(
112116
context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
113117
queryMaxAnalyzedOffset
@@ -119,7 +123,8 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
119123
for (Object textToHighlight : textsToHighlight) {
120124
String text = convertFieldValue(fieldType, textToHighlight);
121125
int textLength = text.length();
122-
if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset) && (textLength > maxAnalyzedOffset)) {
126+
if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
127+
&& (textLength > maxAnalyzedOffset)) {
123128
throw new IllegalArgumentException(
124129
"The length ["
125130
+ textLength
@@ -241,9 +246,9 @@ private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer an
241246
}
242247
}
243248

244-
private static Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
249+
private static Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
245250
if (maxAnalyzedOffset != null) {
246-
return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
251+
return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
247252
}
248253
return analyzer;
249254
}

‎server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ private void assertHighlightOneDoc(
157157
noMatchSize,
158158
expectedPassages.length,
159159
maxAnalyzedOffset,
160-
queryMaxAnalyzedOffset,
160+
QueryMaxAnalyzedOffset.create(queryMaxAnalyzedOffset, maxAnalyzedOffset),
161161
true,
162162
true
163163
);

0 commit comments

Comments
 (0)