Skip to content

Commit cdc01a6

Browse files
svilen-mihaylov-elasticsvilen-mihaylov-elastic
committed
Add ability to set "max_analyzed_offet" implicitly to "index.highlight
.max_analyzed_offset", by setting it excplicitly to "-1".
1 parent 8d1f456 commit cdc01a6

File tree

8 files changed

+83
-29
lines changed

8 files changed

+83
-29
lines changed

‎plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighter.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
1818
import org.elasticsearch.index.query.SearchExecutionContext;
1919
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
20+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
2021
import org.elasticsearch.search.fetch.FetchSubPhase.HitContext;
2122
import org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter;
2223
import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
@@ -52,7 +53,7 @@ protected List<Object> loadFieldValues(
5253
}
5354

5455
@Override
55-
protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
56+
protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
5657
return new AnnotatedHighlighterAnalyzer(super.wrapAnalyzer(analyzer, maxAnalyzedOffset));
5758
}
5859

‎plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextHighlighterTests.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText;
4040
import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper;
4141
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
42+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
4243
import org.elasticsearch.lucene.search.uhighlight.Snippet;
4344
import org.elasticsearch.search.fetch.subphase.highlight.LimitTokenOffsetAnalyzer;
4445
import org.elasticsearch.test.ESTestCase;
@@ -85,7 +86,7 @@ private void assertHighlightOneDoc(
8586
int noMatchSize,
8687
String[] expectedPassages,
8788
int maxAnalyzedOffset,
88-
Integer queryMaxAnalyzedOffset
89+
Integer queryMaxAnalyzedOffsetIn
8990
) throws Exception {
9091

9192
try (Directory dir = newDirectory()) {
@@ -116,8 +117,9 @@ private void assertHighlightOneDoc(
116117
for (int i = 0; i < markedUpInputs.length; i++) {
117118
annotations[i] = AnnotatedText.parse(markedUpInputs[i]);
118119
}
119-
if (queryMaxAnalyzedOffset != null) {
120-
wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset);
120+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = new QueryMaxAnalyzedOffset(queryMaxAnalyzedOffsetIn, maxAnalyzedOffset);
121+
if (queryMaxAnalyzedOffset.isNull() == false) {
122+
wrapperAnalyzer = new LimitTokenOffsetAnalyzer(wrapperAnalyzer, queryMaxAnalyzedOffset.getNotNull());
121123
}
122124
AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer);
123125
hiliteAnalyzer.setAnnotations(annotations);
@@ -311,6 +313,19 @@ public void testExceedMaxAnalyzedOffset() throws Exception {
311313
e.getMessage()
312314
);
313315

316+
// Same as before, but force using index maxOffset (20) as queryMaxOffset by passing -1.
317+
assertHighlightOneDoc(
318+
"text",
319+
new String[] { "[Long Text exceeds](Long+Text+exceeds) MAX analyzed offset)" },
320+
query,
321+
Locale.ROOT,
322+
breakIterator,
323+
0,
324+
new String[] {},
325+
20,
326+
-1
327+
);
328+
314329
assertHighlightOneDoc(
315330
"text",
316331
new String[] { "[Long Text Exceeds](Long+Text+Exceeds) MAX analyzed offset [Long Text Exceeds](Long+Text+Exceeds)" },

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomFieldHighlighter.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
3434
private final Locale breakIteratorLocale;
3535
private final int noMatchSize;
3636
private String fieldValue;
37-
private final Integer queryMaxAnalyzedOffset;
37+
private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
3838

3939
CustomFieldHighlighter(
4040
String field,
@@ -47,7 +47,7 @@ class CustomFieldHighlighter extends FieldHighlighter {
4747
PassageFormatter passageFormatter,
4848
Comparator<Passage> passageSortComparator,
4949
int noMatchSize,
50-
Integer queryMaxAnalyzedOffset
50+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset
5151
) {
5252
super(
5353
field,
@@ -112,8 +112,8 @@ protected Passage[] getSummaryPassagesNoHighlight(int maxPassages) {
112112

113113
@Override
114114
protected Passage[] highlightOffsetsEnums(OffsetsEnum off) throws IOException {
115-
if (queryMaxAnalyzedOffset != null) {
116-
off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset);
115+
if (queryMaxAnalyzedOffset.isNull() == false) {
116+
off = new LimitedOffsetsEnum(off, queryMaxAnalyzedOffset.getNotNull());
117117
}
118118
return super.highlightOffsetsEnums(off);
119119
}

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighter.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public final class CustomUnifiedHighlighter extends UnifiedHighlighter {
6666
private final int noMatchSize;
6767
private final CustomFieldHighlighter fieldHighlighter;
6868
private final int maxAnalyzedOffset;
69-
private final Integer queryMaxAnalyzedOffset;
69+
private final QueryMaxAnalyzedOffset queryMaxAnalyzedOffset;
7070

7171
/**
7272
* Creates a new instance of {@link CustomUnifiedHighlighter}
@@ -94,7 +94,7 @@ public CustomUnifiedHighlighter(
9494
int noMatchSize,
9595
int maxPassages,
9696
int maxAnalyzedOffset,
97-
Integer queryMaxAnalyzedOffset,
97+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset,
9898
boolean requireFieldMatch,
9999
boolean weightMatchesEnabled
100100
) {
@@ -125,9 +125,9 @@ public Snippet[] highlightField(LeafReader reader, int docId, CheckedSupplier<St
125125
return null;
126126
}
127127
int fieldValueLength = fieldValue.length();
128-
if (((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset)
128+
if ((queryMaxAnalyzedOffset.isNull() || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
129129
&& (getOffsetSource(field) == OffsetSource.ANALYSIS)
130-
&& (fieldValueLength > maxAnalyzedOffset))) {
130+
&& (fieldValueLength > maxAnalyzedOffset)) {
131131
throw new IllegalArgumentException(
132132
"The length ["
133133
+ fieldValueLength

‎server/src/main/java/org/elasticsearch/lucene/search/uhighlight/QueryMaxAnalyzedOffset.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene.search.uhighlight;
11+
12+
public class QueryMaxAnalyzedOffset {
13+
private final Integer queryMaxAnalyzedOffset;
14+
15+
public QueryMaxAnalyzedOffset(final Integer queryMaxAnalyzedOffset, final int indexMaxAnalyzedOffset) {
16+
// If we have a negative value, grab value for the actual maximum from the index.
17+
this.queryMaxAnalyzedOffset = (queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset >= 0)
18+
? queryMaxAnalyzedOffset
19+
: Integer.valueOf(indexMaxAnalyzedOffset);
20+
}
21+
22+
public boolean isNull() {
23+
return queryMaxAnalyzedOffset == null;
24+
}
25+
26+
public int getNotNull() {
27+
return queryMaxAnalyzedOffset;
28+
}
29+
}

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.elasticsearch.lucene.search.uhighlight.BoundedBreakIteratorScanner;
3232
import org.elasticsearch.lucene.search.uhighlight.CustomPassageFormatter;
3333
import org.elasticsearch.lucene.search.uhighlight.CustomUnifiedHighlighter;
34+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
3435
import org.elasticsearch.lucene.search.uhighlight.Snippet;
3536
import org.elasticsearch.search.fetch.FetchContext;
3637
import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -121,7 +122,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
121122
int maxAnalyzedOffset = indexSettings.getHighlightMaxAnalyzedOffset();
122123
boolean weightMatchesEnabled = indexSettings.isWeightMatchesEnabled();
123124
int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments();
124-
Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
125+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = new QueryMaxAnalyzedOffset(
126+
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
127+
maxAnalyzedOffset
128+
);
125129
Analyzer analyzer = wrapAnalyzer(
126130
fieldContext.context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
127131
queryMaxAnalyzedOffset
@@ -171,7 +175,7 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) {
171175
fieldContext.field.fieldOptions().noMatchSize(),
172176
highlighterNumberOfFragments,
173177
maxAnalyzedOffset,
174-
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
178+
queryMaxAnalyzedOffset,
175179
fieldContext.field.fieldOptions().requireFieldMatch(),
176180
weightMatchesEnabled
177181
);
@@ -186,9 +190,9 @@ protected PassageFormatter getPassageFormatter(SearchHighlightContext.Field fiel
186190
);
187191
}
188192

189-
protected Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
190-
if (maxAnalyzedOffset != null) {
191-
analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
193+
protected Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
194+
if (maxAnalyzedOffset.isNull() == false) {
195+
analyzer = new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
192196
}
193197
return analyzer;
194198
}

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/PlainHighlighter.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.elasticsearch.common.text.Text;
2828
import org.elasticsearch.index.IndexSettings;
2929
import org.elasticsearch.index.mapper.MappedFieldType;
30+
import org.elasticsearch.lucene.search.uhighlight.QueryMaxAnalyzedOffset;
3031
import org.elasticsearch.search.fetch.FetchContext;
3132
import org.elasticsearch.search.fetch.FetchSubPhase;
3233

@@ -107,7 +108,10 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
107108
ArrayList<OrderedTextFragment> fragsList = new ArrayList<>();
108109
List<Object> textsToHighlight;
109110
final int maxAnalyzedOffset = context.getSearchExecutionContext().getIndexSettings().getHighlightMaxAnalyzedOffset();
110-
Integer queryMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzedOffset();
111+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset = new QueryMaxAnalyzedOffset(
112+
fieldContext.field.fieldOptions().maxAnalyzedOffset(),
113+
maxAnalyzedOffset
114+
);
111115
Analyzer analyzer = wrapAnalyzer(
112116
context.getSearchExecutionContext().getIndexAnalyzer(f -> Lucene.KEYWORD_ANALYZER),
113117
queryMaxAnalyzedOffset
@@ -119,7 +123,8 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
119123
for (Object textToHighlight : textsToHighlight) {
120124
String text = convertFieldValue(fieldType, textToHighlight);
121125
int textLength = text.length();
122-
if ((queryMaxAnalyzedOffset == null || queryMaxAnalyzedOffset > maxAnalyzedOffset) && (textLength > maxAnalyzedOffset)) {
126+
if ((queryMaxAnalyzedOffset.isNull() || queryMaxAnalyzedOffset.getNotNull() > maxAnalyzedOffset)
127+
&& (textLength > maxAnalyzedOffset)) {
123128
throw new IllegalArgumentException(
124129
"The length ["
125130
+ textLength
@@ -241,9 +246,9 @@ private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer an
241246
}
242247
}
243248

244-
private static Analyzer wrapAnalyzer(Analyzer analyzer, Integer maxAnalyzedOffset) {
245-
if (maxAnalyzedOffset != null) {
246-
return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset);
249+
private static Analyzer wrapAnalyzer(Analyzer analyzer, QueryMaxAnalyzedOffset maxAnalyzedOffset) {
250+
if (maxAnalyzedOffset.isNull() == false) {
251+
return new LimitTokenOffsetAnalyzer(analyzer, maxAnalyzedOffset.getNotNull());
247252
}
248253
return analyzer;
249254
}

‎server/src/test/java/org/elasticsearch/lucene/search/uhighlight/CustomUnifiedHighlighterTests.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ private void assertHighlightOneDoc(
9393
int noMatchSize,
9494
String[] expectedPassages,
9595
int maxAnalyzedOffset,
96-
Integer queryMaxAnalyzedOffset
96+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset
9797
) throws Exception {
9898
assertHighlightOneDoc(
9999
fieldName,
@@ -120,7 +120,7 @@ private void assertHighlightOneDoc(
120120
int noMatchSize,
121121
String[] expectedPassages,
122122
int maxAnalyzedOffset,
123-
Integer queryMaxAnalyzedOffset,
123+
QueryMaxAnalyzedOffset queryMaxAnalyzedOffset,
124124
UnifiedHighlighter.OffsetSource offsetSource
125125
) throws Exception {
126126
try (Directory dir = newDirectory()) {
@@ -453,7 +453,7 @@ public void testExceedMaxAnalyzedOffset() throws Exception {
453453
0,
454454
new String[] {},
455455
10,
456-
queryMaxAnalyzedOffset
456+
new QueryMaxAnalyzedOffset(queryMaxAnalyzedOffset, 10)
457457
);
458458
});
459459
assertEquals(
@@ -473,7 +473,7 @@ public void testExceedMaxAnalyzedOffset() throws Exception {
473473
1,
474474
new String[] { "exceeds" },
475475
10,
476-
10
476+
new QueryMaxAnalyzedOffset(10, 10)
477477
);
478478
}
479479

@@ -491,7 +491,7 @@ public void testExceedMaxAnalyzedOffsetWithRepeatedWords() throws Exception {
491491
0,
492492
new String[] { "Testing <b>Fun</b> Testing Fun" },
493493
29,
494-
10,
494+
new QueryMaxAnalyzedOffset(10, 29),
495495
UnifiedHighlighter.OffsetSource.ANALYSIS
496496
);
497497
assertHighlightOneDoc(
@@ -504,7 +504,7 @@ public void testExceedMaxAnalyzedOffsetWithRepeatedWords() throws Exception {
504504
0,
505505
new String[] { "Testing <b>Fun</b> Testing Fun" },
506506
29,
507-
10,
507+
new QueryMaxAnalyzedOffset(10, 29),
508508
UnifiedHighlighter.OffsetSource.POSTINGS
509509
);
510510
}
@@ -540,7 +540,7 @@ public void testExceedMaxAnalyzedOffsetRandomOffset() throws Exception {
540540
0,
541541
new String[] { output },
542542
47,
543-
randomOffset,
543+
new QueryMaxAnalyzedOffset(randomOffset, 47),
544544
offsetSource
545545
);
546546
}

0 commit comments

Comments
 (0)