Skip to content

Commit 3d1fa86

Browse files
Kubik42Kubik42
authored
[9.1] Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (#134253) (#134516)
* Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading (#134253)
1 parent 25c27f9 commit 3d1fa86

File tree

25 files changed

+1404
-382
lines changed

25 files changed

+1404
-382
lines changed

‎docs/changelog/134253.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134253
2+
summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading
3+
area: Mapping
4+
type: bug
5+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
253253
String parentField = searchExecutionContext.parentPath(name());
254254
var parent = searchExecutionContext.lookup().fieldType(parentField);
255255

256-
if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
257-
&& keywordParent.ignoreAbove() != Integer.MAX_VALUE) {
256+
if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
258257
if (parent.isStored()) {
259258
return storedFieldFetcher(parentField, keywordParent.originalName());
260259
} else if (parent.hasDocValues()) {
@@ -278,7 +277,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
278277
if (kwd != null) {
279278
var fieldType = kwd.fieldType();
280279

281-
if (fieldType.ignoreAbove() != Integer.MAX_VALUE) {
280+
if (fieldType.ignoreAbove().isSet()) {
282281
if (fieldType.isStored()) {
283282
return storedFieldFetcher(fieldType.name(), fieldType.originalName());
284283
} else if (fieldType.hasDocValues()) {

‎plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -250,12 +250,10 @@ public static class Builder extends FieldMapper.Builder {
250250
false
251251
).acceptsNull();
252252

253-
final Parameter<Integer> ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
254-
.addValidator(v -> {
255-
if (v < 0) {
256-
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
257-
}
258-
});
253+
final Parameter<Integer> ignoreAbove = Parameter.ignoreAboveParam(
254+
m -> toType(m).ignoreAbove,
255+
IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE
256+
);
259257
final Parameter<String> nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();
260258

261259
public Builder(String name) {

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -814,20 +814,21 @@ public Iterator<Setting<?>> settings() {
814814

815815
public static final Setting<Integer> IGNORE_ABOVE_SETTING = Setting.intSetting(
816816
"index.mapping.ignore_above",
817-
IndexSettings::getIgnoreAboveDefaultValue,
817+
settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)),
818818
0,
819819
Integer.MAX_VALUE,
820820
Property.IndexScope,
821821
Property.ServerlessPublic
822822
);
823823

824-
private static String getIgnoreAboveDefaultValue(final Settings settings) {
825-
if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
826-
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) {
827-
return "8191";
828-
} else {
829-
return String.valueOf(Integer.MAX_VALUE);
824+
private static int getIgnoreAboveDefaultValue(final Settings settings) {
825+
if (settings == null) {
826+
return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE;
830827
}
828+
return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(
829+
IndexSettings.MODE.get(settings),
830+
IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)
831+
);
831832
}
832833

833834
public static final Setting<SeqNoFieldMapper.SeqNoIndexOptions> SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting(

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1314,6 +1314,14 @@ public static Parameter<Boolean> docValuesParam(Function<FieldMapper, Boolean> i
13141314
return Parameter.boolParam("doc_values", false, initializer, defaultValue);
13151315
}
13161316

1317+
public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, int defaultValue) {
1318+
return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
1319+
if (v < 0) {
1320+
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
1321+
}
1322+
});
1323+
}
1324+
13171325
/**
13181326
* Defines a script parameter
13191327
* @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
9393
import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
9494
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
95+
import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue;
9596

9697
/**
9798
* A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -230,15 +231,14 @@ public Builder(final String name, final MappingParserContext mappingParserContex
230231
String name,
231232
IndexAnalyzers indexAnalyzers,
232233
ScriptCompiler scriptCompiler,
233-
int ignoreAboveDefault,
234234
IndexVersion indexCreatedVersion,
235235
SourceKeepMode sourceKeepMode
236236
) {
237237
this(
238238
name,
239239
indexAnalyzers,
240240
scriptCompiler,
241-
ignoreAboveDefault,
241+
getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
242242
indexCreatedVersion,
243243
IndexMode.STANDARD,
244244
null,
@@ -285,20 +285,15 @@ private Builder(
285285
}
286286
}).precludesParameters(normalizer);
287287
this.ignoreAboveDefault = ignoreAboveDefault;
288-
this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault)
289-
.addValidator(v -> {
290-
if (v < 0) {
291-
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
292-
}
293-
});
288+
this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault);
294289
this.indexSortConfig = indexSortConfig;
295290
this.indexMode = indexMode;
296291
this.useDocValuesSkipper = useDocValuesSkipper;
297292
this.indexSourceKeepMode = indexSourceKeepMode;
298293
}
299294

300295
public Builder(String name, IndexVersion indexCreatedVersion) {
301-
this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE);
296+
this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE);
302297
}
303298

304299
public Builder ignoreAbove(int ignoreAbove) {
@@ -503,14 +498,15 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort
503498

504499
public static final class KeywordFieldType extends StringFieldType {
505500

506-
private final int ignoreAbove;
501+
private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD);
502+
503+
private final IgnoreAbove ignoreAbove;
507504
private final String nullValue;
508505
private final NamedAnalyzer normalizer;
509506
private final boolean eagerGlobalOrdinals;
510507
private final FieldValues<String> scriptValues;
511508
private final boolean isDimension;
512509
private final boolean isSyntheticSource;
513-
private final IndexMode indexMode;
514510
private final IndexSortConfig indexSortConfig;
515511
private final boolean hasDocValuesSkipper;
516512
private final String originalName;
@@ -534,36 +530,34 @@ public KeywordFieldType(
534530
);
535531
this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
536532
this.normalizer = normalizer;
537-
this.ignoreAbove = builder.ignoreAbove.getValue();
533+
this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion);
538534
this.nullValue = builder.nullValue.getValue();
539535
this.scriptValues = builder.scriptValues();
540536
this.isDimension = builder.dimension.getValue();
541537
this.isSyntheticSource = isSyntheticSource;
542-
this.indexMode = builder.indexMode;
543538
this.indexSortConfig = builder.indexSortConfig;
544539
this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
545540
this.originalName = isSyntheticSource ? name + "._original" : null;
546541
}
547542

543+
public KeywordFieldType(String name) {
544+
this(name, true, true, Collections.emptyMap());
545+
}
546+
548547
public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
549548
super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
550549
this.normalizer = Lucene.KEYWORD_ANALYZER;
551-
this.ignoreAbove = Integer.MAX_VALUE;
550+
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
552551
this.nullValue = null;
553552
this.eagerGlobalOrdinals = false;
554553
this.scriptValues = null;
555554
this.isDimension = false;
556555
this.isSyntheticSource = false;
557-
this.indexMode = IndexMode.STANDARD;
558556
this.indexSortConfig = null;
559557
this.hasDocValuesSkipper = false;
560558
this.originalName = null;
561559
}
562560

563-
public KeywordFieldType(String name) {
564-
this(name, true, true, Collections.emptyMap());
565-
}
566-
567561
public KeywordFieldType(String name, FieldType fieldType) {
568562
super(
569563
name,
@@ -574,13 +568,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
574568
Collections.emptyMap()
575569
);
576570
this.normalizer = Lucene.KEYWORD_ANALYZER;
577-
this.ignoreAbove = Integer.MAX_VALUE;
571+
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
578572
this.nullValue = null;
579573
this.eagerGlobalOrdinals = false;
580574
this.scriptValues = null;
581575
this.isDimension = false;
582576
this.isSyntheticSource = false;
583-
this.indexMode = IndexMode.STANDARD;
584577
this.indexSortConfig = null;
585578
this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
586579
this.originalName = null;
@@ -589,13 +582,12 @@ public KeywordFieldType(String name, FieldType fieldType) {
589582
public KeywordFieldType(String name, NamedAnalyzer analyzer) {
590583
super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
591584
this.normalizer = Lucene.KEYWORD_ANALYZER;
592-
this.ignoreAbove = Integer.MAX_VALUE;
585+
this.ignoreAbove = IGNORE_ABOVE_DEFAULT;
593586
this.nullValue = null;
594587
this.eagerGlobalOrdinals = false;
595588
this.scriptValues = null;
596589
this.isDimension = false;
597590
this.isSyntheticSource = false;
598-
this.indexMode = IndexMode.STANDARD;
599591
this.indexSortConfig = null;
600592
this.hasDocValuesSkipper = false;
601593
this.originalName = null;
@@ -900,10 +892,7 @@ protected String parseSourceValue(Object value) {
900892
}
901893

902894
private String applyIgnoreAboveAndNormalizer(String value) {
903-
if (value.length() > ignoreAbove) {
904-
return null;
905-
}
906-
895+
if (ignoreAbove.isIgnored(value)) return null;
907896
return normalizeValue(normalizer(), name(), value);
908897
}
909898

@@ -1022,7 +1011,7 @@ public CollapseType collapseType() {
10221011

10231012
/** Values that have more chars than the return value of this method will
10241013
* be skipped at parsing time. */
1025-
public int ignoreAbove() {
1014+
public IgnoreAbove ignoreAbove() {
10261015
return ignoreAbove;
10271016
}
10281017

@@ -1040,10 +1029,6 @@ public boolean hasNormalizer() {
10401029
return normalizer != Lucene.KEYWORD_ANALYZER;
10411030
}
10421031

1043-
public IndexMode getIndexMode() {
1044-
return indexMode;
1045-
}
1046-
10471032
public IndexSortConfig getIndexSortConfig() {
10481033
return indexSortConfig;
10491034
}
@@ -1177,7 +1162,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
11771162
return false;
11781163
}
11791164

1180-
if (value.stringLength() > fieldType().ignoreAbove()) {
1165+
if (fieldType().ignoreAbove().isIgnored(value)) {
11811166
context.addIgnoredField(fullPath());
11821167
if (isSyntheticSource) {
11831168
// Save a copy of the field so synthetic source can load it
@@ -1341,7 +1326,7 @@ protected BytesRef preserve(BytesRef value) {
13411326
}
13421327
}
13431328

1344-
if (fieldType().ignoreAbove != Integer.MAX_VALUE) {
1329+
if (fieldType().ignoreAbove.isSet()) {
13451330
layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
13461331
@Override
13471332
protected void writeValue(Object value, XContentBuilder b) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/Mapper.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.index.IndexVersions;
2020
import org.elasticsearch.xcontent.ToXContentFragment;
2121
import org.elasticsearch.xcontent.XContentBuilder;
22+
import org.elasticsearch.xcontent.XContentString;
2223

2324
import java.io.IOException;
2425
import java.util.Arrays;
@@ -131,6 +132,78 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) {
131132
}
132133
}
133134

135+
/**
136+
* This class models the ignore_above parameter in indices.
137+
*/
138+
public static final class IgnoreAbove {
139+
140+
public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE;
141+
public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191;
142+
143+
private final Integer value;
144+
private final Integer defaultValue;
145+
146+
public IgnoreAbove(Integer value) {
147+
this(Objects.requireNonNull(value), IndexMode.STANDARD, IndexVersion.current());
148+
}
149+
150+
public IgnoreAbove(Integer value, IndexMode indexMode) {
151+
this(value, indexMode, IndexVersion.current());
152+
}
153+
154+
public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) {
155+
if (value != null && value < 0) {
156+
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]");
157+
}
158+
159+
this.value = value;
160+
this.defaultValue = getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion);
161+
}
162+
163+
public int get() {
164+
return value != null ? value : defaultValue;
165+
}
166+
167+
/**
168+
* Returns whether ignore_above is set; at field or index level.
169+
*/
170+
public boolean isSet() {
171+
// if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value
172+
return Integer.valueOf(get()).equals(defaultValue) == false;
173+
}
174+
175+
/**
176+
* Returns whether the given string will be ignored.
177+
*/
178+
public boolean isIgnored(final String s) {
179+
if (s == null) return false;
180+
return lengthExceedsIgnoreAbove(s.length());
181+
}
182+
183+
public boolean isIgnored(final XContentString s) {
184+
if (s == null) return false;
185+
return lengthExceedsIgnoreAbove(s.stringLength());
186+
}
187+
188+
private boolean lengthExceedsIgnoreAbove(int strLength) {
189+
return strLength > get();
190+
}
191+
192+
public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
193+
if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) {
194+
return IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES;
195+
} else {
196+
return IGNORE_ABOVE_DEFAULT_VALUE;
197+
}
198+
}
199+
200+
private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) {
201+
return indexMode == IndexMode.LOGSDB
202+
&& (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB));
203+
}
204+
205+
}
206+
134207
private final String leafName;
135208

136209
@SuppressWarnings("this-escape")

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,15 +1003,15 @@ public boolean isAggregatable() {
10031003
* A delegate by definition must have doc_values or be stored so most of the time it can be used for loading.
10041004
*/
10051005
public boolean canUseSyntheticSourceDelegateForLoading() {
1006-
return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE;
1006+
return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false;
10071007
}
10081008

10091009
/**
10101010
* Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
10111011
*/
10121012
public boolean canUseSyntheticSourceDelegateForQuerying() {
10131013
return syntheticSourceDelegate != null
1014-
&& syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE
1014+
&& syntheticSourceDelegate.ignoreAbove().isSet() == false
10151015
&& syntheticSourceDelegate.isIndexed();
10161016
}
10171017

@@ -1027,7 +1027,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) {
10271027
return false;
10281028
}
10291029
// Can't push equality if the field we're checking for is so big we'd ignore it.
1030-
return str.length() <= syntheticSourceDelegate.ignoreAbove();
1030+
return syntheticSourceDelegate.ignoreAbove().isIgnored(str) == false;
10311031
}
10321032

10331033
@Override

0 commit comments

Comments
 (0)