Skip to content

Commit 0966bf4

Browse files
Kubik42Kubik42
committed
Added IgnoreAbove record, addressed index-level ignore above
1 parent 0003f72 commit 0966bf4

File tree

11 files changed

+241
-150
lines changed

11 files changed

+241
-150
lines changed

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
253253
String parentField = searchExecutionContext.parentPath(name());
254254
var parent = searchExecutionContext.lookup().fieldType(parentField);
255255

256-
if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent
257-
&& keywordParent.ignoreAbove() != Integer.MAX_VALUE) {
256+
if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) {
258257
if (parent.isStored()) {
259258
return storedFieldFetcher(parentField, keywordParent.originalName());
260259
} else if (parent.hasDocValues()) {
@@ -278,7 +277,7 @@ private IOFunction<LeafReaderContext, CheckedIntFunction<List<Object>, IOExcepti
278277
if (kwd != null) {
279278
var fieldType = kwd.fieldType();
280279

281-
if (fieldType.ignoreAbove() != Integer.MAX_VALUE) {
280+
if (fieldType.ignoreAbove().isSet()) {
282281
if (fieldType.isStored()) {
283282
return storedFieldFetcher(fieldType.name(), fieldType.originalName());
284283
} else if (fieldType.hasDocValues()) {

‎server/src/main/java/org/elasticsearch/index/IgnoreAbove.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index;
11+
12+
import org.elasticsearch.xcontent.XContentString;
13+
14+
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES;
15+
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
16+
17+
public record IgnoreAbove(Integer value, int defaultValue) {
18+
19+
public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = IgnoreAbove.builder()
20+
.defaultValue(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES)
21+
.build();
22+
23+
public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = IgnoreAbove.builder()
24+
.defaultValue(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES)
25+
.build();
26+
27+
public int get() {
28+
return value != null ? value : defaultValue;
29+
}
30+
31+
public boolean isSet() {
32+
return value != defaultValue;
33+
}
34+
35+
public boolean isIgnored(final String s) {
36+
if (s == null) return false;
37+
return lengthExceedsIgnoreAbove(s.length());
38+
}
39+
40+
public boolean isIgnored(final XContentString s) {
41+
if (s == null) return false;
42+
return lengthExceedsIgnoreAbove(s.stringLength());
43+
}
44+
45+
private boolean lengthExceedsIgnoreAbove(int strLength) {
46+
return strLength > get();
47+
}
48+
49+
public static Builder builder() {
50+
return new Builder();
51+
}
52+
53+
public static final class Builder {
54+
55+
private Integer value;
56+
private int defaultValue; // cannot be null, hence int
57+
58+
private Builder() {}
59+
60+
public Builder value(Integer value) {
61+
this.value = value;
62+
return this;
63+
}
64+
65+
public Builder defaultValue(int defaultValue) {
66+
this.defaultValue = defaultValue;
67+
return this;
68+
}
69+
70+
public IgnoreAbove build() {
71+
return new IgnoreAbove(value, defaultValue);
72+
}
73+
}
74+
}

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1338,10 +1338,15 @@ public static Parameter<Boolean> normsParam(Function<FieldMapper, Boolean> initi
13381338
}
13391339

13401340
public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer) {
1341-
return ignoreAboveParam(initializer, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES);
1341+
return ignoreAboveParam(initializer, null, null);
13421342
}
13431343

1344-
public static Parameter<Integer> ignoreAboveParam(Function<FieldMapper, Integer> initializer, final int defaultValue) {
1344+
public static Parameter<Integer> ignoreAboveParam(
1345+
Function<FieldMapper, Integer> initializer,
1346+
final IndexMode indexMode,
1347+
final IndexVersion indexCreatedVersion
1348+
) {
1349+
final int defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion);
13451350
return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> {
13461351
if (v < 0) {
13471352
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 17 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@
4141
import org.elasticsearch.common.lucene.search.AutomatonQueries;
4242
import org.elasticsearch.common.unit.Fuzziness;
4343
import org.elasticsearch.core.Nullable;
44+
import org.elasticsearch.index.IgnoreAbove;
4445
import org.elasticsearch.index.IndexMode;
45-
import org.elasticsearch.index.IndexSettings;
4646
import org.elasticsearch.index.IndexSortConfig;
4747
import org.elasticsearch.index.IndexVersion;
4848
import org.elasticsearch.index.IndexVersions;
@@ -90,7 +90,6 @@
9090

9191
import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH;
9292
import static org.elasticsearch.core.Strings.format;
93-
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
9493
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
9594
import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
9695
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
@@ -181,7 +180,6 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
181180
false
182181
);
183182
private final Parameter<Integer> ignoreAbove;
184-
private final int ignoreAboveDefault;
185183
private final IndexSortConfig indexSortConfig;
186184
private final IndexMode indexMode;
187185
private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
@@ -220,14 +218,15 @@ public Builder(final String name, final MappingParserContext mappingParserContex
220218
name,
221219
mappingParserContext.getIndexAnalyzers(),
222220
mappingParserContext.scriptCompiler(),
223-
IGNORE_ABOVE_DEFAULT_STANDARD_INDICES,
224221
mappingParserContext.getIndexSettings().getIndexVersionCreated(),
225222
mappingParserContext.getIndexSettings().getMode(),
226223
mappingParserContext.getIndexSettings().getIndexSortConfig(),
227224
USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()),
228225
false,
229226
mappingParserContext.getIndexSettings().sourceKeepMode()
230227
);
228+
229+
// if ignore_above is configured at index-level, then set it now
231230
if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) {
232231
this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()));
233232
}
@@ -240,25 +239,13 @@ public Builder(final String name, final MappingParserContext mappingParserContex
240239
IndexVersion indexCreatedVersion,
241240
SourceKeepMode sourceKeepMode
242241
) {
243-
this(
244-
name,
245-
indexAnalyzers,
246-
scriptCompiler,
247-
IndexSettings.getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion),
248-
indexCreatedVersion,
249-
IndexMode.STANDARD,
250-
null,
251-
false,
252-
false,
253-
sourceKeepMode
254-
);
242+
this(name, indexAnalyzers, scriptCompiler, indexCreatedVersion, IndexMode.STANDARD, null, false, false, sourceKeepMode);
255243
}
256244

257245
private Builder(
258246
String name,
259247
IndexAnalyzers indexAnalyzers,
260248
ScriptCompiler scriptCompiler,
261-
int ignoreAboveDefault,
262249
IndexVersion indexCreatedVersion,
263250
IndexMode indexMode,
264251
IndexSortConfig indexSortConfig,
@@ -292,8 +279,7 @@ private Builder(
292279
);
293280
}
294281
}).precludesParameters(normalizer);
295-
this.ignoreAboveDefault = ignoreAboveDefault;
296-
this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault);
282+
this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), indexMode, indexCreatedVersion);
297283
this.indexSortConfig = indexSortConfig;
298284
this.indexMode = indexMode;
299285
this.enableDocValuesSkipper = enableDocValuesSkipper;
@@ -315,7 +301,6 @@ public static Builder buildWithDocValuesSkipper(
315301
name,
316302
null,
317303
ScriptCompiler.NONE,
318-
IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion),
319304
indexCreatedVersion,
320305
indexMode,
321306
// Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed.
@@ -536,8 +521,7 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort
536521

537522
public static final class KeywordFieldType extends StringFieldType {
538523

539-
private final int ignoreAbove;
540-
private final int ignoreAboveDefaultValue;
524+
private final IgnoreAbove ignoreAbove;
541525
private final String nullValue;
542526
private final NamedAnalyzer normalizer;
543527
private final boolean eagerGlobalOrdinals;
@@ -567,8 +551,10 @@ public KeywordFieldType(
567551
);
568552
this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue();
569553
this.normalizer = normalizer;
570-
this.ignoreAbove = builder.ignoreAbove.getValue();
571-
this.ignoreAboveDefaultValue = builder.ignoreAbove.getDefaultValue();
554+
this.ignoreAbove = IgnoreAbove.builder()
555+
.value(builder.ignoreAbove.getValue())
556+
.defaultValue(builder.ignoreAbove.getDefaultValue())
557+
.build();
572558
this.nullValue = builder.nullValue.getValue();
573559
this.scriptValues = builder.scriptValues();
574560
this.isDimension = builder.dimension.getValue();
@@ -585,7 +571,7 @@ public KeywordFieldType(String name) {
585571
public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
586572
super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
587573
this.normalizer = Lucene.KEYWORD_ANALYZER;
588-
this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
574+
this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
589575
this.nullValue = null;
590576
this.eagerGlobalOrdinals = false;
591577
this.scriptValues = null;
@@ -606,7 +592,7 @@ public KeywordFieldType(String name, FieldType fieldType) {
606592
Collections.emptyMap()
607593
);
608594
this.normalizer = Lucene.KEYWORD_ANALYZER;
609-
this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
595+
this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
610596
this.nullValue = null;
611597
this.eagerGlobalOrdinals = false;
612598
this.scriptValues = null;
@@ -620,7 +606,7 @@ public KeywordFieldType(String name, FieldType fieldType) {
620606
public KeywordFieldType(String name, NamedAnalyzer analyzer) {
621607
super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap());
622608
this.normalizer = Lucene.KEYWORD_ANALYZER;
623-
this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES;
609+
this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES;
624610
this.nullValue = null;
625611
this.eagerGlobalOrdinals = false;
626612
this.scriptValues = null;
@@ -934,10 +920,7 @@ protected String parseSourceValue(Object value) {
934920
}
935921

936922
private String applyIgnoreAboveAndNormalizer(String value) {
937-
if (value.length() > ignoreAbove) {
938-
return null;
939-
}
940-
923+
if (ignoreAbove.isIgnored(value)) return null;
941924
return normalizeValue(normalizer(), name(), value);
942925
}
943926

@@ -1056,14 +1039,10 @@ public CollapseType collapseType() {
10561039

10571040
/** Values that have more chars than the return value of this method will
10581041
* be skipped at parsing time. */
1059-
public int ignoreAbove() {
1042+
public IgnoreAbove ignoreAbove() {
10601043
return ignoreAbove;
10611044
}
10621045

1063-
public boolean isIgnoreAboveSet() {
1064-
return ignoreAbove != ignoreAboveDefaultValue;
1065-
}
1066-
10671046
@Override
10681047
public boolean isDimension() {
10691048
return isDimension;
@@ -1119,7 +1098,6 @@ public String originalName() {
11191098
private final boolean isSyntheticSource;
11201099

11211100
private final IndexAnalyzers indexAnalyzers;
1122-
private final int ignoreAboveDefault;
11231101
private final IndexMode indexMode;
11241102
private final IndexSortConfig indexSortConfig;
11251103
private final boolean enableDocValuesSkipper;
@@ -1151,7 +1129,6 @@ private KeywordFieldMapper(
11511129
this.scriptCompiler = builder.scriptCompiler;
11521130
this.indexCreatedVersion = builder.indexCreatedVersion;
11531131
this.isSyntheticSource = isSyntheticSource;
1154-
this.ignoreAboveDefault = builder.ignoreAboveDefault;
11551132
this.indexMode = builder.indexMode;
11561133
this.indexSortConfig = builder.indexSortConfig;
11571134
this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
@@ -1212,7 +1189,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value)
12121189
return false;
12131190
}
12141191

1215-
if (value.stringLength() > fieldType().ignoreAbove()) {
1192+
if (fieldType().ignoreAbove().isIgnored(value)) {
12161193
context.addIgnoredField(fullPath());
12171194
if (isSyntheticSource) {
12181195
// Save a copy of the field so synthetic source can load it
@@ -1307,7 +1284,6 @@ public FieldMapper.Builder getMergeBuilder() {
13071284
leafName(),
13081285
indexAnalyzers,
13091286
scriptCompiler,
1310-
ignoreAboveDefault,
13111287
indexCreatedVersion,
13121288
indexMode,
13131289
indexSortConfig,
@@ -1381,7 +1357,7 @@ protected BytesRef preserve(BytesRef value) {
13811357
}
13821358
}
13831359

1384-
if (fieldType().isIgnoreAboveSet()) {
1360+
if (fieldType().ignoreAbove.isSet()) {
13851361
layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) {
13861362
@Override
13871363
protected void writeValue(Object value, XContentBuilder b) throws IOException {

‎server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1021,15 +1021,15 @@ public boolean isAggregatable() {
10211021
* A delegate by definition must have doc_values or be stored so most of the time it can be used for loading.
10221022
*/
10231023
public boolean canUseSyntheticSourceDelegateForLoading() {
1024-
return syntheticSourceDelegate != null && syntheticSourceDelegate.isIgnoreAboveSet() == false;
1024+
return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false;
10251025
}
10261026

10271027
/**
10281028
* Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
10291029
*/
10301030
public boolean canUseSyntheticSourceDelegateForQuerying() {
10311031
return syntheticSourceDelegate != null
1032-
&& syntheticSourceDelegate.isIgnoreAboveSet() == false
1032+
&& syntheticSourceDelegate.ignoreAbove().isSet() == false
10331033
&& syntheticSourceDelegate.isIndexed();
10341034
}
10351035

@@ -1045,7 +1045,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) {
10451045
return false;
10461046
}
10471047
// Can't push equality if the field we're checking for is so big we'd ignore it.
1048-
return str.length() <= syntheticSourceDelegate.ignoreAbove();
1048+
return syntheticSourceDelegate.ignoreAbove().isIgnored(str) == false;
10491049
}
10501050

10511051
@Override

0 commit comments

Comments
 (0)