Skip to content

Gate disable norms on text fields for logsdb/tsdb indices #133493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 25, 2025
Merged

Gate disable norms on text fields for logsdb/tsdb indices #133493

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d4f41f7
Gate disable norms on text fields for logsdb/tsdb indices
Kubik42 Aug 25, 2025
67080d8
Update docs/changelog/133493.yaml
Kubik42 Aug 25, 2025
edc3fec
Delete docs/changelog/133493.yaml
Kubik42 Aug 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/index/IndexVersions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ private static Version parseUnchecked(String version) {
public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES = def(9_033_0_00, Version.LUCENE_10_2_2);
public static final IndexVersion IGNORED_SOURCE_FIELDS_PER_ENTRY_WITH_FF = def(9_034_0_00, Version.LUCENE_10_2_2);
public static final IndexVersion EXCLUDE_SOURCE_VECTORS_DEFAULT = def(9_035_0_00, Version.LUCENE_10_2_2);
public static final IndexVersion DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB = def(9_036_0_00, Version.LUCENE_10_2_2);

/*
* STOP! READ THIS FIRST! No, really,
Expand Down
15 changes: 10 additions & 5 deletions server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,7 @@ public Builder(
this.isSyntheticSourceEnabled = isSyntheticSourceEnabled;
this.withinMultiField = withinMultiField;

// don't enable norms by default if the index is LOGSDB or TSDB based
this.norms = Parameter.normsParam(
m -> ((TextFieldMapper) m).norms,
() -> indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES
);
this.norms = Parameter.normsParam(m -> ((TextFieldMapper) m).norms, this::normsDefault);

// If synthetic source is used we need to either store this field
// to recreate the source or use keyword multi-fields for that.
Expand All @@ -341,6 +337,15 @@ public Builder(
);
}

private boolean normsDefault() {
if (indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_NORMS_BY_DEFAULT_FOR_LOGSDB_AND_TSDB)) {
// don't enable norms by default if the index is LOGSDB or TSDB based
return indexMode != IndexMode.LOGSDB && indexMode != IndexMode.TIME_SERIES;
}
// bwc - historically, norms were enabled by default on text fields regardless of which index mode was used
return true;
}

public static boolean multiFieldsNotStoredByDefaultIndexVersionCheck(IndexVersion indexCreatedVersion) {
return indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED)
|| indexCreatedVersion.between(
Expand Down
76 changes: 71 additions & 5 deletions server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1436,7 +1436,7 @@ public void testEmpty() throws Exception {
});
}

public void testNormalizeByDefault() throws IOException {
public void testNormsEnabledByDefault() throws IOException {
// given
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName());
Expand All @@ -1461,7 +1461,7 @@ public void testNormalizeByDefault() throws IOException {
assertThat(fieldType.omitNorms(), is(false));
}

public void testNormalizeWhenIndexModeIsNotGiven() throws IOException {
public void testNormsEnabledWhenIndexModeIsNotGiven() throws IOException {
// given
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
Settings indexSettings = indexSettingsBuilder.build();
Expand All @@ -1485,7 +1485,7 @@ public void testNormalizeWhenIndexModeIsNotGiven() throws IOException {
assertThat(fieldType.omitNorms(), is(false));
}

public void testNormalizeWhenIndexModeIsNull() throws IOException {
public void textNormsEnabledWhenIndexModeIsNull() throws IOException {
// given
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), (String) null);
Expand All @@ -1510,7 +1510,7 @@ public void testNormalizeWhenIndexModeIsNull() throws IOException {
assertThat(fieldType.omitNorms(), is(false));
}

public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException {
public void testNormsDisabledWhenIndexModeIsLogsDb() throws IOException {
// given
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
Expand Down Expand Up @@ -1538,7 +1538,7 @@ public void testDontNormalizeWhenIndexModeIsLogsDB() throws IOException {
assertThat(fieldType.omitNorms(), is(true));
}

public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException {
public void testNormsDisabledWhenIndexModeIsTsdb() throws IOException {
// given
Instant currentTime = Instant.now();
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
Expand Down Expand Up @@ -1574,4 +1574,70 @@ public void testDontNormalizeWhenIndexModeIsTSDB() throws IOException {
assertThat(fieldType.omitNorms(), is(true));
}

public void testNormsEnabledWhenIndexModeIsLogsDb_bwcCheck() throws IOException {
// given
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
Settings indexSettings = indexSettingsBuilder.build();

XContentBuilder mapping = mapping(b -> {
b.startObject("potato");
b.field("type", "text");
b.endObject();
});

var source = source(b -> {
b.field("@timestamp", Instant.now());
b.field("potato", "a potato flew around my room");
});

// when
IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT;
DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper();
ParsedDocument doc = mapper.parse(source);

List<IndexableField> fields = doc.rootDoc().getFields("potato");
IndexableFieldType fieldType = fields.get(0).fieldType();

// then
assertThat(fieldType.omitNorms(), is(false));
}

public void testNormsEnabledWhenIndexModeIsTsdb_bwcCheck() throws IOException {
// given
Instant currentTime = Instant.now();
Settings.Builder indexSettingsBuilder = getIndexSettingsBuilder();
indexSettingsBuilder.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), currentTime.minus(1, ChronoUnit.HOURS).toEpochMilli())
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), currentTime.plus(1, ChronoUnit.HOURS).toEpochMilli())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dimension");
Settings indexSettings = indexSettingsBuilder.build();

XContentBuilder mapping = mapping(b -> {
b.startObject("potato");
b.field("type", "text");
b.endObject();

b.startObject("@timestamp");
b.field("type", "date");
b.endObject();
});

var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
b.field("@timestamp", Instant.now());
b.field("potato", "a potato flew around my room");
}, null);

// when
IndexVersion bwcIndexVersion = IndexVersions.EXCLUDE_SOURCE_VECTORS_DEFAULT;
DocumentMapper mapper = createMapperService(bwcIndexVersion, indexSettings, mapping).documentMapper();
ParsedDocument doc = mapper.parse(source);

List<IndexableField> fields = doc.rootDoc().getFields("potato");
IndexableFieldType fieldType = fields.get(0).fieldType();

// then
assertThat(fieldType.omitNorms(), is(false));
}

}