Skip to content

Commit be680df

Browse files
salvatore-campagnajavanna
salvatore-campagna
authored and
javanna
committed
Inject the host.name field mapping only if required for logsdb index mode (elastic#114573)
Here we check for the existence of a `host.name` field in index sort settings when the index mode is `logsdb` and decide to inject the field in the mapping depending on whether it exists or not. By default `host.name` is required for sorting in LogsDB. This reduces the chances for errors at mapping or template composition time as a result of injecting the `host.name` field only if strictly required. A user who wants to override index sort settings without including a `host.name` field would be able to do so without finding an additional `host.name` field in the mappings (injected automatically). If users override the sort settings and a `host.name` field is not included we don't need to inject such field since sorting does not require it anymore. As a result of this change we have the following: * the user does not provide any index sorting configuration: we are responsible for injecting the default sort fields and their mapping (for `logsdb`) * the user explicitly provides non-empty index sorting configuration: the user is also responsible for providing correct mappings and we do not modify index sorting or mappings Note also that all sort settings `index.sort.*` are `final` which means doing this check once, when mappings are merged at template composition time, is enough.
1 parent 6b4291b commit be680df

File tree

6 files changed

+856
-46
lines changed

6 files changed

+856
-46
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1373,7 +1373,7 @@ private static void updateIndexMappingsAndBuildSortOrder(
13731373
MapperService mapperService = indexService.mapperService();
13741374
IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD;
13751375
List<CompressedXContent> allMappings = new ArrayList<>();
1376-
final CompressedXContent defaultMapping = indexMode.getDefaultMapping();
1376+
final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings());
13771377
if (defaultMapping != null) {
13781378
allMappings.add(defaultMapping);
13791379
}

‎server/src/main/java/org/elasticsearch/index/IndexMode.java‎

Lines changed: 30 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
7575
}
7676

7777
@Override
78-
public CompressedXContent getDefaultMapping() {
78+
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
7979
return null;
8080
}
8181

@@ -171,7 +171,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
171171
}
172172

173173
@Override
174-
public CompressedXContent getDefaultMapping() {
174+
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
175175
return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
176176
}
177177

@@ -249,8 +249,10 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
249249
}
250250

251251
@Override
252-
public CompressedXContent getDefaultMapping() {
253-
return DEFAULT_LOGS_TIMESTAMP_MAPPING;
252+
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
253+
return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME)
254+
? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME
255+
: DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
254256
}
255257

256258
@Override
@@ -308,6 +310,8 @@ public String getDefaultCodec() {
308310
}
309311
};
310312

313+
private static final String HOST_NAME = "host.name";
314+
311315
private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
312316
settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
313317
settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
@@ -324,48 +328,33 @@ protected static String tsdbMode() {
324328
return "[" + IndexSettings.MODE.getKey() + "=time_series]";
325329
}
326330

327-
public static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
331+
private static CompressedXContent createDefaultMapping(boolean includeHostName) throws IOException {
332+
return new CompressedXContent((builder, params) -> {
333+
builder.startObject(MapperService.SINGLE_MAPPING_NAME)
334+
.startObject(DataStreamTimestampFieldMapper.NAME)
335+
.field("enabled", true)
336+
.endObject()
337+
.startObject("properties")
338+
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
339+
.field("type", DateFieldMapper.CONTENT_TYPE)
340+
.endObject();
341+
342+
if (includeHostName) {
343+
builder.startObject(HOST_NAME).field("type", KeywordFieldMapper.CONTENT_TYPE).field("ignore_above", 1024).endObject();
344+
}
328345

329-
static {
330-
try {
331-
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = new CompressedXContent(
332-
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
333-
.startObject(DataStreamTimestampFieldMapper.NAME)
334-
.field("enabled", true)
335-
.endObject()
336-
.startObject("properties")
337-
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
338-
.field("type", DateFieldMapper.CONTENT_TYPE)
339-
.field("ignore_malformed", "false")
340-
.endObject()
341-
.endObject()
342-
.endObject())
343-
);
344-
} catch (IOException e) {
345-
throw new AssertionError(e);
346-
}
346+
return builder.endObject().endObject();
347+
});
347348
}
348349

349-
public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
350+
private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
351+
352+
private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME;
350353

351354
static {
352355
try {
353-
DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
354-
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
355-
.startObject(DataStreamTimestampFieldMapper.NAME)
356-
.field("enabled", true)
357-
.endObject()
358-
.startObject("properties")
359-
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
360-
.field("type", DateFieldMapper.CONTENT_TYPE)
361-
.endObject()
362-
.startObject("host.name")
363-
.field("type", KeywordFieldMapper.CONTENT_TYPE)
364-
.field("ignore_above", 1024)
365-
.endObject()
366-
.endObject()
367-
.endObject())
368-
);
356+
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false);
357+
DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true);
369358
} catch (IOException e) {
370359
throw new AssertionError(e);
371360
}
@@ -421,7 +410,7 @@ public String getName() {
421410
* Get default mapping for this index or {@code null} if there is none.
422411
*/
423412
@Nullable
424-
public abstract CompressedXContent getDefaultMapping();
413+
public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings);
425414

426415
/**
427416
* Build the {@link FieldMapper} for {@code _id}.

‎server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,24 @@
1313
import org.elasticsearch.common.settings.Settings;
1414
import org.elasticsearch.test.ESTestCase;
1515

16+
import static org.hamcrest.Matchers.containsString;
1617
import static org.hamcrest.Matchers.equalTo;
18+
import static org.hamcrest.Matchers.not;
1719

1820
public class LogsIndexModeTests extends ESTestCase {
1921
public void testLogsIndexModeSetting() {
2022
assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB));
2123
}
2224

23-
public void testSortField() {
25+
public void testDefaultHostNameSortField() {
26+
final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
27+
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
28+
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
29+
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
30+
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
31+
}
32+
33+
public void testCustomSortField() {
2434
final Settings sortSettings = Settings.builder()
2535
.put(buildSettings())
2636
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
@@ -29,7 +39,9 @@ public void testSortField() {
2939
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
3040
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
3141
assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB));
32-
assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
42+
assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id"));
43+
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false));
44+
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host")));
3345
}
3446

3547
public void testSortMode() {

‎test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -302,8 +302,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
302302
mapperMetrics
303303
);
304304

305-
if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
306-
mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
305+
if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping(indexSettings) != null) {
306+
mapperService.merge(
307+
null,
308+
indexSettings.getMode().getDefaultMapping(indexSettings),
309+
MapperService.MergeReason.MAPPING_UPDATE
310+
);
307311
}
308312

309313
return mapperService;

0 commit comments

Comments
 (0)