Skip to content

Commit 6a543e3

Browse files
jdconradjdconrad
authored
Merge branch 'main' into mtv24
2 parents a27a736 + 035c48f commit 6a543e3

File tree

9 files changed

+203
-86
lines changed

9 files changed

+203
-86
lines changed

‎docs/changelog/135653.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 135653
2+
summary: Reset health status on successful empty checkpoint
3+
area: Machine Learning
4+
type: bug
5+
issues:
6+
- 135650

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -606,6 +606,9 @@ tests:
606606
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
607607
method: test
608608
issue: https://github.com/elastic/elasticsearch/issues/134407
609+
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
610+
method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry) snapshot version}
611+
issue: https://github.com/elastic/elasticsearch/issues/135584
609612

610613
# Examples:
611614
#

‎test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import org.elasticsearch.TransportVersion;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.common.settings.Setting;
1415
import org.elasticsearch.common.settings.Settings;
1516
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
1617
import org.elasticsearch.env.Environment;
@@ -61,14 +62,25 @@ public static MapperService newMapperService(
6162
Settings settings,
6263
IndicesModule indicesModule,
6364
String indexName
65+
) throws IOException {
66+
return newMapperService(xContentRegistry, tempDir, settings, indicesModule, indexName, new Setting<?>[0]);
67+
}
68+
69+
public static MapperService newMapperService(
70+
NamedXContentRegistry xContentRegistry,
71+
Path tempDir,
72+
Settings settings,
73+
IndicesModule indicesModule,
74+
String indexName,
75+
Setting<?>[] additionalSettings
6476
) throws IOException {
6577
Settings.Builder settingsBuilder = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), tempDir).put(settings);
6678
if (settings.get(IndexMetadata.SETTING_VERSION_CREATED) == null) {
6779
settingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current());
6880
}
6981
Settings finalSettings = settingsBuilder.build();
7082
MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
71-
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings);
83+
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexName, finalSettings, additionalSettings);
7284
IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, finalSettings).indexAnalyzers;
7385
SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
7486
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.apache.logging.log4j.Logger;
1212
import org.apache.lucene.util.SetOnce;
1313
import org.elasticsearch.Version;
14+
import org.elasticsearch.action.admin.cluster.stats.MappingVisitor;
1415
import org.elasticsearch.cluster.metadata.IndexMetadata;
1516
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
1617
import org.elasticsearch.cluster.metadata.ProjectMetadata;
@@ -19,6 +20,7 @@
1920
import org.elasticsearch.common.regex.Regex;
2021
import org.elasticsearch.common.settings.Settings;
2122
import org.elasticsearch.common.xcontent.XContentHelper;
23+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
2224
import org.elasticsearch.core.CheckedFunction;
2325
import org.elasticsearch.core.Strings;
2426
import org.elasticsearch.index.IndexMode;
@@ -35,12 +37,15 @@
3537
import org.elasticsearch.index.mapper.SourceFieldMapper;
3638
import org.elasticsearch.xcontent.XContentType;
3739
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
40+
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldType;
3841

3942
import java.io.IOException;
4043
import java.time.Instant;
4144
import java.util.ArrayList;
4245
import java.util.List;
4346
import java.util.Locale;
47+
import java.util.Map;
48+
import java.util.Objects;
4449
import java.util.Set;
4550
import java.util.function.Supplier;
4651

@@ -192,13 +197,13 @@ && matchesLogsPattern(dataStreamName)) {
192197
}
193198
}
194199

195-
if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false) {
200+
if (licenseService.allowPatternTextTemplating(isTemplateValidation) == false && mappingHints.maybeUsesPatternText) {
196201
additionalSettings.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true);
197202
}
198203
}
199204

200-
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
201-
static MappingHints EMPTY = new MappingHints(false, false, false);
205+
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) {
206+
static MappingHints EMPTY = new MappingHints(false, false, false, false);
202207
}
203208

204209
private static boolean matchesLogsPattern(final String name) {
@@ -237,16 +242,17 @@ MappingHints getMappingHints(
237242
hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
238243
if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) {
239244
// Custom sort config, no point for further checks on [host.name] field.
240-
return new MappingHints(hasSyntheticSourceUsage, false, false);
245+
return new MappingHints(hasSyntheticSourceUsage, false, false, true);
241246
}
242247
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
243248
&& IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) {
244249
// Settings for adding and sorting on [host.name] are already set, propagate them.
245-
return new MappingHints(hasSyntheticSourceUsage, true, true);
250+
return new MappingHints(hasSyntheticSourceUsage, true, true, true);
246251
}
247252
}
248253

249254
try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) {
255+
boolean maybeUsesPatternText = PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(indexTemplateAndCreateRequestSettings);
250256
// combinedTemplateMappings can be null when creating system indices
251257
// combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
252258
if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
@@ -258,9 +264,17 @@ MappingHints getMappingHints(
258264
// The _doc.properties.host* is needed to determine whether host.name field can be injected.
259265
// The _doc.subobjects is needed to determine whether subobjects is enabled.
260266
List<CompressedXContent> filteredMappings = new ArrayList<>(combinedTemplateMappings.size());
267+
String[] mappingIncludesArray = MAPPING_INCLUDES.toArray(String[]::new);
261268
for (CompressedXContent mappingSource : combinedTemplateMappings) {
262269
var ref = mappingSource.compressedReference();
263-
var map = XContentHelper.convertToMap(ref, true, XContentType.JSON, MAPPING_INCLUDES, Set.of()).v2();
270+
Map<String, Object> map;
271+
if (maybeUsesPatternText == false) {
272+
var fullMap = XContentHelper.convertToMap(ref, true, XContentType.JSON).v2();
273+
maybeUsesPatternText = checkMappingForPatternText(fullMap);
274+
map = XContentMapValues.filter(fullMap, mappingIncludesArray, new String[0]);
275+
} else {
276+
map = XContentHelper.convertToMap(ref, true, XContentType.JSON, MAPPING_INCLUDES, Set.of()).v2();
277+
}
264278
filteredMappings.add(new CompressedXContent(map));
265279
}
266280
combinedTemplateMappings = filteredMappings;
@@ -277,7 +291,7 @@ MappingHints getMappingHints(
277291
|| addHostNameField
278292
|| (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
279293
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues());
280-
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
294+
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, maybeUsesPatternText);
281295
}
282296
} catch (AssertionError | Exception e) {
283297
// In case invalid mappings or setting are provided, then mapper service creation can fail.
@@ -288,6 +302,21 @@ MappingHints getMappingHints(
288302
}
289303
}
290304

305+
@SuppressWarnings("unchecked")
306+
private boolean checkMappingForPatternText(Map<String, Object> mapping) {
307+
var docMapping = mapping.get("_doc");
308+
if ((docMapping instanceof Map) == false) {
309+
return false;
310+
}
311+
boolean[] usesPatternText = { false };
312+
MappingVisitor.visitMapping((Map<String, Object>) docMapping, (field, fieldMapping) -> {
313+
if (Objects.equals(fieldMapping.get("type"), PatternTextFieldType.CONTENT_TYPE)) {
314+
usesPatternText[0] = true;
315+
}
316+
});
317+
return usesPatternText[0];
318+
}
319+
291320
// Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
292321
private IndexMetadata buildIndexMetadataForMapperService(
293322
String indexName,

‎x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java‎

Lines changed: 53 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1919
import org.elasticsearch.cluster.metadata.Template;
2020
import org.elasticsearch.common.compress.CompressedXContent;
21+
import org.elasticsearch.common.settings.Setting;
2122
import org.elasticsearch.common.settings.Settings;
2223
import org.elasticsearch.core.Tuple;
2324
import org.elasticsearch.index.IndexMode;
@@ -26,9 +27,12 @@
2627
import org.elasticsearch.index.IndexVersion;
2728
import org.elasticsearch.index.MapperTestUtils;
2829
import org.elasticsearch.index.mapper.SourceFieldMapper;
30+
import org.elasticsearch.indices.IndicesModule;
2931
import org.elasticsearch.license.License;
3032
import org.elasticsearch.license.LicenseService;
3133
import org.elasticsearch.license.MockLicenseState;
34+
import org.elasticsearch.license.XPackLicenseState;
35+
import org.elasticsearch.license.internal.XPackLicenseStatus;
3236
import org.elasticsearch.test.ESTestCase;
3337
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
3438
import org.junit.Before;
@@ -72,6 +76,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase {
7276
""";
7377

7478
private LogsdbLicenseService logsdbLicenseService;
79+
private LogsdbLicenseService basicLogsdbLicenseService;
7580
private final AtomicInteger newMapperServiceCounter = new AtomicInteger();
7681

7782
@Before
@@ -84,6 +89,13 @@ public void setup() throws Exception {
8489
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
8590
logsdbLicenseService.setLicenseState(licenseState);
8691
logsdbLicenseService.setLicenseService(mockLicenseService);
92+
93+
var basicLicenseState = new XPackLicenseState(() -> 0L, new XPackLicenseStatus(License.OperationMode.BASIC, true, null));
94+
var basicLicenseService = mock(LicenseService.class);
95+
when(basicLicenseService.getLicense()).thenReturn(null);
96+
basicLogsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
97+
basicLogsdbLicenseService.setLicenseState(basicLicenseState);
98+
basicLogsdbLicenseService.setLicenseService(basicLicenseService);
8799
}
88100

89101
private LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) {
@@ -121,13 +133,23 @@ private Settings generateLogsdbSettings(Settings settings, String mapping) throw
121133
}
122134

123135
private Settings generateLogsdbSettings(Settings settings, String mapping, Version version) throws IOException {
124-
var provider = new LogsdbIndexModeSettingsProvider(
125-
logsdbLicenseService,
126-
Settings.builder().put("cluster.logsdb.enabled", true).build()
127-
);
136+
return generateLogsdbSettings(settings, mapping, version, logsdbLicenseService);
137+
}
138+
139+
private Settings generateLogsdbSettings(Settings settings, String mapping, Version version, LogsdbLicenseService licenseService)
140+
throws IOException {
141+
var provider = new LogsdbIndexModeSettingsProvider(licenseService, Settings.builder().put("cluster.logsdb.enabled", true).build());
142+
var logsdbPlugin = new LogsDBPlugin(settings);
128143
provider.init(im -> {
129144
newMapperServiceCounter.incrementAndGet();
130-
return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
145+
return MapperTestUtils.newMapperService(
146+
xContentRegistry(),
147+
createTempDir(),
148+
im.getSettings(),
149+
new IndicesModule(List.of(logsdbPlugin)),
150+
im.getIndex().getName(),
151+
logsdbPlugin.getSettings().stream().filter(Setting::hasIndexScope).toArray(Setting<?>[]::new)
152+
);
131153
}, IndexVersion::current, () -> version, true, true);
132154
Settings.Builder settingsBuilder = builder();
133155
provider.provideAdditionalSettings(
@@ -959,19 +981,34 @@ public void testExplicitRoutingPathNotAllowedByLicense() throws Exception {
959981
assertThat(IndexMetadata.INDEX_ROUTING_PATH.get(result), empty());
960982
}
961983

962-
public void testPatternTextNotAllowedByLicense() throws Exception {
963-
MockLicenseState licenseState = MockLicenseState.createMock();
964-
when(licenseState.copyCurrentLicenseState()).thenReturn(licenseState);
965-
when(licenseState.isAllowed(same(LogsdbLicenseService.PATTERN_TEXT_TEMPLATING_FEATURE))).thenReturn(false);
966-
logsdbLicenseService = new LogsdbLicenseService(Settings.EMPTY);
967-
logsdbLicenseService.setLicenseState(licenseState);
984+
public void testPatternTextNotAllowedByLicense() throws IOException {
985+
String[] patternTextLicenceCheckedFieldMappings = {
986+
"{\"_doc\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}",
987+
"{\"_doc\":{\"properties\":{\"error\":{\"properties\":{\"message\":{\"type\":\"pattern_text\"}}}}}}",
988+
"{\"_doc\":{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}}",
989+
"{\"_doc\":{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}}" };
968990

969-
var settings = Settings.builder()
970-
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host,message")
971-
.put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), true)
991+
var expectedSettings = Settings.builder()
992+
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
993+
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
994+
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
972995
.build();
973-
Settings result = generateLogsdbSettings(settings);
974-
assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result));
996+
997+
for (String mapping : patternTextLicenceCheckedFieldMappings) {
998+
var result = generateLogsdbSettings(Settings.EMPTY, mapping, Version.CURRENT, basicLogsdbLicenseService);
999+
assertEquals(expectedSettings, result);
1000+
}
1001+
}
1002+
1003+
public void testPatternTextNotAllowedByLicenseAlreadyDisallowed() throws IOException {
1004+
Settings settings = Settings.builder().put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), "true").build();
1005+
var result = generateLogsdbSettings(settings, null, Version.CURRENT, basicLogsdbLicenseService);
1006+
var expected = Settings.builder()
1007+
.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true)
1008+
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
1009+
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
1010+
.build();
1011+
assertEquals(expected, result);
9751012
}
9761013

9771014
public void testSortAndHostNamePropagateValue() throws Exception {

‎x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java‎

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import org.elasticsearch.license.XPackLicenseState;
2020
import org.elasticsearch.license.internal.XPackLicenseStatus;
2121
import org.elasticsearch.test.ESTestCase;
22-
import org.elasticsearch.xpack.logsdb.patterntext.PatternTextFieldMapper;
2322
import org.junit.Before;
2423

2524
import java.io.IOException;
@@ -77,10 +76,7 @@ public void testGetAdditionalIndexSettingsDefault() {
7776
builder
7877
);
7978
var result = builder.build();
80-
var expected = Settings.builder()
81-
.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED")
82-
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
83-
.build();
79+
var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build();
8480
assertEquals(expected, result);
8581
}
8682

@@ -101,17 +97,11 @@ public void testGetAdditionalIndexSettingsApm() throws IOException {
10197
builder
10298
);
10399
var result = builder.build();
104-
Settings expectedAdditionalSettings = Settings.builder()
105-
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
106-
.build();
107-
assertEquals(expectedAdditionalSettings, result);
100+
assertEquals(Settings.EMPTY, result);
108101
}
109102

110103
public void testGetAdditionalIndexSettingsProfiling() throws IOException {
111104
Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build();
112-
Settings expectedAdditionalSettings = Settings.builder()
113-
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
114-
.build();
115105
for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) {
116106
String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0);
117107
Settings.Builder builder = Settings.builder();
@@ -127,14 +117,14 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException {
127117
builder
128118
);
129119
var result = builder.build();
130-
assertEquals(expectedAdditionalSettings, result);
120+
assertEquals(Settings.EMPTY, result);
131121
}
132122

133123
for (String indexName : new String[] { ".profiling-sq-executables", ".profiling-sq-leafframes", ".profiling-stacktraces" }) {
134124
Settings.Builder builder = Settings.builder();
135125
provider.provideAdditionalSettings(indexName, null, null, null, null, settings, List.of(), IndexVersion.current(), builder);
136126
var result = builder.build();
137-
assertEquals(expectedAdditionalSettings, result);
127+
assertEquals(Settings.EMPTY, result);
138128
}
139129
}
140130

@@ -155,10 +145,7 @@ public void testGetAdditionalIndexSettingsTsdb() throws IOException {
155145
builder
156146
);
157147
var result = builder.build();
158-
Settings expectedAdditionalSettings = Settings.builder()
159-
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
160-
.build();
161-
assertEquals(expectedAdditionalSettings, result);
148+
assertEquals(Settings.EMPTY, result);
162149
}
163150

164151
public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception {
@@ -202,10 +189,7 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception
202189
);
203190

204191
var result = builder.build();
205-
var expected = Settings.builder()
206-
.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED")
207-
.put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true)
208-
.build();
192+
var expected = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "STORED").build();
209193
assertEquals(expected, result);
210194
}
211195
}

0 commit comments

Comments
 (0)