Skip to content

Commit a5345fe

Browse files
kkrik-eskkrik-es
authored
[8.18] Licensing controls for logsdb routing on sort fields (#120276) (#122584)
* Licensing controls for logsdb routing on sort fields (#120276) * Restrict routing on sort fields to enterprise license * sync * bypass checking for serverless * Node deprecation warning for indexes and component templates with source mode in mapping * Revert "Node deprecation warning for indexes and component templates with source mode in mapping" This reverts commit 0fd4ca7. * address comments (cherry picked from commit 37f9745) # Conflicts: # x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java # x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java * Update LogsDBPlugin.java * Update LogsDBPlugin.java * Update LogsDBUsageTransportAction.java
1 parent ed36694 commit a5345fe

File tree

10 files changed

+192
-123
lines changed

10 files changed

+192
-123
lines changed

‎x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ public void testLogsdbRouteOnSortFields() throws IOException {
257257
var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
258258
assertEquals("logsdb", settings.get("index.mode"));
259259
assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
260-
assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
261-
assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
260+
assertEquals("false", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
261+
assertNull(settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
262262
}
263263
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.List;
2626
import java.util.Map;
2727

28+
import static org.hamcrest.Matchers.anyOf;
2829
import static org.hamcrest.Matchers.equalTo;
2930

3031
public class LogsdbRestIT extends ESRestTestCase {
@@ -73,9 +74,15 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
7374
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
7475
logger.info("response's features: {}", features);
7576
assertThat(features, Matchers.not(Matchers.empty()));
76-
Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
77-
assertThat(feature.get("name"), equalTo("synthetic-source"));
78-
assertThat(feature.get("license_level"), equalTo("enterprise"));
77+
boolean found = false;
78+
for (var feature : features) {
79+
if (feature.get("family") != null) {
80+
assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields")));
81+
assertThat(feature.get("license_level"), equalTo("enterprise"));
82+
found = true;
83+
}
84+
}
85+
assertTrue(found);
7986

8087
var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings("test-index").get("test-index")).get("settings");
8188
assertNull(settings.get("index.mapping.source.mode")); // Default, no downgrading.

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,12 @@
3131

3232
import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.LOGSDB_PRIOR_LOGS_USAGE;
3333
import static org.elasticsearch.xpack.logsdb.LogsPatternUsageService.USAGE_CHECK_MAX_PERIOD;
34-
import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
34+
import static org.elasticsearch.xpack.logsdb.LogsdbLicenseService.FALLBACK_SETTING;
3535

3636
public class LogsDBPlugin extends Plugin implements ActionPlugin {
3737

3838
private final Settings settings;
39-
private final SyntheticSourceLicenseService licenseService;
39+
private final LogsdbLicenseService licenseService;
4040
public static final Setting<Boolean> CLUSTER_LOGSDB_ENABLED = Setting.boolSetting(
4141
"cluster.logsdb.enabled",
4242
false,
@@ -48,7 +48,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin {
4848

4949
public LogsDBPlugin(Settings settings) {
5050
this.settings = settings;
51-
this.licenseService = new SyntheticSourceLicenseService(settings);
51+
this.licenseService = new LogsdbLicenseService(settings);
5252
this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings);
5353
}
5454

@@ -87,6 +87,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
8787
IndexVersion.current(),
8888
parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
8989
),
90+
DiscoveryNode.isStateless(settings) == false,
9091
DiscoveryNode.isStateless(settings) == false
9192
);
9293
return List.of(logsdbIndexModeSettingsProvider);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ protected void masterOperation(
6969
}
7070
}
7171
final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
72-
final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
72+
final boolean hasCustomCutoffDate = System.getProperty(LogsdbLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
7373
if (featureService.clusterHasFeature(state, XPackFeatures.LOGSDB_TELMETRY_STATS)) {
7474
final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
7575
final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java‎

Lines changed: 53 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,16 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
4949
static final String LOGS_PATTERN = "logs-*-*";
5050
private static final Set<String> MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects");
5151

52-
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
52+
private final LogsdbLicenseService licenseService;
5353
private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
5454
private final SetOnce<Supplier<IndexVersion>> createdIndexVersion = new SetOnce<>();
5555
private final SetOnce<Boolean> supportFallbackToStoredSource = new SetOnce<>();
56+
private final SetOnce<Boolean> supportFallbackLogsdbRouting = new SetOnce<>();
5657

5758
private volatile boolean isLogsdbEnabled;
5859

59-
LogsdbIndexModeSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService, final Settings settings) {
60-
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
60+
LogsdbIndexModeSettingsProvider(LogsdbLicenseService licenseService, final Settings settings) {
61+
this.licenseService = licenseService;
6162
this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings);
6263
}
6364

@@ -68,11 +69,13 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
6869
void init(
6970
CheckedFunction<IndexMetadata, MapperService, IOException> factory,
7071
Supplier<IndexVersion> indexVersion,
71-
boolean supportFallbackToStoredSource
72+
boolean supportFallbackToStoredSource,
73+
boolean supportFallbackLogsdbRouting
7274
) {
7375
this.mapperServiceFactory.set(factory);
7476
this.createdIndexVersion.set(indexVersion);
7577
this.supportFallbackToStoredSource.set(supportFallbackToStoredSource);
78+
this.supportFallbackLogsdbRouting.set(supportFallbackLogsdbRouting);
7679
}
7780

7881
@Override
@@ -93,6 +96,7 @@ public Settings getAdditionalIndexSettings(
9396
) {
9497
Settings.Builder settingsBuilder = null;
9598
boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB;
99+
boolean isTemplateValidation = "validate-index-name".equals(indexName);
96100

97101
// Inject logsdb index mode, based on the logs pattern.
98102
if (isLogsdbEnabled
@@ -110,76 +114,74 @@ && matchesLogsPattern(dataStreamName)) {
110114
if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) {
111115
// This index name is used when validating component and index templates, we should skip this check in that case.
112116
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
113-
boolean isTemplateValidation = "validate-index-name".equals(indexName);
114117
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
115118
templateIndexMode,
116119
indexName,
117120
dataStreamName
118121
);
119-
if (syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
122+
if (licenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
120123
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
121-
if (settingsBuilder == null) {
122-
settingsBuilder = Settings.builder();
123-
}
124-
settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString());
124+
settingsBuilder = getBuilder(settingsBuilder).put(
125+
IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
126+
SourceFieldMapper.Mode.STORED.toString()
127+
);
125128
}
126129
}
127130

128131
if (isLogsDB) {
129132
// Inject sorting on [host.name], in addition to [@timestamp].
130133
if (mappingHints.sortOnHostName) {
131-
if (settingsBuilder == null) {
132-
settingsBuilder = Settings.builder();
133-
}
134134
if (mappingHints.addHostNameField) {
135135
// Inject keyword field [host.name] too.
136-
settingsBuilder.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true);
136+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true);
137137
}
138-
settingsBuilder.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true);
138+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true);
139139
}
140140

141141
// Inject routing path matching sort fields.
142142
if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) {
143-
List<String> sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()));
144-
sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH));
145-
if (sortFields.size() < 2) {
146-
throw new IllegalStateException(
147-
String.format(
148-
Locale.ROOT,
149-
"data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields "
150-
+ "(excluding timestamp), needs at least 2",
151-
dataStreamName,
152-
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
153-
sortFields.size()
154-
)
155-
);
156-
}
157-
if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) {
158-
List<String> routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey());
159-
if (routingPaths.equals(sortFields) == false) {
143+
if (supportFallbackLogsdbRouting.get() == false || licenseService.allowLogsdbRoutingOnSortField(isTemplateValidation)) {
144+
List<String> sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()));
145+
sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH));
146+
if (sortFields.size() < 2) {
160147
throw new IllegalStateException(
161148
String.format(
162149
Locale.ROOT,
163-
"data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort "
164-
+ "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]",
150+
"data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields "
151+
+ "(excluding timestamp), needs at least 2",
165152
dataStreamName,
166153
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
167-
routingPaths,
168-
sortFields
154+
sortFields.size()
169155
)
170156
);
171157
}
172-
} else {
173-
if (settingsBuilder == null) {
174-
settingsBuilder = Settings.builder();
158+
if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) {
159+
List<String> routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey());
160+
if (routingPaths.equals(sortFields) == false) {
161+
throw new IllegalStateException(
162+
String.format(
163+
Locale.ROOT,
164+
"data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort "
165+
+ "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]",
166+
dataStreamName,
167+
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
168+
routingPaths,
169+
sortFields
170+
)
171+
);
172+
}
173+
} else {
174+
settingsBuilder = getBuilder(settingsBuilder).putList(INDEX_ROUTING_PATH.getKey(), sortFields);
175175
}
176-
settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields);
176+
} else {
177+
// Routing on sort fields is not allowed, reset the corresponding index setting.
178+
LOGGER.debug("creation of index [{}] with logsdb mode and routing on sort fields without it being allowed", indexName);
179+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false);
177180
}
178181
}
179182
}
180183

181184
return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build();
182-
183185
}
184186

185187
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
@@ -194,6 +196,14 @@ private static IndexMode resolveIndexMode(final String mode) {
194196
return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
195197
}
196198

199+
// Returned value needs to be reassigned to the passed arg, to track the created builder.
200+
private static Settings.Builder getBuilder(Settings.Builder builder) {
201+
if (builder == null) {
202+
return Settings.builder();
203+
}
204+
return builder;
205+
}
206+
197207
MappingHints getMappingHints(
198208
String indexName,
199209
IndexMode templateIndexMode,
@@ -260,8 +270,8 @@ MappingHints getMappingHints(
260270
|| mapperService.mappingLookup().getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED));
261271
boolean sortOnHostName = IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
262272
|| addHostNameField
263-
|| ((hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
264-
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()));
273+
|| (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
274+
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues());
265275
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
266276
}
267277
} catch (AssertionError | Exception e) {

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java‎ renamed to ‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@
2323
/**
2424
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
2525
*/
26-
final class SyntheticSourceLicenseService {
26+
final class LogsdbLicenseService {
2727

2828
static final String MAPPINGS_FEATURE_FAMILY = "mappings";
2929
// You can only override this property if you received explicit approval from Elastic.
3030
static final String CUTOFF_DATE_SYS_PROP_NAME = "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
31-
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class);
31+
private static final Logger LOGGER = LogManager.getLogger(LogsdbLicenseService.class);
3232
static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2025, 2, 4, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
3333

3434
/**
@@ -53,16 +53,22 @@ final class SyntheticSourceLicenseService {
5353
License.OperationMode.GOLD
5454
);
5555

56+
static final LicensedFeature.Momentary LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE = LicensedFeature.momentary(
57+
MAPPINGS_FEATURE_FAMILY,
58+
"logsdb-routing-on-sort-fields",
59+
License.OperationMode.ENTERPRISE
60+
);
61+
5662
private final long cutoffDate;
5763
private LicenseService licenseService;
5864
private XPackLicenseState licenseState;
5965
private volatile boolean syntheticSourceFallback;
6066

61-
SyntheticSourceLicenseService(Settings settings) {
67+
LogsdbLicenseService(Settings settings) {
6268
this(settings, System.getProperty(CUTOFF_DATE_SYS_PROP_NAME));
6369
}
6470

65-
SyntheticSourceLicenseService(Settings settings, String cutoffDate) {
71+
LogsdbLicenseService(Settings settings, String cutoffDate) {
6672
this.syntheticSourceFallback = FALLBACK_SETTING.get(settings);
6773
this.cutoffDate = getCutoffDate(cutoffDate);
6874
}
@@ -97,6 +103,13 @@ && checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplat
97103
return true;
98104
}
99105

106+
/**
107+
* @return whether indexes in logsdb mode can use routing on sort fields.
108+
*/
109+
public boolean allowLogsdbRoutingOnSortField(boolean isTemplateValidation) {
110+
return checkFeature(LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE, licenseState.copyCurrentLicenseState(), isTemplateValidation);
111+
}
112+
100113
private static boolean checkFeature(
101114
LicensedFeature.Momentary licensedFeature,
102115
XPackLicenseState licenseStateSnapshot,

0 commit comments

Comments
 (0)