Skip to content

Commit 37f9745

Browse files
kkrik-eskkrik-es
authored
Licensing controls for logsdb routing on sort fields (#120276)
* Restrict routing on sort fields to enterprise license * sync * bypass checking for serverless * Node deprecation warning for indexes and component templates with source mode in mapping * Revert "Node deprecation warning for indexes and component templates with source mode in mapping" This reverts commit 0fd4ca7. * address comments
1 parent 7781345 commit 37f9745

File tree

10 files changed

+192
-124
lines changed

10 files changed

+192
-124
lines changed

‎x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ public void testLogsdbRouteOnSortFields() throws IOException {
235235
var settings = (Map<?, ?>) ((Map<?, ?>) getIndexSettings(index).get(index)).get("settings");
236236
assertEquals("logsdb", settings.get("index.mode"));
237237
assertEquals(SourceFieldMapper.Mode.STORED.toString(), settings.get("index.mapping.source.mode"));
238-
assertEquals("true", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
239-
assertEquals(List.of("host.name", "message"), settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
238+
assertEquals("false", settings.get(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey()));
239+
assertNull(settings.get(IndexMetadata.INDEX_ROUTING_PATH.getKey()));
240240
}
241241
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.List;
2626
import java.util.Map;
2727

28+
import static org.hamcrest.Matchers.anyOf;
2829
import static org.hamcrest.Matchers.equalTo;
2930

3031
public class LogsdbRestIT extends ESRestTestCase {
@@ -66,9 +67,15 @@ public void testFeatureUsageWithLogsdbIndex() throws IOException {
6667
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
6768
logger.info("response's features: {}", features);
6869
assertThat(features, Matchers.not(Matchers.empty()));
69-
Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
70-
assertThat(feature.get("name"), equalTo("synthetic-source"));
71-
assertThat(feature.get("license_level"), equalTo("enterprise"));
70+
boolean found = false;
71+
for (var feature : features) {
72+
if (feature.get("family") != null) {
73+
assertThat(feature.get("name"), anyOf(equalTo("synthetic-source"), equalTo("logsdb-routing-on-sort-fields")));
74+
assertThat(feature.get("license_level"), equalTo("enterprise"));
75+
found = true;
76+
}
77+
}
78+
assertTrue(found);
7279

7380
var indexResponse = (Map<?, ?>) getIndexSettings("test-index", true).get("test-index");
7481
logger.info("indexResponse: {}", indexResponse);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,12 @@
4040
import java.util.function.Predicate;
4141
import java.util.function.Supplier;
4242

43-
import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
43+
import static org.elasticsearch.xpack.logsdb.LogsdbLicenseService.FALLBACK_SETTING;
4444

4545
public class LogsDBPlugin extends Plugin implements ActionPlugin {
4646

4747
private final Settings settings;
48-
private final SyntheticSourceLicenseService licenseService;
48+
private final LogsdbLicenseService licenseService;
4949
private static final Setting<Boolean> LOGSDB_PRIOR_LOGS_USAGE = Setting.boolSetting(
5050
"logsdb.prior_logs_usage",
5151
false,
@@ -63,7 +63,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin {
6363

6464
public LogsDBPlugin(Settings settings) {
6565
this.settings = settings;
66-
this.licenseService = new SyntheticSourceLicenseService(settings);
66+
this.licenseService = new LogsdbLicenseService(settings);
6767
this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings);
6868
}
6969

@@ -82,7 +82,6 @@ public Collection<?> createComponents(PluginServices services) {
8282
CLUSTER_LOGSDB_ENABLED,
8383
logsdbIndexModeSettingsProvider::updateClusterIndexModeLogsdbEnabled
8484
);
85-
8685
// Nothing to share here:
8786
return super.createComponents(services);
8887
}
@@ -95,6 +94,7 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
9594
IndexVersion.current(),
9695
parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
9796
),
97+
DiscoveryNode.isStateless(settings) == false,
9898
DiscoveryNode.isStateless(settings) == false
9999
);
100100
return List.of(logsdbIndexModeSettingsProvider);

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ protected void masterOperation(
6464
}
6565
}
6666
final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
67-
final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
67+
final boolean hasCustomCutoffDate = System.getProperty(LogsdbLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
6868
final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
6969
final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);
7070
final int finalNumIndices = numIndices;

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java‎

Lines changed: 53 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,16 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
4949
static final String LOGS_PATTERN = "logs-*-*";
5050
private static final Set<String> MAPPING_INCLUDES = Set.of("_doc._source.*", "_doc.properties.host**", "_doc.subobjects");
5151

52-
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
52+
private final LogsdbLicenseService licenseService;
5353
private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
5454
private final SetOnce<Supplier<IndexVersion>> createdIndexVersion = new SetOnce<>();
5555
private final SetOnce<Boolean> supportFallbackToStoredSource = new SetOnce<>();
56+
private final SetOnce<Boolean> supportFallbackLogsdbRouting = new SetOnce<>();
5657

5758
private volatile boolean isLogsdbEnabled;
5859

59-
LogsdbIndexModeSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService, final Settings settings) {
60-
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
60+
LogsdbIndexModeSettingsProvider(LogsdbLicenseService licenseService, final Settings settings) {
61+
this.licenseService = licenseService;
6162
this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings);
6263
}
6364

@@ -68,11 +69,13 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
6869
void init(
6970
CheckedFunction<IndexMetadata, MapperService, IOException> factory,
7071
Supplier<IndexVersion> indexVersion,
71-
boolean supportFallbackToStoredSource
72+
boolean supportFallbackToStoredSource,
73+
boolean supportFallbackLogsdbRouting
7274
) {
7375
this.mapperServiceFactory.set(factory);
7476
this.createdIndexVersion.set(indexVersion);
7577
this.supportFallbackToStoredSource.set(supportFallbackToStoredSource);
78+
this.supportFallbackLogsdbRouting.set(supportFallbackLogsdbRouting);
7679
}
7780

7881
@Override
@@ -93,6 +96,7 @@ public Settings getAdditionalIndexSettings(
9396
) {
9497
Settings.Builder settingsBuilder = null;
9598
boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB;
99+
boolean isTemplateValidation = "validate-index-name".equals(indexName);
96100

97101
// Inject logsdb index mode, based on the logs pattern.
98102
if (isLogsdbEnabled
@@ -110,76 +114,74 @@ && matchesLogsPattern(dataStreamName)) {
110114
if (mappingHints.hasSyntheticSourceUsage && supportFallbackToStoredSource.get()) {
111115
// This index name is used when validating component and index templates, we should skip this check in that case.
112116
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
113-
boolean isTemplateValidation = "validate-index-name".equals(indexName);
114117
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
115118
templateIndexMode,
116119
indexName,
117120
dataStreamName
118121
);
119-
if (syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
122+
if (licenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) {
120123
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
121-
if (settingsBuilder == null) {
122-
settingsBuilder = Settings.builder();
123-
}
124-
settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString());
124+
settingsBuilder = getBuilder(settingsBuilder).put(
125+
IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(),
126+
SourceFieldMapper.Mode.STORED.toString()
127+
);
125128
}
126129
}
127130

128131
if (isLogsDB) {
129132
// Inject sorting on [host.name], in addition to [@timestamp].
130133
if (mappingHints.sortOnHostName) {
131-
if (settingsBuilder == null) {
132-
settingsBuilder = Settings.builder();
133-
}
134134
if (mappingHints.addHostNameField) {
135135
// Inject keyword field [host.name] too.
136-
settingsBuilder.put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true);
136+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true);
137137
}
138-
settingsBuilder.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true);
138+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true);
139139
}
140140

141141
// Inject routing path matching sort fields.
142142
if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) {
143-
List<String> sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()));
144-
sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH));
145-
if (sortFields.size() < 2) {
146-
throw new IllegalStateException(
147-
String.format(
148-
Locale.ROOT,
149-
"data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields "
150-
+ "(excluding timestamp), needs at least 2",
151-
dataStreamName,
152-
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
153-
sortFields.size()
154-
)
155-
);
156-
}
157-
if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) {
158-
List<String> routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey());
159-
if (routingPaths.equals(sortFields) == false) {
143+
if (supportFallbackLogsdbRouting.get() == false || licenseService.allowLogsdbRoutingOnSortField(isTemplateValidation)) {
144+
List<String> sortFields = new ArrayList<>(settings.getAsList(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()));
145+
sortFields.removeIf(s -> s.equals(DataStreamTimestampFieldMapper.DEFAULT_PATH));
146+
if (sortFields.size() < 2) {
160147
throw new IllegalStateException(
161148
String.format(
162149
Locale.ROOT,
163-
"data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort "
164-
+ "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]",
150+
"data stream [%s] in logsdb mode and with [%s] index setting has only %d sort fields "
151+
+ "(excluding timestamp), needs at least 2",
165152
dataStreamName,
166153
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
167-
routingPaths,
168-
sortFields
154+
sortFields.size()
169155
)
170156
);
171157
}
172-
} else {
173-
if (settingsBuilder == null) {
174-
settingsBuilder = Settings.builder();
158+
if (settings.hasValue(IndexMetadata.INDEX_ROUTING_PATH.getKey())) {
159+
List<String> routingPaths = settings.getAsList(IndexMetadata.INDEX_ROUTING_PATH.getKey());
160+
if (routingPaths.equals(sortFields) == false) {
161+
throw new IllegalStateException(
162+
String.format(
163+
Locale.ROOT,
164+
"data stream [%s] in logsdb mode and with [%s] index setting has mismatching sort "
165+
+ "and routing fields, [index.routing_path:%s], [index.sort.fields:%s]",
166+
dataStreamName,
167+
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(),
168+
routingPaths,
169+
sortFields
170+
)
171+
);
172+
}
173+
} else {
174+
settingsBuilder = getBuilder(settingsBuilder).putList(INDEX_ROUTING_PATH.getKey(), sortFields);
175175
}
176-
settingsBuilder.putList(INDEX_ROUTING_PATH.getKey(), sortFields);
176+
} else {
177+
// Routing on sort fields is not allowed, reset the corresponding index setting.
178+
LOGGER.debug("creation of index [{}] with logsdb mode and routing on sort fields without it being allowed", indexName);
179+
settingsBuilder = getBuilder(settingsBuilder).put(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false);
177180
}
178181
}
179182
}
180183

181184
return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build();
182-
183185
}
184186

185187
record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField) {
@@ -194,6 +196,14 @@ private static IndexMode resolveIndexMode(final String mode) {
194196
return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
195197
}
196198

199+
// Returned value needs to be reassigned to the passed arg, to track the created builder.
200+
private static Settings.Builder getBuilder(Settings.Builder builder) {
201+
if (builder == null) {
202+
return Settings.builder();
203+
}
204+
return builder;
205+
}
206+
197207
MappingHints getMappingHints(
198208
String indexName,
199209
IndexMode templateIndexMode,
@@ -260,8 +270,8 @@ MappingHints getMappingHints(
260270
|| mapperService.mappingLookup().getMapping().getRoot().subobjects() == ObjectMapper.Subobjects.DISABLED));
261271
boolean sortOnHostName = IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings)
262272
|| addHostNameField
263-
|| ((hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
264-
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()));
273+
|| (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues())
274+
|| (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues());
265275
return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField);
266276
}
267277
} catch (AssertionError | Exception e) {

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java renamed to x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbLicenseService.java

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@
2323
/**
2424
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
2525
*/
26-
final class SyntheticSourceLicenseService {
26+
final class LogsdbLicenseService {
2727

2828
static final String MAPPINGS_FEATURE_FAMILY = "mappings";
2929
// You can only override this property if you received explicit approval from Elastic.
3030
static final String CUTOFF_DATE_SYS_PROP_NAME = "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
31-
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class);
31+
private static final Logger LOGGER = LogManager.getLogger(LogsdbLicenseService.class);
3232
static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2025, 2, 4, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
3333

3434
/**
@@ -53,16 +53,22 @@ final class SyntheticSourceLicenseService {
5353
License.OperationMode.GOLD
5454
);
5555

56+
static final LicensedFeature.Momentary LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE = LicensedFeature.momentary(
57+
MAPPINGS_FEATURE_FAMILY,
58+
"logsdb-routing-on-sort-fields",
59+
License.OperationMode.ENTERPRISE
60+
);
61+
5662
private final long cutoffDate;
5763
private LicenseService licenseService;
5864
private XPackLicenseState licenseState;
5965
private volatile boolean syntheticSourceFallback;
6066

61-
SyntheticSourceLicenseService(Settings settings) {
67+
LogsdbLicenseService(Settings settings) {
6268
this(settings, System.getProperty(CUTOFF_DATE_SYS_PROP_NAME));
6369
}
6470

65-
SyntheticSourceLicenseService(Settings settings, String cutoffDate) {
71+
LogsdbLicenseService(Settings settings, String cutoffDate) {
6672
this.syntheticSourceFallback = FALLBACK_SETTING.get(settings);
6773
this.cutoffDate = getCutoffDate(cutoffDate);
6874
}
@@ -97,6 +103,13 @@ && checkFeature(SYNTHETIC_SOURCE_FEATURE_LEGACY, licenseStateSnapshot, isTemplat
97103
return true;
98104
}
99105

106+
/**
107+
* @return whether indexes in logsdb mode can use routing on sort fields.
108+
*/
109+
public boolean allowLogsdbRoutingOnSortField(boolean isTemplateValidation) {
110+
return checkFeature(LOGSDB_ROUTING_ON_SORT_FIELDS_FEATURE, licenseState.copyCurrentLicenseState(), isTemplateValidation);
111+
}
112+
100113
private static boolean checkFeature(
101114
LicensedFeature.Momentary licensedFeature,
102115
XPackLicenseState licenseStateSnapshot,

0 commit comments

Comments
 (0)