Skip to content

Commit 74785f3

Browse files
martijnvgmartijnvg
authored
Add initial synthetic source fallback logic (#112994) (#113519)
Add initial code required to fallback synthetic source mode to stored source mode using an index settings provider. Note that the final version relies on a new index setting that determines source mode, which is currently controlled by `mode` mapping attribute in `_source` meta field mapper. Additionally index modes should not enforce synthetic source mode.
1 parent 5ef062c commit 74785f3

File tree

9 files changed

+396
-0
lines changed

9 files changed

+396
-0
lines changed

‎x-pack/plugin/logsdb/build.gradle‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import org.elasticsearch.gradle.internal.info.BuildParams
9+
10+
evaluationDependsOn(xpackModule('core'))
11+
12+
apply plugin: 'elasticsearch.internal-es-plugin'
13+
apply plugin: 'elasticsearch.internal-java-rest-test'
14+
15+
esplugin {
16+
name 'logsdb'
17+
description 'A plugin for logsdb related functionality'
18+
classname 'org.elasticsearch.xpack.logsdb.LogsDBPlugin'
19+
extendedPlugins = ['x-pack-core']
20+
}
21+
base {
22+
archivesName = 'x-pack-logsdb'
23+
}
24+
25+
dependencies {
26+
compileOnly project(path: xpackModule('core'))
27+
testImplementation(testArtifact(project(xpackModule('core'))))
28+
}
29+
30+
tasks.named("javaRestTest").configure {
31+
usesDefaultDistribution()
32+
}

‎x-pack/plugin/logsdb/qa/build.gradle‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+

‎x-pack/plugin/logsdb/qa/with-basic/build.gradle‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import org.elasticsearch.gradle.internal.info.BuildParams
9+
10+
apply plugin: 'elasticsearch.internal-java-rest-test'
11+
12+
dependencies {
13+
javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
14+
}
15+
16+
tasks.named("javaRestTest").configure {
17+
// This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
18+
BuildParams.withFipsEnabledOnly(it)
19+
20+
usesDefaultDistribution()
21+
}

‎x-pack/plugin/logsdb/qa/with-basic/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
12+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
13+
import org.elasticsearch.test.rest.ESRestTestCase;
14+
import org.hamcrest.Matchers;
15+
import org.junit.ClassRule;
16+
17+
import java.io.IOException;
18+
import java.util.List;
19+
import java.util.Map;
20+
21+
public class LogsdbRestIT extends ESRestTestCase {
22+
23+
@ClassRule
24+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
25+
.distribution(DistributionType.DEFAULT)
26+
.setting("xpack.license.self_generated.type", "basic")
27+
.setting("xpack.security.enabled", "false")
28+
.build();
29+
30+
@Override
31+
protected String getTestRestCluster() {
32+
return cluster.getHttpAddresses();
33+
}
34+
35+
public void testFeatureUsageWithLogsdbIndex() throws IOException {
36+
{
37+
var response = getAsMap("/_license/feature_usage");
38+
@SuppressWarnings("unchecked")
39+
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
40+
assertThat(features, Matchers.empty());
41+
}
42+
{
43+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
44+
var response = getAsMap("/_license/feature_usage");
45+
@SuppressWarnings("unchecked")
46+
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
47+
assertThat(features, Matchers.empty());
48+
}
49+
}
50+
51+
}

‎x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbRestIT.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
12+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
13+
import org.elasticsearch.test.rest.ESRestTestCase;
14+
import org.hamcrest.Matchers;
15+
import org.junit.ClassRule;
16+
17+
import java.io.IOException;
18+
import java.util.List;
19+
import java.util.Map;
20+
21+
import static org.hamcrest.Matchers.equalTo;
22+
23+
public class LogsdbRestIT extends ESRestTestCase {
24+
25+
@ClassRule
26+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
27+
.distribution(DistributionType.DEFAULT)
28+
.setting("xpack.security.enabled", "false")
29+
.setting("xpack.license.self_generated.type", "trial")
30+
.build();
31+
32+
@Override
33+
protected String getTestRestCluster() {
34+
return cluster.getHttpAddresses();
35+
}
36+
37+
public void testFeatureUsageWithLogsdbIndex() throws IOException {
38+
{
39+
var response = getAsMap("/_license/feature_usage");
40+
@SuppressWarnings("unchecked")
41+
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
42+
assertThat(features, Matchers.empty());
43+
}
44+
{
45+
createIndex("test-index", Settings.builder().put("index.mode", "logsdb").build());
46+
var response = getAsMap("/_license/feature_usage");
47+
@SuppressWarnings("unchecked")
48+
List<Map<?, ?>> features = (List<Map<?, ?>>) response.get("features");
49+
logger.info("response's features: {}", features);
50+
assertThat(features, Matchers.not(Matchers.empty()));
51+
Map<?, ?> feature = features.stream().filter(map -> "mappings".equals(map.get("family"))).findFirst().get();
52+
assertThat(feature.get("name"), equalTo("synthetic-source"));
53+
assertThat(feature.get("license_level"), equalTo("enterprise"));
54+
}
55+
}
56+
57+
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.elasticsearch.cluster.node.DiscoveryNode;
11+
import org.elasticsearch.common.settings.Setting;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.index.IndexSettingProvider;
14+
import org.elasticsearch.plugins.Plugin;
15+
import org.elasticsearch.xpack.core.XPackPlugin;
16+
17+
import java.util.Collection;
18+
import java.util.List;
19+
20+
import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
21+
22+
public class LogsDBPlugin extends Plugin {
23+
24+
private final Settings settings;
25+
private final SyntheticSourceLicenseService licenseService;
26+
27+
public LogsDBPlugin(Settings settings) {
28+
this.settings = settings;
29+
this.licenseService = new SyntheticSourceLicenseService(settings);
30+
}
31+
32+
@Override
33+
public Collection<?> createComponents(PluginServices services) {
34+
licenseService.setLicenseState(XPackPlugin.getSharedLicenseState());
35+
var clusterSettings = services.clusterService().getClusterSettings();
36+
clusterSettings.addSettingsUpdateConsumer(FALLBACK_SETTING, licenseService::setSyntheticSourceFallback);
37+
// Nothing to share here:
38+
return super.createComponents(services);
39+
}
40+
41+
@Override
42+
public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
43+
if (DiscoveryNode.isStateless(settings)) {
44+
return List.of();
45+
}
46+
return List.of(new SyntheticSourceIndexSettingsProvider(licenseService));
47+
}
48+
49+
@Override
50+
public List<Setting<?>> getSettings() {
51+
return List.of(FALLBACK_SETTING);
52+
}
53+
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.apache.logging.log4j.LogManager;
11+
import org.apache.logging.log4j.Logger;
12+
import org.elasticsearch.cluster.metadata.Metadata;
13+
import org.elasticsearch.common.compress.CompressedXContent;
14+
import org.elasticsearch.common.settings.Settings;
15+
import org.elasticsearch.index.IndexMode;
16+
import org.elasticsearch.index.IndexSettingProvider;
17+
import org.elasticsearch.index.IndexSettings;
18+
19+
import java.time.Instant;
20+
import java.util.List;
21+
import java.util.Locale;
22+
23+
/**
24+
* An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used.
25+
*/
26+
public class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider {
27+
28+
private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class);
29+
30+
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
31+
32+
public SyntheticSourceIndexSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService) {
33+
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
34+
}
35+
36+
@Override
37+
public Settings getAdditionalIndexSettings(
38+
String indexName,
39+
String dataStreamName,
40+
boolean isTimeSeries,
41+
Metadata metadata,
42+
Instant resolvedAt,
43+
Settings indexTemplateAndCreateRequestSettings,
44+
List<CompressedXContent> combinedTemplateMappings
45+
) {
46+
if (newIndexHasSyntheticSourceUsage(indexTemplateAndCreateRequestSettings)
47+
&& syntheticSourceLicenseService.fallbackToStoredSource()) {
48+
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
49+
// TODO: handle falling back to stored source
50+
}
51+
return Settings.EMPTY;
52+
}
53+
54+
boolean newIndexHasSyntheticSourceUsage(Settings indexTemplateAndCreateRequestSettings) {
55+
// TODO: build tmp MapperService and check whether SourceFieldMapper#isSynthetic() to determine synthetic source usage.
56+
// Not using IndexSettings.MODE.get() to avoid validation that may fail at this point.
57+
var rawIndexMode = indexTemplateAndCreateRequestSettings.get(IndexSettings.MODE.getKey());
58+
IndexMode indexMode = rawIndexMode != null ? Enum.valueOf(IndexMode.class, rawIndexMode.toUpperCase(Locale.ROOT)) : null;
59+
return indexMode != null && indexMode.isSyntheticSourceEnabled();
60+
}
61+
}

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.logsdb;
9+
10+
import org.elasticsearch.common.settings.Setting;
11+
import org.elasticsearch.common.settings.Settings;
12+
import org.elasticsearch.license.License;
13+
import org.elasticsearch.license.LicensedFeature;
14+
import org.elasticsearch.license.XPackLicenseState;
15+
16+
/**
17+
* Determines based on license and fallback setting whether synthetic source usages should fallback to stored source.
18+
*/
19+
public final class SyntheticSourceLicenseService {
20+
21+
private static final String MAPPINGS_FEATURE_FAMILY = "mappings";
22+
23+
/**
24+
* A setting that determines whether source mode should always be stored source. Regardless of licence.
25+
*/
26+
public static final Setting<Boolean> FALLBACK_SETTING = Setting.boolSetting(
27+
"xpack.mapping.synthetic_source_fallback_to_stored_source",
28+
false,
29+
Setting.Property.NodeScope,
30+
Setting.Property.Dynamic
31+
);
32+
33+
private static final LicensedFeature.Momentary SYNTHETIC_SOURCE_FEATURE = LicensedFeature.momentary(
34+
MAPPINGS_FEATURE_FAMILY,
35+
"synthetic-source",
36+
License.OperationMode.ENTERPRISE
37+
);
38+
39+
private XPackLicenseState licenseState;
40+
private volatile boolean syntheticSourceFallback;
41+
42+
public SyntheticSourceLicenseService(Settings settings) {
43+
syntheticSourceFallback = FALLBACK_SETTING.get(settings);
44+
}
45+
46+
/**
47+
* @return whether synthetic source mode should fallback to stored source.
48+
*/
49+
public boolean fallbackToStoredSource() {
50+
if (syntheticSourceFallback) {
51+
return true;
52+
}
53+
54+
return SYNTHETIC_SOURCE_FEATURE.check(licenseState) == false;
55+
}
56+
57+
void setSyntheticSourceFallback(boolean syntheticSourceFallback) {
58+
this.syntheticSourceFallback = syntheticSourceFallback;
59+
}
60+
61+
void setLicenseState(XPackLicenseState licenseState) {
62+
this.licenseState = licenseState;
63+
}
64+
}

0 commit comments

Comments
 (0)