Skip to content

Commit f66f5bb

Browse files
jimczijimczi
authored
[inference_metadata_fields] Introduce a final index setting for semantic text format selection (#119070)
This change replaces the system property used for enabling the new semantic text format with a dedicated index setting resolved at index creation. The new format is tied to the index version, so this approach ensures that indices created with the new version but using the legacy format retain their original configuration. By sealing the format at index creation, we prevent unexpected changes and provide flexibility in tests to explicitly choose the desired format for execution.
1 parent 8f37562 commit f66f5bb

File tree

20 files changed

+369
-371
lines changed

20 files changed

+369
-371
lines changed

‎server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ static boolean executeBulkItemRequest(
405405
}
406406

407407
private static String[] getStoredFieldsSpec(IndexShard indexShard) {
408-
if (InferenceMetadataFieldsMapper.isEnabled(indexShard.indexSettings().getIndexVersionCreated())) {
408+
if (InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) {
409409
if (indexShard.mapperService().mappingLookup().inferenceFields().size() > 0) {
410410
// Retrieves the inference metadata field containing the inference results for all semantic fields defined in the mapping.
411411
return new String[] { RoutingFieldMapper.NAME, InferenceMetadataFieldsMapper.NAME };

‎server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -375,8 +375,7 @@ private static UpdateHelper.Result deleteInferenceResults(
375375
IndexMetadata indexMetadata,
376376
MappingLookup mappingLookup
377377
) {
378-
if (result.getResponseResult() != DocWriteResponse.Result.UPDATED
379-
|| InferenceMetadataFieldsMapper.isEnabled(indexMetadata.getCreationVersion())) {
378+
if (result.getResponseResult() != DocWriteResponse.Result.UPDATED || InferenceMetadataFieldsMapper.isEnabled(mappingLookup)) {
380379
return result;
381380
}
382381

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.index.fielddata.IndexFieldDataService;
3535
import org.elasticsearch.index.mapper.FieldMapper;
3636
import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
37+
import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
3738
import org.elasticsearch.index.mapper.MapperService;
3839
import org.elasticsearch.index.mapper.SourceFieldMapper;
3940
import org.elasticsearch.index.similarity.SimilarityService;
@@ -190,6 +191,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
190191
IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
191192
SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING,
192193
IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
194+
InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
193195

194196
// validate that built-in similarities don't get redefined
195197
Setting.groupSetting("index.similarity.", (s) -> {

‎server/src/main/java/org/elasticsearch/index/get/ShardGetService.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,8 @@ private GetResult innerGetFetch(
303303
boolean hasInferenceMetadataFields = false;
304304
if (storedFields != null) {
305305
for (String field : storedFields) {
306-
if (field.equals(InferenceMetadataFieldsMapper.NAME) && InferenceMetadataFieldsMapper.isEnabled(indexVersion)) {
306+
if (field.equals(InferenceMetadataFieldsMapper.NAME)
307+
&& InferenceMetadataFieldsMapper.isEnabled(indexShard.mapperService().mappingLookup())) {
307308
hasInferenceMetadataFields = true;
308309
continue;
309310
}

‎server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java‎

Lines changed: 46 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,12 @@
99

1010
package org.elasticsearch.index.mapper;
1111

12-
import org.apache.logging.log4j.LogManager;
13-
import org.apache.logging.log4j.Logger;
1412
import org.apache.lucene.search.IndexSearcher;
1513
import org.apache.lucene.search.Query;
1614
import org.apache.lucene.search.join.BitSetProducer;
17-
import org.elasticsearch.core.Booleans;
18-
import org.elasticsearch.index.IndexVersion;
15+
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.common.settings.Setting;
17+
import org.elasticsearch.common.settings.Settings;
1918
import org.elasticsearch.index.IndexVersions;
2019
import org.elasticsearch.index.query.SearchExecutionContext;
2120

@@ -28,37 +27,20 @@
2827
* the field name for removal from _source.
2928
*/
3029
public abstract class InferenceMetadataFieldsMapper extends MetadataFieldMapper {
31-
public static class SystemProperty {
32-
private static final Logger logger = LogManager.getLogger(SystemProperty.class);
33-
34-
private final boolean enabled;
35-
36-
private SystemProperty(String name) {
37-
String propertyName = "es." + name;
38-
this.enabled = parseSystemProperty(propertyName, false);
39-
if (this.enabled) {
40-
logger.info("Feature " + name + " (via system property '" + propertyName + "') is enabled");
41-
} else {
42-
logger.debug("Feature " + name + " (via system property '" + propertyName + "') is disabled");
43-
}
44-
}
45-
46-
private boolean parseSystemProperty(String propertyName, boolean defaultValue) {
47-
final String propertyValue = System.getProperty(propertyName);
48-
logger.trace("System property [{}] is set to [{}]", propertyName, propertyValue);
49-
try {
50-
return Booleans.parseBoolean(propertyValue, defaultValue);
51-
} catch (IllegalArgumentException e) {
52-
throw new IllegalArgumentException("Invalid value [" + propertyValue + "] for system property [" + propertyName + "]", e);
53-
}
54-
}
55-
56-
public boolean isEnabled() {
57-
return enabled;
58-
}
59-
}
60-
61-
public static final SystemProperty INFERENCE_METADATA_FIELDS_SYSTEM_PROPERTY = new SystemProperty("inference_metadata_fields");
30+
/**
31+
* Internal index setting to control the format used for semantic text fields.
32+
* Determines whether to use the legacy format (default: true).
33+
* This setting is immutable and can only be defined at index creation
34+
* to ensure the internal format remains consistent throughout the index's lifecycle.
35+
*/
36+
public static final Setting<Boolean> USE_LEGACY_SEMANTIC_TEXT_FORMAT = Setting.boolSetting(
37+
"index.mapping.semantic_text.use_legacy_format",
38+
// don't use the new format by default yet
39+
true,
40+
Setting.Property.Final,
41+
Setting.Property.IndexScope,
42+
Setting.Property.InternalIndex
43+
);
6244

6345
public static final String NAME = "_inference_fields";
6446
public static final String CONTENT_TYPE = "_inference_fields";
@@ -92,7 +74,34 @@ public abstract ValueFetcher valueFetcher(
9274
);
9375
}
9476

95-
public static boolean isEnabled(IndexVersion indexVersion) {
96-
return indexVersion.onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS) && INFERENCE_METADATA_FIELDS_SYSTEM_PROPERTY.isEnabled();
77+
/**
78+
* Checks if the {@link InferenceMetadataFieldsMapper} is enabled for the given {@link Settings}.
79+
*
80+
* This indicates whether the new format for semantic text fields is active.
81+
* The new format is enabled if:
82+
* 1. The index version is on or after {@link IndexVersions#INFERENCE_METADATA_FIELDS}, and
83+
* 2. The legacy semantic text format is disabled.
84+
*
85+
* @param settings the index settings to evaluate
86+
* @return {@code true} if the new format is enabled; {@code false} otherwise
87+
*/
88+
public static boolean isEnabled(Settings settings) {
89+
return IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS)
90+
&& USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false;
91+
}
92+
93+
/**
94+
* Checks if the {@link InferenceMetadataFieldsMapper} is enabled based on the provided {@link Mapping}.
95+
*
96+
* This indicates whether the new format for semantic text fields is active by verifying the existence
97+
* of the {@link InferenceMetadataFieldsMapper} in the mapping's metadata.
98+
*
99+
* @param mappingLookup the mapping to evaluate
100+
* @return {@code true} if the {@link InferenceMetadataFieldsMapper} is present; {@code false} otherwise
101+
*/
102+
public static boolean isEnabled(MappingLookup mappingLookup) {
103+
return mappingLookup != null
104+
&& mappingLookup.getMapping()
105+
.getMetadataMapperByName(InferenceMetadataFieldsMapper.NAME) instanceof InferenceMetadataFieldsMapper;
97106
}
98107
}

‎server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,10 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
245245
this.indexSettings = indexSettings;
246246
}
247247

248+
public IndexSettings indexSettings() {
249+
return indexSettings;
250+
}
251+
248252
public Query parentTypeFilter() {
249253
return parentTypeFilter;
250254
}

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,7 @@ public BytesReference applyFilters(
443443
}
444444
var modSourceFilter = sourceFilter;
445445
if (context != null
446-
&& InferenceMetadataFieldsMapper.isEnabled(context.indexSettings().getIndexVersionCreated())
446+
&& InferenceMetadataFieldsMapper.isEnabled(context.mappingLookup())
447447
&& context.mappingLookup().inferenceFields().isEmpty() == false) {
448448
String[] modExcludes = new String[excludes != null ? excludes.length + 1 : 1];
449449
if (excludes != null) {

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,7 @@ private void hitExecute(HitContext hitContext) {
9090
* to the original _source if it has been requested.
9191
*/
9292
private Source replaceInferenceMetadataFields(SearchHit hit, Source source) {
93-
if (InferenceMetadataFieldsMapper.isEnabled(
94-
fetchContext.getSearchExecutionContext().getIndexSettings().getIndexVersionCreated()
95-
) == false) {
93+
if (InferenceMetadataFieldsMapper.isEnabled(fetchContext.getSearchExecutionContext().getMappingLookup()) == false) {
9694
return source;
9795
}
9896

‎x-pack/plugin/inference/build.gradle‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -399,17 +399,7 @@ tasks.named("thirdPartyAudit").configure {
399399
)
400400
}
401401

402-
tasks.named('test') {
403-
if (buildParams.isSnapshotBuild()) {
404-
systemProperty('es.inference_metadata_fields', 'true')
405-
systemProperty('tests.jvm.argline', '-Des.inference_metadata_fields=true')
406-
}
407-
}
408-
409402
tasks.named('yamlRestTest') {
410403
usesDefaultDistribution()
411-
if (buildParams.isSnapshotBuild()) {
412-
systemProperty('tests.jvm.argline', '-Des.inference_metadata_fields=true')
413-
}
414404
}
415405

‎x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
package org.elasticsearch.xpack.inference.action.filter;
99

10+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
11+
1012
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
1113
import org.elasticsearch.action.bulk.BulkItemResponse;
1214
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -19,13 +21,12 @@
1921
import org.elasticsearch.cluster.metadata.IndexMetadata;
2022
import org.elasticsearch.common.settings.Settings;
2123
import org.elasticsearch.index.IndexVersion;
22-
import org.elasticsearch.index.IndexVersions;
24+
import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
2325
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
2426
import org.elasticsearch.inference.SimilarityMeasure;
2527
import org.elasticsearch.plugins.Plugin;
2628
import org.elasticsearch.search.builder.SearchSourceBuilder;
2729
import org.elasticsearch.test.ESIntegTestCase;
28-
import org.elasticsearch.test.index.IndexVersionUtils;
2930
import org.elasticsearch.xpack.inference.Utils;
3031
import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension;
3132
import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension;
@@ -35,6 +36,7 @@
3536
import java.util.Collection;
3637
import java.util.HashMap;
3738
import java.util.HashSet;
39+
import java.util.List;
3840
import java.util.Locale;
3941
import java.util.Map;
4042
import java.util.Set;
@@ -45,7 +47,16 @@
4547
public class ShardBulkInferenceActionFilterIT extends ESIntegTestCase {
4648
public static final String INDEX_NAME = "test-index";
4749

48-
private IndexVersion indexVersion;
50+
private final boolean useLegacyFormat;
51+
52+
public ShardBulkInferenceActionFilterIT(boolean useLegacyFormat) {
53+
this.useLegacyFormat = useLegacyFormat;
54+
}
55+
56+
@ParametersFactory
57+
public static Iterable<Object[]> parameters() throws Exception {
58+
return List.of(new Object[] { true }, new Object[] { false });
59+
}
4960

5061
@Before
5162
public void setup() throws Exception {
@@ -68,16 +79,13 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
6879
@Override
6980
public Settings indexSettings() {
7081
return Settings.builder()
71-
.put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion)
82+
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
7283
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
84+
.put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
7385
.build();
7486
}
7587

7688
public void testBulkOperations() throws Exception {
77-
this.indexVersion = randomFrom(
78-
IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS),
79-
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersion.current())
80-
);
8189
indicesAdmin().prepareCreate(INDEX_NAME)
8290
.setMapping(
8391
String.format(

0 commit comments

Comments
 (0)