Skip to content

Commit de13467

Browse files
Mikep86Mikep86
authored
Add inference metadata fields feature flag and fix texts (#118190)
1 parent cc0f394 commit de13467

File tree

33 files changed

+1053
-465
lines changed

33 files changed

+1053
-465
lines changed

‎server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import org.elasticsearch.core.Strings;
4444
import org.elasticsearch.core.TimeValue;
4545
import org.elasticsearch.core.Tuple;
46-
import org.elasticsearch.index.IndexVersions;
4746
import org.elasticsearch.index.IndexingPressure;
4847
import org.elasticsearch.index.engine.Engine;
4948
import org.elasticsearch.index.engine.VersionConflictEngineException;
@@ -406,7 +405,7 @@ static boolean executeBulkItemRequest(
406405
}
407406

408407
private static String[] getStoredFieldsSpec(IndexShard indexShard) {
409-
if (indexShard.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS)) {
408+
if (InferenceMetadataFieldsMapper.isEnabled(indexShard.indexSettings().getIndexVersionCreated())) {
410409
if (indexShard.mapperService().mappingLookup().inferenceFields().size() > 0) {
411410
// Retrieves the inference metadata field containing the inference results for all semantic fields defined in the mapping.
412411
return new String[] { RoutingFieldMapper.NAME, InferenceMetadataFieldsMapper.NAME };

‎server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
import org.elasticsearch.core.Tuple;
4343
import org.elasticsearch.index.IndexNotFoundException;
4444
import org.elasticsearch.index.IndexService;
45-
import org.elasticsearch.index.IndexVersions;
4645
import org.elasticsearch.index.engine.VersionConflictEngineException;
4746
import org.elasticsearch.index.mapper.InferenceFieldMapper;
47+
import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
4848
import org.elasticsearch.index.mapper.Mapper;
4949
import org.elasticsearch.index.mapper.MappingLookup;
5050
import org.elasticsearch.index.shard.IndexShard;
@@ -376,7 +376,7 @@ private static UpdateHelper.Result deleteInferenceResults(
376376
MappingLookup mappingLookup
377377
) {
378378
if (result.getResponseResult() != DocWriteResponse.Result.UPDATED
379-
|| indexMetadata.getCreationVersion().onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS)) {
379+
|| InferenceMetadataFieldsMapper.isEnabled(indexMetadata.getCreationVersion())) {
380380
return result;
381381
}
382382

@@ -445,6 +445,7 @@ private static UpdateHelper.Result deleteInferenceResults(
445445
* @return The field's original value, or {@code null} if none was provided
446446
*/
447447
private static Object getOriginalValueLegacy(String fullPath, Map<String, Object> sourceAsMap) {
448+
// TODO: Fix bug here when semantic text field is in an object
448449
Object fieldValue = sourceAsMap.get(fullPath);
449450
if (fieldValue == null) {
450451
return null;

‎server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ public UpdateHelper(ScriptService scriptService) {
6060
* Prepares an update request by converting it into an index or delete request or an update response (no action).
6161
*/
6262
public Result prepare(UpdateRequest request, IndexShard indexShard, LongSupplier nowInMillis) throws IOException {
63+
// TODO: Don't hard-code gFields
6364
return prepare(request, indexShard, nowInMillis, new String[] { RoutingFieldMapper.NAME });
6465
}
6566

‎server/src/main/java/org/elasticsearch/index/get/ShardGetService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@ private GetResult innerGetFetch(
303303
boolean hasInferenceMetadataFields = false;
304304
if (storedFields != null) {
305305
for (String field : storedFields) {
306-
if (field.equals(InferenceMetadataFieldsMapper.NAME) && indexVersion.onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS)) {
306+
if (field.equals(InferenceMetadataFieldsMapper.NAME) && InferenceMetadataFieldsMapper.isEnabled(indexVersion)) {
307307
hasInferenceMetadataFields = true;
308308
continue;
309309
}

‎server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
import org.apache.lucene.search.IndexSearcher;
1313
import org.apache.lucene.search.Query;
1414
import org.apache.lucene.search.join.BitSetProducer;
15+
import org.elasticsearch.common.util.FeatureFlag;
16+
import org.elasticsearch.index.IndexVersion;
17+
import org.elasticsearch.index.IndexVersions;
1518
import org.elasticsearch.index.query.SearchExecutionContext;
1619

1720
import java.util.Map;
@@ -23,6 +26,8 @@
2326
* the field name for removal from _source.
2427
*/
2528
public abstract class InferenceMetadataFieldsMapper extends MetadataFieldMapper {
29+
public static final FeatureFlag INFERENCE_METADATA_FIELDS_FEATURE_FLAG = new FeatureFlag("inference_metadata_fields");
30+
2631
public static final String NAME = "_inference_fields";
2732
public static final String CONTENT_TYPE = "_inference_fields";
2833

@@ -54,4 +59,8 @@ public abstract ValueFetcher valueFetcher(
5459
IndexSearcher searcher
5560
);
5661
}
62+
63+
public static boolean isEnabled(IndexVersion indexVersion) {
64+
return indexVersion.onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS) && INFERENCE_METADATA_FIELDS_FEATURE_FLAG.isEnabled();
65+
}
5766
}

‎server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ public BytesReference applyFilters(
442442
}
443443
var modSourceFilter = sourceFilter;
444444
if (context != null
445-
&& context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.INFERENCE_METADATA_FIELDS)
445+
&& InferenceMetadataFieldsMapper.isEnabled(context.indexSettings().getIndexVersionCreated())
446446
&& context.mappingLookup().inferenceFields().isEmpty() == false) {
447447
String[] modExcludes = new String[excludes != null ? excludes.length + 1 : 1];
448448
if (excludes != null) {

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ public Builder(String name) {
7878
super(name);
7979
}
8080

81+
public Builder setStored(boolean value) {
82+
stored.setValue(value);
83+
return this;
84+
}
85+
8186
@Override
8287
protected Parameter<?>[] getParameters() {
8388
return new Parameter<?>[] { stored, meta };

‎server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhase.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
package org.elasticsearch.search.fetch.subphase;
1111

1212
import org.apache.lucene.index.LeafReaderContext;
13-
import org.elasticsearch.index.IndexVersions;
1413
import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
1514
import org.elasticsearch.search.SearchHit;
1615
import org.elasticsearch.search.fetch.FetchContext;
@@ -91,12 +90,12 @@ private void hitExecute(HitContext hitContext) {
9190
* to the original _source if it has been requested.
9291
*/
9392
private Source replaceInferenceMetadataFields(SearchHit hit, Source source) {
94-
if (fetchContext.getSearchExecutionContext()
95-
.getIndexSettings()
96-
.getIndexVersionCreated()
97-
.before(IndexVersions.INFERENCE_METADATA_FIELDS)) {
93+
if (InferenceMetadataFieldsMapper.isEnabled(
94+
fetchContext.getSearchExecutionContext().getIndexSettings().getIndexVersionCreated()
95+
) == false) {
9896
return source;
9997
}
98+
10099
var field = hit.removeMetadataFields(InferenceMetadataFieldsMapper.NAME);
101100
if (field == null || field.getValues().isEmpty()) {
102101
return source;

‎server/src/test/java/org/elasticsearch/action/bulk/TransportShardBulkActionTests.java‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,16 @@ public class TransportShardBulkActionTests extends IndexShardTestCase {
9292
private final ShardId shardId = new ShardId("index", "_na_", 0);
9393
private final Settings idxSettings = indexSettings(IndexVersion.current(), 1, 0).build();
9494

95-
private IndexMetadata indexMetadata() throws IOException {
96-
return IndexMetadata.builder("index").putMapping("""
95+
private IndexMetadata indexMetadata(String mapping) {
96+
IndexMetadata.Builder builder = IndexMetadata.builder("index").settings(idxSettings).primaryTerm(0, 1);
97+
if (mapping != null) {
98+
builder.putMapping(mapping);
99+
}
100+
return builder.build();
101+
}
102+
103+
private IndexMetadata indexMetadata() {
104+
return indexMetadata("""
97105
{
98106
"properties": {
99107
"foo": {
@@ -106,7 +114,7 @@ private IndexMetadata indexMetadata() throws IOException {
106114
}
107115
}
108116
}
109-
}""").settings(idxSettings).primaryTerm(0, 1).build();
117+
}""");
110118
}
111119

112120
public void testExecuteBulkIndexRequest() throws Exception {
@@ -502,7 +510,7 @@ public void testNoopUpdateRequest() throws Exception {
502510
IndexShard shard = mockShard(null, null);
503511

504512
UpdateHelper updateHelper = mock(UpdateHelper.class);
505-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
513+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
506514
new UpdateHelper.Result(
507515
noopUpdateResponse,
508516
DocWriteResponse.Result.NOOP,
@@ -557,7 +565,7 @@ public void testUpdateRequestWithFailure() throws Exception {
557565
);
558566

559567
UpdateHelper updateHelper = mock(UpdateHelper.class);
560-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
568+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
561569
new UpdateHelper.Result(
562570
updateResponse,
563571
randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -618,7 +626,7 @@ public void testUpdateRequestWithConflictFailure() throws Exception {
618626
);
619627

620628
UpdateHelper updateHelper = mock(UpdateHelper.class);
621-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
629+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
622630
new UpdateHelper.Result(
623631
updateResponse,
624632
randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -681,7 +689,7 @@ public void testUpdateRequestWithSuccess() throws Exception {
681689
);
682690

683691
UpdateHelper updateHelper = mock(UpdateHelper.class);
684-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
692+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
685693
new UpdateHelper.Result(
686694
updateResponse,
687695
created ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -738,7 +746,7 @@ public void testUpdateWithDelete() throws Exception {
738746
when(shard.applyDeleteOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong())).thenReturn(deleteResult);
739747

740748
UpdateHelper updateHelper = mock(UpdateHelper.class);
741-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
749+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
742750
new UpdateHelper.Result(
743751
updateResponse,
744752
DocWriteResponse.Result.DELETED,
@@ -784,7 +792,7 @@ public void testFailureDuringUpdateProcessing() throws Exception {
784792

785793
UpdateHelper updateHelper = mock(UpdateHelper.class);
786794
final ElasticsearchException err = new ElasticsearchException("oops");
787-
when(updateHelper.prepare(any(), eq(shard), any())).thenThrow(err);
795+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenThrow(err);
788796
BulkItemRequest[] items = new BulkItemRequest[] { primaryRequest };
789797
BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
790798

@@ -916,7 +924,7 @@ public void testRetries() throws Exception {
916924
});
917925

918926
UpdateHelper updateHelper = mock(UpdateHelper.class);
919-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
927+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
920928
new UpdateHelper.Result(
921929
updateResponse,
922930
randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1129,7 +1137,7 @@ public void testNoopMappingUpdateInfiniteLoopPrevention() throws Exception {
11291137
);
11301138

11311139
UpdateHelper updateHelper = mock(UpdateHelper.class);
1132-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
1140+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
11331141
new UpdateHelper.Result(
11341142
new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"),
11351143
randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1195,7 +1203,7 @@ public void testNoopMappingUpdateSuccessOnRetry() throws Exception {
11951203
);
11961204

11971205
UpdateHelper updateHelper = mock(UpdateHelper.class);
1198-
when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(
1206+
when(updateHelper.prepare(any(), eq(shard), any(), any())).thenReturn(
11991207
new UpdateHelper.Result(
12001208
new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value"),
12011209
randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED,
@@ -1235,6 +1243,9 @@ private IndexShard mockShard(IndexSettings indexSettings, MapperService mapperSe
12351243

12361244
if (indexSettings != null) {
12371245
when(shard.indexSettings()).thenReturn(indexSettings);
1246+
} else {
1247+
IndexSettings defaultIndexSettings = new IndexSettings(indexMetadata(null), Settings.EMPTY);
1248+
when(shard.indexSettings()).thenReturn(defaultIndexSettings);
12381249
}
12391250

12401251
if (mapperService != null) {

‎server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,12 @@
1111

1212
import org.apache.lucene.index.LeafReaderContext;
1313
import org.apache.lucene.index.memory.MemoryIndex;
14+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1415
import org.elasticsearch.common.Strings;
1516
import org.elasticsearch.common.bytes.BytesReference;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.index.IndexSettings;
19+
import org.elasticsearch.index.IndexVersion;
1620
import org.elasticsearch.index.query.SearchExecutionContext;
1721
import org.elasticsearch.search.SearchHit;
1822
import org.elasticsearch.search.fetch.FetchContext;
@@ -190,6 +194,12 @@ private HitContext hitExecuteMultiple(
190194
when(fetchContext.getIndexName()).thenReturn("index");
191195
SearchExecutionContext sec = mock(SearchExecutionContext.class);
192196
when(sec.isSourceEnabled()).thenReturn(sourceBuilder != null);
197+
IndexSettings indexSettings = new IndexSettings(
198+
IndexMetadata.builder("index").settings(indexSettings(IndexVersion.current(), 1, 0)).build(),
199+
Settings.EMPTY
200+
);
201+
when(sec.indexVersionCreated()).thenReturn(indexSettings.getIndexVersionCreated());
202+
when(sec.getIndexSettings()).thenReturn(indexSettings);
193203
when(fetchContext.getSearchExecutionContext()).thenReturn(sec);
194204

195205
final SearchHit searchHit = SearchHit.unpooled(1, null, nestedIdentity);

0 commit comments

Comments
 (0)