Skip to content

Commit 148c558

Browse files
parkertimminsparkertimmins
authored
Merge branch 'main' into parker/random-test-numeric-cast-null-value
2 parents ee0d142 + 8f42478 commit 148c558

File tree

8 files changed

+103
-24
lines changed

8 files changed

+103
-24
lines changed

‎docs/changelog/133080.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 133080
2+
summary: "Disallow creating `semantic_text` fields in indices created prior to 8.11.0"
3+
area: Relevance
4+
type: bug
5+
issues: []

‎muted-tests.yml‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -489,21 +489,12 @@ tests:
489489
- class: org.elasticsearch.search.CCSDuelIT
490490
method: testTermsAggsWithProfile
491491
issue: https://github.com/elastic/elasticsearch/issues/132880
492-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
493-
method: testFetchMany
494-
issue: https://github.com/elastic/elasticsearch/issues/132948
495-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
496-
method: testFetch
497-
issue: https://github.com/elastic/elasticsearch/issues/132956
498492
- class: org.elasticsearch.cluster.ClusterInfoServiceIT
499493
method: testMaxQueueLatenciesInClusterInfo
500494
issue: https://github.com/elastic/elasticsearch/issues/132957
501495
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
502496
method: test {p0=search/400_synthetic_source/_doc_count}
503497
issue: https://github.com/elastic/elasticsearch/issues/132965
504-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
505-
method: testSyntheticSourceWithTranslogSnapshot
506-
issue: https://github.com/elastic/elasticsearch/issues/132964
507498
- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
508499
method: test {p0=search/160_exists_query/Test exists query on unmapped float field}
509500
issue: https://github.com/elastic/elasticsearch/issues/132984
@@ -594,15 +585,9 @@ tests:
594585
- class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
595586
method: testCCSClusterDetailsWhereAllShardsSkippedInCanMatch
596587
issue: https://github.com/elastic/elasticsearch/issues/133370
597-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
598-
method: testSyntheticSourceMany
599-
issue: https://github.com/elastic/elasticsearch/issues/133394
600588
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
601589
method: test {p0=search.vectors/42_knn_search_int4_flat/kNN search with filter}
602590
issue: https://github.com/elastic/elasticsearch/issues/133420
603-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
604-
method: testSyntheticSourceInNestedObject
605-
issue: https://github.com/elastic/elasticsearch/issues/133426
606591
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
607592
method: test {p0=search/160_exists_query/Test exists query on date field in empty index}
608593
issue: https://github.com/elastic/elasticsearch/issues/133439
@@ -627,9 +612,6 @@ tests:
627612
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
628613
method: testStopQueryLocal
629614
issue: https://github.com/elastic/elasticsearch/issues/133481
630-
- class: org.elasticsearch.index.mapper.LongFieldMapperTests
631-
method: testSyntheticSource
632-
issue: https://github.com/elastic/elasticsearch/issues/133496
633615
- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
634616
method: test {csv-spec:spatial.ConvertFromStringParseError}
635617
issue: https://github.com/elastic/elasticsearch/issues/133507
@@ -645,6 +627,9 @@ tests:
645627
- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
646628
method: test {p0=search/10_source_filtering/no filtering}
647629
issue: https://github.com/elastic/elasticsearch/issues/133561
630+
- class: org.elasticsearch.compute.lucene.LuceneTopNSourceOperatorTests
631+
method: testShardDataPartitioning
632+
issue: https://github.com/elastic/elasticsearch/issues/133574
648633

649634
# Examples:
650635
#

‎server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.cluster.metadata.ProjectId;
3030
import org.elasticsearch.cluster.metadata.ProjectMetadata;
3131
import org.elasticsearch.cluster.project.ProjectResolver;
32+
import org.elasticsearch.cluster.project.ProjectStateRegistry;
3233
import org.elasticsearch.cluster.routing.GlobalRoutingTable;
3334
import org.elasticsearch.cluster.routing.RoutingTable;
3435
import org.elasticsearch.cluster.service.ClusterService;
@@ -162,13 +163,19 @@ private ClusterState filterClusterState(final ClusterState inputState) {
162163
}
163164
final Metadata.Builder mdBuilder = Metadata.builder(inputState.metadata());
164165
final GlobalRoutingTable.Builder rtBuilder = GlobalRoutingTable.builder(inputState.globalRoutingTable());
166+
final ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder(inputState);
165167
for (var projectId : metadata.projects().keySet()) {
166168
if (projectIds.contains(projectId) == false) {
167169
mdBuilder.removeProject(projectId);
168170
rtBuilder.removeProject(projectId);
171+
psBuilder.removeProject(projectId);
169172
}
170173
}
171-
return ClusterState.builder(inputState).metadata(mdBuilder.build()).routingTable(rtBuilder.build()).build();
174+
return ClusterState.builder(inputState)
175+
.metadata(mdBuilder.build())
176+
.routingTable(rtBuilder.build())
177+
.putCustom(ProjectStateRegistry.TYPE, psBuilder.build())
178+
.build();
172179
}
173180

174181
@SuppressForbidden(reason = "exposing ClusterState#compatibilityVersions requires reading them")

‎server/src/test/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateActionTests.java‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.cluster.node.VersionInformation;
2323
import org.elasticsearch.cluster.project.DefaultProjectResolver;
2424
import org.elasticsearch.cluster.project.ProjectResolver;
25+
import org.elasticsearch.cluster.project.ProjectStateRegistry;
2526
import org.elasticsearch.cluster.project.TestProjectResolvers;
2627
import org.elasticsearch.cluster.routing.GlobalRoutingTableTestHelper;
2728
import org.elasticsearch.cluster.routing.RoutingTable;
@@ -169,6 +170,13 @@ public void testGetClusterStateForManyProjects() throws Exception {
169170
assertThat(routingTable, notNullValue());
170171
assertThat(routingTable.indicesRouting().keySet(), containsInAnyOrder(expectedIndices));
171172
}
173+
if (request.customs()) {
174+
ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(response.getState());
175+
assertThat(projectStateRegistry.size(), equalTo(numberOfProjects));
176+
Settings projectSettings = projectStateRegistry.getProjectSettings(projectId);
177+
assertThat(projectSettings, notNullValue());
178+
assertThat(projectSettings.keySet(), contains("setting_1"));
179+
}
172180
}
173181
}
174182

@@ -194,6 +202,13 @@ private static void assertSingleProjectResponse(
194202
} else {
195203
assertThat(routingTables.get(projectId).indicesRouting(), anEmptyMap());
196204
}
205+
if (request.customs()) {
206+
ProjectStateRegistry projectStateRegistry = ProjectStateRegistry.get(response.getState());
207+
assertThat(projectStateRegistry.size(), equalTo(1));
208+
Settings projectSettings = projectStateRegistry.getProjectSettings(projectId);
209+
assertThat(projectSettings, notNullValue());
210+
assertThat(projectSettings.keySet(), contains("setting_1"));
211+
}
197212
}
198213

199214
private ClusterStateResponse executeAction(ProjectResolver projectResolver, ClusterStateRequest request, ClusterState state)
@@ -232,7 +247,7 @@ private static ClusterStateRequest buildRandomRequest(Set<String> indexNames) {
232247
request.nodes(randomBoolean());
233248
request.routingTable(randomBoolean());
234249
request.blocks(randomBoolean());
235-
request.customs(randomBoolean());
250+
request.customs(true);
236251
return request;
237252
}
238253

@@ -241,9 +256,14 @@ private static ClusterState buildClusterState(ProjectMetadata.Builder... project
241256
Arrays.stream(projects).forEach(metadataBuilder::put);
242257
final var metadata = metadataBuilder.build();
243258

244-
return ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12)))
245-
.metadata(metadata)
259+
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName(randomAlphaOfLengthBetween(4, 12)));
260+
ProjectStateRegistry.Builder psBuilder = ProjectStateRegistry.builder();
261+
for (ProjectMetadata.Builder project : projects) {
262+
psBuilder.putProjectSettings(project.getId(), Settings.builder().put("setting_1", randomIdentifier()).build());
263+
}
264+
return csBuilder.metadata(metadata)
246265
.routingTable(GlobalRoutingTableTestHelper.buildRoutingTable(metadata, RoutingTable.Builder::addAsNew))
266+
.putCustom(ProjectStateRegistry.TYPE, psBuilder.build())
247267
.build();
248268
}
249269

‎server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ public void testLongIndexingCoercesIntoRange() throws Exception {
103103
assertThat(doc.rootDoc().getFields("field"), hasSize(1));
104104
}
105105

106+
// This is the biggest long that double can represent exactly
107+
public static final long MAX_SAFE_LONG_FOR_DOUBLE = 1L << 53;
108+
106109
@Override
107110
protected Number randomNumber() {
108111
if (randomBoolean()) {
@@ -111,7 +114,8 @@ protected Number randomNumber() {
111114
if (randomBoolean()) {
112115
return randomDouble();
113116
}
114-
return randomDoubleBetween(Long.MIN_VALUE, Long.MAX_VALUE, true);
117+
// TODO: increase the range back to full LONG range once https://github.com/elastic/elasticsearch/issues/132893 is fixed
118+
return randomDoubleBetween(-MAX_SAFE_LONG_FOR_DOUBLE, MAX_SAFE_LONG_FOR_DOUBLE, true);
115119
}
116120

117121
public void testFetchCoerced() throws IOException {

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import static org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase.randomNormalizedVector;
6868
import static org.elasticsearch.index.codec.vectors.IVFVectorsFormat.DYNAMIC_VISIT_RATIO;
6969
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DEFAULT_OVERSAMPLE;
70+
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.INDEXED_BY_DEFAULT_INDEX_VERSION;
7071
import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.IVF_FORMAT;
7172
import static org.hamcrest.Matchers.containsString;
7273
import static org.hamcrest.Matchers.equalTo;
@@ -106,7 +107,7 @@ private void indexMapping(XContentBuilder b, IndexVersion indexVersion) throws I
106107
if (elementType != ElementType.FLOAT) {
107108
b.field("element_type", elementType.toString());
108109
}
109-
if (indexVersion.onOrAfter(DenseVectorFieldMapper.INDEXED_BY_DEFAULT_INDEX_VERSION) || indexed) {
110+
if (indexVersion.onOrAfter(INDEXED_BY_DEFAULT_INDEX_VERSION) || indexed) {
110111
// Serialize if it's new index version, or it was not the default for previous indices
111112
b.field("index", indexed);
112113
}

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
import java.util.function.Function;
102102
import java.util.function.Supplier;
103103

104+
import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR;
104105
import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ;
105106
import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ_BACKPORT_8_X;
106107
import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING;
@@ -127,6 +128,7 @@
127128
*/
128129
public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper {
129130
private static final Logger logger = LogManager.getLogger(SemanticTextFieldMapper.class);
131+
public static final String UNSUPPORTED_INDEX_MESSAGE = "[semantic_text] is available on indices created with 8.11 or higher.";
130132
public static final NodeFeature SEMANTIC_TEXT_IN_OBJECT_FIELD_FIX = new NodeFeature("semantic_text.in_object_field_fix");
131133
public static final NodeFeature SEMANTIC_TEXT_SINGLE_FIELD_UPDATE_FIX = new NodeFeature("semantic_text.single_field_update_fix");
132134
public static final NodeFeature SEMANTIC_TEXT_DELETE_FIX = new NodeFeature("semantic_text.delete_fix");
@@ -165,6 +167,9 @@ public static final TypeParser parser(Supplier<ModelRegistry> modelRegistry) {
165167

166168
public static BiConsumer<String, MappingParserContext> validateParserContext(String type) {
167169
return (n, c) -> {
170+
if (c.getIndexSettings().getIndexVersionCreated().before(NEW_SPARSE_VECTOR)) {
171+
throw new UnsupportedOperationException(UNSUPPORTED_INDEX_MESSAGE);
172+
}
168173
if (InferenceMetadataFieldsMapper.isEnabled(c.getIndexSettings().getSettings()) == false) {
169174
notInMultiFields(type).accept(n, c);
170175
}

‎x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_ELSER_2_INFERENCE_ID;
112112
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE;
113113
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.INDEX_OPTIONS_FIELD;
114+
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper.UNSUPPORTED_INDEX_MESSAGE;
114115
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettings;
115116
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettingsOtherThan;
116117
import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomSemanticText;
@@ -414,6 +415,57 @@ public void testInvalidTaskTypes() {
414415
}
415416
}
416417

418+
@Override
419+
protected IndexVersion boostNotAllowedIndexVersion() {
420+
return IndexVersions.NEW_SPARSE_VECTOR;
421+
}
422+
423+
public void testOldIndexSemanticTextDenseVectorRaisesError() throws IOException {
424+
final String fieldName = "field";
425+
final XContentBuilder fieldMapping = fieldMapping(b -> {
426+
b.field("type", "semantic_text");
427+
b.field(INFERENCE_ID_FIELD, "test_inference_id");
428+
b.startObject("model_settings");
429+
b.field("task_type", "text_embedding");
430+
b.field("dimensions", 384);
431+
b.field("similarity", "cosine");
432+
b.field("element_type", "float");
433+
b.endObject();
434+
});
435+
assertOldIndexUnsupported(fieldMapping);
436+
}
437+
438+
public void testOldIndexSemanticTextMinimalMappingRaisesError() throws IOException {
439+
final XContentBuilder fieldMapping = fieldMapping(this::minimalMapping);
440+
assertOldIndexUnsupported(fieldMapping);
441+
}
442+
443+
public void testOldIndexSemanticTextSparseVersionRaisesError() throws IOException {
444+
final XContentBuilder fieldMapping = fieldMapping(b -> {
445+
b.field("type", "semantic_text");
446+
b.field("inference_id", "another_inference_id");
447+
b.startObject("model_settings");
448+
b.field("task_type", "sparse_embedding");
449+
b.endObject();
450+
});
451+
assertOldIndexUnsupported(fieldMapping);
452+
}
453+
454+
private void assertOldIndexUnsupported(XContentBuilder fieldMapping) {
455+
456+
MapperParsingException exception = assertThrows(
457+
MapperParsingException.class,
458+
() -> createMapperService(
459+
fieldMapping,
460+
true,
461+
IndexVersions.V_8_0_0,
462+
IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR)
463+
)
464+
);
465+
assertTrue(exception.getMessage().contains(UNSUPPORTED_INDEX_MESSAGE));
466+
assertTrue(exception.getRootCause() instanceof UnsupportedOperationException);
467+
}
468+
417469
public void testMultiFieldsSupport() throws IOException {
418470
if (useLegacyFormat) {
419471
Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(fieldMapping(b -> {

0 commit comments

Comments
 (0)