Skip to content

Commit 18a029f

Browse files
gmarouligmarouli
committed
Merge branch 'main' into failures-lifecycle-config
2 parents f5bd96b + 85d375c commit 18a029f

File tree

26 files changed

+1139
-250
lines changed

26 files changed

+1139
-250
lines changed

‎docs/changelog/126629.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126629
2+
summary: Default new `semantic_text` fields to use BBQ when models are compatible
3+
area: Relevance
4+
type: enhancement
5+
issues: []

‎docs/changelog/127285.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 127285
2+
summary: Restore model registry validation for the semantic text field
3+
area: Search
4+
type: enhancement
5+
issues: []

‎server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/XContentMeteringParserDecoratorIT.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.plugins.internal;
1111

1212
import org.elasticsearch.action.index.IndexRequest;
13+
import org.elasticsearch.index.Index;
1314
import org.elasticsearch.index.IndexSettings;
1415
import org.elasticsearch.index.engine.EngineFactory;
1516
import org.elasticsearch.index.engine.InternalEngine;
@@ -104,7 +105,7 @@ public IndexResult index(Index index) throws IOException {
104105
IndexResult result = super.index(index);
105106

106107
DocumentSizeReporter documentParsingReporter = documentParsingProvider.newDocumentSizeReporter(
107-
shardId.getIndexName(),
108+
shardId.getIndex(),
108109
config().getMapperService(),
109110
DocumentSizeAccumulator.EMPTY_INSTANCE
110111
);
@@ -131,22 +132,22 @@ public <T> XContentMeteringParserDecorator newMeteringParserDecorator(IndexReque
131132

132133
@Override
133134
public DocumentSizeReporter newDocumentSizeReporter(
134-
String indexName,
135+
Index index,
135136
MapperService mapperService,
136137
DocumentSizeAccumulator documentSizeAccumulator
137138
) {
138-
return new TestDocumentSizeReporter(indexName);
139+
return new TestDocumentSizeReporter(index);
139140
}
140141
};
141142
}
142143
}
143144

144145
public static class TestDocumentSizeReporter implements DocumentSizeReporter {
145146

146-
private final String indexName;
147+
private final Index index;
147148

148-
public TestDocumentSizeReporter(String indexName) {
149-
this.indexName = indexName;
149+
public TestDocumentSizeReporter(Index index) {
150+
this.index = index;
150151
}
151152

152153
@Override
@@ -155,7 +156,7 @@ public void onIndexingCompleted(ParsedDocument parsedDocument) {
155156
if (delta > XContentMeteringParserDecorator.UNKNOWN_SIZE) {
156157
COUNTER.addAndGet(delta);
157158
}
158-
assertThat(indexName, equalTo(TEST_INDEX_NAME));
159+
assertThat(index.getName(), equalTo(TEST_INDEX_NAME));
159160
}
160161
}
161162

‎server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import org.elasticsearch.cluster.routing.ShardIterator;
2626
import org.elasticsearch.cluster.routing.ShardRouting;
2727
import org.elasticsearch.cluster.service.ClusterService;
28-
import org.elasticsearch.core.Predicates;
2928
import org.elasticsearch.index.shard.ShardId;
3029
import org.elasticsearch.indices.IndicesService;
3130
import org.elasticsearch.injection.guice.Inject;
@@ -105,14 +104,7 @@ protected void masterOperation(
105104
Set<ResolvedExpression> indicesAndAliases = indexNameExpressionResolver.resolveExpressions(project.metadata(), request.indices());
106105
for (String index : concreteIndices) {
107106
final AliasFilter aliasFilter = indicesService.buildAliasFilter(project, index, indicesAndAliases);
108-
final String[] aliases = indexNameExpressionResolver.indexAliases(
109-
project.metadata(),
110-
index,
111-
Predicates.always(),
112-
Predicates.always(),
113-
true,
114-
indicesAndAliases
115-
);
107+
final String[] aliases = indexNameExpressionResolver.allIndexAliases(project.metadata(), index, indicesAndAliases);
116108
indicesAndFilters.put(index, AliasFilter.of(aliasFilter.getQueryBuilder(), aliases));
117109
}
118110

‎server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
import org.elasticsearch.common.util.Maps;
6363
import org.elasticsearch.common.util.concurrent.CountDown;
6464
import org.elasticsearch.common.util.concurrent.EsExecutors;
65-
import org.elasticsearch.core.Predicates;
6665
import org.elasticsearch.core.TimeValue;
6766
import org.elasticsearch.index.Index;
6867
import org.elasticsearch.index.IndexNotFoundException;
@@ -232,14 +231,7 @@ private Map<String, OriginalIndices> buildPerIndexOriginalIndices(
232231
blocks.indexBlockedRaiseException(projectState.projectId(), ClusterBlockLevel.READ, index);
233232
}
234233

235-
String[] aliases = indexNameExpressionResolver.indexAliases(
236-
projectState.metadata(),
237-
index,
238-
Predicates.always(),
239-
Predicates.always(),
240-
true,
241-
indicesAndAliases
242-
);
234+
String[] aliases = indexNameExpressionResolver.allIndexAliases(projectState.metadata(), index, indicesAndAliases);
243235
String[] finalIndices = Strings.EMPTY_ARRAY;
244236
if (aliases == null
245237
|| aliases.length == 0

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import java.util.Set;
6060
import java.util.SortedMap;
6161
import java.util.function.BiFunction;
62+
import java.util.function.BiPredicate;
6263
import java.util.function.Function;
6364
import java.util.function.LongSupplier;
6465
import java.util.function.Predicate;
@@ -80,6 +81,13 @@ public class IndexNameExpressionResolver {
8081
public static final String EXCLUDED_DATA_STREAMS_KEY = "es.excluded_ds";
8182
public static final IndexVersion SYSTEM_INDEX_ENFORCEMENT_INDEX_VERSION = IndexVersions.V_8_0_0;
8283

84+
private static final BiPredicate<DataStreamAlias, Boolean> ALL_DATA_STREAM_ALIASES = (ignoredAlias, ignoredIsData) -> true;
85+
// Alias filters are not applied against indices in an abstraction's failure component.
86+
// They do not match the mapping of the data stream nor are the documents mapped for searching.
87+
private static final BiPredicate<DataStreamAlias, Boolean> ONLY_FILTERING_DATA_STREAM_ALIASES = (
88+
dataStreamAlias,
89+
isData) -> dataStreamAlias.filteringRequired() && isData;
90+
8391
private final ThreadContext threadContext;
8492
private final SystemIndices systemIndices;
8593
private final ProjectResolver projectResolver;
@@ -1054,12 +1062,21 @@ public String[] filteringAliases(ProjectMetadata project, String index, Set<Reso
10541062
project,
10551063
index,
10561064
AliasMetadata::filteringRequired,
1057-
DataStreamAlias::filteringRequired,
1065+
ONLY_FILTERING_DATA_STREAM_ALIASES,
10581066
false,
10591067
resolvedExpressions
10601068
);
10611069
}
10621070

1071+
/**
1072+
* Iterates through the list of indices and selects the effective list of all aliases for the
1073+
* given index. Aliases are returned even if the index is included in the resolved expressions.
1074+
* <b>NOTE</b>: The provided expressions must have been resolved already via {@link #resolveExpressions}.
1075+
*/
1076+
public String[] allIndexAliases(ProjectMetadata project, String index, Set<ResolvedExpression> resolvedExpressions) {
1077+
return indexAliases(project, index, Predicates.always(), ALL_DATA_STREAM_ALIASES, true, resolvedExpressions);
1078+
}
1079+
10631080
/**
10641081
* Whether to generate the candidate set from index aliases, or from the set of resolved expressions.
10651082
* @param indexAliasesSize the number of aliases of the index
@@ -1080,7 +1097,7 @@ public String[] indexAliases(
10801097
ProjectMetadata project,
10811098
String index,
10821099
Predicate<AliasMetadata> requiredAlias,
1083-
Predicate<DataStreamAlias> requiredDataStreamAlias,
1100+
BiPredicate<DataStreamAlias, Boolean> requiredDataStreamAlias,
10841101
boolean skipIdentity,
10851102
Set<ResolvedExpression> resolvedExpressions
10861103
) {
@@ -1107,13 +1124,8 @@ public String[] indexAliases(
11071124
IndexAbstraction ia = project.getIndicesLookup().get(index);
11081125
DataStream dataStream = ia.getParentDataStream();
11091126
if (dataStream != null) {
1110-
if (dataStream.getFailureComponent().containsIndex(index)) {
1111-
// Alias filters are not applied against indices in an abstraction's failure component.
1112-
// They do not match the mapping of the data stream nor are the documents mapped for searching.
1113-
return null;
1114-
}
1115-
1116-
if (skipIdentity == false && resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStream.getName())) {
1127+
boolean isData = dataStream.isFailureStoreIndex(index) == false;
1128+
if (skipIdentity == false && resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStream.getName(), isData)) {
11171129
// skip the filters when the request targets the data stream name + selector directly
11181130
return null;
11191131
}
@@ -1122,12 +1134,14 @@ public String[] indexAliases(
11221134
if (iterateIndexAliases(dataStreamAliases.size(), resolvedExpressions.size())) {
11231135
aliasesForDataStream = dataStreamAliases.values()
11241136
.stream()
1125-
.filter(dataStreamAlias -> resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStreamAlias.getName()))
1137+
.filter(
1138+
dataStreamAlias -> resolvedExpressionsContainsAbstraction(resolvedExpressions, dataStreamAlias.getName(), isData)
1139+
)
11261140
.filter(dataStreamAlias -> dataStreamAlias.getDataStreams().contains(dataStream.getName()))
11271141
.toList();
11281142
} else {
11291143
aliasesForDataStream = resolvedExpressions.stream()
1130-
.filter(expression -> expression.selector() == null || expression.selector().shouldIncludeData())
1144+
.filter(expression -> (expression.selector() == null || expression.selector().shouldIncludeData()) == isData)
11311145
.map(ResolvedExpression::resource)
11321146
.map(dataStreamAliases::get)
11331147
.filter(dataStreamAlias -> dataStreamAlias != null && dataStreamAlias.getDataStreams().contains(dataStream.getName()))
@@ -1136,11 +1150,12 @@ public String[] indexAliases(
11361150

11371151
List<String> requiredAliases = null;
11381152
for (DataStreamAlias dataStreamAlias : aliasesForDataStream) {
1139-
if (requiredDataStreamAlias.test(dataStreamAlias)) {
1153+
if (requiredDataStreamAlias.test(dataStreamAlias, isData)) {
11401154
if (requiredAliases == null) {
11411155
requiredAliases = new ArrayList<>(aliasesForDataStream.size());
11421156
}
1143-
requiredAliases.add(dataStreamAlias.getName());
1157+
String alias = isData ? dataStreamAlias.getName() : dataStreamAlias.getName() + "::failures";
1158+
requiredAliases.add(alias);
11441159
} else {
11451160
// we have a non-required alias for this data stream so no need to check further
11461161
return null;
@@ -1162,7 +1177,7 @@ public String[] indexAliases(
11621177
// Indices can only be referenced with a data selector, or a null selector if selectors are disabled
11631178
for (AliasMetadata aliasMetadata : indexAliases.values()) {
11641179
var alias = aliasMetadata.alias();
1165-
if (resolvedExpressionsContainsAbstraction(resolvedExpressions, alias)) {
1180+
if (resolvedExpressionsContainsAbstraction(resolvedExpressions, alias, true)) {
11661181
if (requiredAlias.test(aliasMetadata) == false) {
11671182
return null;
11681183
}
@@ -1185,9 +1200,16 @@ public String[] indexAliases(
11851200
}
11861201
}
11871202

1188-
private static boolean resolvedExpressionsContainsAbstraction(Set<ResolvedExpression> resolvedExpressions, String abstractionName) {
1189-
return resolvedExpressions.contains(new ResolvedExpression(abstractionName))
1190-
|| resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA));
1203+
private static boolean resolvedExpressionsContainsAbstraction(
1204+
Set<ResolvedExpression> resolvedExpressions,
1205+
String abstractionName,
1206+
boolean isData
1207+
) {
1208+
if (isData) {
1209+
return resolvedExpressions.contains(new ResolvedExpression(abstractionName))
1210+
|| resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.DATA));
1211+
}
1212+
return resolvedExpressions.contains(new ResolvedExpression(abstractionName, IndexComponentSelector.FAILURES));
11911213
}
11921214

11931215
/**

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ private static Version parseUnchecked(String version) {
162162
public static final IndexVersion UPGRADE_TO_LUCENE_10_2_0 = def(9_022_00_0, Version.LUCENE_10_2_0);
163163
public static final IndexVersion UPGRADE_TO_LUCENE_10_2_1 = def(9_023_00_0, Version.LUCENE_10_2_1);
164164
public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1);
165+
public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1);
165166
/*
166167
* STOP! READ THIS FIRST! No, really,
167168
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,11 @@ public Builder elementType(ElementType elementType) {
295295
return this;
296296
}
297297

298+
public Builder indexOptions(IndexOptions indexOptions) {
299+
this.indexOptions.setValue(indexOptions);
300+
return this;
301+
}
302+
298303
@Override
299304
public DenseVectorFieldMapper build(MapperBuilderContext context) {
300305
// Validate again here because the dimensions or element type could have been set programmatically,
@@ -1226,7 +1231,7 @@ public final String toString() {
12261231
public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType);
12271232
}
12281233

1229-
abstract static class IndexOptions implements ToXContent {
1234+
public abstract static class IndexOptions implements ToXContent {
12301235
final VectorIndexType type;
12311236

12321237
IndexOptions(VectorIndexType type) {
@@ -1235,21 +1240,36 @@ abstract static class IndexOptions implements ToXContent {
12351240

12361241
abstract KnnVectorsFormat getVectorsFormat(ElementType elementType);
12371242

1238-
final void validateElementType(ElementType elementType) {
1239-
if (type.supportsElementType(elementType) == false) {
1243+
public boolean validate(ElementType elementType, int dim, boolean throwOnError) {
1244+
return validateElementType(elementType, throwOnError) && validateDimension(dim, throwOnError);
1245+
}
1246+
1247+
public boolean validateElementType(ElementType elementType) {
1248+
return validateElementType(elementType, true);
1249+
}
1250+
1251+
final boolean validateElementType(ElementType elementType, boolean throwOnError) {
1252+
boolean validElementType = type.supportsElementType(elementType);
1253+
if (throwOnError && validElementType == false) {
12401254
throw new IllegalArgumentException(
12411255
"[element_type] cannot be [" + elementType.toString() + "] when using index type [" + type + "]"
12421256
);
12431257
}
1258+
return validElementType;
12441259
}
12451260

12461261
abstract boolean updatableTo(IndexOptions update);
12471262

1248-
public void validateDimension(int dim) {
1249-
if (type.supportsDimension(dim)) {
1250-
return;
1263+
public boolean validateDimension(int dim) {
1264+
return validateDimension(dim, true);
1265+
}
1266+
1267+
public boolean validateDimension(int dim, boolean throwOnError) {
1268+
boolean supportsDimension = type.supportsDimension(dim);
1269+
if (throwOnError && supportsDimension == false) {
1270+
throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
12511271
}
1252-
throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
1272+
return supportsDimension;
12531273
}
12541274

12551275
abstract boolean doEquals(IndexOptions other);
@@ -1758,12 +1778,12 @@ boolean updatableTo(IndexOptions update) {
17581778

17591779
}
17601780

1761-
static class Int8HnswIndexOptions extends QuantizedIndexOptions {
1781+
public static class Int8HnswIndexOptions extends QuantizedIndexOptions {
17621782
private final int m;
17631783
private final int efConstruction;
17641784
private final Float confidenceInterval;
17651785

1766-
Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) {
1786+
public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval, RescoreVector rescoreVector) {
17671787
super(VectorIndexType.INT8_HNSW, rescoreVector);
17681788
this.m = m;
17691789
this.efConstruction = efConstruction;
@@ -1901,11 +1921,11 @@ public String toString() {
19011921
}
19021922
}
19031923

1904-
static class BBQHnswIndexOptions extends QuantizedIndexOptions {
1924+
public static class BBQHnswIndexOptions extends QuantizedIndexOptions {
19051925
private final int m;
19061926
private final int efConstruction;
19071927

1908-
BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) {
1928+
public BBQHnswIndexOptions(int m, int efConstruction, RescoreVector rescoreVector) {
19091929
super(VectorIndexType.BBQ_HNSW, rescoreVector);
19101930
this.m = m;
19111931
this.efConstruction = efConstruction;
@@ -1947,11 +1967,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
19471967
}
19481968

19491969
@Override
1950-
public void validateDimension(int dim) {
1951-
if (type.supportsDimension(dim)) {
1952-
return;
1970+
public boolean validateDimension(int dim, boolean throwOnError) {
1971+
boolean supportsDimension = type.supportsDimension(dim);
1972+
if (throwOnError && supportsDimension == false) {
1973+
throw new IllegalArgumentException(
1974+
type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
1975+
);
19531976
}
1954-
throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
1977+
return supportsDimension;
19551978
}
19561979
}
19571980

@@ -1995,15 +2018,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
19952018
}
19962019

19972020
@Override
1998-
public void validateDimension(int dim) {
1999-
if (type.supportsDimension(dim)) {
2000-
return;
2021+
public boolean validateDimension(int dim, boolean throwOnError) {
2022+
boolean supportsDimension = type.supportsDimension(dim);
2023+
if (throwOnError && supportsDimension == false) {
2024+
throw new IllegalArgumentException(
2025+
type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
2026+
);
20012027
}
2002-
throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
2028+
return supportsDimension;
20032029
}
2030+
20042031
}
20052032

2006-
record RescoreVector(float oversample) implements ToXContentObject {
2033+
public record RescoreVector(float oversample) implements ToXContentObject {
20072034
static final String NAME = "rescore_vector";
20082035
static final String OVERSAMPLE = "oversample";
20092036

@@ -2323,7 +2350,7 @@ ElementType getElementType() {
23232350
return elementType;
23242351
}
23252352

2326-
IndexOptions getIndexOptions() {
2353+
public IndexOptions getIndexOptions() {
23272354
return indexOptions;
23282355
}
23292356
}

0 commit comments

Comments
 (0)