Skip to content

Commit d203b06

Browse files
joegallojoegallo
authored
Merge branch 'main' into fix/test/lookupJoinOnTimeSeriesIndex
2 parents c06ba35 + 47d4b98 commit d203b06

File tree

54 files changed

+3607
-3258
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+3607
-3258
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@
1515
import org.gradle.api.Project;
1616
import org.gradle.api.artifacts.Configuration;
1717
import org.gradle.api.attributes.Category;
18+
import org.gradle.api.attributes.Usage;
1819
import org.gradle.api.plugins.JavaPlugin;
1920

2021
public class DependenciesInfoPlugin implements Plugin<Project> {
22+
23+
public static String USAGE_ATTRIBUTE = "DependenciesInfo";
24+
2125
@Override
2226
public void apply(final Project project) {
2327
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
@@ -43,6 +47,9 @@ public void apply(final Project project) {
4347
)
4448
);
4549

50+
dependenciesInfoFilesConfiguration.attributes(
51+
attributes -> attributes.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, USAGE_ATTRIBUTE))
52+
);
4653
project.getArtifacts().add("dependenciesInfoFiles", depsInfo);
4754

4855
}

‎distribution/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ configurations {
3131
attributes {
3232
attribute(Category.CATEGORY_ATTRIBUTE, project.getObjects().named(Category.class, Category.DOCUMENTATION))
3333
}
34+
attributes {
35+
attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, DependenciesInfoPlugin.USAGE_ATTRIBUTE))
36+
}
3437
}
3538
featuresMetadata {
3639
attributes {

‎docs/changelog/129126.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129126
2+
summary: "Synthetic source: avoid storing multi fields of type text and `match_only_text`\
3+
\ by default"
4+
area: Mapping
5+
type: bug
6+
issues: []

‎docs/changelog/129140.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129140
2+
summary: Increment inference stats counter for shard bulk inference calls
3+
area: Machine Learning
4+
type: enhancement
5+
issues: []

‎docs/changelog/129181.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 129181
2+
summary: Add Cluster Feature for L2 Norm
3+
area: "Search"
4+
type: bug
5+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.elasticsearch.common.lucene.Lucene;
3434
import org.elasticsearch.common.unit.Fuzziness;
3535
import org.elasticsearch.index.IndexVersion;
36+
import org.elasticsearch.index.IndexVersions;
3637
import org.elasticsearch.index.analysis.IndexAnalyzers;
3738
import org.elasticsearch.index.analysis.NamedAnalyzer;
3839
import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -101,12 +102,9 @@ public static class Builder extends FieldMapper.Builder {
101102
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
102103

103104
private final TextParams.Analyzers analyzers;
105+
private final boolean withinMultiField;
104106

105-
public Builder(String name, IndexAnalyzers indexAnalyzers) {
106-
this(name, IndexVersion.current(), indexAnalyzers);
107-
}
108-
109-
public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
107+
public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
110108
super(name);
111109
this.indexCreatedVersion = indexCreatedVersion;
112110
this.analyzers = new TextParams.Analyzers(
@@ -115,6 +113,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
115113
m -> ((MatchOnlyTextFieldMapper) m).positionIncrementGap,
116114
indexCreatedVersion
117115
);
116+
this.withinMultiField = withinMultiField;
118117
}
119118

120119
@Override
@@ -140,18 +139,21 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
140139
@Override
141140
public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
142141
MatchOnlyTextFieldType tft = buildFieldType(context);
143-
return new MatchOnlyTextFieldMapper(
144-
leafName(),
145-
Defaults.FIELD_TYPE,
146-
tft,
147-
builderParams(this, context),
148-
context.isSourceSynthetic(),
149-
this
150-
);
142+
final boolean storeSource;
143+
if (indexCreatedVersion.onOrAfter(IndexVersions.MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED)) {
144+
storeSource = context.isSourceSynthetic()
145+
&& withinMultiField == false
146+
&& multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false;
147+
} else {
148+
storeSource = context.isSourceSynthetic();
149+
}
150+
return new MatchOnlyTextFieldMapper(leafName(), Defaults.FIELD_TYPE, tft, builderParams(this, context), storeSource, this);
151151
}
152152
}
153153

154-
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
154+
public static final TypeParser PARSER = new TypeParser(
155+
(n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
156+
);
155157

156158
public static class MatchOnlyTextFieldType extends StringFieldType {
157159

@@ -406,6 +408,7 @@ private String storedFieldNameForSyntheticSource() {
406408
private final int positionIncrementGap;
407409
private final boolean storeSource;
408410
private final FieldType fieldType;
411+
private final boolean withinMultiField;
409412

410413
private MatchOnlyTextFieldMapper(
411414
String simpleName,
@@ -424,6 +427,7 @@ private MatchOnlyTextFieldMapper(
424427
this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
425428
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
426429
this.storeSource = storeSource;
430+
this.withinMultiField = builder.withinMultiField;
427431
}
428432

429433
@Override
@@ -433,7 +437,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
433437

434438
@Override
435439
public FieldMapper.Builder getMergeBuilder() {
436-
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this);
440+
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
437441
}
438442

439443
@Override

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.lucene.tests.index.RandomIndexWriter;
2424
import org.elasticsearch.common.Strings;
2525
import org.elasticsearch.core.Tuple;
26+
import org.elasticsearch.index.IndexSettings;
2627
import org.elasticsearch.index.mapper.DocumentMapper;
2728
import org.elasticsearch.index.mapper.KeywordFieldMapper;
2829
import org.elasticsearch.index.mapper.LuceneDocument;
@@ -46,8 +47,10 @@
4647
import java.util.stream.Collectors;
4748

4849
import static org.hamcrest.Matchers.containsString;
50+
import static org.hamcrest.Matchers.empty;
4951
import static org.hamcrest.Matchers.equalTo;
5052
import static org.hamcrest.Matchers.instanceOf;
53+
import static org.hamcrest.core.Is.is;
5154

5255
public class MatchOnlyTextFieldMapperTests extends MapperTestCase {
5356

@@ -255,4 +258,91 @@ public void testDocValuesLoadedFromSynthetic() throws IOException {
255258
protected IngestScriptSupport ingestScriptSupport() {
256259
throw new AssumptionViolatedException("not supported");
257260
}
261+
262+
public void testStoreParameterDefaultsSyntheticSource() throws IOException {
263+
var indexSettingsBuilder = getIndexSettingsBuilder();
264+
indexSettingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic");
265+
var indexSettings = indexSettingsBuilder.build();
266+
267+
var mapping = mapping(b -> {
268+
b.startObject("name");
269+
b.field("type", "match_only_text");
270+
b.endObject();
271+
});
272+
DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper();
273+
274+
var source = source(b -> b.field("name", "quick brown fox"));
275+
ParsedDocument doc = mapper.parse(source);
276+
277+
{
278+
List<IndexableField> fields = doc.rootDoc().getFields("name");
279+
IndexableFieldType fieldType = fields.get(0).fieldType();
280+
assertThat(fieldType.stored(), is(false));
281+
}
282+
{
283+
List<IndexableField> fields = doc.rootDoc().getFields("name._original");
284+
IndexableFieldType fieldType = fields.get(0).fieldType();
285+
assertThat(fieldType.stored(), is(true));
286+
}
287+
}
288+
289+
public void testStoreParameterDefaultsSyntheticSourceWithKeywordMultiField() throws IOException {
290+
var indexSettingsBuilder = getIndexSettingsBuilder();
291+
indexSettingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic");
292+
var indexSettings = indexSettingsBuilder.build();
293+
294+
var mapping = mapping(b -> {
295+
b.startObject("name");
296+
b.field("type", "match_only_text");
297+
b.startObject("fields");
298+
b.startObject("keyword");
299+
b.field("type", "keyword");
300+
b.endObject();
301+
b.endObject();
302+
b.endObject();
303+
});
304+
DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper();
305+
306+
var source = source(b -> b.field("name", "quick brown fox"));
307+
ParsedDocument doc = mapper.parse(source);
308+
{
309+
List<IndexableField> fields = doc.rootDoc().getFields("name");
310+
IndexableFieldType fieldType = fields.get(0).fieldType();
311+
assertThat(fieldType.stored(), is(false));
312+
}
313+
{
314+
List<IndexableField> fields = doc.rootDoc().getFields("name._original");
315+
assertThat(fields, empty());
316+
}
317+
}
318+
319+
public void testStoreParameterDefaultsSyntheticSourceTextFieldIsMultiField() throws IOException {
320+
var indexSettingsBuilder = getIndexSettingsBuilder();
321+
indexSettingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic");
322+
var indexSettings = indexSettingsBuilder.build();
323+
324+
var mapping = mapping(b -> {
325+
b.startObject("name");
326+
b.field("type", "keyword");
327+
b.startObject("fields");
328+
b.startObject("text");
329+
b.field("type", "match_only_text");
330+
b.endObject();
331+
b.endObject();
332+
b.endObject();
333+
});
334+
DocumentMapper mapper = createMapperService(indexSettings, mapping).documentMapper();
335+
336+
var source = source(b -> b.field("name", "quick brown fox"));
337+
ParsedDocument doc = mapper.parse(source);
338+
{
339+
List<IndexableField> fields = doc.rootDoc().getFields("name.text");
340+
IndexableFieldType fieldType = fields.get(0).fieldType();
341+
assertThat(fieldType.stored(), is(false));
342+
}
343+
{
344+
List<IndexableField> fields = doc.rootDoc().getFields("name.text._original");
345+
assertThat(fields, empty());
346+
}
347+
}
258348
}

‎qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,12 +276,11 @@ TopDocs doVectorQuery(byte[] vector, IndexSearcher searcher) throws IOException
276276
TopDocs doVectorQuery(float[] vector, IndexSearcher searcher) throws IOException {
277277
Query knnQuery;
278278
int topK = this.topK;
279-
int efSearch = this.efSearch;
280279
if (overSamplingFactor > 1f) {
281280
// oversample the topK results to get more candidates for the final result
282281
topK = (int) Math.ceil(topK * overSamplingFactor);
283-
efSearch = Math.max(topK, efSearch);
284282
}
283+
int efSearch = Math.max(topK, this.efSearch);
285284
if (indexType == KnnIndexTester.IndexType.IVF) {
286285
knnQuery = new IVFKnnFloatVectorQuery(VECTOR_FIELD, vector, topK, efSearch, null, nProbe);
287286
} else {

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ private static Version parseUnchecked(String version) {
171171
public static final IndexVersion DEFAULT_TO_ACORN_HNSW_FILTER_HEURISTIC = def(9_026_0_00, Version.LUCENE_10_2_1);
172172
public static final IndexVersion SEQ_NO_WITHOUT_POINTS = def(9_027_0_00, Version.LUCENE_10_2_1);
173173
public static final IndexVersion INDEX_INT_SORT_INT_TYPE = def(9_028_0_00, Version.LUCENE_10_2_1);
174+
public static final IndexVersion MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED = def(9_029_0_00, Version.LUCENE_10_2_1);
174175

175176
/*
176177
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/index/codec/vectors/CentroidAssignments.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.codec.vectors;
11+
12+
import org.apache.lucene.internal.hppc.IntArrayList;
13+
14+
final class CentroidAssignments {
15+
16+
private final int numCentroids;
17+
private final float[][] cachedCentroids;
18+
private final IntArrayList[] assignmentsByCluster;
19+
20+
private CentroidAssignments(int numCentroids, float[][] cachedCentroids, IntArrayList[] assignmentsByCluster) {
21+
this.numCentroids = numCentroids;
22+
this.cachedCentroids = cachedCentroids;
23+
this.assignmentsByCluster = assignmentsByCluster;
24+
}
25+
26+
CentroidAssignments(float[][] centroids, IntArrayList[] assignmentsByCluster) {
27+
this(centroids.length, centroids, assignmentsByCluster);
28+
}
29+
30+
CentroidAssignments(int numCentroids, IntArrayList[] assignmentsByCluster) {
31+
this(numCentroids, null, assignmentsByCluster);
32+
}
33+
34+
// Getters and setters
35+
public int numCentroids() {
36+
return numCentroids;
37+
}
38+
39+
public float[][] cachedCentroids() {
40+
return cachedCentroids;
41+
}
42+
43+
public IntArrayList[] assignmentsByCluster() {
44+
return assignmentsByCluster;
45+
}
46+
}

0 commit comments

Comments
 (0)