Skip to content

Commit 5daefe7

Browse files
dnhatndnhatn
committed
Merge remote-tracking branch 'elastic/main' into partial-results-ccs
2 parents 88ded0e + 45f0471 commit 5daefe7

File tree

24 files changed

+368
-54
lines changed

24 files changed

+368
-54
lines changed

‎docs/changelog/120952.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 120952
2+
summary: Add `_metric_names_hash` field to OTel metric mappings
3+
area: Data streams
4+
type: bug
5+
issues: []

‎docs/changelog/122637.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 122637
2+
summary: Use `FallbackSyntheticSourceBlockLoader` for `unsigned_long` and `scaled_float`
3+
fields
4+
area: Mapping
5+
type: enhancement
6+
issues: []

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java‎

Lines changed: 78 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.elasticsearch.index.mapper.BlockLoader;
3434
import org.elasticsearch.index.mapper.BlockSourceReader;
3535
import org.elasticsearch.index.mapper.DocumentParserContext;
36+
import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
3637
import org.elasticsearch.index.mapper.FieldMapper;
3738
import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues;
3839
import org.elasticsearch.index.mapper.MapperBuilderContext;
@@ -195,7 +196,9 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) {
195196
scalingFactor.getValue(),
196197
nullValue.getValue(),
197198
metric.getValue(),
198-
indexMode
199+
indexMode,
200+
coerce.getValue().value(),
201+
context.isSourceSynthetic()
199202
);
200203
return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this);
201204
}
@@ -209,6 +212,8 @@ public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
209212
private final Double nullValue;
210213
private final TimeSeriesParams.MetricType metricType;
211214
private final IndexMode indexMode;
215+
private final boolean coerce;
216+
private final boolean isSyntheticSource;
212217

213218
public ScaledFloatFieldType(
214219
String name,
@@ -219,21 +224,25 @@ public ScaledFloatFieldType(
219224
double scalingFactor,
220225
Double nullValue,
221226
TimeSeriesParams.MetricType metricType,
222-
IndexMode indexMode
227+
IndexMode indexMode,
228+
boolean coerce,
229+
boolean isSyntheticSource
223230
) {
224231
super(name, indexed, stored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta);
225232
this.scalingFactor = scalingFactor;
226233
this.nullValue = nullValue;
227234
this.metricType = metricType;
228235
this.indexMode = indexMode;
236+
this.coerce = coerce;
237+
this.isSyntheticSource = isSyntheticSource;
229238
}
230239

231240
public ScaledFloatFieldType(String name, double scalingFactor) {
232241
this(name, scalingFactor, true);
233242
}
234243

235244
public ScaledFloatFieldType(String name, double scalingFactor, boolean indexed) {
236-
this(name, indexed, false, true, Collections.emptyMap(), scalingFactor, null, null, null);
245+
this(name, indexed, false, true, Collections.emptyMap(), scalingFactor, null, null, null, false, false);
237246
}
238247

239248
public double getScalingFactor() {
@@ -315,13 +324,73 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
315324
double scalingFactorInverse = 1d / scalingFactor;
316325
return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse);
317326
}
327+
if (isSyntheticSource) {
328+
return new FallbackSyntheticSourceBlockLoader(fallbackSyntheticSourceBlockLoaderReader(), name()) {
329+
@Override
330+
public Builder builder(BlockFactory factory, int expectedCount) {
331+
return factory.doubles(expectedCount);
332+
}
333+
};
334+
}
335+
318336
ValueFetcher valueFetcher = sourceValueFetcher(blContext.sourcePaths(name()));
319337
BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
320338
? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
321339
: BlockSourceReader.lookupMatchingAll();
322340
return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
323341
}
324342

343+
private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBlockLoaderReader() {
344+
var nullValueAdjusted = nullValue != null ? adjustSourceValue(nullValue, scalingFactor) : null;
345+
346+
return new FallbackSyntheticSourceBlockLoader.ReaderWithNullValueSupport<>(nullValue) {
347+
@Override
348+
public void convertValue(Object value, List<Double> accumulator) {
349+
if (coerce && value.equals("")) {
350+
if (nullValueAdjusted != null) {
351+
accumulator.add(nullValueAdjusted);
352+
}
353+
}
354+
355+
try {
356+
// Convert to doc_values format
357+
var converted = adjustSourceValue(NumberFieldMapper.NumberType.objectToDouble(value), scalingFactor);
358+
accumulator.add(converted);
359+
} catch (Exception e) {
360+
// Malformed value, skip it
361+
}
362+
}
363+
364+
@Override
365+
protected void parseNonNullValue(XContentParser parser, List<Double> accumulator) throws IOException {
366+
// Aligned with implementation of `parseCreateField(XContentParser)`
367+
if (coerce && parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0) {
368+
if (nullValueAdjusted != null) {
369+
accumulator.add(nullValueAdjusted);
370+
}
371+
}
372+
373+
try {
374+
double value = parser.doubleValue(coerce);
375+
// Convert to doc_values format
376+
var converted = adjustSourceValue(value, scalingFactor);
377+
accumulator.add(converted);
378+
} catch (Exception e) {
379+
// Malformed value, skip it
380+
}
381+
}
382+
383+
@Override
384+
public void writeToBlock(List<Double> values, BlockLoader.Builder blockBuilder) {
385+
var longBuilder = (BlockLoader.DoubleBuilder) blockBuilder;
386+
387+
for (var value : values) {
388+
longBuilder.appendDouble(value);
389+
}
390+
}
391+
};
392+
}
393+
325394
@Override
326395
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
327396
FielddataOperation operation = fieldDataContext.fielddataOperation();
@@ -386,12 +455,16 @@ protected Double parseSourceValue(Object value) {
386455
doubleValue = NumberFieldMapper.NumberType.objectToDouble(value);
387456
}
388457

389-
double factor = getScalingFactor();
390-
return Math.round(doubleValue * factor) / factor;
458+
return adjustSourceValue(doubleValue, getScalingFactor());
391459
}
392460
};
393461
}
394462

463+
// Adjusts precision of a double value so that it looks like it came from doc_values.
464+
private static Double adjustSourceValue(double value, double scalingFactor) {
465+
return Math.round(value * scalingFactor) / scalingFactor;
466+
}
467+
395468
@Override
396469
public Object valueForDisplay(Object value) {
397470
if (value == null) {

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldBlockLoaderTests.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index.mapper.extras;
11+
12+
import org.elasticsearch.index.mapper.NumberFieldBlockLoaderTestCase;
13+
import org.elasticsearch.logsdb.datageneration.FieldType;
14+
import org.elasticsearch.plugins.Plugin;
15+
16+
import java.util.Collection;
17+
import java.util.List;
18+
import java.util.Map;
19+
20+
public class ScaledFloatFieldBlockLoaderTests extends NumberFieldBlockLoaderTestCase<Double> {
21+
public ScaledFloatFieldBlockLoaderTests() {
22+
super(FieldType.SCALED_FLOAT);
23+
}
24+
25+
@Override
26+
protected Double convert(Number value, Map<String, Object> fieldMapping) {
27+
var scalingFactor = ((Number) fieldMapping.get("scaling_factor")).doubleValue();
28+
29+
var docValues = (boolean) fieldMapping.getOrDefault("doc_values", false);
30+
31+
// There is a slight inconsistency between values that are read from doc_values and from source.
32+
// Due to how precision reduction is applied to source values so that they are consistent with doc_values.
33+
// See #122547.
34+
if (docValues) {
35+
var reverseScalingFactor = 1d / scalingFactor;
36+
return Math.round(value.doubleValue() * scalingFactor) * reverseScalingFactor;
37+
}
38+
39+
// Adjust values coming from source to the way they are stored in doc_values.
40+
// See mapper implementation.
41+
return Math.round(value.doubleValue() * scalingFactor) / scalingFactor;
42+
}
43+
44+
@Override
45+
protected Collection<? extends Plugin> getPlugins() {
46+
return List.of(new MapperExtrasPlugin());
47+
}
48+
}

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ public void testRangeQuery() throws IOException {
9595
0.1 + randomDouble() * 100,
9696
null,
9797
null,
98-
null
98+
null,
99+
false,
100+
false
99101
);
100102
Directory dir = newDirectory();
101103
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));

‎muted-tests.yml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -227,18 +227,6 @@ tests:
227227
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
228228
method: test {p0=ml/*}
229229
issue: https://github.com/elastic/elasticsearch/issues/120816
230-
- class: org.elasticsearch.upgrades.VectorSearchIT
231-
method: testBBQVectorSearch {upgradedNodes=0}
232-
issue: https://github.com/elastic/elasticsearch/issues/121253
233-
- class: org.elasticsearch.upgrades.VectorSearchIT
234-
method: testBBQVectorSearch {upgradedNodes=1}
235-
issue: https://github.com/elastic/elasticsearch/issues/121271
236-
- class: org.elasticsearch.upgrades.VectorSearchIT
237-
method: testBBQVectorSearch {upgradedNodes=2}
238-
issue: https://github.com/elastic/elasticsearch/issues/121272
239-
- class: org.elasticsearch.upgrades.VectorSearchIT
240-
method: testBBQVectorSearch {upgradedNodes=3}
241-
issue: https://github.com/elastic/elasticsearch/issues/121273
242230
- class: org.elasticsearch.xpack.security.authc.ldap.ActiveDirectorySessionFactoryTests
243231
issue: https://github.com/elastic/elasticsearch/issues/121285
244232
- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
import org.elasticsearch.client.Request;
1515
import org.elasticsearch.client.Response;
16+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1617
import org.elasticsearch.common.settings.Settings;
1718
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1819

@@ -456,7 +457,11 @@ public void testBBQVectorSearch() throws Exception {
456457
}
457458
""";
458459
// create index and index 10 random floating point vectors
459-
createIndex(BBQ_INDEX_NAME, Settings.EMPTY, mapping);
460+
createIndex(
461+
BBQ_INDEX_NAME,
462+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(),
463+
mapping
464+
);
460465
index64DimVectors(BBQ_INDEX_NAME);
461466
// force merge the index
462467
client().performRequest(new Request("POST", "/" + BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1"));
@@ -485,8 +490,8 @@ public void testBBQVectorSearch() throws Exception {
485490
Map<String, Object> response = search(searchRequest);
486491
assertThat(extractValue(response, "hits.total.value"), equalTo(7));
487492
List<Map<String, Object>> hits = extractValue(response, "hits.hits");
488-
assertThat(hits.get(0).get("_id"), equalTo("0"));
489-
assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
493+
assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0"));
494+
assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
490495

491496
// search with knn
492497
searchRequest = new Request("POST", "/" + BBQ_INDEX_NAME + "/_search");
@@ -504,8 +509,12 @@ public void testBBQVectorSearch() throws Exception {
504509
response = search(searchRequest);
505510
assertThat(extractValue(response, "hits.total.value"), equalTo(2));
506511
hits = extractValue(response, "hits.hits");
507-
assertThat(hits.get(0).get("_id"), equalTo("0"));
508-
assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005));
512+
assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0"));
513+
assertThat(
514+
"expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response,
515+
(double) hits.get(0).get("_score"),
516+
closeTo(0.9934857, 0.005)
517+
);
509518
}
510519

511520
public void testFlatBBQVectorSearch() throws Exception {
@@ -530,7 +539,11 @@ public void testFlatBBQVectorSearch() throws Exception {
530539
}
531540
""";
532541
// create index and index 10 random floating point vectors
533-
createIndex(FLAT_BBQ_INDEX_NAME, Settings.EMPTY, mapping);
542+
createIndex(
543+
FLAT_BBQ_INDEX_NAME,
544+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build(),
545+
mapping
546+
);
534547
index64DimVectors(FLAT_BBQ_INDEX_NAME);
535548
// force merge the index
536549
client().performRequest(new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_forcemerge?max_num_segments=1"));
@@ -559,8 +572,8 @@ public void testFlatBBQVectorSearch() throws Exception {
559572
Map<String, Object> response = search(searchRequest);
560573
assertThat(extractValue(response, "hits.total.value"), equalTo(7));
561574
List<Map<String, Object>> hits = extractValue(response, "hits.hits");
562-
assertThat(hits.get(0).get("_id"), equalTo("0"));
563-
assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
575+
assertThat("hits: " + response, hits.get(0).get("_id"), equalTo("0"));
576+
assertThat("hits: " + response, (double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001));
564577

565578
// search with knn
566579
searchRequest = new Request("POST", "/" + FLAT_BBQ_INDEX_NAME + "/_search");
@@ -578,8 +591,12 @@ public void testFlatBBQVectorSearch() throws Exception {
578591
response = search(searchRequest);
579592
assertThat(extractValue(response, "hits.total.value"), equalTo(2));
580593
hits = extractValue(response, "hits.hits");
581-
assertThat(hits.get(0).get("_id"), equalTo("0"));
582-
assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005));
594+
assertThat("expected: 0 received" + hits.get(0).get("_id") + " hits: " + response, hits.get(0).get("_id"), equalTo("0"));
595+
assertThat(
596+
"expected_near: 0.99 received" + hits.get(0).get("_score") + "hits: " + response,
597+
(double) hits.get(0).get("_score"),
598+
closeTo(0.9934857, 0.005)
599+
);
583600
}
584601

585602
private void index64DimVectors(String indexName) throws Exception {
@@ -605,6 +622,7 @@ private void index64DimVectors(String indexName) throws Exception {
605622
assertOK(client().performRequest(indexRequest));
606623
}
607624
// always refresh to ensure the data is visible
625+
flush(indexName, true);
608626
refresh(indexName);
609627
}
610628

‎server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1967,7 +1967,14 @@ public void testPostRecoveryMerge() throws Exception {
19671967
internalCluster().startMasterOnlyNode();
19681968
final var dataNode = internalCluster().startDataOnlyNode();
19691969
final var indexName = randomIdentifier();
1970-
createIndex(indexName, indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false).build());
1970+
final var indexSettingsBuilder = indexSettings(1, 0).put(INDEX_MERGE_ENABLED, false);
1971+
if (randomBoolean()) {
1972+
indexSettingsBuilder.put(
1973+
IndexMetadata.SETTING_VERSION_CREATED,
1974+
IndexVersionUtils.randomVersionBetween(random(), IndexVersions.UPGRADE_TO_LUCENE_10_0_0, IndexVersion.current())
1975+
);
1976+
}
1977+
createIndex(indexName, indexSettingsBuilder.build());
19711978

19721979
final var initialSegmentCount = 20;
19731980
for (int i = 0; i < initialSegmentCount; i++) {
@@ -2051,7 +2058,7 @@ public void testPostRecoveryMergeDisabledOnOlderIndices() throws Exception {
20512058
IndexVersionUtils.randomVersionBetween(
20522059
random(),
20532060
IndexVersionUtils.getLowestWriteCompatibleVersion(),
2054-
IndexVersionUtils.getPreviousVersion(IndexVersions.MERGE_ON_RECOVERY_VERSION)
2061+
IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)
20552062
)
20562063
)
20572064
.build()

‎server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,6 +293,6 @@ public void parse(XContentParser parser, List<T> accumulator) throws IOException
293293
parseNonNullValue(parser, accumulator);
294294
}
295295

296-
abstract void parseNonNullValue(XContentParser parser, List<T> accumulator) throws IOException;
296+
protected abstract void parseNonNullValue(XContentParser parser, List<T> accumulator) throws IOException;
297297
}
298298
}

‎server/src/main/java/org/elasticsearch/indices/PostRecoveryMerger.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner;
1919
import org.elasticsearch.core.Releasable;
2020
import org.elasticsearch.core.Strings;
21+
import org.elasticsearch.core.UpdateForV10;
2122
import org.elasticsearch.index.IndexVersions;
2223
import org.elasticsearch.index.shard.IndexShard;
2324
import org.elasticsearch.index.shard.ShardId;
@@ -46,6 +47,7 @@ class PostRecoveryMerger {
4647
private static final boolean TRIGGER_MERGE_AFTER_RECOVERY;
4748

4849
static {
50+
@UpdateForV10(owner = UpdateForV10.Owner.DISTRIBUTED_INDEXING) // remove this escape hatch
4951
final var propertyValue = System.getProperty("es.trigger_merge_after_recovery");
5052
if (propertyValue == null) {
5153
TRIGGER_MERGE_AFTER_RECOVERY = true;
@@ -95,7 +97,7 @@ PeerRecoveryTargetService.RecoveryListener maybeMergeAfterRecovery(
9597
return recoveryListener;
9698
}
9799

98-
if (indexMetadata.getCreationVersion().before(IndexVersions.MERGE_ON_RECOVERY_VERSION)) {
100+
if (indexMetadata.getCreationVersion().before(IndexVersions.UPGRADE_TO_LUCENE_10_0_0)) {
99101
return recoveryListener;
100102
}
101103

0 commit comments

Comments
 (0)