Skip to content

Commit 0a3f611

Browse files
bpinteabpintea
authored
Merge branch 'main' into enh/allow_sort_before_inlinejoin
2 parents b470b00 + 364c70e commit 0a3f611

File tree

37 files changed

+744
-463
lines changed

37 files changed

+744
-463
lines changed

‎docs/changelog/132408.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132408
2+
summary: Correct exception for missing nested path
3+
area: Search
4+
type: bug
5+
issues: []

‎docs/changelog/132414.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132414
2+
summary: Adjust date docvalue formatting to return 4xx instead of 5xx
3+
area: Search
4+
type: bug
5+
issues: []

‎docs/changelog/92568.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 92568
2+
summary: Support nested fields for term vectors API when using artificial documents
3+
area: Search
4+
type: enhancement
5+
issues:
6+
- 91902

‎libs/simdvec/src/main/java/org/elasticsearch/simdvec/ES91OSQVectorsScorer.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ public float score(
141141
*
142142
* <p>The results are stored in the provided scores array.
143143
*/
144-
public void scoreBulk(
144+
public float scoreBulk(
145145
byte[] q,
146146
float queryLowerInterval,
147147
float queryUpperInterval,
@@ -158,6 +158,7 @@ public void scoreBulk(
158158
targetComponentSums[i] = Short.toUnsignedInt(in.readShort());
159159
}
160160
in.readFloats(additionalCorrections, 0, BULK_SIZE);
161+
float maxScore = Float.NEGATIVE_INFINITY;
161162
for (int i = 0; i < BULK_SIZE; i++) {
162163
scores[i] = score(
163164
queryLowerInterval,
@@ -172,6 +173,10 @@ public void scoreBulk(
172173
additionalCorrections[i],
173174
scores[i]
174175
);
176+
if (scores[i] > maxScore) {
177+
maxScore = scores[i];
178+
}
175179
}
180+
return maxScore;
176181
}
177182
}

‎libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentES91OSQVectorsScorer.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ private void quantizeScore256Bulk(byte[] q, int count, float[] scores) throws IO
352352
}
353353

354354
@Override
355-
public void scoreBulk(
355+
public float scoreBulk(
356356
byte[] q,
357357
float queryLowerInterval,
358358
float queryUpperInterval,
@@ -366,7 +366,7 @@ public void scoreBulk(
366366
// 128 / 8 == 16
367367
if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) {
368368
if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) {
369-
score256Bulk(
369+
return score256Bulk(
370370
q,
371371
queryLowerInterval,
372372
queryUpperInterval,
@@ -376,9 +376,8 @@ public void scoreBulk(
376376
centroidDp,
377377
scores
378378
);
379-
return;
380379
} else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) {
381-
score128Bulk(
380+
return score128Bulk(
382381
q,
383382
queryLowerInterval,
384383
queryUpperInterval,
@@ -388,10 +387,9 @@ public void scoreBulk(
388387
centroidDp,
389388
scores
390389
);
391-
return;
392390
}
393391
}
394-
super.scoreBulk(
392+
return super.scoreBulk(
395393
q,
396394
queryLowerInterval,
397395
queryUpperInterval,
@@ -403,7 +401,7 @@ public void scoreBulk(
403401
);
404402
}
405403

406-
private void score128Bulk(
404+
private float score128Bulk(
407405
byte[] q,
408406
float queryLowerInterval,
409407
float queryUpperInterval,
@@ -420,6 +418,7 @@ private void score128Bulk(
420418
float ay = queryLowerInterval;
421419
float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
422420
float y1 = queryComponentSum;
421+
float maxScore = Float.NEGATIVE_INFINITY;
423422
for (; i < limit; i += FLOAT_SPECIES_128.length()) {
424423
var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_128, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN);
425424
var lx = FloatVector.fromMemorySegment(
@@ -453,6 +452,7 @@ private void score128Bulk(
453452
if (similarityFunction == EUCLIDEAN) {
454453
res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f);
455454
res = FloatVector.broadcast(FLOAT_SPECIES_128, 1).div(res).max(0);
455+
maxScore = res.reduceLanes(VectorOperators.MAX);
456456
res.intoArray(scores, i);
457457
} else {
458458
// For cosine and max inner product, we need to apply the additional correction, which is
@@ -463,17 +463,20 @@ private void score128Bulk(
463463
// not sure how to do it better
464464
for (int j = 0; j < FLOAT_SPECIES_128.length(); j++) {
465465
scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]);
466+
maxScore = Math.max(maxScore, scores[i + j]);
466467
}
467468
} else {
468469
res = res.add(1f).mul(0.5f).max(0);
469470
res.intoArray(scores, i);
471+
maxScore = res.reduceLanes(VectorOperators.MAX);
470472
}
471473
}
472474
}
473475
in.seek(offset + 14L * BULK_SIZE);
476+
return maxScore;
474477
}
475478

476-
private void score256Bulk(
479+
private float score256Bulk(
477480
byte[] q,
478481
float queryLowerInterval,
479482
float queryUpperInterval,
@@ -490,6 +493,7 @@ private void score256Bulk(
490493
float ay = queryLowerInterval;
491494
float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
492495
float y1 = queryComponentSum;
496+
float maxScore = Float.NEGATIVE_INFINITY;
493497
for (; i < limit; i += FLOAT_SPECIES_256.length()) {
494498
var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_256, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN);
495499
var lx = FloatVector.fromMemorySegment(
@@ -523,6 +527,7 @@ private void score256Bulk(
523527
if (similarityFunction == EUCLIDEAN) {
524528
res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f);
525529
res = FloatVector.broadcast(FLOAT_SPECIES_256, 1).div(res).max(0);
530+
maxScore = res.reduceLanes(VectorOperators.MAX);
526531
res.intoArray(scores, i);
527532
} else {
528533
// For cosine and max inner product, we need to apply the additional correction, which is
@@ -533,13 +538,16 @@ private void score256Bulk(
533538
// not sure how to do it better
534539
for (int j = 0; j < FLOAT_SPECIES_256.length(); j++) {
535540
scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]);
541+
maxScore = Math.max(maxScore, scores[i + j]);
536542
}
537543
} else {
538544
res = res.add(1f).mul(0.5f).max(0);
545+
maxScore = res.reduceLanes(VectorOperators.MAX);
539546
res.intoArray(scores, i);
540547
}
541548
}
542549
}
543550
in.seek(offset + 14L * BULK_SIZE);
551+
return maxScore;
544552
}
545553
}

‎modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -364,25 +364,24 @@ public void applyClusterState(ClusterChangedEvent event) {
364364
// Skip project clients that have no credentials configured. This should not happen in serverless.
365365
// But it is safer to skip them and is also a more consistent behaviour with the cases when
366366
// project secrets are not present.
367-
final var clientSettings = allClientSettings.entrySet()
367+
final var clientSettingsWithCredentials = allClientSettings.entrySet()
368368
.stream()
369369
.filter(entry -> entry.getValue().getCredential() != null)
370370
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
371371

372-
if (allClientSettings.size() != clientSettings.size()) {
373-
logger.warn(
374-
"Project [{}] has [{}] GCS client settings, but [{}] is usable due to missing credentials for clients {}",
375-
project.id(),
376-
allClientSettings.size(),
377-
clientSettings.size(),
378-
Sets.difference(allClientSettings.keySet(), clientSettings.keySet())
379-
);
380-
}
381-
382372
// TODO: If performance is an issue, we may consider comparing just the relevant project secrets for new or updated clients
383373
// and avoid building the clientSettings
384-
if (newOrUpdated(project.id(), clientSettings)) {
385-
updatedPerProjectClients.put(project.id(), new PerProjectClientsHolder(clientSettings));
374+
if (newOrUpdated(project.id(), clientSettingsWithCredentials)) {
375+
if (allClientSettings.size() != clientSettingsWithCredentials.size()) {
376+
logger.warn(
377+
"Project [{}] has [{}] GCS client settings, but [{}] is usable due to missing credentials for clients {}",
378+
project.id(),
379+
allClientSettings.size(),
380+
clientSettingsWithCredentials.size(),
381+
Sets.difference(allClientSettings.keySet(), clientSettingsWithCredentials.keySet())
382+
);
383+
}
384+
updatedPerProjectClients.put(project.id(), new PerProjectClientsHolder(clientSettingsWithCredentials));
386385
}
387386
}
388387

‎muted-tests.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -473,9 +473,6 @@ tests:
473473
- class: org.elasticsearch.packaging.test.DockerTests
474474
method: test151MachineDependentHeapWithSizeOverride
475475
issue: https://github.com/elastic/elasticsearch/issues/123437
476-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
477-
method: testUpdates
478-
issue: https://github.com/elastic/elasticsearch/issues/131795
479476
- class: org.elasticsearch.xpack.restart.FullClusterRestartIT
480477
method: testWatcherWithApiKey {cluster=UPGRADED}
481478
issue: https://github.com/elastic/elasticsearch/issues/131964
@@ -485,12 +482,6 @@ tests:
485482
- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1EnrichUnavailableRemotesIT
486483
method: testEsqlEnrichWithSkipUnavailable
487484
issue: https://github.com/elastic/elasticsearch/issues/132078
488-
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
489-
method: test {yaml=update/100_synthetic_source/stored text}
490-
issue: https://github.com/elastic/elasticsearch/issues/132108
491-
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
492-
method: test {yaml=update/100_synthetic_source/keyword}
493-
issue: https://github.com/elastic/elasticsearch/issues/132110
494485
- class: org.elasticsearch.index.engine.MergeWithLowDiskSpaceIT
495486
method: testRelocationWhileForceMerging
496487
issue: https://github.com/elastic/elasticsearch/issues/131789

‎rest-api-spec/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,4 +90,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
9090
task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively")
9191
task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed")
9292
task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist")
93+
task.skipTest("update/100_synthetic_source/keyword", "synthetic recovery source means _recovery_source field will not be present")
94+
task.skipTest("update/100_synthetic_source/stored text", "synthetic recovery source means _recovery_source field will not be present")
9395
})

‎server/src/internalClusterTest/java/org/elasticsearch/discovery/single/SingleNodeDiscoveryIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,8 @@ public Path nodeConfigPath(int nodeOrdinal) {
8787
0,
8888
"other",
8989
Arrays.asList(getTestTransportPlugin(), MockHttpTransport.TestPlugin.class),
90-
Function.identity()
90+
Function.identity(),
91+
TEST_ENTITLEMENTS::addEntitledNodePaths
9192
);
9293
try {
9394
other.beforeTest(random());
@@ -137,7 +138,8 @@ public Path nodeConfigPath(int nodeOrdinal) {
137138
0,
138139
"other",
139140
Arrays.asList(getTestTransportPlugin(), MockHttpTransport.TestPlugin.class),
140-
Function.identity()
141+
Function.identity(),
142+
TEST_ENTITLEMENTS::addEntitledNodePaths
141143
);
142144
try (var mockLog = MockLog.capture(JoinHelper.class)) {
143145
mockLog.addExpectation(

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,8 @@ public Path nodeConfigPath(int nodeOrdinal) {
7777
InternalSettingsPlugin.class,
7878
getTestTransportPlugin()
7979
),
80-
Function.identity()
80+
Function.identity(),
81+
TEST_ENTITLEMENTS::addEntitledNodePaths
8182
);
8283
secondCluster.beforeTest(random());
8384
}

0 commit comments

Comments
 (0)