Skip to content

Commit d12f757

Browse files
benwtrentbenwtrent
authored
[8.19] Allow zero for rescore_vector.oversample to indicate by-passing oversample and rescoring (#125599) (#127662)
* Allow zero for rescore_vector.oversample to indicate by-passing oversample and rescoring (#125599) This allows a `rescore_vector: {oversample: 0}` to indicate bypassing oversampling and rescoring. This is useful for: - Updating a quantized mapping to turn off automatic rescoring - Bypassing oversampling at query time in an ad-hoc manner if its on by default in the mapping closes: #125157 (cherry picked from commit 009a86a) * Adding versions for bwc for #125599 (#127663) blocks: #127662 This adds bwc versions for #125599 * fixing versioning for merge
1 parent 0201ce6 commit d12f757

File tree

12 files changed

+835
-73
lines changed

12 files changed

+835
-73
lines changed

‎docs/changelog/125599.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 125599
2+
summary: Allow zero for `rescore_vector.oversample` to indicate by-passing oversample
3+
and rescoring
4+
area: Vector Search
5+
type: enhancement
6+
issues: []

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,3 +340,260 @@ setup:
340340
- match: { hits.hits.0._score: $rescore_score0 }
341341
- match: { hits.hits.1._score: $rescore_score1 }
342342
- match: { hits.hits.2._score: $rescore_score2 }
343+
---
344+
"Test index configured rescore vector updateable and settable to 0":
345+
- requires:
346+
cluster_features: ["mapper.dense_vector.rescore_zero_vector"]
347+
reason: Needs rescore_zero_vector feature
348+
349+
- do:
350+
indices.create:
351+
index: bbq_rescore_0_hnsw
352+
body:
353+
settings:
354+
index:
355+
number_of_shards: 1
356+
mappings:
357+
properties:
358+
vector:
359+
type: dense_vector
360+
index_options:
361+
type: bbq_hnsw
362+
rescore_vector:
363+
oversample: 0
364+
365+
- do:
366+
indices.create:
367+
index: bbq_rescore_update_hnsw
368+
body:
369+
settings:
370+
index:
371+
number_of_shards: 1
372+
mappings:
373+
properties:
374+
vector:
375+
type: dense_vector
376+
index_options:
377+
type: bbq_hnsw
378+
rescore_vector:
379+
oversample: 1
380+
381+
- do:
382+
indices.put_mapping:
383+
index: bbq_rescore_update_hnsw
384+
body:
385+
properties:
386+
vector:
387+
type: dense_vector
388+
index_options:
389+
type: bbq_hnsw
390+
rescore_vector:
391+
oversample: 0
392+
393+
- do:
394+
indices.get_mapping:
395+
index: bbq_rescore_update_hnsw
396+
397+
- match: { .bbq_rescore_update_hnsw.mappings.properties.vector.index_options.rescore_vector.oversample: 0 }
398+
---
399+
"Test index configured rescore vector score consistency":
400+
- requires:
401+
cluster_features: ["mapper.dense_vector.rescore_zero_vector"]
402+
reason: Needs rescore_zero_vector feature
403+
- skip:
404+
features: "headers"
405+
- do:
406+
indices.create:
407+
index: bbq_rescore_zero_hnsw
408+
body:
409+
settings:
410+
index:
411+
number_of_shards: 1
412+
mappings:
413+
properties:
414+
vector:
415+
type: dense_vector
416+
dims: 64
417+
index: true
418+
similarity: max_inner_product
419+
index_options:
420+
type: bbq_hnsw
421+
rescore_vector:
422+
oversample: 0
423+
424+
- do:
425+
bulk:
426+
index: bbq_rescore_zero_hnsw
427+
refresh: true
428+
body: |
429+
{ "index": {"_id": "1"}}
430+
{ "vector": [0.077, 0.32 , -0.205, 0.63 , 0.032, 0.201, 0.167, -0.313, 0.176, 0.531, -0.375, 0.334, -0.046, 0.078, -0.349, 0.272, 0.307, -0.083, 0.504, 0.255, -0.404, 0.289, -0.226, -0.132, -0.216, 0.49 , 0.039, 0.507, -0.307, 0.107, 0.09 , -0.265, -0.285, 0.336, -0.272, 0.369, -0.282, 0.086, -0.132, 0.475, -0.224, 0.203, 0.439, 0.064, 0.246, -0.396, 0.297, 0.242, -0.028, 0.321, -0.022, -0.009, -0.001 , 0.031, -0.533, 0.45, -0.683, 1.331, 0.194, -0.157, -0.1 , -0.279, -0.098, -0.176] }
431+
{ "index": {"_id": "2"}}
432+
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
433+
{ "index": {"_id": "3"}}
434+
{ "vector": [0.196, 0.514, 0.039, 0.555, -0.042, 0.242, 0.463, -0.348, -0.08 , 0.442, -0.067, -0.05 , -0.001, 0.298, -0.377, 0.048, 0.307, 0.159, 0.278, 0.119, -0.057, 0.333, -0.289, -0.438, -0.014, 0.361, -0.169, 0.292, -0.229, 0.123, 0.031, -0.138, -0.139, 0.315, -0.216, 0.322, -0.445, -0.059, 0.071, 0.429, -0.602, -0.142, 0.11 , 0.192, 0.259, -0.241, 0.181, -0.166, 0.082, 0.107, -0.05 , 0.155, 0.011, 0.161, -0.486, 0.569, -0.489, 0.901, 0.208, 0.011, -0.209, -0.153, -0.27 , -0.013] }
435+
436+
- do:
437+
headers:
438+
Content-Type: application/json
439+
search:
440+
rest_total_hits_as_int: true
441+
index: bbq_rescore_zero_hnsw
442+
body:
443+
knn:
444+
field: vector
445+
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
446+
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
447+
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
448+
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
449+
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
450+
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
451+
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
452+
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
453+
k: 3
454+
num_candidates: 3
455+
456+
- match: { hits.total: 3 }
457+
- set: { hits.hits.0._score: raw_score0 }
458+
- set: { hits.hits.1._score: raw_score1 }
459+
- set: { hits.hits.2._score: raw_score2 }
460+
461+
462+
- do:
463+
headers:
464+
Content-Type: application/json
465+
search:
466+
rest_total_hits_as_int: true
467+
index: bbq_rescore_zero_hnsw
468+
body:
469+
knn:
470+
field: vector
471+
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
472+
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
473+
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
474+
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
475+
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
476+
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
477+
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
478+
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
479+
k: 3
480+
num_candidates: 3
481+
rescore_vector:
482+
oversample: 2
483+
484+
- match: { hits.total: 3 }
485+
- set: { hits.hits.0._score: override_score0 }
486+
- set: { hits.hits.1._score: override_score1 }
487+
- set: { hits.hits.2._score: override_score2 }
488+
489+
- do:
490+
indices.put_mapping:
491+
index: bbq_rescore_zero_hnsw
492+
body:
493+
properties:
494+
vector:
495+
type: dense_vector
496+
dims: 64
497+
index: true
498+
similarity: max_inner_product
499+
index_options:
500+
type: bbq_hnsw
501+
rescore_vector:
502+
oversample: 2
503+
504+
- do:
505+
headers:
506+
Content-Type: application/json
507+
search:
508+
rest_total_hits_as_int: true
509+
index: bbq_rescore_zero_hnsw
510+
body:
511+
knn:
512+
field: vector
513+
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
514+
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
515+
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
516+
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
517+
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
518+
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
519+
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
520+
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
521+
k: 3
522+
num_candidates: 3
523+
524+
- match: { hits.total: 3 }
525+
- set: { hits.hits.0._score: default_rescore0 }
526+
- set: { hits.hits.1._score: default_rescore1 }
527+
- set: { hits.hits.2._score: default_rescore2 }
528+
529+
- do:
530+
indices.put_mapping:
531+
index: bbq_rescore_zero_hnsw
532+
body:
533+
properties:
534+
vector:
535+
type: dense_vector
536+
dims: 64
537+
index: true
538+
similarity: max_inner_product
539+
index_options:
540+
type: bbq_hnsw
541+
rescore_vector:
542+
oversample: 0
543+
544+
- do:
545+
headers:
546+
Content-Type: application/json
547+
search:
548+
rest_total_hits_as_int: true
549+
index: bbq_rescore_zero_hnsw
550+
body:
551+
query:
552+
script_score:
553+
query: {match_all: {} }
554+
script:
555+
source: "double similarity = dotProduct(params.query_vector, 'vector'); return similarity < 0 ? 1 / (1 + -1 * similarity) : similarity + 1"
556+
params:
557+
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
558+
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
559+
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
560+
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
561+
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
562+
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
563+
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
564+
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
565+
566+
# Compare scores as hit IDs may change depending on how things are distributed
567+
- match: { hits.total: 3 }
568+
- match: { hits.hits.0._score: $override_score0 }
569+
- match: { hits.hits.0._score: $default_rescore0 }
570+
- match: { hits.hits.1._score: $override_score1 }
571+
- match: { hits.hits.1._score: $default_rescore1 }
572+
- match: { hits.hits.2._score: $override_score2 }
573+
- match: { hits.hits.2._score: $default_rescore2 }
574+
575+
- do:
576+
headers:
577+
Content-Type: application/json
578+
search:
579+
rest_total_hits_as_int: true
580+
index: bbq_rescore_zero_hnsw
581+
body:
582+
knn:
583+
field: vector
584+
query_vector: [0.128, 0.067, -0.08 , 0.395, -0.11 , -0.259, 0.473, -0.393,
585+
0.292, 0.571, -0.491, 0.444, -0.288, 0.198, -0.343, 0.015,
586+
0.232, 0.088, 0.228, 0.151, -0.136, 0.236, -0.273, -0.259,
587+
-0.217, 0.359, -0.207, 0.352, -0.142, 0.192, -0.061, -0.17 ,
588+
-0.343, 0.189, -0.221, 0.32 , -0.301, -0.1 , 0.005, 0.232,
589+
-0.344, 0.136, 0.252, 0.157, -0.13 , -0.244, 0.193, -0.034,
590+
-0.12 , -0.193, -0.102, 0.252, -0.185, -0.167, -0.575, 0.582,
591+
-0.426, 0.983, 0.212, 0.204, 0.03 , -0.276, -0.425, -0.158]
592+
k: 3
593+
num_candidates: 3
594+
595+
# Compare scores as hit IDs may change depending on how things are distributed
596+
- match: { hits.total: 3 }
597+
- match: { hits.hits.0._score: $raw_score0 }
598+
- match: { hits.hits.1._score: $raw_score1 }
599+
- match: { hits.hits.2._score: $raw_score2 }

0 commit comments

Comments
 (0)