Merge branch 'main' into semantic_text_chunks_format

Author: jimczi

docs/changelog/132408.yaml

@@ -0,0 +1,5 @@
+pr: 132408
+summary: Correct exception for missing nested path
+area: Search
+type: bug
+issues: []

docs/changelog/132414.yaml

@@ -0,0 +1,5 @@
+pr: 132414
+summary: Adjust date docvalue formatting to return 4xx instead of 5xx
+area: Search
+type: bug
+issues: []

docs/changelog/92568.yaml

@@ -0,0 +1,6 @@
+pr: 92568
+summary: Support nested fields for term vectors API when using artificial documents
+area: Search
+type: enhancement
+issues:
+  - 91902

docs/reference/query-languages/query-dsl/query-dsl-pinned-query.md

@@ -7,6 +7,10 @@ mapped_pages:
 
 Promotes selected documents to rank higher than those matching a given query. This feature is typically used to guide searchers to curated documents that are promoted over and above any "organic" matches for a search. The promoted or "pinned" documents are identified using the document IDs stored in the [`_id`](/reference/elasticsearch/mapping-reference/mapping-id-field.md) field.
 
+::::{note}
+Pinned queries are designed to work only with relevance-based sorting. Using explicit `sort` criteria overrides the pinned document promotion.
+::::
+
 ## Example request [_example_request]
 
 ```console

libs/simdvec/src/main/java/org/elasticsearch/simdvec/ES91OSQVectorsScorer.java

@@ -141,7 +141,7 @@ public float score(
      *
      * <p>The results are stored in the provided scores array.
      */
-    public void scoreBulk(
+    public float scoreBulk(
         byte[] q,
         float queryLowerInterval,
         float queryUpperInterval,
@@ -158,6 +158,7 @@ public void scoreBulk(
             targetComponentSums[i] = Short.toUnsignedInt(in.readShort());
         }
         in.readFloats(additionalCorrections, 0, BULK_SIZE);
+        float maxScore = Float.NEGATIVE_INFINITY;
         for (int i = 0; i < BULK_SIZE; i++) {
             scores[i] = score(
                 queryLowerInterval,
@@ -172,6 +173,10 @@ public void scoreBulk(
                 additionalCorrections[i],
                 scores[i]
             );
+            if (scores[i] > maxScore) {
+                maxScore = scores[i];
+            }
         }
+        return maxScore;
     }
 }

libs/simdvec/src/main/java/org/elasticsearch/simdvec/internal/vectorization/DefaultESVectorUtilSupport.java

@@ -80,10 +80,11 @@ public void calculateOSQGridPoints(float[] target, int[] quantize, int points, f
         float dbb = 0;
         float dax = 0;
         float dbx = 0;
+        float invPmOnes = 1f / (points - 1f);
         for (int i = 0; i < target.length; ++i) {
             float v = target[i];
             float k = quantize[i];
-            float s = k / (points - 1);
+            float s = k * invPmOnes;
             float ms = 1f - s;
             daa = fma(ms, ms, daa);
             dab = fma(ms, s, dab);

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentES91OSQVectorsScorer.java

@@ -352,7 +352,7 @@ private void quantizeScore256Bulk(byte[] q, int count, float[] scores) throws IO
     }
 
     @Override
-    public void scoreBulk(
+    public float scoreBulk(
         byte[] q,
         float queryLowerInterval,
         float queryUpperInterval,
@@ -366,7 +366,7 @@ public void scoreBulk(
         // 128 / 8 == 16
         if (length >= 16 && PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS) {
             if (PanamaESVectorUtilSupport.VECTOR_BITSIZE >= 256) {
-                score256Bulk(
+                return score256Bulk(
                     q,
                     queryLowerInterval,
                     queryUpperInterval,
@@ -376,9 +376,8 @@ public void scoreBulk(
                     centroidDp,
                     scores
                 );
-                return;
             } else if (PanamaESVectorUtilSupport.VECTOR_BITSIZE == 128) {
-                score128Bulk(
+                return score128Bulk(
                     q,
                     queryLowerInterval,
                     queryUpperInterval,
@@ -388,10 +387,9 @@ public void scoreBulk(
                     centroidDp,
                     scores
                 );
-                return;
             }
         }
-        super.scoreBulk(
+        return super.scoreBulk(
             q,
             queryLowerInterval,
             queryUpperInterval,
@@ -403,7 +401,7 @@ public void scoreBulk(
         );
     }
 
-    private void score128Bulk(
+    private float score128Bulk(
         byte[] q,
         float queryLowerInterval,
         float queryUpperInterval,
@@ -420,6 +418,7 @@ private void score128Bulk(
         float ay = queryLowerInterval;
         float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
         float y1 = queryComponentSum;
+        float maxScore = Float.NEGATIVE_INFINITY;
         for (; i < limit; i += FLOAT_SPECIES_128.length()) {
             var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_128, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN);
             var lx = FloatVector.fromMemorySegment(
@@ -453,6 +452,7 @@ private void score128Bulk(
             if (similarityFunction == EUCLIDEAN) {
                 res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f);
                 res = FloatVector.broadcast(FLOAT_SPECIES_128, 1).div(res).max(0);
+                maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX));
                 res.intoArray(scores, i);
             } else {
                 // For cosine and max inner product, we need to apply the additional correction, which is
@@ -463,17 +463,20 @@ private void score128Bulk(
                     // not sure how to do it better
                     for (int j = 0; j < FLOAT_SPECIES_128.length(); j++) {
                         scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]);
+                        maxScore = Math.max(maxScore, scores[i + j]);
                     }
                 } else {
                     res = res.add(1f).mul(0.5f).max(0);
                     res.intoArray(scores, i);
+                    maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX));
                 }
             }
         }
         in.seek(offset + 14L * BULK_SIZE);
+        return maxScore;
     }
 
-    private void score256Bulk(
+    private float score256Bulk(
         byte[] q,
         float queryLowerInterval,
         float queryUpperInterval,
@@ -490,6 +493,7 @@ private void score256Bulk(
         float ay = queryLowerInterval;
         float ly = (queryUpperInterval - ay) * FOUR_BIT_SCALE;
         float y1 = queryComponentSum;
+        float maxScore = Float.NEGATIVE_INFINITY;
         for (; i < limit; i += FLOAT_SPECIES_256.length()) {
             var ax = FloatVector.fromMemorySegment(FLOAT_SPECIES_256, memorySegment, offset + i * Float.BYTES, ByteOrder.LITTLE_ENDIAN);
             var lx = FloatVector.fromMemorySegment(
@@ -523,6 +527,7 @@ private void score256Bulk(
             if (similarityFunction == EUCLIDEAN) {
                 res = res.mul(-2).add(additionalCorrections).add(queryAdditionalCorrection).add(1f);
                 res = FloatVector.broadcast(FLOAT_SPECIES_256, 1).div(res).max(0);
+                maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX));
                 res.intoArray(scores, i);
             } else {
                 // For cosine and max inner product, we need to apply the additional correction, which is
@@ -533,13 +538,16 @@ private void score256Bulk(
                     // not sure how to do it better
                     for (int j = 0; j < FLOAT_SPECIES_256.length(); j++) {
                         scores[i + j] = VectorUtil.scaleMaxInnerProductScore(scores[i + j]);
+                        maxScore = Math.max(maxScore, scores[i + j]);
                     }
                 } else {
                     res = res.add(1f).mul(0.5f).max(0);
+                    maxScore = Math.max(maxScore, res.reduceLanes(VectorOperators.MAX));
                     res.intoArray(scores, i);
                 }
             }
         }
         in.seek(offset + 14L * BULK_SIZE);
+        return maxScore;
     }
 }

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorUtilSupport.java

@@ -132,7 +132,7 @@ public void centerAndCalculateOSQStatsEuclidean(float[] vector, float[] centroid
                 FloatVector centeredVec = v.sub(c);
                 FloatVector deltaVec = centeredVec.sub(vecMeanVec);
                 norm2Vec = fma(centeredVec, centeredVec, norm2Vec);
-                vecMeanVec = vecMeanVec.add(deltaVec.div(count));
+                vecMeanVec = vecMeanVec.add(deltaVec.mul(1f / count));
                 FloatVector delta2Vec = centeredVec.sub(vecMeanVec);
                 m2Vec = fma(deltaVec, delta2Vec, m2Vec);
                 minVec = minVec.min(centeredVec);
@@ -214,7 +214,7 @@ public void centerAndCalculateOSQStatsDp(float[] vector, float[] centroid, float
                 FloatVector centeredVec = v.sub(c);
                 FloatVector deltaVec = centeredVec.sub(vecMeanVec);
                 norm2Vec = fma(centeredVec, centeredVec, norm2Vec);
-                vecMeanVec = vecMeanVec.add(deltaVec.div(count));
+                vecMeanVec = vecMeanVec.add(deltaVec.mul(1f / count));
                 FloatVector delta2Vec = centeredVec.sub(vecMeanVec);
                 m2Vec = fma(deltaVec, delta2Vec, m2Vec);
                 minVec = minVec.min(centeredVec);
@@ -278,6 +278,7 @@ public void calculateOSQGridPoints(float[] target, int[] quantize, int points, f
         float dbb = 0;
         float dax = 0;
         float dbx = 0;
+        float invPmOnes = 1f / (points - 1f);
         // if the array size is large (> 2x platform vector size), it's worth the overhead to vectorize
         if (target.length > 2 * FLOAT_SPECIES.length()) {
             FloatVector daaVec = FloatVector.zero(FLOAT_SPECIES);
@@ -286,11 +287,11 @@ public void calculateOSQGridPoints(float[] target, int[] quantize, int points, f
             FloatVector daxVec = FloatVector.zero(FLOAT_SPECIES);
             FloatVector dbxVec = FloatVector.zero(FLOAT_SPECIES);
             FloatVector ones = FloatVector.broadcast(FLOAT_SPECIES, 1f);
-            FloatVector pmOnes = FloatVector.broadcast(FLOAT_SPECIES, points - 1f);
+            FloatVector invPmOnesVec = FloatVector.broadcast(FLOAT_SPECIES, invPmOnes);
             for (; i < FLOAT_SPECIES.loopBound(target.length); i += FLOAT_SPECIES.length()) {
                 FloatVector v = FloatVector.fromArray(FLOAT_SPECIES, target, i);
                 FloatVector oVec = IntVector.fromArray(INTEGER_SPECIES, quantize, i).convert(VectorOperators.I2F, 0).reinterpretAsFloats();
-                FloatVector sVec = oVec.div(pmOnes);
+                FloatVector sVec = oVec.mul(invPmOnesVec);
                 FloatVector smVec = ones.sub(sVec);
                 daaVec = fma(smVec, smVec, daaVec);
                 dabVec = fma(smVec, sVec, dabVec);
@@ -307,7 +308,7 @@ public void calculateOSQGridPoints(float[] target, int[] quantize, int points, f
 
         for (; i < target.length; i++) {
             float k = quantize[i];
-            float s = k / (points - 1);
+            float s = k * invPmOnes;
             float ms = 1f - s;
             daa = fma(ms, ms, daa);
             dab = fma(ms, s, dab);
@@ -798,25 +799,26 @@ public static float ipFloatByteImpl(float[] q, byte[] d) {
     @Override
     public int quantizeVectorWithIntervals(float[] vector, int[] destination, float lowInterval, float upperInterval, byte bits) {
         float nSteps = ((1 << bits) - 1);
-        float step = (upperInterval - lowInterval) / nSteps;
+        float invStep = nSteps / (upperInterval - lowInterval);
         int sumQuery = 0;
         int i = 0;
         if (vector.length > 2 * FLOAT_SPECIES.length()) {
             int limit = FLOAT_SPECIES.loopBound(vector.length);
             FloatVector lowVec = FloatVector.broadcast(FLOAT_SPECIES, lowInterval);
             FloatVector upperVec = FloatVector.broadcast(FLOAT_SPECIES, upperInterval);
-            FloatVector stepVec = FloatVector.broadcast(FLOAT_SPECIES, step);
+            FloatVector invStepVec = FloatVector.broadcast(FLOAT_SPECIES, invStep);
             for (; i < limit; i += FLOAT_SPECIES.length()) {
                 FloatVector v = FloatVector.fromArray(FLOAT_SPECIES, vector, i);
                 FloatVector xi = v.max(lowVec).min(upperVec); // clamp
-                IntVector assignment = xi.sub(lowVec).div(stepVec).add(0.5f).convert(VectorOperators.F2I, 0).reinterpretAsInts(); // round
+                // round
+                IntVector assignment = xi.sub(lowVec).mul(invStepVec).add(0.5f).convert(VectorOperators.F2I, 0).reinterpretAsInts();
                 sumQuery += assignment.reduceLanes(ADD);
                 assignment.intoArray(destination, i);
             }
         }
         for (; i < vector.length; i++) {
             float xi = Math.min(Math.max(vector[i], lowInterval), upperInterval);
-            int assignment = Math.round((xi - lowInterval) / step);
+            int assignment = Math.round((xi - lowInterval) * invStep);
             sumQuery += assignment;
             destination[i] = assignment;
         }