Merge branch 'main' into cs-remove-error-parser

Author: jonathan-buttner

benchmarks/src/main/java/org/elasticsearch/benchmark/search/query/range/DateFieldMapperDocValuesSkipperBenchmark.java

@@ -21,14 +21,14 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -295,7 +295,7 @@ public void rangeQueryWithDocValuesSkipper(final Blackhole bh) throws IOExceptio
     /**
      * Runs the actual Lucene range query, optionally combining a {@link LongPoint} index query
      * with doc values ({@link SortedNumericDocValuesField}) via {@link IndexOrDocValuesQuery},
-     * and then wrapping it with an {@link IndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
+     * and then wrapping it with an {@link XIndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
      *
      * @param searcher            the Lucene {@link IndexSearcher}
      * @param rangeStartTimestamp lower bound of the timestamp range
@@ -316,7 +316,7 @@ private long rangeQuery(final IndexSearcher searcher, long rangeStartTimestamp,
             )
             : SortedNumericDocValuesField.newSlowRangeQuery(TIMESTAMP_FIELD, rangeStartTimestamp, rangeEndTimestamp);
 
-        final Query query = new IndexSortSortedNumericDocValuesRangeQuery(
+        final Query query = new XIndexSortSortedNumericDocValuesRangeQuery(
             TIMESTAMP_FIELD,
             rangeStartTimestamp,
             rangeEndTimestamp,

build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt

@@ -61,3 +61,11 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
 
 @defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
 org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
+
+# This is a temporary patch as there is a low level Lucene bug in certain scenarios
+# this should be fixed in the new Lucene release 10.3+
+org.apache.lucene.document.LongField#newExactQuery(java.lang.String, long) @ Use org.elasticsearch.lucene.document.NumericField#newExactLongQuery(java.lang.String, long) instead.
+org.apache.lucene.document.LongField#newRangeQuery(java.lang.String, long, long) @ Use org.elasticsearch.lucene.document.NumericField#newRangeLongQuery(java.lang.String, long, long) instead.
+org.apache.lucene.document.IntField#newExactQuery(java.lang.String, int) @ Use org.elasticsearch.lucene.document.NumericField#newExactIntQuery(java.lang.String, int) instead.
+org.apache.lucene.document.IntField#newRangeQuery(java.lang.String, int, int) @ Use org.elasticsearch.lucene.document.NumericField#newRangeIntQuery(java.lang.String, int, int) instead.
+org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery @ use org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery instead.

build-tools-internal/src/main/resources/templates/index.md

@@ -28,16 +28,17 @@ for(bundle in changelogBundles) {
     def nonNotableHighlights = bundle.nonNotableHighlights
     def unqualifiedVersion = bundle.unqualifiedVersion
     def coming = !bundle.bundle.released
-
-    if (coming) {
-        print "\n"
-        print "```{applies_to}\n"
-        print "stack: coming ${version}\n"
-        print "```"
-    }
 %>
 ## ${unqualifiedVersion} [elasticsearch-${versionForIds}-release-notes]
 <%
+
+if (coming) {
+    print "```{applies_to}\n"
+    print "stack: coming ${version}\n"
+    print "```"
+    print "\n"
+}
+
 if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) {
     print "\n### Highlights [elasticsearch-${versionForIds}-highlights]\n"
 }

build.gradle

@@ -433,6 +433,10 @@ tasks.named("updateDaemonJvm") {
       org.gradle.platform.Architecture.X86_64,
       org.gradle.platform.OperatingSystem.WINDOWS
     ),
+    BuildPlatformFactory.of(
+      org.gradle.platform.Architecture.AARCH64,
+      org.gradle.platform.OperatingSystem.WINDOWS
+    ),
     // anyone still using x86 osx?
     BuildPlatformFactory.of(
       org.gradle.platform.Architecture.X86_64,

docs/changelog/128504.yaml

@@ -0,0 +1,5 @@
+pr: 128504
+summary: Add l2_norm normalization support to linear retriever
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/128509.yaml

@@ -0,0 +1,5 @@
+pr: 128509
+summary: Use default Lucene postings format when index mode is standard.
+area: Codec
+type: enhancement
+issues: []

docs/changelog/128689.yaml

@@ -0,0 +1,5 @@
+pr: 128689
+summary: Fix minmax normalizer handling of single-doc result sets
+area: Search
+type: bug
+issues: []

docs/changelog/128740.yaml

@@ -0,0 +1,5 @@
+pr: 128740
+summary: Optimize sparse vector stats collection
+area: Stats
+type: enhancement
+issues: []

docs/changelog/128746.yaml

@@ -0,0 +1,5 @@
+pr: 128746
+summary: Fix computation of last block size in Azure concurrent multipart uploads
+area: Snapshot/Restore
+type: bug
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -276,7 +276,7 @@ Each entry specifies the following parameters:
 `normalizer`
 :   (Optional, String)
 
-    Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, and `none`. Defaults to `none`.
+    - Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
 
     * `none`
     * `minmax` : A `MinMaxScoreNormalizer` that normalizes scores based on the following formula
@@ -285,6 +285,7 @@ Each entry specifies the following parameters:
         score = (score - min) / (max - min)
         ```
 
+    * `l2_norm` : An `L2ScoreNormalizer` that normalizes scores using the L2 norm of the score values.
 
 See also [this hybrid search example](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.