Merge branch 'main' into ivf_hkmeans

Author: john-wagster

benchmarks/src/main/java/org/elasticsearch/benchmark/search/query/range/DateFieldMapperDocValuesSkipperBenchmark.java

@@ -21,14 +21,14 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.SortedNumericSortField;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -295,7 +295,7 @@ public void rangeQueryWithDocValuesSkipper(final Blackhole bh) throws IOExceptio
     /**
      * Runs the actual Lucene range query, optionally combining a {@link LongPoint} index query
      * with doc values ({@link SortedNumericDocValuesField}) via {@link IndexOrDocValuesQuery},
-     * and then wrapping it with an {@link IndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
+     * and then wrapping it with an {@link XIndexSortSortedNumericDocValuesRangeQuery} to utilize the index sort.
      *
      * @param searcher            the Lucene {@link IndexSearcher}
      * @param rangeStartTimestamp lower bound of the timestamp range
@@ -316,7 +316,7 @@ private long rangeQuery(final IndexSearcher searcher, long rangeStartTimestamp,
             )
             : SortedNumericDocValuesField.newSlowRangeQuery(TIMESTAMP_FIELD, rangeStartTimestamp, rangeEndTimestamp);
 
-        final Query query = new IndexSortSortedNumericDocValuesRangeQuery(
+        final Query query = new XIndexSortSortedNumericDocValuesRangeQuery(
             TIMESTAMP_FIELD,
             rangeStartTimestamp,
             rangeEndTimestamp,

build-tools-internal/src/main/resources/templates/index.md

@@ -28,16 +28,17 @@ for(bundle in changelogBundles) {
     def nonNotableHighlights = bundle.nonNotableHighlights
     def unqualifiedVersion = bundle.unqualifiedVersion
     def coming = !bundle.bundle.released
-
-    if (coming) {
-        print "\n"
-        print "```{applies_to}\n"
-        print "stack: coming ${version}\n"
-        print "```"
-    }
 %>
 ## ${unqualifiedVersion} [elasticsearch-${versionForIds}-release-notes]
 <%
+
+if (coming) {
+    print "```{applies_to}\n"
+    print "stack: coming ${version}\n"
+    print "```"
+    print "\n"
+}
+
 if (!notableHighlights.isEmpty() || !nonNotableHighlights.isEmpty()) {
     print "\n### Highlights [elasticsearch-${versionForIds}-highlights]\n"
 }

build.gradle

@@ -433,6 +433,10 @@ tasks.named("updateDaemonJvm") {
       org.gradle.platform.Architecture.X86_64,
       org.gradle.platform.OperatingSystem.WINDOWS
     ),
+    BuildPlatformFactory.of(
+      org.gradle.platform.Architecture.AARCH64,
+      org.gradle.platform.OperatingSystem.WINDOWS
+    ),
     // anyone still using x86 osx?
     BuildPlatformFactory.of(
       org.gradle.platform.Architecture.X86_64,

docs/changelog/128504.yaml

@@ -0,0 +1,5 @@
+pr: 128504
+summary: Add l2_norm normalization support to linear retriever
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/128509.yaml

@@ -0,0 +1,5 @@
+pr: 128509
+summary: Use default Lucene postings format when index mode is standard.
+area: Codec
+type: enhancement
+issues: []

docs/changelog/128740.yaml

@@ -0,0 +1,5 @@
+pr: 128740
+summary: Optimize sparse vector stats collection
+area: Stats
+type: enhancement
+issues: []

docs/changelog/128746.yaml

@@ -0,0 +1,5 @@
+pr: 128746
+summary: Fix computation of last block size in Azure concurrent multipart uploads
+area: Snapshot/Restore
+type: bug
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers.md

@@ -276,7 +276,7 @@ Each entry specifies the following parameters:
 `normalizer`
 :   (Optional, String)
 
-    Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, and `none`. Defaults to `none`.
+    - Specifies how we will normalize the retriever’s scores, before applying the specified `weight`. Available values are: `minmax`, `l2_norm`, and `none`. Defaults to `none`.
 
     * `none`
     * `minmax` : A `MinMaxScoreNormalizer` that normalizes scores based on the following formula
@@ -285,6 +285,7 @@ Each entry specifies the following parameters:
         score = (score - min) / (max - min)
         ```
 
+    * `l2_norm` : An `L2ScoreNormalizer` that normalizes scores using the L2 norm of the score values.
 
 See also [this hybrid search example](docs-content://solutions/search/retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.
 

gradle/gradle-daemon-jvm.properties

@@ -1,8 +1,9 @@
 #This file is generated by updateDaemonJvm
-toolchainUrl.LINUX.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/linux/aarch64/jdk/hotspot/normal/eclipse?project\=jdk
-toolchainUrl.LINUX.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/linux/x64/jdk/hotspot/normal/eclipse?project\=jdk
-toolchainUrl.MAC_OS.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/mac/aarch64/jdk/hotspot/normal/eclipse?project\=jdk
-toolchainUrl.MAC_OS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/mac/x64/jdk/hotspot/normal/eclipse?project\=jdk
-toolchainUrl.WINDOWS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.6+7/windows/x64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.LINUX.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/linux/aarch64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.LINUX.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/linux/x64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.MAC_OS.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/mac/aarch64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.MAC_OS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/mac/x64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.WINDOWS.AARCH64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/windows/aarch64/jdk/hotspot/normal/eclipse?project\=jdk
+toolchainUrl.WINDOWS.X86_64=https\://api.adoptium.net/v3/binary/version/jdk-21.0.7+6/windows/x64/jdk/hotspot/normal/eclipse?project\=jdk
 toolchainVendor=ADOPTIUM
 toolchainVersion=21

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java

@@ -11,7 +11,6 @@
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DoublePoint;
-import org.apache.lucene.document.LongField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
@@ -31,6 +30,7 @@
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.lucene.document.NumericField;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -48,7 +48,7 @@ public void testTermQuery() {
         );
         double value = (randomDouble() * 2 - 1) * 10000;
         long scaledValue = Math.round(value * ft.getScalingFactor());
-        assertEquals(LongField.newExactQuery("scaled_float", scaledValue), ft.termQuery(value, MOCK_CONTEXT));
+        assertEquals(NumericField.newExactLongQuery("scaled_float", scaledValue), ft.termQuery(value, MOCK_CONTEXT));
 
         MappedFieldType ft2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 0.1 + randomDouble() * 100, false);
         ElasticsearchException e2 = expectThrows(ElasticsearchException.class, () -> ft2.termQuery("42", MOCK_CONTEXT_DISALLOW_EXPENSIVE));