Upgrade elasticsearch to lucene 10.3.0

.buildkite/pipelines/lucene-snapshot/build-snapshot.yml

@@ -2,7 +2,9 @@ steps:
   - trigger: apache-lucene-build-snapshot
     label: Trigger pipeline to build lucene snapshot
     key: lucene-build
-    if: build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == ""
+    if: (build.env("LUCENE_BUILD_ID") == null || build.env("LUCENE_BUILD_ID") == "")
+    build:
+      branch: branch_10x
   - wait
   - label: Upload and update lucene snapshot
     command: .buildkite/scripts/lucene-snapshot/upload-snapshot.sh

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -26,7 +26,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
+import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -63,7 +63,6 @@
 public class TSDBDocValuesMergeBenchmark {
 
     static {
-        // For Elasticsearch900Lucene101Codec:
         LogConfigurator.loadLog4jPlugins();
         LogConfigurator.configureESLogging();
         LogConfigurator.setNodeName("test");
@@ -259,8 +258,7 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
         var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
-        config.setCodec(new Elasticsearch900Lucene101Codec() {
-
+        config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {
                 return docValuesFormat;

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.2.0
-lucene            = 10.2.2
+lucene            = 10.3.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24.0.2+12@fdc5d0102fe0414db21410ad5834341f

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -60,13 +60,6 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dio.netty.noUnsafe=true",
                 "-Dio.netty.noKeySetOptimization=true",
                 "-Dio.netty.recycler.maxCapacityPerThread=0",
-                // temporary until we get off-heap vector stats in Lucene 10.3
-                "--add-opens=org.apache.lucene.core/org.apache.lucene.codecs.lucene99=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene90=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene91=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene92=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene94=org.elasticsearch.server",
-                "--add-opens=org.apache.lucene.backward_codecs/org.apache.lucene.backward_codecs.lucene95=org.elasticsearch.server",
                 // log4j 2
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.2.2
-:lucene_version_path:   10_2_2
+:lucene_version:        10.3.0
+:lucene_version_path:   10_3_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/133980.yaml

@@ -0,0 +1,9 @@
+pr: 133980
+summary: Upgrade elasticsearch to lucene 10.3.0
+area: Search
+type: upgrade
+issues: []
+highlight:
+  title: "New lucene 10.3.0 release"
+  body: |-
+    - Improved performance for lexical, vector and primary-key searches

gradle/verification-metadata.xml

@@ -3170,129 +3170,129 @@
             <sha256 value="015d5c229f3cd5c0ebf175c1da08d596d94043362ae9d92637d88848c90537c8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-common" version="10.2.2">
-         <artifact name="lucene-analysis-common-10.2.2.jar">
-            <sha256 value="4dc1acc6b5a43be0049057bc99772dd24717cf0084c7ff79c68c984f120c140b" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-common" version="10.3.0">
+         <artifact name="lucene-analysis-common-10.3.0.jar">
+            <sha256 value="124bddea3fa0683d3e174552fba5fcf0f24a2f57f1ccf6957010d010d246db8b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-icu" version="10.2.2">
-         <artifact name="lucene-analysis-icu-10.2.2.jar">
-            <sha256 value="7baba617b706e2fd35db5bae73117ae77837de29ebabf74204ef5d0bdbb60440" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-icu" version="10.3.0">
+         <artifact name="lucene-analysis-icu-10.3.0.jar">
+            <sha256 value="25839bc8dc2b24c0fb3b7babb17d4b953a5ba179f0ec78237fcb6fb5be8925ca" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="10.2.2">
-         <artifact name="lucene-analysis-kuromoji-10.2.2.jar">
-            <sha256 value="26d5d03b3a123da2547e08d53cc9674eb1a13fe1a03be90e09acee30003a781f" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-kuromoji" version="10.3.0">
+         <artifact name="lucene-analysis-kuromoji-10.3.0.jar">
+            <sha256 value="5a2b5ae58529657417f3223dae26e5643d0d941064aa8a7e8256bc1ab8150680" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="10.2.2">
-         <artifact name="lucene-analysis-morfologik-10.2.2.jar">
-            <sha256 value="a219ed3645be3fb153f35ca258dafd8b7db6c76f750adf1a666f562c4d3c9b64" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-morfologik" version="10.3.0">
+         <artifact name="lucene-analysis-morfologik-10.3.0.jar">
+            <sha256 value="2bfb28600f650eed1a75154ef64ef0187b34d4cc771ceb1d3139f6df6ae80b22" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-nori" version="10.2.2">
-         <artifact name="lucene-analysis-nori-10.2.2.jar">
-            <sha256 value="977682e997f3cdd48dc192dc74204d483c967c15dc6ab57e02e2582dcc3035e7" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-nori" version="10.3.0">
+         <artifact name="lucene-analysis-nori-10.3.0.jar">
+            <sha256 value="34ba4d86d838fb8c3d65d58a58b6631a4502b7225272654a26237824316dd71c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="10.2.2">
-         <artifact name="lucene-analysis-phonetic-10.2.2.jar">
-            <sha256 value="5393550db5fbd64bfee2076f27017f19a3993b48704943b60a48d3a37ed7b0ac" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-phonetic" version="10.3.0">
+         <artifact name="lucene-analysis-phonetic-10.3.0.jar">
+            <sha256 value="b9ac996687602c98331c1c60db810c28bffa1fe0e507f12180ac6caee0bfdea8" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="10.2.2">
-         <artifact name="lucene-analysis-smartcn-10.2.2.jar">
-            <sha256 value="caa9f83dd0cef9720a94ada29bcc2a9a839608e1fb665ac9bfb8cef6cf57f5b9" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-smartcn" version="10.3.0">
+         <artifact name="lucene-analysis-smartcn-10.3.0.jar">
+            <sha256 value="7f4dbf438a8651450b19e96792f7db89e8686259098b6df1c97a722d40b7556c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="10.2.2">
-         <artifact name="lucene-analysis-stempel-10.2.2.jar">
-            <sha256 value="5bc1cca148d22cde9c485fee5a3861e92d02405eadb3c590dcae91786c94378a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-analysis-stempel" version="10.3.0">
+         <artifact name="lucene-analysis-stempel-10.3.0.jar">
+            <sha256 value="d0abf7d4c95a3bc29228eab8a9211c024694dfa340cf094c5fd38db9ff0a2518" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-backward-codecs" version="10.2.2">
-         <artifact name="lucene-backward-codecs-10.2.2.jar">
-            <sha256 value="2abe4c2d92382805938ae5d43eca4d894c33846df01e49225ce0234d9af49a1d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-backward-codecs" version="10.3.0">
+         <artifact name="lucene-backward-codecs-10.3.0.jar">
+            <sha256 value="6ba67dbdca534cd4411249a99d1154c8b44acd44c94a44aa97086840d069568b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-codecs" version="10.2.2">
-         <artifact name="lucene-codecs-10.2.2.jar">
-            <sha256 value="c3bee894d8132d9268283e73998a02e17d71a3b730515b0597f50a615e087679" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-codecs" version="10.3.0">
+         <artifact name="lucene-codecs-10.3.0.jar">
+            <sha256 value="0628a26040706fea7a222a91c8ca339aef9ea8ceda6d24527a817dc7e0ad5c92" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-core" version="10.2.2">
-         <artifact name="lucene-core-10.2.2.jar">
-            <sha256 value="9c219c702449a41c16ad60edc58d12f6d2f4989ef555e875e942904209e25029" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-core" version="10.3.0">
+         <artifact name="lucene-core-10.3.0.jar">
+            <sha256 value="9d70afe194568ca0328b7f762e64eb8094a5e5a76ea3a30b7182f0e7d5f0dd35" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-expressions" version="10.2.2">
-         <artifact name="lucene-expressions-10.2.2.jar">
-            <sha256 value="b93e825f004de92a7ad792ed862387e9b8abd30fc67b044353717d067587194d" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-expressions" version="10.3.0">
+         <artifact name="lucene-expressions-10.3.0.jar">
+            <sha256 value="0291da16292ab2f6b3a23746387e9820095ca5e6e319dc8a6497bc0bc9d3e61c" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-facet" version="10.2.2">
-         <artifact name="lucene-facet-10.2.2.jar">
-            <sha256 value="0c47089ec9301e15bcabcf2c3dd8d3a4cdb46eec1e1d5a49f4bdba771f3123f5" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-facet" version="10.3.0">
+         <artifact name="lucene-facet-10.3.0.jar">
+            <sha256 value="9ccb6f76538c0540680ba591e7b2afdbae7902d62d42054561ca6bc34152ebc4" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-grouping" version="10.2.2">
-         <artifact name="lucene-grouping-10.2.2.jar">
-            <sha256 value="a2d6def4857428f9fb799819d00a059727824f916d5a45029ea0d6dd7f6d27c8" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-grouping" version="10.3.0">
+         <artifact name="lucene-grouping-10.3.0.jar">
+            <sha256 value="1466f4ed29a6137054d577c32a7f69af5fd8047e5dd6fbf6f3ec44014405a69e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-highlighter" version="10.2.2">
-         <artifact name="lucene-highlighter-10.2.2.jar">
-            <sha256 value="c3ba585eb9dc6a159931a96fb9235d84254e244b3aaaa05b16f71811ae73a887" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-highlighter" version="10.3.0">
+         <artifact name="lucene-highlighter-10.3.0.jar">
+            <sha256 value="acbf67631c42197791363ce8e8f155f62aeef51ade809e0847a89eba44785c7d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-join" version="10.2.2">
-         <artifact name="lucene-join-10.2.2.jar">
-            <sha256 value="a203901be5ed1ef3540c349d73df5d813c2056f7b9b000dd2bb9217750875a3a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-join" version="10.3.0">
+         <artifact name="lucene-join-10.3.0.jar">
+            <sha256 value="f0fb5e1c8252437cb9f53b584916d47829cd08e2ab4ca13815832b6090fd8ac2" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-memory" version="10.2.2">
-         <artifact name="lucene-memory-10.2.2.jar">
-            <sha256 value="929aa2f96dac2cf03ca7bcfc008aaa3af5a7075b7dad67cb4774972d1ededc97" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-memory" version="10.3.0">
+         <artifact name="lucene-memory-10.3.0.jar">
+            <sha256 value="ea77d0b2eb0b3e1f7fa04f2fbe671f5af3ae836cc40320411e195bdc9421f17b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-misc" version="10.2.2">
-         <artifact name="lucene-misc-10.2.2.jar">
-            <sha256 value="894e835d7b8285d5a23527df73cc78d7698b86a14cc87c088a86a7c832f86e64" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-misc" version="10.3.0">
+         <artifact name="lucene-misc-10.3.0.jar">
+            <sha256 value="723cf1ec4f7497a761ea33404a11def2799e7ab4345fc704e333d61e08908441" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queries" version="10.2.2">
-         <artifact name="lucene-queries-10.2.2.jar">
-            <sha256 value="7a3c9b59164d5c494c5243dd86814f34c7721f613401d4c7e30cc80c88afe733" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queries" version="10.3.0">
+         <artifact name="lucene-queries-10.3.0.jar">
+            <sha256 value="c0c5891b390759a0cf503c7864e6eb4d8447cc8df20806a4254bfdf3d2f43a91" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-queryparser" version="10.2.2">
-         <artifact name="lucene-queryparser-10.2.2.jar">
-            <sha256 value="010e1478b6649a5f62aa411a512519cf1f849b6bc48961fd117c73a8dc591c67" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-queryparser" version="10.3.0">
+         <artifact name="lucene-queryparser-10.3.0.jar">
+            <sha256 value="b4b57bc3d56c5a1ccc674f55a16086c37700eb151fa749b45887e391ec0f26f9" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-sandbox" version="10.2.2">
-         <artifact name="lucene-sandbox-10.2.2.jar">
-            <sha256 value="27eb0da8367fcc81663ff760fc98a1606ed45ac043eb32c7f575815b6e56b2f5" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-sandbox" version="10.3.0">
+         <artifact name="lucene-sandbox-10.3.0.jar">
+            <sha256 value="00aa7be5c214bff31d62b489fbb5f0c389545e9f455324b2b92681956d37c94f" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial-extras" version="10.2.2">
-         <artifact name="lucene-spatial-extras-10.2.2.jar">
-            <sha256 value="06996ef7d2bd05f98743ff5413c37008ee56da9e26b60bb622abfa125b7dfda1" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial-extras" version="10.3.0">
+         <artifact name="lucene-spatial-extras-10.3.0.jar">
+            <sha256 value="ad5a5991acee4a92e813c7577ad6a3cfc03061076ca6f2489084830fec804f37" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-spatial3d" version="10.2.2">
-         <artifact name="lucene-spatial3d-10.2.2.jar">
-            <sha256 value="f5c94bbbc9b6d2c5f7d0e053fe4fe49304c1ccf51cdbdb27c25730837675d819" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-spatial3d" version="10.3.0">
+         <artifact name="lucene-spatial3d-10.3.0.jar">
+            <sha256 value="5d26e7e4192eb7159a980b8c330ae2aa09c598314160fcf959fc0b13949d1e78" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-suggest" version="10.2.2">
-         <artifact name="lucene-suggest-10.2.2.jar">
-            <sha256 value="8019b895f247c07de828cc44f7dc584a9fad45bebdcce8a8cefcfea9b5a3c859" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-suggest" version="10.3.0">
+         <artifact name="lucene-suggest-10.3.0.jar">
+            <sha256 value="4b53bf7160e8cdf2ae6bf1a27018bb4fbfdb1841fbdad988c5ff71c7920f4a84" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.lucene" name="lucene-test-framework" version="10.2.2">
-         <artifact name="lucene-test-framework-10.2.2.jar">
-            <sha256 value="0681d876cbcf5a6050514edf70c09652d30134bef0cdeb2db90cc807fc71918a" origin="Generated by Gradle"/>
+      <component group="org.apache.lucene" name="lucene-test-framework" version="10.3.0">
+         <artifact name="lucene-test-framework-10.3.0.jar">
+            <sha256 value="2befd6caadd920a9cb5883e5b62f8f19080d38e3debb32aae280a9d6eb033e68" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.maven" name="maven-api-annotations" version="4.0.0-rc-3">

plugins/store-smb/src/main/java/org/elasticsearch/index/store/smb/SmbMmapFsDirectoryFactory.java

@@ -25,7 +25,7 @@ public final class SmbMmapFsDirectoryFactory extends FsDirectoryFactory {
     @Override
     protected Directory newFSDirectory(Path location, LockFactory lockFactory, IndexSettings indexSettings) throws IOException {
         return new SmbDirectoryWrapper(
-            setPreload(
+            setMMapFunctions(
                 new MMapDirectory(location, lockFactory),
                 new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING))
             )

qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java

@@ -13,7 +13,7 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
@@ -130,7 +130,7 @@ static Codec createCodec(CmdLineArgs args) {
                 format = new Lucene99HnswVectorsFormat(args.hnswM(), args.hnswEfConstruction(), 1, null);
             }
         }
-        return new Lucene101Codec() {
+        return new Lucene103Codec() {
             @Override
             public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 return format;