Upgrade to Lucene 10.1.0

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 10.0.0
+lucene            = 10.1.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 23+37@3c5b90190c68498b986a97f276efd28a

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.0.0
-:lucene_version_path:   10_0_0
+:lucene_version:        10.1.0
+:lucene_version_path:   10_1_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/119308.yaml

@@ -0,0 +1,5 @@
+pr: 119308
+summary: Upgrade to Lucene 10.1.0
+area: Search
+type: upgrade
+issues: []

docs/reference/search/profile.asciidoc

@@ -176,7 +176,7 @@ The API returns the following result:
                 "time_in_nanos": 775274,
                 "children" : [
                   {
-                    "name": "SimpleTopScoreDocCollector",
+                    "name": "TopScoreDocCollector",
                     "reason": "search_top_hits",
                     "time_in_nanos": 775274
                   }
@@ -537,7 +537,7 @@ Looking at the previous example:
     "time_in_nanos": 775274,
     "children" : [
       {
-        "name": "SimpleTopScoreDocCollector",
+        "name": "TopScoreDocCollector",
         "reason": "search_top_hits",
         "time_in_nanos": 775274
       }
@@ -551,7 +551,7 @@ Looking at the previous example:
 
 
 We see a top-level collector named `QueryPhaseCollector` which holds a child
-`SimpleTopScoreDocCollector`. `SimpleTopScoreDocCollector` is the  default
+`TopScoreDocCollector`. `TopScoreDocCollector` is the  default
 "scoring and sorting" `Collector` used by {es}. The `reason` field attempts
 to give a plain English description of the class name. The `time_in_nanos`
 is similar to the time in the Query tree: a wall-clock time inclusive of all
@@ -751,7 +751,7 @@ The API returns the following result:
                 "time_in_nanos": 1945072,
                 "children": [
                   {
-                    "name": "SimpleTopScoreDocCollector",
+                    "name": "TopScoreDocCollector",
                     "reason": "search_top_hits",
                     "time_in_nanos": 22577
                   },
@@ -788,7 +788,7 @@ major portions of the query are represented:
 2. The second `TermQuery` (message:search) represents the `post_filter` query.
 
 The Collector tree is fairly straightforward, showing how a single
-QueryPhaseCollector that holds the normal scoring SimpleTopScoreDocCollector
+QueryPhaseCollector that holds the normal scoring TopScoreDocCollector
 used to collect top hits, as well as BucketCollectorWrapper to run all scoped
 aggregations.
 
@@ -1332,7 +1332,7 @@ One of the `dfs.knn` sections for a shard looks like the following:
         "rewrite_time" : 1275732,
         "collector" : [
             {
-                "name" : "SimpleTopScoreDocCollector",
+                "name" : "TopScoreDocCollector",
                 "reason" : "search_top_hits",
                 "time_in_nanos" : 17163
             }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermStates;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -23,7 +24,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafSimScorer;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Matches;
@@ -214,7 +214,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
             // No need to ever look at the _source for non-scoring term queries
             return in.createWeight(searcher, scoreMode, boost);
         }
-
         // We use a LinkedHashSet here to preserve the ordering of terms to ensure that
         // later summing of float scores per term is consistent
         final Set<Term> terms = new LinkedHashSet<>();
@@ -267,6 +266,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
 
             @Override
             public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                NumericDocValues norms = context.reader().getNormValues(field);
                 RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0);
                 if (scorer == null) {
                     return Explanation.noMatch("No matching phrase");
@@ -277,8 +277,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
                 float phraseFreq = scorer.freq();
                 Explanation freqExplanation = Explanation.match(phraseFreq, "phraseFreq=" + phraseFreq);
-                final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
-                Explanation scoreExplanation = leafSimScorer.explain(doc, freqExplanation);
+                Explanation scoreExplanation = simScorer.explain(freqExplanation, getNormValue(norms, doc));
                 return Explanation.match(
                     scoreExplanation.getValue(),
                     "weight(" + getQuery() + " in " + doc + ") [" + searcher.getSimilarity().getClass().getSimpleName() + "], result of:",
@@ -297,9 +296,9 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
                     public Scorer get(long leadCost) throws IOException {
                         final Scorer approximationScorer = approximationSupplier.get(leadCost);
                         final DocIdSetIterator approximation = approximationScorer.iterator();
-                        final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
                         final CheckedIntFunction<List<Object>, IOException> valueFetcher = valueFetcherProvider.apply(context);
-                        return new RuntimePhraseScorer(approximation, leafSimScorer, valueFetcher, field, in);
+                        NumericDocValues norms = context.reader().getNormValues(field);
+                        return new RuntimePhraseScorer(approximation, simScorer, norms, valueFetcher, field, in);
                     }
 
                     @Override
@@ -335,12 +334,23 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException {
         };
     }
 
+    private static long getNormValue(NumericDocValues norms, int doc) throws IOException {
+        if (norms != null) {
+            boolean found = norms.advanceExact(doc);
+            assert found;
+            return norms.longValue();
+        } else {
+            return 1L; // default norm
+        }
+    }
+
     private class RuntimePhraseScorer extends Scorer {
-        private final LeafSimScorer scorer;
+        private final SimScorer scorer;
         private final CheckedIntFunction<List<Object>, IOException> valueFetcher;
         private final String field;
         private final Query query;
         private final TwoPhaseIterator twoPhase;
+        private final NumericDocValues norms;
 
         private final MemoryIndexEntry cacheEntry = new MemoryIndexEntry();
 
@@ -349,12 +359,14 @@ private class RuntimePhraseScorer extends Scorer {
 
         private RuntimePhraseScorer(
             DocIdSetIterator approximation,
-            LeafSimScorer scorer,
+            SimScorer scorer,
+            NumericDocValues norms,
             CheckedIntFunction<List<Object>, IOException> valueFetcher,
             String field,
             Query query
         ) {
             this.scorer = scorer;
+            this.norms = norms;
             this.valueFetcher = valueFetcher;
             this.field = field;
             this.query = query;
@@ -386,12 +398,12 @@ public TwoPhaseIterator twoPhaseIterator() {
 
         @Override
         public float getMaxScore(int upTo) throws IOException {
-            return scorer.getSimScorer().score(Float.MAX_VALUE, 1L);
+            return scorer.score(Float.MAX_VALUE, 1L);
         }
 
         @Override
         public float score() throws IOException {
-            return scorer.score(docID(), freq());
+            return scorer.score(freq(), getNormValue(norms, doc));
         }
 
         @Override

rest-api-spec/build.gradle

@@ -57,6 +57,8 @@ tasks.named("precommit").configure {
 tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling")
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.collector.0.name", "TopScoreDocCollector", "dfs knn vector profiling")
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.collector.0.name", "TopScoreDocCollector", "dfs knn vector profiling with vector_operations_count")
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml

@@ -230,7 +230,6 @@ dfs knn vector profiling:
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 }
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 }
   - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 }
-  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" }
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 
@@ -275,6 +274,47 @@ dfs knn vector profiling description:
   - match: { hits.total.value: 1 }
   - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScoreQuery[0,...][0.009673266,...],0.009673266" }
 
+---
+dfs knn vector profiling collector name:
+  - requires:
+      cluster_features: "lucene_10_1_upgrade"
+      reason: collector name changed with lucene 10.1
+
+  - do:
+      indices.create:
+        index: images
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              image:
+                type: "dense_vector"
+                dims: 3
+                index: true
+                similarity: "l2_norm"
+
+  - do:
+      index:
+        index: images
+        id: "1"
+        refresh: true
+        body:
+          image: [1, 5, -20]
+
+  - do:
+      search:
+        index: images
+        body:
+          profile: true
+          knn:
+            field: "image"
+            query_vector: [-5, 9, -12]
+            k: 1
+            num_candidates: 100
+
+  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "TopScoreDocCollector" }
+
 ---
 dfs knn vector profiling with vector_operations_count:
   - requires:
@@ -335,7 +375,6 @@ dfs knn vector profiling with vector_operations_count:
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 }
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 }
   - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 }
-  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" }
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 

server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java

@@ -105,7 +105,7 @@ public void testFieldUsageStats() throws ExecutionException, InterruptedExceptio
         assertTrue(stats.hasField("field2"));
         // positions because of span query
         assertEquals(
-            Set.of(UsageContext.TERMS, UsageContext.POSTINGS, UsageContext.FREQS, UsageContext.POSITIONS),
+            Set.of(UsageContext.TERMS, UsageContext.POSTINGS, UsageContext.FREQS, UsageContext.POSITIONS, UsageContext.NORMS),
             stats.get("field2").keySet()
         );
         assertEquals(1L * numShards, stats.get("field2").getTerms());

server/src/main/java/module-info.java

@@ -461,7 +461,8 @@
         with
             org.elasticsearch.index.codec.Elasticsearch814Codec,
             org.elasticsearch.index.codec.Elasticsearch816Codec,
-            org.elasticsearch.index.codec.Elasticsearch900Codec;
+            org.elasticsearch.index.codec.Elasticsearch900Codec,
+            org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
 
     provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;
 

server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzer.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat;
 import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
 import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
+import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
 import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.FieldsProducer;
@@ -21,7 +22,7 @@
 import org.apache.lucene.codecs.PointsReader;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.ByteVectorValues;
 import org.apache.lucene.index.DirectoryReader;
@@ -306,6 +307,9 @@ private static void readProximity(Terms terms, PostingsEnum postings) throws IOE
     private static BlockTermState getBlockTermState(TermsEnum termsEnum, BytesRef term) throws IOException {
         if (term != null && termsEnum.seekExact(term)) {
             final TermState termState = termsEnum.termState();
+            if (termState instanceof final Lucene101PostingsFormat.IntBlockTermState blockTermState) {
+                return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
+            }
             if (termState instanceof final Lucene912PostingsFormat.IntBlockTermState blockTermState) {
                 return new BlockTermState(blockTermState.docStartFP, blockTermState.posStartFP, blockTermState.payStartFP);
             }

server/src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -70,7 +70,6 @@
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -89,10 +88,9 @@
 import java.util.Map;
 import java.util.Objects;
 
-import static org.apache.lucene.util.Version.LUCENE_10_0_0;
-
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene100";
+
+    public static final String LATEST_CODEC = "Lucene101";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 
@@ -156,25 +154,7 @@ public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOExcepti
      * Reads the segments infos from the given segments file name, failing if it fails to load
      */
     private static SegmentInfos readSegmentInfos(String segmentsFileName, Directory directory) throws IOException {
-        // TODO Use readCommit(Directory directory, String segmentFileName, int minSupportedMajorVersion) once Lucene 10.1 is available
-        // and remove the try-catch block for IndexFormatTooOldException
-        assert IndexVersion.current().luceneVersion().equals(LUCENE_10_0_0) : "remove the try-catch block below";
-        try {
-            return SegmentInfos.readCommit(directory, segmentsFileName);
-        } catch (IndexFormatTooOldException e) {
-            try {
-                // Temporary workaround until Lucene 10.1 is available: try to leverage min. read-only compatibility to read the last commit
-                // and then check if this is the commit we want. This should always work for the case we are interested in (archive and
-                // searchable snapshots indices in N-2 version) as no newer commit should be ever written.
-                var segmentInfos = readSegmentInfos(directory);
-                if (segmentsFileName.equals(segmentInfos.getSegmentsFileName())) {
-                    return segmentInfos;
-                }
-            } catch (Exception suppressed) {
-                e.addSuppressed(suppressed);
-            }
-            throw e;
-        }
+        return SegmentInfos.readCommit(directory, segmentsFileName, IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major);
     }
 
     /**

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -144,6 +144,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME = def(9_006_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP = def(9_007_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion HOSTNAME_DOC_VALUES_SPARSE_INDEX = def(9_008_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion UPGRADE_TO_LUCENE_10_1_0 = def(9_009_00_0, Version.LUCENE_10_1_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -12,7 +12,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene100.Lucene100Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.Nullable;
@@ -46,7 +46,7 @@ public class CodecService implements CodecProvider {
     public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) {
         final var codecs = new HashMap<String, Codec>();
 
-        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_SPEED, mapperService, bigArrays);
+        Codec legacyBestSpeedCodec = new LegacyPerFieldMapperCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, bigArrays);
         if (ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) {
             codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays));
         } else {
@@ -58,7 +58,7 @@ public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays)
             BEST_COMPRESSION_CODEC,
             new PerFieldMapperCodec(Zstd814StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays)
         );
-        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene100Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
+        Codec legacyBestCompressionCodec = new LegacyPerFieldMapperCodec(Lucene101Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays);
         codecs.put(LEGACY_BEST_COMPRESSION_CODEC, legacyBestCompressionCodec);
 
         codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());