Upgrade to Lucene 10.1.0 (#119308)

This commit upgrades to Lucene 10.1.0.

Author: ChrisHegarty

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 10.0.0
+lucene            = 10.1.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 23+37@3c5b90190c68498b986a97f276efd28a

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.0.0
-:lucene_version_path:   10_0_0
+:lucene_version:        10.1.0
+:lucene_version_path:   10_1_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/119308.yaml

@@ -0,0 +1,5 @@
+pr: 119308
+summary: Upgrade to Lucene 10.1.0
+area: Search
+type: upgrade
+issues: []

docs/reference/search/profile.asciidoc

@@ -176,7 +176,7 @@ The API returns the following result:
                 "time_in_nanos": 775274,
                 "children" : [
                   {
-                    "name": "SimpleTopScoreDocCollector",
+                    "name": "TopScoreDocCollector",
                     "reason": "search_top_hits",
                     "time_in_nanos": 775274
                   }
@@ -537,7 +537,7 @@ Looking at the previous example:
     "time_in_nanos": 775274,
     "children" : [
       {
-        "name": "SimpleTopScoreDocCollector",
+        "name": "TopScoreDocCollector",
         "reason": "search_top_hits",
         "time_in_nanos": 775274
       }
@@ -551,7 +551,7 @@ Looking at the previous example:
 
 
 We see a top-level collector named `QueryPhaseCollector` which holds a child
-`SimpleTopScoreDocCollector`. `SimpleTopScoreDocCollector` is the  default
+`TopScoreDocCollector`. `TopScoreDocCollector` is the  default
 "scoring and sorting" `Collector` used by {es}. The `reason` field attempts
 to give a plain English description of the class name. The `time_in_nanos`
 is similar to the time in the Query tree: a wall-clock time inclusive of all
@@ -751,7 +751,7 @@ The API returns the following result:
                 "time_in_nanos": 1945072,
                 "children": [
                   {
-                    "name": "SimpleTopScoreDocCollector",
+                    "name": "TopScoreDocCollector",
                     "reason": "search_top_hits",
                     "time_in_nanos": 22577
                   },
@@ -788,7 +788,7 @@ major portions of the query are represented:
 2. The second `TermQuery` (message:search) represents the `post_filter` query.
 
 The Collector tree is fairly straightforward, showing how a single
-QueryPhaseCollector that holds the normal scoring SimpleTopScoreDocCollector
+QueryPhaseCollector that holds the normal scoring TopScoreDocCollector
 used to collect top hits, as well as BucketCollectorWrapper to run all scoped
 aggregations.
 
@@ -1332,7 +1332,7 @@ One of the `dfs.knn` sections for a shard looks like the following:
         "rewrite_time" : 1275732,
         "collector" : [
             {
-                "name" : "SimpleTopScoreDocCollector",
+                "name" : "TopScoreDocCollector",
                 "reason" : "search_top_hits",
                 "time_in_nanos" : 17163
             }

gradle/verification-metadata.xml

@@ -12,6 +12,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.FieldInvertState;
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermStates;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -23,7 +24,6 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafSimScorer;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Matches;
@@ -214,7 +214,6 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo
             // No need to ever look at the _source for non-scoring term queries
             return in.createWeight(searcher, scoreMode, boost);
         }
-
         // We use a LinkedHashSet here to preserve the ordering of terms to ensure that
         // later summing of float scores per term is consistent
         final Set<Term> terms = new LinkedHashSet<>();
@@ -267,6 +266,7 @@ public boolean isCacheable(LeafReaderContext ctx) {
 
             @Override
             public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                NumericDocValues norms = context.reader().getNormValues(field);
                 RuntimePhraseScorer scorer = (RuntimePhraseScorer) scorerSupplier(context).get(0);
                 if (scorer == null) {
                     return Explanation.noMatch("No matching phrase");
@@ -277,8 +277,7 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio
                 }
                 float phraseFreq = scorer.freq();
                 Explanation freqExplanation = Explanation.match(phraseFreq, "phraseFreq=" + phraseFreq);
-                final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
-                Explanation scoreExplanation = leafSimScorer.explain(doc, freqExplanation);
+                Explanation scoreExplanation = simScorer.explain(freqExplanation, getNormValue(norms, doc));
                 return Explanation.match(
                     scoreExplanation.getValue(),
                     "weight(" + getQuery() + " in " + doc + ") [" + searcher.getSimilarity().getClass().getSimpleName() + "], result of:",
@@ -297,9 +296,9 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
                     public Scorer get(long leadCost) throws IOException {
                         final Scorer approximationScorer = approximationSupplier.get(leadCost);
                         final DocIdSetIterator approximation = approximationScorer.iterator();
-                        final LeafSimScorer leafSimScorer = new LeafSimScorer(simScorer, context.reader(), field, scoreMode.needsScores());
                         final CheckedIntFunction<List<Object>, IOException> valueFetcher = valueFetcherProvider.apply(context);
-                        return new RuntimePhraseScorer(approximation, leafSimScorer, valueFetcher, field, in);
+                        NumericDocValues norms = context.reader().getNormValues(field);
+                        return new RuntimePhraseScorer(approximation, simScorer, norms, valueFetcher, field, in);
                     }
 
                     @Override
@@ -335,12 +334,23 @@ public Matches matches(LeafReaderContext context, int doc) throws IOException {
         };
     }
 
+    private static long getNormValue(NumericDocValues norms, int doc) throws IOException {
+        if (norms != null) {
+            boolean found = norms.advanceExact(doc);
+            assert found;
+            return norms.longValue();
+        } else {
+            return 1L; // default norm
+        }
+    }
+
     private class RuntimePhraseScorer extends Scorer {
-        private final LeafSimScorer scorer;
+        private final SimScorer scorer;
         private final CheckedIntFunction<List<Object>, IOException> valueFetcher;
         private final String field;
         private final Query query;
         private final TwoPhaseIterator twoPhase;
+        private final NumericDocValues norms;
 
         private final MemoryIndexEntry cacheEntry = new MemoryIndexEntry();
 
@@ -349,12 +359,14 @@ private class RuntimePhraseScorer extends Scorer {
 
         private RuntimePhraseScorer(
             DocIdSetIterator approximation,
-            LeafSimScorer scorer,
+            SimScorer scorer,
+            NumericDocValues norms,
             CheckedIntFunction<List<Object>, IOException> valueFetcher,
             String field,
             Query query
         ) {
             this.scorer = scorer;
+            this.norms = norms;
             this.valueFetcher = valueFetcher;
             this.field = field;
             this.query = query;
@@ -386,12 +398,12 @@ public TwoPhaseIterator twoPhaseIterator() {
 
         @Override
         public float getMaxScore(int upTo) throws IOException {
-            return scorer.getSimScorer().score(Float.MAX_VALUE, 1L);
+            return scorer.score(Float.MAX_VALUE, 1L);
         }
 
         @Override
         public float score() throws IOException {
-            return scorer.score(docID(), freq());
+            return scorer.score(freq(), getNormValue(norms, doc));
         }
 
         @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java

@@ -57,6 +57,8 @@ tasks.named("precommit").configure {
 tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling")
   task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.collector.0.name", "TopScoreDocCollector", "dfs knn vector profiling")
+  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.collector.0.name", "TopScoreDocCollector", "dfs knn vector profiling with vector_operations_count")
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")

rest-api-spec/build.gradle

@@ -230,7 +230,6 @@ dfs knn vector profiling:
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 }
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 }
   - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 }
-  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" }
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 
@@ -275,6 +274,47 @@ dfs knn vector profiling description:
   - match: { hits.total.value: 1 }
   - match: { profile.shards.0.dfs.knn.0.query.0.description: "DocAndScoreQuery[0,...][0.009673266,...],0.009673266" }
 
+---
+dfs knn vector profiling collector name:
+  - requires:
+      cluster_features: "lucene_10_1_upgrade"
+      reason: collector name changed with lucene 10.1
+
+  - do:
+      indices.create:
+        index: images
+        body:
+          settings:
+            index.number_of_shards: 1
+          mappings:
+            properties:
+              image:
+                type: "dense_vector"
+                dims: 3
+                index: true
+                similarity: "l2_norm"
+
+  - do:
+      index:
+        index: images
+        id: "1"
+        refresh: true
+        body:
+          image: [1, 5, -20]
+
+  - do:
+      search:
+        index: images
+        body:
+          profile: true
+          knn:
+            field: "image"
+            query_vector: [-5, 9, -12]
+            k: 1
+            num_candidates: 100
+
+  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "TopScoreDocCollector" }
+
 ---
 dfs knn vector profiling with vector_operations_count:
   - requires:
@@ -335,7 +375,6 @@ dfs knn vector profiling with vector_operations_count:
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight: 0 }
   - gt: { profile.shards.0.dfs.knn.0.query.0.breakdown.create_weight_count: 0 }
   - gt: { profile.shards.0.dfs.knn.0.rewrite_time: 0 }
-  - match: { profile.shards.0.dfs.knn.0.collector.0.name: "SimpleTopScoreDocCollector" }
   - match: { profile.shards.0.dfs.knn.0.collector.0.reason: "search_top_hits" }
   - gt: { profile.shards.0.dfs.knn.0.collector.0.time_in_nanos: 0 }
 

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml

@@ -105,7 +105,7 @@ public void testFieldUsageStats() throws ExecutionException, InterruptedExceptio
         assertTrue(stats.hasField("field2"));
         // positions because of span query
         assertEquals(
-            Set.of(UsageContext.TERMS, UsageContext.POSTINGS, UsageContext.FREQS, UsageContext.POSITIONS),
+            Set.of(UsageContext.TERMS, UsageContext.POSTINGS, UsageContext.FREQS, UsageContext.POSITIONS, UsageContext.NORMS),
             stats.get("field2").keySet()
         );
         assertEquals(1L * numShards, stats.get("field2").getTerms());

server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java

@@ -461,7 +461,8 @@
         with
             org.elasticsearch.index.codec.Elasticsearch814Codec,
             org.elasticsearch.index.codec.Elasticsearch816Codec,
-            org.elasticsearch.index.codec.Elasticsearch900Codec;
+            org.elasticsearch.index.codec.Elasticsearch900Codec,
+            org.elasticsearch.index.codec.Elasticsearch900Lucene101Codec;
 
     provides org.apache.logging.log4j.core.util.ContextDataProvider with org.elasticsearch.common.logging.DynamicContextDataProvider;