Upgrade to Lucene 10 (elastic#114741)

The most relevant ES changes that upgrading to Lucene 10 requires are:

- use the appropriate IOContext
- Scorer / ScorerSupplier breaking changes
- Regex automaton are no longer determinized by default
- minimize moved to test classes
- introduce Elasticsearch900Codec
- adjust slicing code according to the added support for intra-segment concurrency
- disable intra-segment concurrency in tests
- adjust accessor methods for many Lucene classes that became a record
- adapt to breaking changes in the analysis area

Co-authored-by: Christoph Büscher <[email protected]>
Co-authored-by: Mayya Sharipova <[email protected]>
Co-authored-by: ChrisHegarty <[email protected]>
Co-authored-by: Brian Seeders <[email protected]>
Co-authored-by: Armin Braun <[email protected]>
Co-authored-by: Panagiotis Bailis <[email protected]>
Co-authored-by: Benjamin Trent <[email protected]>

Author: 8 people

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -56,7 +56,6 @@ steps:
         matrix:
           setup:
             BWC_VERSION:
-              - 7.17.13
               - 8.9.1
               - 8.10.0
         agents:

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -19,7 +19,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
-import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
@@ -217,19 +217,17 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
-    RandomAccessQuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
+    QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());
         return new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(dims, size, sq, false, sim, null, slice);
     }
 
-    RandomVectorScorerSupplier luceneScoreSupplier(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim)
-        throws IOException {
+    RandomVectorScorerSupplier luceneScoreSupplier(QuantizedByteVectorValues values, VectorSimilarityFunction sim) throws IOException {
         return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorerSupplier(sim, values);
     }
 
-    RandomVectorScorer luceneScorer(RandomAccessQuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec)
-        throws IOException {
+    RandomVectorScorer luceneScorer(QuantizedByteVectorValues values, VectorSimilarityFunction sim, float[] queryVec) throws IOException {
         return new Lucene99ScalarQuantizedVectorScorer(null).getRandomVectorScorer(sim, values, queryVec);
     }
 

build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt

@@ -59,10 +59,6 @@ org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
 org.apache.lucene.index.NoMergePolicy#INSTANCE @ explicit use of NoMergePolicy risks forgetting to configure NoMergeScheduler; use org.elasticsearch.common.lucene.Lucene#indexWriterConfigWithNoMerging() instead.
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
-org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
-org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.0.0
-lucene            = 9.12.0
+lucene            = 10.0.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4

distribution/src/config/jvm.options

@@ -62,6 +62,9 @@
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.setAsTypeCache
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
 
+# Lucene 10: apply MADV_NORMAL advice to enable more aggressive readahead
+-Dorg.apache.lucene.store.defaultReadAdvice=normal
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.12.0
-:lucene_version_path:   9_12_0
+:lucene_version:        10.0.0
+:lucene_version_path:   10_0_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/113482.yaml

@@ -0,0 +1,27 @@
+pr: 113482
+summary: The 'persian' analyzer has stemmer by default
+area: Analysis
+type: breaking
+issues:
+- 113050
+breaking:
+  title: The 'persian' analyzer has stemmer by default
+  area: Analysis
+  details: >-
+    Lucene 10 has added a final stemming step to its PersianAnalyzer that Elasticsearch
+    exposes as 'persian' analyzer. Existing indices will keep the old
+    non-stemming behaviour while new indices will see the updated behaviour with
+    added stemming.
+    Users that wish to maintain the non-stemming behaviour need to define their
+    own analyzer as outlined in
+    https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer.
+    Users that wish to use the new stemming behaviour for existing indices will
+    have to reindex their data.
+  impact: >-
+    Indexing with the 'persian' analyzer will produce slightly different tokens.
+    Users should check if this impacts their search results. If they wish to
+    maintain the legacy non-stemming behaviour they can define their own
+    analyzer equivalent as explained in
+    https://www.elastic.co/guide/en/elasticsearch/reference/8.15/analysis-lang-analyzer.html#persian-analyzer.
+  notable: false
+

docs/changelog/113614.yaml

@@ -0,0 +1,18 @@
+pr: 113614
+summary: The 'german2' stemmer is now an alias for the 'german' snowball stemmer
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: The "german2" snowball stemmer is now an alias for the "german" stemmer
+  area: Analysis
+  details: >-
+    Lucene 10 has merged the improved "german2" snowball language stemmer with the
+    "german" stemmer. For Elasticsearch, "german2" is now a deprecated alias for
+    "german". This may results in slightly different tokens being generated for
+    terms with umlaut substitution (like "ue" for "ü" etc...)
+  impact: >-
+    Replace usages of "german2" with "german" in analysis configuration. Old
+    indices that use the "german" stemmer should be reindexed if possible.
+  notable: false
+

docs/changelog/114124.yaml

@@ -0,0 +1,18 @@
+pr: 114124
+summary: The Korean dictionary for Nori has been updated 
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: The Korean dictionary for Nori has been updated
+  area: Analysis
+  details: >-
+    Lucene 10 ships with an updated Korean dictionary (mecab-ko-dic-2.1.1). 
+    For details see https://github.com/apache/lucene/issues/11452. Users
+    experiencing changes in search behaviour on existing data are advised to
+    reindex.
+  impact: >-
+    The change is small and should generally provide better analysis results.
+    Existing indices for full-text use cases should be reindexed though.
+  notable: false
+

docs/changelog/114146.yaml

@@ -0,0 +1,20 @@
+pr: 114146 
+summary: Snowball stemmers have been upgraded 
+area: Analysis
+type: breaking
+issues: []
+breaking:
+  title: Snowball stemmers have been upgraded
+  area: Analysis
+  details: >-
+    Lucene 10 ships with an upgrade of its Snowball stemmers. 
+    For details see https://github.com/apache/lucene/issues/13209. Users using
+    Snowball stemmers that are experiencing changes in search behaviour on 
+    existing data are advised to reindex.
+  impact: >-
+    The upgrade should generally provide improved stemming results. Small changes
+    in token analysis can lead to mismatches with previously index data, so
+    existing indices using Snowball stemmers as part of their analysis chain
+    should be reindexed.
+  notable: false
+