Merge branch 'main' into fix/es-10981

Author: JeremyDahlgren

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -19,6 +19,7 @@
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
 import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier;
+import org.apache.lucene.util.hnsw.UpdateableRandomVectorScorer;
 import org.apache.lucene.util.quantization.QuantizedByteVectorValues;
 import org.apache.lucene.util.quantization.ScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -76,10 +77,10 @@ public class VectorScorerBenchmark {
     float vec2Offset;
     float scoreCorrectionConstant;
 
-    RandomVectorScorer luceneDotScorer;
-    RandomVectorScorer luceneSqrScorer;
-    RandomVectorScorer nativeDotScorer;
-    RandomVectorScorer nativeSqrScorer;
+    UpdateableRandomVectorScorer luceneDotScorer;
+    UpdateableRandomVectorScorer luceneSqrScorer;
+    UpdateableRandomVectorScorer nativeDotScorer;
+    UpdateableRandomVectorScorer nativeSqrScorer;
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
@@ -118,12 +119,16 @@ public void setup() throws IOException {
         in = dir.openInput("vector.data", IOContext.DEFAULT);
         var values = vectorValues(dims, 2, in, VectorSimilarityFunction.DOT_PRODUCT);
         scoreCorrectionConstant = values.getScalarQuantizer().getConstantMultiplier();
-        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer(0);
+        luceneDotScorer = luceneScoreSupplier(values, VectorSimilarityFunction.DOT_PRODUCT).scorer();
+        luceneDotScorer.setScoringOrdinal(0);
         values = vectorValues(dims, 2, in, VectorSimilarityFunction.EUCLIDEAN);
-        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer(0);
+        luceneSqrScorer = luceneScoreSupplier(values, VectorSimilarityFunction.EUCLIDEAN).scorer();
+        luceneSqrScorer.setScoringOrdinal(0);
 
-        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer(0);
-        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer(0);
+        nativeDotScorer = factory.getInt7SQVectorScorerSupplier(DOT_PRODUCT, in, values, scoreCorrectionConstant).get().scorer();
+        nativeDotScorer.setScoringOrdinal(0);
+        nativeSqrScorer = factory.getInt7SQVectorScorerSupplier(EUCLIDEAN, in, values, scoreCorrectionConstant).get().scorer();
+        nativeSqrScorer.setScoringOrdinal(0);
 
         // setup for getInt7SQVectorScorer / query vector scoring
         float[] queryVec = new float[dims];

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.1.0
-lucene            = 10.1.0
+lucene            = 10.2.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 24+36@1f9ff9062db4449d8ca828c504ffae90
@@ -8,7 +8,7 @@ spatial4j         = 0.7
 jts               = 1.15.0
 jackson           = 2.15.0
 snakeyaml         = 2.0
-icu4j             = 68.2
+icu4j             = 77.1
 supercsv          = 2.4.0
 log4j             = 2.19.0
 slf4j             = 2.0.6

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java

@@ -19,13 +19,10 @@
 import org.elasticsearch.core.CheckedFunction;
 import org.elasticsearch.env.Environment;
 
-import java.io.BufferedReader;
 import java.io.CharArrayWriter;
 import java.io.Closeable;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
+import java.io.Reader;
 import java.util.Arrays;
 import java.util.List;
 
@@ -47,11 +44,6 @@ class AddStringKeyStoreCommand extends BaseKeyStoreCommand {
         this.arguments = parser.nonOptions("setting names");
     }
 
-    // pkg private so tests can manipulate
-    InputStream getStdin() {
-        return System.in;
-    }
-
     @Override
     protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
         final List<String> settings = arguments.values(options);
@@ -64,7 +56,7 @@ protected void executeCommand(Terminal terminal, OptionSet options, Environment
         final Closeable closeable;
         final CheckedFunction<String, char[], IOException> valueSupplier;
         if (options.has(stdinOption)) {
-            final BufferedReader stdinReader = new BufferedReader(new InputStreamReader(getStdin(), StandardCharsets.UTF_8));
+            final Reader stdinReader = terminal.getReader();
             valueSupplier = s -> {
                 try (CharArrayWriter writer = new CharArrayWriter()) {
                     int c;

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java

@@ -18,30 +18,20 @@
 import org.elasticsearch.common.settings.KeyStoreWrapper;
 import org.elasticsearch.env.Environment;
 
-import java.io.ByteArrayInputStream;
 import java.io.CharArrayWriter;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasToString;
 
 public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
-    InputStream input;
-
     @Override
     protected Command newCommand() {
         return new AddStringKeyStoreCommand() {
             @Override
             protected Environment createEnv(OptionSet options, ProcessInfo processInfo) throws UserException {
                 return env;
             }
-
-            @Override
-            InputStream getStdin() {
-                return input;
-            }
         };
     }
 
@@ -167,7 +157,7 @@ public void testStdinShort() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("secret value 1");
+        terminal.addSecretInput("secret value 1");
         execute("-x", "foo");
         assertSecureString("foo", "secret value 1", password);
     }
@@ -176,7 +166,7 @@ public void testStdinLong() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("secret value 2");
+        terminal.addSecretInput("secret value 2");
         execute("--stdin", "foo");
         assertSecureString("foo", "secret value 2", password);
     }
@@ -185,7 +175,7 @@ public void testStdinNoInput() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("");
+        terminal.addSecretInput("");
         execute("-x", "foo");
         assertSecureString("foo", "", password);
     }
@@ -194,7 +184,7 @@ public void testStdinInputWithLineBreaks() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("Typedthisandhitenter\n");
+        terminal.addSecretInput("Typedthisandhitenter\n");
         execute("-x", "foo");
         assertSecureString("foo", "Typedthisandhitenter", password);
     }
@@ -203,7 +193,7 @@ public void testStdinInputWithCarriageReturn() throws Exception {
         String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("Typedthisandhitenter\r");
+        terminal.addSecretInput("Typedthisandhitenter\r");
         execute("-x", "foo");
         assertSecureString("foo", "Typedthisandhitenter", password);
     }
@@ -212,7 +202,9 @@ public void testStdinWithMultipleValues() throws Exception {
         final String password = "keystorepassword";
         KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
-        setInput("bar1\nbar2\nbar3");
+        terminal.addSecretInput("bar1");
+        terminal.addSecretInput("bar2");
+        terminal.addSecretInput("bar3");
         execute(randomFrom("-x", "--stdin"), "foo1", "foo2", "foo3");
         assertSecureString("foo1", "bar1", password);
         assertSecureString("foo2", "bar2", password);
@@ -228,7 +220,7 @@ public void testAddUtf8String() throws Exception {
             for (int i = 0; i < stringSize; i++) {
                 secretChars.write((char) randomIntBetween(129, 2048));
             }
-            setInput(secretChars.toString());
+            terminal.addSecretInput(secretChars.toString());
             execute("-x", "foo");
             assertSecureString("foo", secretChars.toString(), password);
         }
@@ -265,8 +257,4 @@ public void testAddToUnprotectedKeystore() throws Exception {
         execute("foo");
         assertSecureString("foo", "bar", password);
     }
-
-    void setInput(String inputStr) {
-        input = new ByteArrayInputStream(inputStr.getBytes(StandardCharsets.UTF_8));
-    }
 }

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.1.0
-:lucene_version_path:   10_1_0
+:lucene_version:        10.2.0
+:lucene_version_path:   10_2_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/125517.yaml

@@ -0,0 +1,5 @@
+pr: 125517
+summary: Semantic Text Chunking Indexing Pressure
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/126594.yaml

@@ -0,0 +1,5 @@
+pr: 126594
+summary: Upgrade to Lucene 10.2.0
+area: Search
+type: upgrade
+issues: []

docs/changelog/126729.yaml

@@ -0,0 +1,6 @@
+pr: 126729
+summary: Use terminal reader in keystore add command
+area: Infra/CLI
+type: bug
+issues:
+ - 98115

docs/changelog/126778.yaml

@@ -0,0 +1,5 @@
+pr: 126778
+summary: Fix bbq quantization algorithm but for differently distributed components
+area: Vector Search
+type: bug
+issues: []

docs/changelog/126792.yaml

@@ -0,0 +1,5 @@
+pr: 126792
+summary: Revert endpoint creation validation for ELSER and E5
+area: Machine Learning
+type: bug
+issues: []