Merge branch 'main' into fix-test-mtermvectors

Author: elasticmachine

docs/changelog/120852.yaml

@@ -0,0 +1,5 @@
+pr: 120852
+summary: Correct line and column numbers of missing named parameters
+area: ES|QL
+type: bug
+issues: []

docs/changelog/121324.yaml

@@ -0,0 +1,6 @@
+pr: 121324
+summary: Support duplicate suggestions in completion field
+area: Suggesters
+type: bug
+issues:
+ - 82432

docs/changelog/121428.yaml

@@ -0,0 +1,6 @@
+pr: 121428
+summary: Fix infer on and elasticsearch service endpoint created with a deployment
+  id
+area: Machine Learning
+type: bug
+issues: []

libs/core/src/main/java/org/elasticsearch/core/CheckedSupplier.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+/**
+ * A {@link java.util.function.Supplier}-like interface which allows throwing checked exceptions.
+ */
+@FunctionalInterface
+public interface CheckedSupplier<T, E extends Exception> {
+    T get() throws E;
+}

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java

@@ -52,10 +52,9 @@
  * <p>
  * A bit like Mockito but way more painful.
  */
-class DummyImplementations {
-
-    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
+public class DummyImplementations {
 
+    public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
         @Override
         public Locale[] getAvailableLocales() {
             throw unexpected();

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -96,6 +96,9 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
 
     private static final Map<String, CheckAction> checkActions = Stream.concat(
         Stream.<Entry<String, CheckAction>>of(
+            entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
+            entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
+            entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
             entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
             entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
             entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -338,6 +341,11 @@ private static void systemExit() {
         System.exit(123);
     }
 
+    private static void staticMethodNeverEntitledViaReflection() throws Exception {
+        Method systemExit = System.class.getMethod("exit", int.class);
+        systemExit.invoke(null, 123);
+    }
+
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -348,6 +356,11 @@ private static void processBuilder_start() throws IOException {
         new ProcessBuilder("").start();
     }
 
+    private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
+        Method processBuilderStart = ProcessBuilder.class.getMethod("start");
+        processBuilderStart.invoke(new ProcessBuilder(""));
+    }
+
     private static void processBuilder_startPipeline() throws IOException {
         ProcessBuilder.startPipeline(List.of());
     }
@@ -386,6 +399,10 @@ private static void setHttpsConnectionProperties() {
         new DummyLocaleServiceProvider();
     }
 
+    private static void constructorNeverEntitledViaReflection() throws Exception {
+        DummyLocaleServiceProvider.class.getConstructor().newInstance();
+    }
+
     private static void breakIteratorProvider$() {
         new DummyBreakIteratorProvider();
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -14,6 +14,8 @@
 import com.sun.tools.attach.AttachNotSupportedException;
 import com.sun.tools.attach.VirtualMachine;
 
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.CheckedSupplier;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.entitlement.runtime.api.NotEntitledException;
@@ -22,8 +24,10 @@
 import org.elasticsearch.logging.Logger;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileAttribute;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -144,30 +148,31 @@ private static String findAgentJar() {
      * @throws IllegalStateException if the entitlements system can't prevent an unauthorized action of our choosing
      */
     private static void selfTest() {
-        ensureCannotStartProcess();
-        ensureCanCreateTempFile();
+        ensureCannotStartProcess(ProcessBuilder::start);
+        ensureCanCreateTempFile(EntitlementBootstrap::createTempFile);
+
+        // Try again with reflection
+        ensureCannotStartProcess(EntitlementBootstrap::reflectiveStartProcess);
+        ensureCanCreateTempFile(EntitlementBootstrap::reflectiveCreateTempFile);
     }
 
-    private static void ensureCannotStartProcess() {
+    private static void ensureCannotStartProcess(CheckedConsumer<ProcessBuilder, ?> startProcess) {
         try {
             // The command doesn't matter; it doesn't even need to exist
-            new ProcessBuilder("").start();
+            startProcess.accept(new ProcessBuilder(""));
         } catch (NotEntitledException e) {
             logger.debug("Success: Entitlement protection correctly prevented process creation");
             return;
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw new IllegalStateException("Failed entitlement protection self-test", e);
         }
         throw new IllegalStateException("Entitlement protection self-test was incorrectly permitted");
     }
 
-    /**
-     * Originally {@code Security.selfTest}.
-     */
     @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
-    private static void ensureCanCreateTempFile() {
+    private static void ensureCanCreateTempFile(CheckedSupplier<Path, ?> createTempFile) {
         try {
-            Path p = Files.createTempFile(null, null);
+            Path p = createTempFile.get();
             p.toFile().deleteOnExit();
 
             // Make an effort to clean up the file immediately; also, deleteOnExit leaves the file if the JVM exits abnormally.
@@ -184,5 +189,24 @@ private static void ensureCanCreateTempFile() {
         logger.debug("Success: Entitlement protection correctly permitted temp file creation");
     }
 
+    @SuppressForbidden(reason = "accesses jvm default tempdir as a self-test")
+    private static Path createTempFile() throws Exception {
+        return Files.createTempFile(null, null);
+    }
+
+    private static void reflectiveStartProcess(ProcessBuilder pb) throws Exception {
+        try {
+            var start = ProcessBuilder.class.getMethod("start");
+            start.invoke(pb);
+        } catch (InvocationTargetException e) {
+            throw (Exception) e.getCause();
+        }
+    }
+
+    private static Path reflectiveCreateTempFile() throws Exception {
+        return (Path) Files.class.getMethod("createTempFile", String.class, String.class, FileAttribute[].class)
+            .invoke(null, null, null, new FileAttribute<?>[0]);
+    }
+
     private static final Logger logger = LogManager.getLogger(EntitlementBootstrap.class);
 }

muted-tests.yml

@@ -140,9 +140,6 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
   issue: https://github.com/elastic/elasticsearch/issues/118217
-- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
-  method: testBottomFieldSort
-  issue: https://github.com/elastic/elasticsearch/issues/118214
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
@@ -389,6 +386,67 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testDependentVariableIsAliasToNested
   issue: https://github.com/elastic/elasticsearch/issues/121415
+- class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT
+  issue: https://github.com/elastic/elasticsearch/issues/121464
+- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
+  method: testLookupExplosionBigStringManyMatches
+  issue: https://github.com/elastic/elasticsearch/issues/121465
+- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
+  method: testClientSecretRotation
+  issue: https://github.com/elastic/elasticsearch/issues/120985
+- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
+  method: testGrantApiKeyForJWT
+  issue: https://github.com/elastic/elasticsearch/issues/121039
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cluster.health/10_basic/cluster health basic test}
+  issue: https://github.com/elastic/elasticsearch/issues/121478
+- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
+  method: testLookupExplosionManyMatches
+  issue: https://github.com/elastic/elasticsearch/issues/121481
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testGetUsersWithProfileUid
+  issue: https://github.com/elastic/elasticsearch/issues/121483
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cat.aliases/10_basic/Empty cluster}
+  issue: https://github.com/elastic/elasticsearch/issues/121484
+- class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
+  method: testTransformLifecycle_RangeQueryThatMatchesNoShards
+  issue: https://github.com/elastic/elasticsearch/issues/121480
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testStopQueryLocal
+  issue: https://github.com/elastic/elasticsearch/issues/121487
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testSuccessfulPathways
+  issue: https://github.com/elastic/elasticsearch/issues/121488
+- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
+  method: testAsyncQueriesWithLimit0
+  issue: https://github.com/elastic/elasticsearch/issues/121489
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testSuggestProfilesWithHint
+  issue: https://github.com/elastic/elasticsearch/issues/121116
+- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT
+  method: test {docs.testFilterToday}
+  issue: https://github.com/elastic/elasticsearch/issues/121474
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testSuggestProfileWithData
+  issue: https://github.com/elastic/elasticsearch/issues/121258
+- class: org.elasticsearch.ingest.geoip.FullClusterRestartIT
+  method: testGeoIpSystemFeaturesMigration {cluster=UPGRADED}
+  issue: https://github.com/elastic/elasticsearch/issues/121115
+- class: org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStepTests
+  method: testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs
+  issue: https://github.com/elastic/elasticsearch/issues/121495
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias}
+  issue: https://github.com/elastic/elasticsearch/issues/121128
+- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
+  method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
+  issue: https://github.com/elastic/elasticsearch/issues/121412
+- class: org.elasticsearch.xpack.inference.common.InferenceServiceNodeLocalRateLimitCalculatorTests
+  issue: https://github.com/elastic/elasticsearch/issues/121294
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testDependentVariableIsAliasToKeyword
+  issue: https://github.com/elastic/elasticsearch/issues/121492
 
 # Examples:
 #

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/suggest/30_context.yml

@@ -395,3 +395,75 @@ setup:
                 field: suggest_multi_contexts
                 contexts:
                   location: []
+
+---
+"Duplicate suggestions in different contexts":
+  - requires:
+      cluster_features: [ "search.completion_field.duplicate.support" ]
+      reason: "Support for duplicate suggestions in different contexts"
+
+  - do:
+      index:
+        refresh: true
+        index: test
+        id:    "1"
+        body:
+          suggest_context:
+            -
+              input: "foox"
+              weight: 2
+              contexts:
+                color: ["red", "yellow"]
+            -
+              input: "foox"
+              weight: 3
+              contexts:
+                color: ["blue", "green", "yellow"]
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "red"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foox" }
+  - match:  { suggest.result.0.options.0._score: 2 }
+
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "yellow"
+
+  - length: { suggest.result: 1  }
+  - length: { suggest.result.0.options: 1  }
+  - match:  { suggest.result.0.options.0.text: "foox" }
+  # the highest weight wins
+  - match:  { suggest.result.0.options.0._score: 3 }
+
+  - do:
+      search:
+        body:
+          suggest:
+            result:
+              text: "foo"
+              completion:
+                field: suggest_context
+                contexts:
+                  color: "blue"
+
+  - length: { suggest.result: 1 }
+  - length: { suggest.result.0.options: 1 }
+  - match: { suggest.result.0.options.0.text: "foox" }
+  - match: { suggest.result.0.options.0._score: 3 }