Merge branch 'main' into ml-proxy-action

Author: jonathan-buttner

docs/changelog/121667.yaml

@@ -0,0 +1,11 @@
+pr: 121667
+summary: Add deprecation warning for flush API
+area: Machine Learning
+type: deprecation
+issues:
+ - 121506
+deprecation:
+  title: Add deprecation warning for flush API
+  area: REST API
+  details: The anomaly detection job flush API is deprecated since it is only required for the post data API, which was deprecated since 7.11.0.
+  impact: This should have a minimal impact on users as the flush API is only required for the post data API, which was deprecated since 7.11.0.

docs/changelog/121911.yaml

@@ -0,0 +1,5 @@
+pr: 121911
+summary: Fix ENRICH validation for use of wildcards
+area: ES|QL
+type: bug
+issues: []

muted-tests.yml

@@ -378,8 +378,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/cat/health/cat-health-no-timestamp-example}
   issue: https://github.com/elastic/elasticsearch/issues/121867
-- class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT
-  issue: https://github.com/elastic/elasticsearch/issues/121882
 - class: org.elasticsearch.xpack.searchablesnapshots.FrozenSearchableSnapshotsIntegTests
   method: testCreateAndRestorePartialSearchableSnapshot
   issue: https://github.com/elastic/elasticsearch/issues/121927
@@ -397,9 +395,12 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/121966
 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/121967
-- class: org.elasticsearch.xpack.esql.parser.StatementParserTests
-  method: testInvalidJoinPatterns
-  issue: https://github.com/elastic/elasticsearch/issues/121968
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/stored text}
+  issue: https://github.com/elastic/elasticsearch/issues/121991
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/keyword}
+  issue: https://github.com/elastic/elasticsearch/issues/121992
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -174,12 +174,17 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
+    public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
+    public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);
+    public static final TransportVersion REMOVE_DESIRED_NODE_VERSION_90 = def(9_000_0_03);
+    public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION_90 = def(9_000_0_04);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00);
     public static final TransportVersion REMOVE_DESIRED_NODE_VERSION = def(9_004_0_00);
     public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION = def(9_005_0_00);
     public static final TransportVersion ESQL_RETRY_ON_SHARD_LEVEL_FAILURE = def(9_006_0_00);
+    public static final TransportVersion ESQL_PROFILE_ASYNC_NANOS = def(9_007_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java

@@ -22,14 +22,12 @@
 import org.elasticsearch.cluster.desirednodes.VersionConflictException;
 import org.elasticsearch.cluster.metadata.DesiredNodes;
 import org.elasticsearch.cluster.metadata.DesiredNodesMetadata;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.RerouteService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -42,18 +40,15 @@
 public class TransportUpdateDesiredNodesAction extends TransportMasterNodeAction<UpdateDesiredNodesRequest, UpdateDesiredNodesResponse> {
     private static final Logger logger = LogManager.getLogger(TransportUpdateDesiredNodesAction.class);
 
-    private final FeatureService featureService;
     private final MasterServiceTaskQueue<UpdateDesiredNodesTask> taskQueue;
 
     @Inject
     public TransportUpdateDesiredNodesAction(
         TransportService transportService,
         ClusterService clusterService,
         RerouteService rerouteService,
-        FeatureService featureService,
         ThreadPool threadPool,
         ActionFilters actionFilters,
-        IndexNameExpressionResolver indexNameExpressionResolver,
         AllocationService allocationService
     ) {
         super(
@@ -67,7 +62,6 @@ public TransportUpdateDesiredNodesAction(
             UpdateDesiredNodesResponse::new,
             EsExecutors.DIRECT_EXECUTOR_SERVICE
         );
-        this.featureService = featureService;
         this.taskQueue = clusterService.createTaskQueue(
             "update-desired-nodes",
             Priority.URGENT,

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java

@@ -49,7 +49,8 @@ public GetSnapshotsResponse(List<SnapshotInfo> snapshots, @Nullable String next,
 
     public GetSnapshotsResponse(StreamInput in) throws IOException {
         this.snapshots = in.readCollectionAsImmutableList(SnapshotInfo::readFrom);
-        if (in.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)) {
+        if (in.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)
+            && in.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_SNAPSHOT_FAILURES_90) == false) {
             // Deprecated `failures` field
             in.readMap(StreamInput::readException);
         }
@@ -83,7 +84,8 @@ public int remaining() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(snapshots);
-        if (out.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)) {
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)
+            && out.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_SNAPSHOT_FAILURES_90) == false) {
             // Deprecated `failures` field
             out.writeMap(Map.of(), StreamOutput::writeException);
         }

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -246,32 +246,6 @@ void skipShard(SearchShardIterator iterator) {
         successfulShardExecution();
     }
 
-    private static boolean assertExecuteOnStartThread() {
-        // Ensure that the current code has the following stacktrace:
-        // AbstractSearchAsyncAction#start -> AbstractSearchAsyncAction#executePhase -> AbstractSearchAsyncAction#performPhaseOnShard
-        final StackTraceElement[] stackTraceElements = Thread.currentThread().getStackTrace();
-        assert stackTraceElements.length >= 6 : stackTraceElements;
-        int index = 0;
-        assert stackTraceElements[index++].getMethodName().equals("getStackTrace");
-        assert stackTraceElements[index++].getMethodName().equals("assertExecuteOnStartThread");
-        assert stackTraceElements[index++].getMethodName().equals("failOnUnavailable");
-        if (stackTraceElements[index].getMethodName().equals("performPhaseOnShard")) {
-            assert stackTraceElements[index].getClassName().endsWith("CanMatchPreFilterSearchPhase");
-            index++;
-        }
-        assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
-        assert stackTraceElements[index++].getMethodName().equals("run");
-
-        assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
-        assert stackTraceElements[index++].getMethodName().equals("executePhase");
-
-        assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
-        assert stackTraceElements[index++].getMethodName().equals("start");
-
-        assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction") == false;
-        return true;
-    }
-
     private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final SearchShardTarget shard) {
         if (throttleConcurrentRequests) {
             var pendingExecutions = pendingExecutionsPerNode.computeIfAbsent(
@@ -313,7 +287,6 @@ public void onFailure(Exception e) {
     }
 
     private void failOnUnavailable(int shardIndex, SearchShardIterator shardIt) {
-        assert assertExecuteOnStartThread();
         SearchShardTarget unassignedShard = new SearchShardTarget(null, shardIt.shardId(), shardIt.getClusterAlias());
         onShardFailure(shardIndex, unassignedShard, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
     }

server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java

@@ -371,7 +371,7 @@ private static void blockForbiddenVersions(TransportVersion joiningTransportVers
      * that are also present across the whole cluster as a result.
      */
     private Set<String> calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map<String, Set<String>> nodeFeatures) {
-        if (featureService.featuresCanBeAssumedForNodes(nodes)) {
+        if (FeatureService.featuresCanBeAssumedForNodes(nodes)) {
             Set<String> assumedFeatures = featureService.getNodeFeatures()
                 .values()
                 .stream()
@@ -382,7 +382,7 @@ private Set<String> calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map<
             // add all assumed features to the featureset of all nodes of the next major version
             nodeFeatures = new HashMap<>(nodeFeatures);
             for (var node : nodes.getNodes().entrySet()) {
-                if (featureService.featuresCanBeAssumedForNode(node.getValue())) {
+                if (FeatureService.featuresCanBeAssumedForNode(node.getValue())) {
                     assert nodeFeatures.containsKey(node.getKey()) : "Node " + node.getKey() + " does not have any features";
                     nodeFeatures.computeIfPresent(node.getKey(), (k, v) -> {
                         var newFeatures = new HashSet<>(v);
@@ -525,7 +525,7 @@ private Set<String> enforceNodeFeatureBarrier(DiscoveryNode node, Set<String> ef
             return newNodeFeatures;
         }
 
-        if (featureService.featuresCanBeAssumedForNode(node)) {
+        if (FeatureService.featuresCanBeAssumedForNode(node)) {
             // it might still be ok for this node to join if this node can have assumed features,
             // and all the missing features are assumed
             // we can get the NodeFeature object direct from this node's registered features

server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java

@@ -161,7 +161,8 @@ public static DesiredNode readFrom(StreamInput in) throws IOException {
         }
         final var memory = ByteSizeValue.readFrom(in);
         final var storage = ByteSizeValue.readFrom(in);
-        if (in.getTransportVersion().before(TransportVersions.REMOVE_DESIRED_NODE_VERSION)) {
+        if (in.getTransportVersion().before(TransportVersions.REMOVE_DESIRED_NODE_VERSION)
+            && in.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_DESIRED_NODE_VERSION_90) == false) {
             in.readOptionalString();
         }
         return new DesiredNode(settings, processors, processorsRange, memory, storage);
@@ -180,7 +181,8 @@ public void writeTo(StreamOutput out) throws IOException {
         }
         memory.writeTo(out);
         storage.writeTo(out);
-        if (out.getTransportVersion().before(TransportVersions.REMOVE_DESIRED_NODE_VERSION)) {
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_DESIRED_NODE_VERSION)
+            && out.getTransportVersion().isPatchFrom(TransportVersions.REMOVE_DESIRED_NODE_VERSION_90) == false) {
             out.writeOptionalString(null);
         }
     }

server/src/main/java/org/elasticsearch/features/FeatureService.java

@@ -53,14 +53,14 @@ public Map<String, NodeFeature> getNodeFeatures() {
     /**
      * Returns {@code true} if {@code node} can have assumed features.
      */
-    public boolean featuresCanBeAssumedForNode(DiscoveryNode node) {
+    public static boolean featuresCanBeAssumedForNode(DiscoveryNode node) {
         return ClusterFeatures.featuresCanBeAssumedForNode(node);
     }
 
     /**
     * Returns {@code true} if one or more nodes in {@code nodes} can have assumed features.
     */
-    public boolean featuresCanBeAssumedForNodes(DiscoveryNodes nodes) {
+    public static boolean featuresCanBeAssumedForNodes(DiscoveryNodes nodes) {
         return ClusterFeatures.featuresCanBeAssumedForNodes(nodes);
     }