Merge branch 'main' into fork_profile_info

Author: ioanatia

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a9e07d97c55f5b854b1c203509e57b6f466eb26318540b2e1b59734355f747b6 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a9e07d97c55f5b854b1c203509e57b6f466eb26318540b2e1b59734355f747b6
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:55ee1dca9780931b0929d6eb734f455790c06ddbb59f55008e0cddebfbfd1e2e AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -79,7 +79,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:55ee1dca9780931b0929d6eb734f455790c06ddbb59f55008e0cddebfbfd1e2e
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/changelog/127564.yaml

@@ -0,0 +1,6 @@
+pr: 127564
+summary: Consider inlinestats when having `field_caps` check for field names
+area: ES|QL
+type: bug
+issues:
+ - 127236

docs/changelog/127966.yaml

@@ -0,0 +1,5 @@
+pr: 127966
+summary: "[ML] Add Rerank support to the Inference Plugin"
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/128327.yaml

@@ -0,0 +1,5 @@
+pr: 128327
+summary: Use internal user for internal inference action
+area: Machine Learning
+type: bug
+issues: []

plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java

@@ -11,7 +11,7 @@
 
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.text.Text;
+import org.elasticsearch.xcontent.Text;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.search.suggest.Suggester;
 

plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java

@@ -11,7 +11,7 @@
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.text.Text;
+import org.elasticsearch.xcontent.Text;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentBuilder;

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -179,6 +179,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion V_8_19_FIELD_CAPS_ADD_CLUSTER_ALIAS = def(8_841_0_32);
     public static final TransportVersion ESQL_HASH_OPERATOR_STATUS_OUTPUT_TIME_8_19 = def(8_841_0_34);
     public static final TransportVersion RERANKER_FAILURES_ALLOWED_8_19 = def(8_841_0_35);
+    public static final TransportVersion ML_INFERENCE_HUGGING_FACE_RERANK_ADDED_8_19 = def(8_841_0_36);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);
@@ -261,7 +262,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_HASH_OPERATOR_STATUS_OUTPUT_TIME = def(9_077_0_00);
     public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED = def(9_078_0_00);
     public static final TransportVersion NODES_STATS_SUPPORTS_MULTI_PROJECT = def(9_079_0_00);
-
+    public static final TransportVersion ML_INFERENCE_HUGGING_FACE_RERANK_ADDED = def(9_080_0_00);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java

@@ -65,6 +65,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexReshardService;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.index.shard.ShardId;
@@ -352,6 +353,12 @@ static ClusterState addIndexClosedBlocks(
             );
         }
 
+        // Check if index closing conflicts with ongoing resharding
+        Set<Index> reshardingIndices = IndexReshardService.reshardingIndices(currentProjectState, indicesToClose);
+        if (reshardingIndices.isEmpty() == false) {
+            throw new IllegalArgumentException("Cannot close indices that are being resharded: " + reshardingIndices);
+        }
+
         final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks());
 
         for (Index index : indicesToClose) {
@@ -940,6 +947,22 @@ static Tuple<ClusterState, List<IndexResult>> closeRoutingTable(
                     continue;
                 }
 
+                // Check if index closing conflicts with ongoing resharding
+                Set<Index> reshardingIndices = IndexReshardService.reshardingIndices(currentProjectState, Set.of(index));
+                if (reshardingIndices.isEmpty() == false) {
+                    closingResults.put(
+                        result.getKey(),
+                        new IndexResult(
+                            result.getKey(),
+                            new IllegalStateException(
+                                "verification of shards before closing " + index + " succeeded but index is being resharded in the meantime"
+                            )
+                        )
+                    );
+                    logger.debug("verification of shards before closing {} succeeded but index is being resharded in the meantime", index);
+                    continue;
+                }
+
                 blocks.removeIndexBlockWithId(projectId, index.getName(), INDEX_CLOSED_BLOCK_ID);
                 blocks.addIndexBlock(projectId, index.getName(), INDEX_CLOSED_BLOCK);
                 final IndexMetadata.Builder updatedMetadata = IndexMetadata.builder(indexMetadata).state(IndexMetadata.State.CLOSE);

server/src/main/java/org/elasticsearch/index/IndexReshardService.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.cluster.ProjectState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Resharding is currently only implemented in Serverless. This service only exposes minimal metadata about resharding
+ * needed by other services.
+ */
+public class IndexReshardService {
+    /**
+     * Returns the indices from the provided set that are currently being resharded.
+     */
+    public static Set<Index> reshardingIndices(final ProjectState projectState, final Set<Index> indicesToCheck) {
+        final Set<Index> indices = new HashSet<>();
+        for (Index index : indicesToCheck) {
+            IndexMetadata indexMetadata = projectState.metadata().index(index);
+
+            if (indexMetadata != null && indexMetadata.getReshardingMetadata() != null) {
+                indices.add(index);
+            }
+        }
+        return indices;
+    }
+}