Merge branch 'main' of github.com:elastic/elasticsearch into ia-enable-ccm-by-default

Author: jonathan-buttner

docs/changelog/138047.yaml

@@ -0,0 +1,6 @@
+pr: 138047
+summary: Add configurable `max_batch_size` for `GoogleVertexAI` embedding service
+  settings
+area: Machine Learning
+type: bug
+issues: []

muted-tests.yml

@@ -460,6 +460,12 @@ tests:
 - class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
   method: test {yaml=search.retrievers/result-diversification/10_mmr_result_diversification_retriever/Test MMR result diversification single index float type}
   issue: https://github.com/elastic/elasticsearch/issues/139335
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  method: test {p0=search/140_pre_filter_search_shards/prefilter on non-indexed date fields}
+  issue: https://github.com/elastic/elasticsearch/issues/139381
+- class: org.elasticsearch.xpack.downsample.DownsampleIT
+  method: testAggregateMethod
+  issue: https://github.com/elastic/elasticsearch/issues/139382
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/engine/ThreadPoolMergeExecutorService.java

@@ -175,6 +175,9 @@ public Iterator<Setting<?>> settings() {
      * Initial value for IO write rate limit of individual merge tasks when doAutoIOThrottle is true
      */
     static final ByteSizeValue START_IO_RATE = ByteSizeValue.ofMb(20L);
+
+    private static final Logger logger = LogManager.getLogger(ThreadPoolMergeExecutorService.class);
+
     /**
      * Total number of submitted merge tasks that support IO auto throttling and that have not yet been run (or aborted).
      * This includes merge tasks that are currently running and that are backlogged (by their respective merge schedulers).
@@ -863,13 +866,15 @@ private static long newTargetIORateBytesPerSec(
                 MIN_IO_RATE.getBytes(),
                 currentTargetIORateBytesPerSec - currentTargetIORateBytesPerSec / 10L
             );
+            logger.debug("Decreasing target IO rate for merges to {}", newTargetIORateBytesPerSec);
         } else if (currentlySubmittedIOThrottledMergeTasks > concurrentMergesCeilLimitForThrottling
             && currentTargetIORateBytesPerSec < MAX_IO_RATE.getBytes()) {
                 // increase target IO rate by 20% (capped)
                 newTargetIORateBytesPerSec = Math.min(
                     MAX_IO_RATE.getBytes(),
                     currentTargetIORateBytesPerSec + currentTargetIORateBytesPerSec / 5L
                 );
+                logger.debug("Increasing target IO rate for merges to {}", newTargetIORateBytesPerSec);
             } else {
                 newTargetIORateBytesPerSec = currentTargetIORateBytesPerSec;
             }

server/src/main/java/org/elasticsearch/inference/ServiceSettings.java

@@ -14,6 +14,8 @@
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.xcontent.ToXContentObject;
 
+import java.util.Map;
+
 public interface ServiceSettings extends ToXContentObject, VersionedNamedWriteable, FilteredXContent {
 
     /**
@@ -61,4 +63,8 @@ default DenseVectorFieldMapper.ElementType elementType() {
      */
     @Nullable
     String modelId();
+
+    default ServiceSettings updateServiceSettings(Map<String, Object> serviceSettings) {
+        return this;
+    }
 }

server/src/main/resources/transport/definitions/referable/google_vertex_ai_configurable_max_batch_size.csv

@@ -0,0 +1 @@
+9241000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-esql_exponential_histogram_supported_version,9240000
+google_vertex_ai_configurable_max_batch_size,9241000

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java

@@ -223,10 +223,7 @@ private Model combineExistingModelWithNewSettings(
             newSecretSettings = existingSecretSettings.newSecretSettings(settingsToUpdate.serviceSettings());
         }
         if (settingsToUpdate.serviceSettings() != null) {
-            // In cluster services can have their deployment settings updated, so this is a special case
-            if (newServiceSettings instanceof ElasticsearchInternalServiceSettings elasticServiceSettings) {
-                newServiceSettings = elasticServiceSettings.updateServiceSettings(settingsToUpdate.serviceSettings());
-            }
+            newServiceSettings = newServiceSettings.updateServiceSettings(settingsToUpdate.serviceSettings());
         }
         if (settingsToUpdate.taskSettings() != null && existingTaskSettings != null) {
             newTaskSettings = existingTaskSettings.updatedTaskSettings(settingsToUpdate.taskSettings());

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java

@@ -748,6 +748,22 @@ public static Integer extractOptionalPositiveInteger(
         return extractOptionalInteger(map, settingName, scope, validationException, true);
     }
 
+    public static Integer extractOptionalPositiveIntegerLessThanOrEqualToMax(
+        Map<String, Object> map,
+        String settingName,
+        int maxValue,
+        String scope,
+        ValidationException validationException
+    ) {
+        Integer optionalField = extractOptionalPositiveInteger(map, settingName, scope, validationException);
+
+        if (optionalField != null && optionalField > maxValue) {
+            validationException.addValidationError(mustBeLessThanOrEqualNumberErrorMessage(settingName, scope, optionalField, maxValue));
+        }
+
+        return optionalField;
+    }
+
     public static Integer extractOptionalInteger(
         Map<String, Object> map,
         String settingName,

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalServiceSettings.java

@@ -227,7 +227,8 @@ public TransportVersion getMinimalSupportedVersion() {
         return TransportVersion.minimumCompatible();
     }
 
-    public ElasticsearchInternalServiceSettings updateServiceSettings(Map<String, Object> serviceSettings) {
+    @Override
+    public ServiceSettings updateServiceSettings(Map<String, Object> serviceSettings) {
         var validationException = new ValidationException();
         var mutableServiceSettings = new HashMap<>(serviceSettings);
 

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/googlevertexai/GoogleVertexAiService.java

@@ -280,11 +280,13 @@ protected void doChunkedInfer(
         ActionListener<List<ChunkedInference>> listener
     ) {
         GoogleVertexAiModel googleVertexAiModel = (GoogleVertexAiModel) model;
+        GoogleVertexAiEmbeddingsServiceSettings serviceSettings = (GoogleVertexAiEmbeddingsServiceSettings) googleVertexAiModel
+            .getServiceSettings();
         var actionCreator = new GoogleVertexAiActionCreator(getSender(), getServiceComponents());
 
         List<EmbeddingRequestChunker.BatchRequestAndListener> batchedRequests = new EmbeddingRequestChunker<>(
             inputs,
-            EMBEDDING_MAX_BATCH_SIZE,
+            serviceSettings.maxBatchSize() == null ? EMBEDDING_MAX_BATCH_SIZE : serviceSettings.maxBatchSize(),
             googleVertexAiModel.getConfigurations().getChunkingSettings()
         ).batchRequestsWithListeners(listener);
 
@@ -306,6 +308,7 @@ public Model updateModelWithEmbeddingDetails(Model model, int embeddingSize) {
                 serviceSettings.dimensionsSetByUser(),
                 serviceSettings.maxInputTokens(),
                 embeddingSize,
+                serviceSettings.maxBatchSize(),
                 serviceSettings.similarity(),
                 serviceSettings.rateLimitSettings()
             );