Merge commit '83fe2eda5cabce7a1a6e8191d63a33ae25b40291' into 2025/05/23/ES-11815

Author: tlrx

docs/changelog/126581.yaml

@@ -0,0 +1,10 @@
+pr: 126581
+summary: "Optimize shared blob cache evictions on shard removal
+  Shared blob cache evictions occur on the cluster applier thread when shards are
+  removed from a node. These can be expensive if a large number of shards are
+  being removed. This change uses the context of the removal to avoid unnecessary
+  evictions that might hold up the applier thread.
+  "
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/reference/aggregations/search-aggregations-bucket-significantterms-aggregation.md

@@ -253,8 +253,8 @@ Like most design decisions, this is the basis of a trade-off in which we have ch
 The JLH score can be used as a significance score by adding the parameter
 
 ```js
-	 "jlh": {
-	 }
+   "jlh": {
+   }
 ```
 
 The scores are derived from the doc frequencies in *foreground* and *background* sets. The *absolute* change in popularity (foregroundPercent - backgroundPercent) would favor common terms whereas the *relative* change in popularity (foregroundPercent/ backgroundPercent) would favor rare terms. Rare vs common is essentially a precision vs recall balance and so the absolute and relative changes are multiplied to provide a sweet spot between precision and recall.
@@ -265,9 +265,9 @@ The scores are derived from the doc frequencies in *foreground* and *background*
 Mutual information as described in "Information Retrieval", Manning et al., Chapter 13.5.1 can be used as significance score by adding the parameter
 
 ```js
-	 "mutual_information": {
-	      "include_negatives": true
-	 }
+   "mutual_information": {
+        "include_negatives": true
+   }
 ```
 
 Mutual information does not differentiate between terms that are descriptive for the subset or for documents outside the subset. The significant terms therefore can contain terms that appear more or less frequent in the subset than outside the subset. To filter out the terms that appear less often in the subset than in documents outside the subset, `include_negatives` can be set to `false`.
@@ -284,8 +284,8 @@ Per default, the assumption is that the documents in the bucket are also contain
 Chi square as described in "Information Retrieval", Manning et al., Chapter 13.5.2 can be used as significance score by adding the parameter
 
 ```js
-	 "chi_square": {
-	 }
+   "chi_square": {
+   }
 ```
 
 Chi square behaves like mutual information and can be configured with the same parameters `include_negatives` and `background_is_superset`.
@@ -296,8 +296,8 @@ Chi square behaves like mutual information and can be configured with the same p
 Google normalized distance as described in ["The Google Similarity Distance", Cilibrasi and Vitanyi, 2007](https://arxiv.org/pdf/cs/0412098v3.pdf) can be used as significance score by adding the parameter
 
 ```js
-	 "gnd": {
-	 }
+   "gnd": {
+   }
 ```
 
 `gnd` also accepts the `background_is_superset` parameter.
@@ -394,8 +394,8 @@ The benefit of this heuristic is that the scoring logic is simple to explain to
 It would be hard for a seasoned boxer to win a championship if the prize was awarded purely on the basis of percentage of fights won - by these rules a newcomer with only one fight under their belt would be impossible to beat. Multiple observations are typically required to reinforce a view so it is recommended in these cases to set both `min_doc_count` and `shard_min_doc_count` to a higher value such as 10 in order to filter out the low-frequency terms that otherwise take precedence.
 
 ```js
-	 "percentage": {
-	 }
+   "percentage": {
+   }
 ```
 
 
@@ -413,11 +413,11 @@ If none of the above measures suits your usecase than another option is to imple
 Customized scores can be implemented via a script:
 
 ```js
-	    "script_heuristic": {
+      "script_heuristic": {
               "script": {
-	        "lang": "painless",
-	        "source": "params._subset_freq/(params._superset_freq - params._subset_freq + 1)"
-	      }
+          "lang": "painless",
+          "source": "params._subset_freq/(params._superset_freq - params._subset_freq + 1)"
+        }
             }
 ```
 

docs/reference/elasticsearch-plugins/discovery-azure-classic-scale.md

@@ -3,7 +3,7 @@ mapped_pages:
   - https://www.elastic.co/guide/en/elasticsearch/plugins/current/discovery-azure-classic-scale.html
 ---
 
-# Scaling out! [discovery-azure-classic-scale]
+# Scaling out [discovery-azure-classic-scale]
 
 You need first to create an image of your previous machine. Disconnect from your machine and run locally the following commands:
 

muted-tests.yml

@@ -474,15 +474,6 @@ tests:
 - class: org.elasticsearch.packaging.test.TemporaryDirectoryConfigTests
   method: test21AcceptsCustomPathInDocker
   issue: https://github.com/elastic/elasticsearch/issues/128114
-- class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
-  method: testSymlinkUpdateTriggerReload
-  issue: https://github.com/elastic/elasticsearch/issues/128369
-- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
-  method: testEqualityAndOther {semantic_text}
-  issue: https://github.com/elastic/elasticsearch/issues/128414
-- class: org.elasticsearch.index.query.MultiMatchQueryBuilderTests
-  method: testToQuery
-  issue: https://github.com/elastic/elasticsearch/issues/128416
 - class: org.elasticsearch.xpack.ml.integration.InferenceIngestIT
   method: testPipelineIngestWithModelAliases
   issue: https://github.com/elastic/elasticsearch/issues/128417

server/src/internalClusterTest/java/org/elasticsearch/plugins/IndexFoldersDeletionListenerIT.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.IndexRemovalReason;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 
@@ -345,12 +346,22 @@ public static class IndexFoldersDeletionListenerPlugin extends Plugin implements
         public List<IndexFoldersDeletionListener> getIndexFoldersDeletionListeners() {
             return List.of(new IndexFoldersDeletionListener() {
                 @Override
-                public void beforeIndexFoldersDeleted(Index index, IndexSettings indexSettings, Path[] indexPaths) {
+                public void beforeIndexFoldersDeleted(
+                    Index index,
+                    IndexSettings indexSettings,
+                    Path[] indexPaths,
+                    IndexRemovalReason reason
+                ) {
                     deletedIndices.add(index);
                 }
 
                 @Override
-                public void beforeShardFoldersDeleted(ShardId shardId, IndexSettings indexSettings, Path[] shardPaths) {
+                public void beforeShardFoldersDeleted(
+                    ShardId shardId,
+                    IndexSettings indexSettings,
+                    Path[] shardPaths,
+                    IndexRemovalReason reason
+                ) {
                     deletedShards.computeIfAbsent(shardId.getIndex(), i -> Collections.synchronizedList(new ArrayList<>())).add(shardId);
                 }
             });

server/src/internalClusterTest/java/org/elasticsearch/reservedstate/service/FileSettingsServiceIT.java

@@ -196,21 +196,18 @@ public void clusterChanged(ClusterChangedEvent event) {
         return new Tuple<>(savedClusterState, metadataVersion);
     }
 
-    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node) {
+    private Tuple<CountDownLatch, AtomicLong> setupClusterStateListener(String node, long version) {
         ClusterService clusterService = internalCluster().clusterService(node);
         CountDownLatch savedClusterState = new CountDownLatch(1);
         AtomicLong metadataVersion = new AtomicLong(-1);
         clusterService.addListener(new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 ReservedStateMetadata reservedState = event.state().metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE);
-                if (reservedState != null) {
-                    ReservedStateHandlerMetadata handlerMetadata = reservedState.handlers().get(ReservedClusterSettingsAction.NAME);
-                    if (handlerMetadata != null && handlerMetadata.keys().contains("indices.recovery.max_bytes_per_sec")) {
-                        clusterService.removeListener(this);
-                        metadataVersion.set(event.state().metadata().version());
-                        savedClusterState.countDown();
-                    }
+                if (reservedState != null && reservedState.version() == version) {
+                    clusterService.removeListener(this);
+                    metadataVersion.set(event.state().metadata().version());
+                    savedClusterState.countDown();
                 }
             }
         });
@@ -258,14 +255,14 @@ public void testSettingsApplied() throws Exception {
         logger.info("--> start master node");
         final String masterNode = internalCluster().startMasterOnlyNode();
         assertMasterNode(internalCluster().nonMasterClient(), masterNode);
-        var savedClusterState = setupClusterStateListener(masterNode);
+        var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
         assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
         assertFalse(dataFileSettingsService.watching());
 
-        writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet());
+        writeJSONFile(masterNode, testJSON, logger, versionCounter.get());
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
     }
 
@@ -276,11 +273,11 @@ public void testSettingsAppliedOnStart() throws Exception {
         FileSettingsService dataFileSettingsService = internalCluster().getInstance(FileSettingsService.class, dataNode);
 
         assertFalse(dataFileSettingsService.watching());
-        var savedClusterState = setupClusterStateListener(dataNode);
+        var savedClusterState = setupClusterStateListener(dataNode, versionCounter.incrementAndGet());
 
         // In internal cluster tests, the nodes share the config directory, so when we write with the data node path
         // the master will pick it up on start
-        writeJSONFile(dataNode, testJSON, logger, versionCounter.incrementAndGet());
+        writeJSONFile(dataNode, testJSON, logger, versionCounter.get());
 
         logger.info("--> start master node");
         final String masterNode = internalCluster().startMasterOnlyNode();
@@ -301,14 +298,14 @@ public void testReservedStatePersistsOnRestart() throws Exception {
             Settings.builder().put(INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s").build()
         );
         assertMasterNode(internalCluster().masterClient(), masterNode);
-        var savedClusterState = setupClusterStateListener(masterNode);
+        var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());
 
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
         assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
 
         logger.info("--> write some settings");
-        writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet());
+        writeJSONFile(masterNode, testJSON, logger, versionCounter.get());
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
 
         logger.info("--> restart master");
@@ -476,12 +473,12 @@ public void testSettingsAppliedOnMasterReElection() throws Exception {
         assertFalse(master1FS.watching());
         assertFalse(master2FS.watching());
 
-        var savedClusterState = setupClusterStateListener(masterNode);
+        var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());
         FileSettingsService masterFileSettingsService = internalCluster().getInstance(FileSettingsService.class, masterNode);
 
         assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
 
-        writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet());
+        writeJSONFile(masterNode, testJSON, logger, versionCounter.get());
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
 
         internalCluster().stopCurrentMasterNode();
@@ -501,8 +498,8 @@ public void testSettingsAppliedOnMasterReElection() throws Exception {
         boolean awaitSuccessful = savedClusterState.v1().await(20, TimeUnit.SECONDS);
         assertTrue(awaitSuccessful);
 
-        savedClusterState = setupClusterStateListener(internalCluster().getMasterName());
-        writeJSONFile(internalCluster().getMasterName(), testJSON43mb, logger, versionCounter.incrementAndGet());
+        savedClusterState = setupClusterStateListener(internalCluster().getMasterName(), versionCounter.incrementAndGet());
+        writeJSONFile(internalCluster().getMasterName(), testJSON43mb, logger, versionCounter.get());
 
         assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb");
     }
@@ -515,21 +512,21 @@ public void testSymlinkUpdateTriggerReload() throws Exception {
         assertBusy(() -> assertTrue(masterFileSettingsService.watching()));
 
         {
-            var savedClusterState = setupClusterStateListener(masterNode);
+            var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());
             // Create the settings.json as a symlink to simulate k8 setup
             // settings.json -> ..data/settings.json
             // ..data -> ..TIMESTAMP_TEMP_FOLDER_1
             var fileDir = Files.createDirectories(baseDir.resolve("..TIMESTAMP_TEMP_FOLDER_1"));
-            writeJSONFile(masterNode, testJSON, logger, versionCounter.incrementAndGet(), fileDir.resolve("settings.json"));
+            writeJSONFile(masterNode, testJSON, logger, versionCounter.get(), fileDir.resolve("settings.json"));
             var dataDir = Files.createSymbolicLink(baseDir.resolve("..data"), fileDir.getFileName());
             Files.createSymbolicLink(baseDir.resolve("settings.json"), dataDir.getFileName().resolve("settings.json"));
             assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "50mb");
         }
         {
-            var savedClusterState = setupClusterStateListener(masterNode);
+            var savedClusterState = setupClusterStateListener(masterNode, versionCounter.incrementAndGet());
             // Update ..data symlink to ..data -> ..TIMESTAMP_TEMP_FOLDER_2 to simulate kubernetes secret update
             var fileDir = Files.createDirectories(baseDir.resolve("..TIMESTAMP_TEMP_FOLDER_2"));
-            writeJSONFile(masterNode, testJSON43mb, logger, versionCounter.incrementAndGet(), fileDir.resolve("settings.json"));
+            writeJSONFile(masterNode, testJSON43mb, logger, versionCounter.get(), fileDir.resolve("settings.json"));
             Files.deleteIfExists(baseDir.resolve("..data"));
             Files.createSymbolicLink(baseDir.resolve("..data"), fileDir.getFileName());
             assertClusterStateSaveOK(savedClusterState.v1(), savedClusterState.v2(), "43mb");

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java

@@ -48,7 +48,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
+import static org.elasticsearch.indices.cluster.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
 /**
  * Service responsible for submitting add and remove aliases requests

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -83,7 +83,7 @@
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
-import static org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
+import static org.elasticsearch.indices.cluster.IndexRemovalReason.NO_LONGER_ASSIGNED;
 
 /**
  * Service responsible for submitting index templates updates

server/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java

@@ -21,7 +21,7 @@
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason;
+import org.elasticsearch.indices.cluster.IndexRemovalReason;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collection;

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -83,6 +83,7 @@
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.cluster.IndexRemovalReason;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
@@ -494,7 +495,12 @@ public synchronized IndexShard createShard(
                         nodeEnv,
                         lock,
                         this.indexSettings,
-                        shardPaths -> indexFoldersDeletionListener.beforeShardFoldersDeleted(shardId, this.indexSettings, shardPaths)
+                        shardPaths -> indexFoldersDeletionListener.beforeShardFoldersDeleted(
+                            shardId,
+                            this.indexSettings,
+                            shardPaths,
+                            IndexRemovalReason.FAILURE
+                        )
                     );
                     path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
                 } catch (Exception inner) {
@@ -704,11 +710,11 @@ private void onShardClose(ShardLock lock) {
                 try {
                     eventListener.beforeIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
                 } finally {
-                    shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings);
+                    shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings, IndexRemovalReason.DELETED);
                     eventListener.afterIndexShardDeleted(lock.getShardId(), indexSettings.getSettings());
                 }
             } catch (IOException e) {
-                shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings);
+                shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings, IndexRemovalReason.DELETED);
                 logger.debug(() -> "[" + lock.getShardId().id() + "] failed to delete shard content - scheduled a retry", e);
             }
         }
@@ -1062,9 +1068,9 @@ public static Function<String, String> dateMathExpressionResolverAt(long instant
     }
 
     public interface ShardStoreDeleter {
-        void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException;
+        void deleteShardStore(String reasonText, ShardLock lock, IndexSettings indexSettings, IndexRemovalReason reason) throws IOException;
 
-        void addPendingDelete(ShardId shardId, IndexSettings indexSettings);
+        void addPendingDelete(ShardId shardId, IndexSettings indexSettings, IndexRemovalReason reason);
     }
 
     public final EngineFactory getEngineFactory() {