Merge branch 'main' into fix_semantic_recovery_test

Author: jimczi

docs/changelog/123569.yaml

@@ -0,0 +1,5 @@
+pr: 123569
+summary: Abort pending deletion on `IndicesService` close
+area: Store
+type: enhancement
+issues: []

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -9,8 +9,6 @@
 
 package org.elasticsearch.core;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicReference;
@@ -21,11 +19,17 @@ public enum Releasables {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Iterable<? extends Releasable> releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
         }
     }
 
@@ -38,7 +42,18 @@ public static void close(@Nullable Releasable releasable) {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Releasable... releasables) {
-        close(true, releasables);
+        RuntimeException firstException = null;
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                firstException = useOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throw firstException;
+        }
     }
 
     /** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */
@@ -63,19 +78,21 @@ public static void closeExpectNoException(Releasable releasable) {
 
     /** Release the provided {@link Releasable}s, ignoring exceptions. */
     public static void closeWhileHandlingException(Releasable... releasables) {
-        close(false, releasables);
+        for (final Releasable releasable : releasables) {
+            try {
+                close(releasable);
+            } catch (RuntimeException e) {
+                // ignored
+            }
+        }
     }
 
-    /** Release the provided {@link Releasable}s, ignoring exceptions if <code>success</code> is {@code false}. */
-    private static void close(boolean success, Releasable... releasables) {
-        try {
-            // this does the right thing with respect to add suppressed and not wrapping errors etc.
-            IOUtils.close(releasables);
-        } catch (IOException e) {
-            if (success) {
-                throw new UncheckedIOException(e);
-            }
+    private static RuntimeException useOrSuppress(RuntimeException firstException, RuntimeException e) {
+        if (firstException == null || firstException == e) {
+            return e;
         }
+        firstException.addSuppressed(e);
+        return firstException;
     }
 
     /** Wrap several releasables into a single one. This is typically useful for use with try-with-resources: for example let's assume

muted-tests.yml

@@ -254,8 +254,6 @@ tests:
 - class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
   method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously}
   issue: https://github.com/elastic/elasticsearch/issues/122102
-- class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT
-  issue: https://github.com/elastic/elasticsearch/issues/121716
 - class: org.elasticsearch.smoketest.SmokeTestMonitoringWithSecurityIT
   method: testHTTPExporterWithSSL
   issue: https://github.com/elastic/elasticsearch/issues/122220
@@ -277,15 +275,9 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_408}
   issue: https://github.com/elastic/elasticsearch/issues/122681
-- class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT
-  method: testScaleWhileShrinking
-  issue: https://github.com/elastic/elasticsearch/issues/122119
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/122707
-- class: org.elasticsearch.snapshots.DedicatedClusterSnapshotRestoreIT
-  method: testRestoreShrinkIndex
-  issue: https://github.com/elastic/elasticsearch/issues/121717
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/cat/allocation/cat-allocation-example}
   issue: https://github.com/elastic/elasticsearch/issues/121976

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -255,6 +255,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories;
     private final IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListeners;
     final AbstractRefCounted indicesRefCount; // pkg-private for testing
+    private final CountDownLatch stopLatch = new CountDownLatch(1);
     private final CountDownLatch closeLatch = new CountDownLatch(1);
     private volatile boolean idFieldDataEnabled;
     private volatile boolean allowExpensiveQueries;
@@ -403,6 +404,7 @@ public ClusterService clusterService() {
 
     @Override
     protected void doStop() {
+        stopLatch.countDown();
         clusterService.removeApplier(timestampFieldMapperService);
         timestampFieldMapperService.doStop();
 
@@ -1440,7 +1442,15 @@ public void processPendingDeletes(Index index, IndexSettings indexSettings, Time
                     }
                     if (remove.isEmpty() == false) {
                         logger.warn("{} still pending deletes present for shards {} - retrying", index, remove.toString());
-                        Thread.sleep(sleepTime);
+                        if (stopLatch.await(sleepTime, TimeUnit.MILLISECONDS)) {
+                            logger.info(
+                                "Indices service stopped. {} aborting pending deletes after [{}] for shards {}",
+                                index,
+                                TimeValue.timeValueNanos(System.nanoTime() - startTimeNS),
+                                remove.toString()
+                            );
+                            break;
+                        }
                         sleepTime = Math.min(maxSleepTimeMs, sleepTime * 2); // increase the sleep time gradually
                         logger.debug("{} schedule pending delete retry after {} ms", index, sleepTime);
                     }

server/src/test/java/org/elasticsearch/reservedstate/service/FileSettingsServiceTests.java

@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
@@ -84,6 +85,7 @@
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
+@LuceneTestCase.SuppressFileSystems("ExtrasFS")
 public class FileSettingsServiceTests extends ESTestCase {
     private static final Logger logger = LogManager.getLogger(FileSettingsServiceTests.class);
     private Environment env;