[Test] Ensure racing happens as expected (elastic#92757)

The test expects the loading work takes long enough so that all racing
threads have time to enter the concurrency-controlled code region.
However, it is possible the loading work is fast enough (or it might be
other threads are too slow) and it finishes before all racing threads
are ready.

This PR makes the loading method wait for a signal before completing.
The signal is sent once all racing threads are actually ready.

Resolves: elastic#92745

Author: ywangd

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwkSetLoaderTests.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings;
+import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -28,14 +29,13 @@
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class JwkSetLoaderTests extends ESTestCase {
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92745")
     public void testConcurrentReloadWillBeQueuedAndShareTheResults() throws IOException, InterruptedException {
         final Path tempDir = createTempDir();
         final Path path = tempDir.resolve("jwkset.json");
@@ -53,23 +53,37 @@ public void testConcurrentReloadWillBeQueuedAndShareTheResults() throws IOExcept
             .mapToObj(i -> new PlainActionFuture<Tuple<Boolean, JwkSetLoader.JwksAlgs>>())
             .toList();
 
-        final var threadsCountDown = new CountDownLatch(nThreads);
+        // Start the first thread for reloading
+        // Ensure it is inside the actual loading method and make it wait there to simulate slow processing
+        final var loadingLatch = new CountDownLatch(1);
         final var readyLatch = new CountDownLatch(1);
         doAnswer(invocation -> {
-            // Mark the thread to be ready when it enters the concurrency controlling area
-            threadsCountDown.countDown();
-            // Wait till the ready flag to start the racing
+            loadingLatch.countDown();
             assertThat(readyLatch.await(10, TimeUnit.SECONDS), is(true));
-            return invocation.callRealMethod();
-        }).when(jwkSetLoader).getFuture();
+            invocation.callRealMethod();
+            return null;
+        }).when(jwkSetLoader).loadInternal(anyActionListener());
 
-        // Start the threads and toggle flags to start racing
-        IntStream.range(0, nThreads).forEach(i -> new Thread(() -> jwkSetLoader.reload(futures.get(i))).start());
+        new Thread(() -> jwkSetLoader.reload(futures.get(0))).start();
+        assertThat(loadingLatch.await(10, TimeUnit.SECONDS), is(true));
+
+        // Start rest of the threads for racing and ensure they are all through the concurrency controlling area
+        Mockito.reset(jwkSetLoader);
+        final var threadsCountDown = new CountDownLatch(nThreads - 1);
+        doAnswer(invocation -> {
+            final Object result = invocation.callRealMethod();
+            threadsCountDown.countDown();
+            return result;
+        }).when(jwkSetLoader).getFuture();
+        IntStream.range(1, nThreads).forEach(i -> new Thread(() -> jwkSetLoader.reload(futures.get(i))).start());
         assertThat(threadsCountDown.await(10, TimeUnit.SECONDS), is(true));
+
+        // Notify the first thread to finish the loading work
         readyLatch.countDown();
 
-        // All concurrent reloading calls will get the same result and the actual reloading work will only happen once
-        futures.forEach(future -> assertThat(future.actionGet(), sameInstance(futures.get(0).actionGet())));
-        verify(jwkSetLoader, times(1)).loadInternal(anyActionListener());
+        // All concurrent reloading calls will get the same result from the first thread and skip the actual loading work
+        final Tuple<Boolean, JwkSetLoader.JwksAlgs> tuple = futures.get(0).actionGet();
+        futures.subList(1, nThreads).forEach(future -> assertThat(future.actionGet(), sameInstance(tuple)));
+        verify(jwkSetLoader, never()).loadInternal(anyActionListener());
     }
 }