kvm,utils/test,config/test: fix flaky unit tests

Signed-off-by: Abhishek Kumar <[email protected]>

Author: shwstppr

framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyScheduledExecutionWrapperTest.java

@@ -25,6 +25,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
@@ -35,6 +37,8 @@
 public class ConfigKeyScheduledExecutionWrapperTest {
     private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, new NamedThreadFactory("TestExecutor"));
 
+    protected static final int DELTA_WAIT_MS = 100;
+
     @Mock
     ConfigKey<Integer> configKey;
 
@@ -76,7 +80,7 @@ public void scheduleOncePerSecondTest() {
 
     private void waitSeconds(int seconds) {
         try {
-            Thread.sleep(seconds * 1000L + 100);
+            Thread.sleep(seconds * 1000L + DELTA_WAIT_MS);
         } catch (InterruptedException e) {
             throw new RuntimeException(e);
         }
@@ -144,8 +148,9 @@ public void temporaryDisableRunsTest() {
     }
 
     static class TestRunnable implements Runnable {
-        private Integer runCount = 0;
+        private final AtomicInteger runCount = new AtomicInteger(0);
         private int waitSeconds = 0;
+        private final AtomicLong resetMs = new AtomicLong(0);
 
         TestRunnable(int waitSeconds) {
             this.waitSeconds = waitSeconds;
@@ -156,7 +161,11 @@ static class TestRunnable implements Runnable {
 
         @Override
         public void run() {
-            runCount++;
+            long resetMsVal = resetMs.get();
+            if (resetMsVal == 0 || System.currentTimeMillis() - resetMsVal > DELTA_WAIT_MS) {
+                runCount.incrementAndGet();
+            }
+            resetMs.set(0);
             if (waitSeconds > 0) {
                 try {
                     Thread.sleep(waitSeconds * 1000L);
@@ -167,11 +176,12 @@ public void run() {
         }
 
         public int getRunCount() {
-            return this.runCount;
+            return this.runCount.get();
         }
 
         public void resetRunCount() {
-            this.runCount = 0;
+            this.runCount.set(0);
+            this.resetMs.set(System.currentTimeMillis());
         }
     }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -6396,18 +6396,41 @@ public void disconnected() {
         for (DisconnectHook hook : _disconnectHooks) {
             hook.start();
         }
+        // compute a shared deadline = now + max(timeout of hooks)
         long start = System.currentTimeMillis();
+        long maxTimeoutMs = 0;
+        for (DisconnectHook hook : _disconnectHooks) {
+            maxTimeoutMs = Math.max(maxTimeoutMs, hook.getTimeoutMs());
+        }
+        System.out.println("Max timeout for disconnect hooks is " + maxTimeoutMs + "ms");
+        final long globalDeadline = start + maxTimeoutMs;
+
+        // join each hook using remaining time until the shared deadline
         for (DisconnectHook hook : _disconnectHooks) {
             try {
-                long elapsed = System.currentTimeMillis() - start;
-                long remaining = hook.getTimeoutMs() - elapsed;
-                long joinWait = remaining > 0 ? remaining : 1;
-                hook.join(joinWait);
-                hook.interrupt();
+                long now = System.currentTimeMillis();
+                long hookDeadline = start + Math.max(0, hook.getTimeoutMs());
+                long effectiveDeadline = Math.min(globalDeadline, hookDeadline);
+                long remaining = effectiveDeadline - now;
+                System.out.println("Joining disconnect hook " + hook + ", remaining time is " + remaining + "ms");
+                if (remaining <= 0) {
+                    // overall timeout already expired
+                    if (hook.isAlive()) {
+                        System.out.println("Interrupting disconnect hook " + hook + " due to timeout");
+                        hook.interrupt();
+                    }
+                    continue;
+                }
+                hook.join(remaining);
+                if (hook.isAlive()) {
+                    hook.interrupt();
+                }
             } catch (InterruptedException ex) {
-                LOGGER.warn("Interrupted disconnect hook: " + ex.getMessage());
+                Thread.currentThread().interrupt();
+                LOGGER.warn("Interrupted while waiting for disconnect hook: " + ex.getMessage());
             }
         }
+
         _disconnectHooks.clear();
     }
 

server/src/test/java/org/apache/cloudstack/vm/schedule/VMSchedulerImplTest.java

@@ -247,8 +247,9 @@ public void testScheduleNextJobScheduleFutureScheduleWithTimeZoneChecks() throws
 
     @Test
     public void testScheduleNextJobScheduleCurrentSchedule() {
-        Date now = DateUtils.setSeconds(new Date(), 0);
-        Date expectedScheduledTime = DateUtils.round(DateUtils.addMinutes(now, 1), Calendar.MINUTE);
+        Date now = DateUtils.round(new Date(), Calendar.MINUTE);
+        Date expectedScheduledTime = DateUtils.addMinutes(now, 1);
+
         UserVm vm = Mockito.mock(UserVm.class);
 
         VMScheduleVO vmSchedule = Mockito.mock(VMScheduleVO.class);
@@ -257,7 +258,8 @@ public void testScheduleNextJobScheduleCurrentSchedule() {
         Mockito.when(vmSchedule.getTimeZoneId()).thenReturn(TimeZone.getTimeZone("UTC").toZoneId());
         Mockito.when(vmSchedule.getStartDate()).thenReturn(DateUtils.addDays(now, -1));
         Mockito.when(userVmManager.getUserVm(Mockito.anyLong())).thenReturn(vm);
-        Date actualScheduledTime = vmScheduler.scheduleNextJob(vmSchedule, new Date());
+
+        Date actualScheduledTime = vmScheduler.scheduleNextJob(vmSchedule, now);
 
         Assert.assertEquals(expectedScheduledTime, actualScheduledTime);
     }

utils/src/test/java/org/apache/cloudstack/utils/cache/LazyCacheTest.java

@@ -30,7 +30,7 @@
 
 @RunWith(MockitoJUnitRunner.class)
 public class LazyCacheTest {
-    private final long expireSeconds = 1;
+    private final long expireSeconds = 2;
     private final String cacheValuePrefix = "ComputedValueFor:";
     private LazyCache<String, String> cache;
     private Function<String, String> mockLoader;