Skip to content

Commit 7a3604c

Browse files
committed
Fix ActivationCollector test flakiness and isolation issues
- Release blocked grains in DisposeAsync to prevent cluster shutdown hangs - Reduce grain counts in ManualCollectionShouldNotCollectBusyActivations to avoid thread pool exhaustion when calling ForceActivationCollection - Wait for all activations to be collected between iterations in ActivationCollectorShouldNotCollectBusyStatelessWorkers to ensure isolation
1 parent 28d7d55 commit 7a3604c

File tree

1 file changed

+23
-2
lines changed

1 file changed

+23
-2
lines changed

test/TesterInternal/ActivationsLifeCycleTests/ActivationCollectorTests.cs

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,11 @@ public async Task DisposeAsync()
110110
{
111111
_diagnosticObserver?.Dispose();
112112

113+
// Release any blocked grains before stopping the cluster to prevent hangs during shutdown.
114+
// This ensures test cleanup completes even if a test times out while grains are blocked.
115+
BusyActivationGcTestGrain1.ReleaseAllBlocked();
116+
StatelessWorkerActivationCollectorTestGrain1.ReleaseAllBlocked();
117+
113118
if (testCluster is null) return;
114119

115120
try
@@ -268,11 +273,15 @@ public async Task ManualCollectionShouldNotCollectBusyActivations()
268273
// 1. Idle activations become stale when fake time is advanced (GetIdleness() > CollectionAgeLimit)
269274
// 2. Busy activations remain not-inactive (IsCurrentlyExecuting = true) regardless of time
270275
// 3. The collector only collects activations that are both inactive AND stale
276+
//
277+
// NOTE: We use smaller grain counts (100 each) compared to the automatic collection test (500 each)
278+
// because we need to call ForceActivationCollection while grains are blocked. Blocking too many grains
279+
// exhausts the silo thread pool and causes the management grain call to timeout.
271280
await Initialize(DEFAULT_IDLE_TIMEOUT, useFakeTimeProvider: true);
272281

273282
TimeSpan shortIdleTimeout = TimeSpan.FromSeconds(1);
274-
const int idleGrainCount = 500;
275-
const int busyGrainCount = 500;
283+
const int idleGrainCount = 100;
284+
const int busyGrainCount = 100;
276285
var idleGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(IdleActivationGcTestGrain1));
277286
var busyGrainTypeName = RuntimeTypeNameFormatter.Format(typeof(BusyActivationGcTestGrain1));
278287

@@ -543,6 +552,18 @@ await _diagnosticObserver.WaitForDeactivationCountAsync(
543552
"ActivationCollectorShouldNotCollectBusyStatelessWorkers: iteration {Iteration} completed successfully. {Remaining} activation(s) remaining.",
544553
iteration,
545554
remainingActivations);
555+
556+
// Wait for all remaining activations to be collected before next iteration
557+
// This ensures test isolation between iterations
558+
if (iteration < 1) // Don't wait after the last iteration
559+
{
560+
_sharedTimeProvider!.Advance(DEFAULT_IDLE_TIMEOUT + TimeSpan.FromSeconds(5));
561+
await _diagnosticObserver.WaitForDeactivationCountAsync(
562+
"StatelessWorkerActivationCollectorTestGrain1",
563+
expectedDeactivations + grainCount, // Total deactivations including the one just released
564+
MAX_WAIT_TIME,
565+
_sharedTimeProvider);
566+
}
546567
}
547568
}
548569

0 commit comments

Comments
 (0)