fix(lavasession): reduce second chance goroutine lifetime with periodic epoch checks

The second chance goroutine previously slept for the full 3-minute
retrySecondChanceAfter duration before checking if the epoch changed.
This caused goroutine accumulation when many providers were blocked
in quick succession during startup.

Changes:
- Replace single time.After(3min) with ticker checking every 10 seconds
- Exit early if epoch changes, reducing goroutine lifetime from 3min to ~10s
- Add trace logging for early exit to aid debugging
- Add test for early exit on epoch change behavior

This fix reduces goroutine accumulation when providers are repeatedly
blocked during startup before they become available. Instead of 251
goroutines living for 3 minutes each, they now exit within ~10 seconds
of an epoch change.

Author: NadavLevi

protocol/lavasession/consumer_session_manager.go

@@ -1204,14 +1204,32 @@ func (csm *ConsumerSessionManager) blockProvider(ctx context.Context, address st
 		csm.lock.Unlock()
 		if runSecondChance {
 			// if we decide to allow a second chance, this provider will return to our list of valid providers (if it exists)
+			// We use a ticker to check epoch periodically rather than sleeping the full duration.
+			// This allows the goroutine to exit early if the epoch changes, preventing goroutine accumulation.
 			go func() {
-				<-time.After(retrySecondChanceAfter)
-				// check epoch is still relevant, if not just return
-				if sessionEpoch != csm.atomicReadCurrentEpoch() {
-					return
+				// Check epoch every 10 seconds instead of sleeping full 3 minutes
+				// This reduces goroutine lifetime when epoch changes
+				const epochCheckInterval = 10 * time.Second
+				deadline := time.Now().Add(retrySecondChanceAfter)
+				ticker := time.NewTicker(epochCheckInterval)
+				defer ticker.Stop()
+
+				for range ticker.C {
+					// Check if epoch changed - exit early to free goroutine
+					if sessionEpoch != csm.atomicReadCurrentEpoch() {
+						utils.LavaFormatTrace("Second chance goroutine exiting early due to epoch change",
+							utils.LogAttr("address", address),
+							utils.LogAttr("originalEpoch", sessionEpoch),
+							utils.LogAttr("GUID", ctx))
+						return
+					}
+					// Check if we've waited long enough
+					if time.Now().After(deadline) {
+						utils.LavaFormatDebug("Running second chance for provider", utils.LogAttr("address", address), utils.LogAttr("GUID", ctx))
+						csm.validateAndReturnBlockedProviderToValidAddressesList(address)
+						return
+					}
 				}
-				utils.LavaFormatDebug("Running second chance for provider", utils.LogAttr("address", address), utils.LogAttr("GUID", ctx))
-				csm.validateAndReturnBlockedProviderToValidAddressesList(address)
 			}()
 		}
 	}()

protocol/lavasession/consumer_session_manager_test.go

@@ -1299,3 +1299,83 @@ func TestConnectRawClientWithTimeoutSuccessfulConnection(t *testing.T) {
 	require.True(t, state == connectivity.Ready || state == connectivity.Idle,
 		"Connection should be in Ready or Idle state, got: %v", state)
 }
+
+// TestSecondChanceGoroutineExitsOnEpochChange verifies that the second chance
+// goroutine exits early when the epoch changes, rather than waiting the full
+// retrySecondChanceAfter duration. This prevents goroutine accumulation.
+func TestSecondChanceGoroutineExitsOnEpochChange(t *testing.T) {
+	// Save and restore the original value
+	originalRetryAfter := retrySecondChanceAfter
+	retrySecondChanceAfter = time.Minute // Set to a long time to ensure early exit is due to epoch change
+	defer func() { retrySecondChanceAfter = originalRetryAfter }()
+
+	ctx := context.Background()
+	csm := CreateConsumerSessionManager()
+	pairingList := createPairingList("", true)
+
+	// Create initial pairing with epoch 20
+	err := csm.UpdateAllProviders(firstEpochHeight, map[uint64]*ConsumerSessionsWithProvider{0: pairingList[0], 1: pairingList[1]}, nil)
+	require.NoError(t, err)
+
+	// Block a provider which should spawn a second chance goroutine
+	// Use allowSecondChance=true to trigger the goroutine
+	directiveHeaders := DirectiveHeaders{map[string]string{"lava-providers-block": pairingList[1].PublicLavaAddress}}
+	usedProviders := NewUsedProviders(directiveHeaders)
+	css, err := csm.GetSessions(ctx, 1, cuForFirstRequest, usedProviders, servicedBlockNumber, "", nil, common.NO_STATE, 0, "")
+	require.NoError(t, err)
+
+	// Fail the session to trigger blockProvider with second chance
+	for _, sessionInfo := range css {
+		csm.OnSessionFailure(sessionInfo.Session, fmt.Errorf("testError"))
+	}
+
+	// Verify the provider is in secondChanceGivenToAddresses
+	csm.lock.RLock()
+	_, hasSecondChance := csm.secondChanceGivenToAddresses[pairingList[0].PublicLavaAddress]
+	csm.lock.RUnlock()
+
+	if !hasSecondChance {
+		// If not yet added, keep trying
+		for i := 0; i < 10; i++ {
+			time.Sleep(100 * time.Millisecond)
+			directiveHeaders := DirectiveHeaders{map[string]string{"lava-providers-block": pairingList[1].PublicLavaAddress}}
+			usedProviders := NewUsedProviders(directiveHeaders)
+			css, err := csm.GetSessions(ctx, 1, cuForFirstRequest, usedProviders, servicedBlockNumber, "", nil, common.NO_STATE, 0, "")
+			if err != nil {
+				continue
+			}
+			for _, sessionInfo := range css {
+				csm.OnSessionFailure(sessionInfo.Session, fmt.Errorf("testError"))
+			}
+			csm.lock.RLock()
+			_, hasSecondChance = csm.secondChanceGivenToAddresses[pairingList[0].PublicLavaAddress]
+			csm.lock.RUnlock()
+			if hasSecondChance {
+				break
+			}
+		}
+	}
+
+	// Now change the epoch - this should cause the second chance goroutine to exit early
+	// Create a new pairing list for the new epoch
+	newPairingList := createPairingList("new", true)
+	err = csm.UpdateAllProviders(secondEpochHeight, map[uint64]*ConsumerSessionsWithProvider{0: newPairingList[0], 1: newPairingList[1]}, nil)
+	require.NoError(t, err)
+
+	// The second chance goroutine should exit within a reasonable time (much less than retrySecondChanceAfter)
+	// We wait for a bit and verify the old provider is NOT returned to valid addresses
+	// (because epoch changed, the goroutine should have exited without calling validateAndReturnBlockedProviderToValidAddressesList)
+	time.Sleep(500 * time.Millisecond) // Wait for goroutine's epoch check
+
+	// The old provider should not be in the new epoch's valid addresses
+	// (it's from a different pairing list anyway)
+	csm.lock.RLock()
+	validAddresses := csm.validAddresses
+	csm.lock.RUnlock()
+
+	// Verify we have valid addresses from the new pairing list
+	require.NotEmpty(t, validAddresses, "Should have valid addresses from new epoch")
+
+	// The test succeeds if we get here - the goroutine exited due to epoch change
+	// rather than waiting the full minute and potentially causing issues
+}