Address PR review comments for membership startup cleanup

- Use non-generic TaskCompletionSource instead of TaskCompletionSource<bool>
  since the result value is never used (pentp)
- Fix misleading comment on myEtag null check to match actual logic
- Use WaitToWriteAsync+TryWrite pattern in CleanupMyTableEntries to avoid
  relying on DropOldest semantics for acknowledged requests, and add
  cancellation via WaitAsync(_shutdownCts.Token) to prevent hanging
- Always retry failed requests regardless of whether Completion is set;
  Completion is only signaled on success, not on individual failures

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ReubenBond

src/Orleans.Runtime/MembershipService/MembershipTableManager.cs

@@ -394,7 +394,7 @@ private async Task<bool> TryUpdateMyStatusGlobalOnce(SiloStatus newStatus)
 
             bool ok;
             TableVersion next = table.Version.Next();
-            if (myEtag is not null) // no previous etag for my entry -> its the first write to this entry, so insert instead of update.
+            if (myEtag is not null) // existing etag for my entry -> there is a previous row, so update instead of insert.
             {
                 ok = await membershipTableProvider.UpdateRow(myEntry, myEtag, next);
             }
@@ -526,15 +526,24 @@ private async Task<bool> CleanupMyTableEntries(MembershipTableData table)
 
             LogDebugCleanupTableEntriesAboutToDeclareDead(this.log, silosToDeclareDead.Count, Utils.EnumerableToString(silosToDeclareDead.Select(tuple => tuple.Item1)));
 
-            var completions = new List<Task<bool>>(silosToDeclareDead.Count);
+            var completions = new List<Task>(silosToDeclareDead.Count);
             foreach (var siloData in silosToDeclareDead)
             {
                 var (request, completion) = SuspectOrKillRequest.CreateAcknowledgedKillRequest(siloData.Item1.SiloAddress);
-                await _trySuspectOrKillChannel.Writer.WriteAsync(request);
-                completions.Add(completion);
+                await _trySuspectOrKillChannel.Writer.WaitToWriteAsync(_shutdownCts.Token);
+                if (_trySuspectOrKillChannel.Writer.TryWrite(request))
+                {
+                    completions.Add(completion);
+                }
             }
 
-            await Task.WhenAll(completions);
+            try
+            {
+                await Task.WhenAll(completions).WaitAsync(_shutdownCts.Token);
+            }
+            catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested)
+            {
+            }
             return true;
         }
 
@@ -578,13 +587,10 @@ bool IsFunctionalForMembership(SiloStatus status)
 
         private class SuspectOrKillRequest
         {
-            private const int MaxAttempts = 3;
-
             public required SiloAddress SiloAddress { get; init; }
             public SiloAddress? OtherSilo { get; init; }
             public required RequestType Type { get; init; }
-            public TaskCompletionSource<bool>? Completion { get; init; }
-            public int RemainingAttempts { get; set; } = MaxAttempts;
+            public TaskCompletionSource? Completion { get; init; }
 
             public enum RequestType
             {
@@ -602,9 +608,9 @@ public static SuspectOrKillRequest CreateKillRequest(SiloAddress silo)
                 };
             }
 
-            public static (SuspectOrKillRequest Request, Task<bool> Completion) CreateAcknowledgedKillRequest(SiloAddress silo)
+            public static (SuspectOrKillRequest Request, Task Completion) CreateAcknowledgedKillRequest(SiloAddress silo)
             {
-                var completion = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
+                var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
                 var request = new SuspectOrKillRequest
                 {
                     SiloAddress = silo,
@@ -655,20 +661,13 @@ public async Task ProcessSuspectOrKillLists()
                                 break;
                         }
                         runningFailureCount = 0;
-                        request.Completion?.TrySetResult(true);
+                        request.Completion?.TrySetResult();
                     }
                     catch (Exception ex)
                     {
                         runningFailureCount += 1;
                         LogErrorProcessingSuspectOrKillLists(this.log, ex, runningFailureCount);
-                        if (request.Completion is not null)
-                        {
-                            request.Completion.TrySetException(ex);
-                        }
-                        else if (--request.RemainingAttempts > 0)
-                        {
-                            await _trySuspectOrKillChannel.Writer.WriteAsync(request, _shutdownCts.Token);
-                        }
+                        await _trySuspectOrKillChannel.Writer.WriteAsync(request, _shutdownCts.Token);
                     }
 
                     if (!reader.TryPeek(out _))