CSHARP-2403: Log any unexpected exceptions that occur in SDAM.

Author: rstam

src/MongoDB.Driver.Core/Core/Clusters/MultiServerCluster.cs

@@ -227,9 +227,31 @@ private async Task MonitorServersAsync()
                     var eventArgs = await _serverDescriptionChangedQueue.DequeueAsync(monitorServersCancellationToken).ConfigureAwait(false); // TODO: add timeout and cancellationToken to DequeueAsync
                     ProcessServerDescriptionChanged(eventArgs);
                 }
-                catch
+                catch (OperationCanceledException) when (monitorServersCancellationToken.IsCancellationRequested)
                 {
-                    // TODO: log this somewhere...
+                    // ignore OperationCanceledException when monitor servers cancellation is requested
+                }
+                catch (Exception unexpectedException)
+                {
+                    // if we catch an exception here it's because of a bug in the driver
+
+                    var handler = _sdamInformationEventHandler;
+                    if (handler != null)
+                    {
+                        try
+                        {
+                            handler.Invoke(new SdamInformationEvent(() =>
+                                string.Format(
+                                    "Unexpected exception in MultiServerCluster.MonitorServersAsync: {0}",
+                                    unexpectedException.ToString())));
+                        }
+                        catch
+                        {
+                            // ignore any exceptions thrown by the handler (note: event handlers aren't supposed to throw exceptions)
+                        }
+                    }
+
+                    // TODO: should we reset the cluster state in some way? (the state is undefined since an unexpected exception was thrown)
                 }
             }
         }

src/MongoDB.Driver.Core/Core/Servers/ServerDescription.cs

@@ -559,5 +559,37 @@ public ServerDescription With(
                 return this;
             }
         }
+
+        /// <summary>
+        /// Returns a new ServerDescription with a new HeartbeatException.
+        /// </summary>
+        /// <param name="heartbeatException">The heartbeat exception.</param>
+        /// <returns>
+        /// A new instance of ServerDescription.
+        /// </returns>
+        public ServerDescription WithHeartbeatException(Exception heartbeatException)
+        {
+            return new ServerDescription(
+                _serverId,
+                _endPoint,
+                averageRoundTripTime: _averageRoundTripTime,
+                canonicalEndPoint: _canonicalEndPoint,
+                electionId: _electionId,
+                heartbeatException: heartbeatException,
+                heartbeatInterval: _heartbeatInterval,
+                lastUpdateTimestamp: _lastUpdateTimestamp,
+                lastWriteTimestamp: _lastWriteTimestamp,
+                logicalSessionTimeout: _logicalSessionTimeout,
+                maxBatchCount: _maxBatchCount,
+                maxDocumentSize: _maxDocumentSize,
+                maxMessageSize: _maxMessageSize,
+                maxWireDocumentSize: _maxWireDocumentSize,
+                replicaSetConfig: _replicaSetConfig,
+                state: _state,
+                tags: _tags,
+                type: _type,
+                version: _version,
+                wireVersionRange: _wireVersionRange);
+        }
     }
 }

src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs

@@ -47,6 +47,7 @@ internal sealed class ServerMonitor : IServerMonitor
         private readonly Action<ServerHeartbeatStartedEvent> _heartbeatStartedEventHandler;
         private readonly Action<ServerHeartbeatSucceededEvent> _heartbeatSucceededEventHandler;
         private readonly Action<ServerHeartbeatFailedEvent> _heartbeatFailedEventHandler;
+        private readonly Action<SdamInformationEvent> _sdamInformationEventHandler;
 
         public event EventHandler<ServerDescriptionChangedEventArgs> DescriptionChanged;
 
@@ -64,6 +65,7 @@ public ServerMonitor(ServerId serverId, EndPoint endPoint, IConnectionFactory co
             eventSubscriber.TryGetEventHandler(out _heartbeatStartedEventHandler);
             eventSubscriber.TryGetEventHandler(out _heartbeatSucceededEventHandler);
             eventSubscriber.TryGetEventHandler(out _heartbeatFailedEventHandler);
+            eventSubscriber.TryGetEventHandler(out _sdamInformationEventHandler);
         }
 
         public ServerDescription Description => Interlocked.CompareExchange(ref _currentDescription, null, null);
@@ -91,7 +93,7 @@ public void Initialize()
 
         public void Invalidate()
         {
-            OnDescriptionChanged(_baseDescription);
+            SetDescriptionIfChanged(_baseDescription);
             RequestHeartbeat();
         }
 
@@ -113,7 +115,47 @@ private async Task MonitorServerAsync()
             {
                 try
                 {
-                    await HeartbeatAsync(heartbeatCancellationToken).ConfigureAwait(false);
+                    try
+                    {
+                        await HeartbeatAsync(heartbeatCancellationToken).ConfigureAwait(false);
+                    }
+                    catch (OperationCanceledException) when (heartbeatCancellationToken.IsCancellationRequested)
+                    {
+                        // ignore OperationCanceledException when heartbeat cancellation is requested
+                    }
+                    catch (Exception unexpectedException)
+                    {
+                        // if we catch an exception here it's because of a bug in the driver (but we need to defend ourselves against that)
+
+                        var handler = _sdamInformationEventHandler;
+                        if (handler != null)
+                        {
+                            try
+                            {
+                                handler.Invoke(new SdamInformationEvent(() =>
+                                    string.Format(
+                                        "Unexpected exception in ServerMonitor.MonitorServerAsync: {0}",
+                                        unexpectedException.ToString())));
+                            }
+                            catch
+                            {
+                                // ignore any exceptions thrown by the handler (note: event handlers aren't supposed to throw exceptions)
+                            }
+                        }
+
+                        // since an unexpected exception was thrown set the server description to Unknown (with the unexpected exception)
+                        try
+                        {
+                            // keep this code as simple as possible to keep the surface area with any remaining possible bugs as small as possible
+                            var newDescription = _baseDescription.WithHeartbeatException(unexpectedException); // not With in case the bug is in With
+                            SetDescription(newDescription); // not SetDescriptionIfChanged in case the bug is in SetDescriptionIfChanged
+                        }
+                        catch
+                        {
+                            // if even the simple code in the try throws just give up (at least we've raised the unexpected exception via an SdamInformationEvent)
+                        }
+                    }
+
                     var newHeartbeatDelay = new HeartbeatDelay(metronome.GetNextTickDelay(), __minHeartbeatInterval);
                     var oldHeartbeatDelay = Interlocked.Exchange(ref _heartbeatDelay, newHeartbeatDelay);
                     if (oldHeartbeatDelay != null)
@@ -198,7 +240,7 @@ private async Task<bool> HeartbeatAsync(CancellationToken cancellationToken)
                 newDescription = newDescription.With(heartbeatException: heartbeatException);
             }
 
-            OnDescriptionChanged(newDescription);
+            SetDescriptionIfChanged(newDescription);
 
             return true;
         }
@@ -257,15 +299,8 @@ private async Task<HeartbeatInfo> GetHeartbeatInfoAsync(IConnection connection,
             }
         }
 
-        private void OnDescriptionChanged(ServerDescription newDescription)
+        private void OnDescriptionChanged(ServerDescription oldDescription, ServerDescription newDescription)
         {
-            var oldDescription = Interlocked.CompareExchange(ref _currentDescription, null, null);
-            if (oldDescription.Equals(newDescription))
-            {
-                return;
-            }
-            Interlocked.Exchange(ref _currentDescription, newDescription);
-
             var handler = DescriptionChanged;
             if (handler != null)
             {
@@ -275,6 +310,29 @@ private void OnDescriptionChanged(ServerDescription newDescription)
             }
         }
 
+        private void SetDescription(ServerDescription newDescription)
+        {
+            var oldDescription = Interlocked.CompareExchange(ref _currentDescription, null, null);
+            SetDescription(oldDescription, newDescription);
+        }
+
+        private void SetDescription(ServerDescription oldDescription, ServerDescription newDescription)
+        {
+            Interlocked.Exchange(ref _currentDescription, newDescription);
+            OnDescriptionChanged(oldDescription, newDescription);
+        }
+
+        private void SetDescriptionIfChanged(ServerDescription newDescription)
+        {
+            var oldDescription = Interlocked.CompareExchange(ref _currentDescription, null, null);
+            if (oldDescription.Equals(newDescription))
+            {
+                return;
+            }
+
+            SetDescription(oldDescription, newDescription);
+        }
+
         private void ThrowIfDisposed()
         {
             if (_state.Value == State.Disposed)