CSHARP-3057: Reduce race conditions in SDAM error handling

Author: vincentkam

src/MongoDB.Driver.Core/Core/Clusters/MultiServerCluster.cs

@@ -383,7 +383,7 @@ private ClusterDescription ProcessReplicaSetChange(ClusterDescription clusterDes
                             lock (_serversLock)
                             {
                                 var server = _servers.SingleOrDefault(x => EndPointHelper.Equals(args.NewServerDescription.EndPoint, x.EndPoint));
-                                server.Invalidate("ReportedPrimaryIsStale");
+                                server.Invalidate("ReportedPrimaryIsStale", args.NewServerDescription.TopologyVersion);
 
                                 _sdamInformationEventHandler?.Invoke(new SdamInformationEvent(() =>
                                     string.Format(
@@ -487,7 +487,7 @@ private ClusterDescription ProcessReplicaSetChange(ClusterDescription clusterDes
                         foreach (var currentPrimary in currentPrimaries)
                         {
                             // kick off the server to invalidate itself
-                            currentPrimary.Invalidate("NoLongerPrimary");
+                            currentPrimary.Invalidate("NoLongerPrimary", args.NewServerDescription.TopologyVersion);
                             // set it to disconnected in the cluster
                             clusterDescription = clusterDescription.WithServerDescription(
                                 new ServerDescription(currentPrimary.ServerId, currentPrimary.EndPoint, "NoLongerPrimary"));

src/MongoDB.Driver.Core/Core/ConnectionPools/ExclusiveConnectionPool.cs

@@ -318,7 +318,7 @@ private async Task EnsureMinSizeAsync(CancellationToken cancellationToken)
 
                     var stopwatch = Stopwatch.StartNew();
                     var connection = CreateNewConnection();
-                    // when adding in a connection, we need to open it because 
+                    // when adding in a connection, we need to open it because
                     // the whole point of having a min pool size is to have
                     // them available and ready...
                     try
@@ -563,6 +563,11 @@ public EndPoint EndPoint
                 get { return _connection.EndPoint; }
             }
 
+            public int Generation
+            {
+                get { return _generation; }
+            }
+
             public bool IsExpired
             {
                 get { return _generation < _connectionPool.Generation || _connection.IsExpired; }
@@ -636,6 +641,11 @@ public EndPoint EndPoint
                 get { return _reference.Instance.EndPoint; }
             }
 
+            public int Generation
+            {
+                get { return _reference.Instance.Generation; }
+            }
+
             public bool IsExpired
             {
                 get

src/MongoDB.Driver.Core/Core/ConnectionPools/IConnectionPool.cs

@@ -31,6 +31,14 @@ namespace MongoDB.Driver.Core.ConnectionPools
     public interface IConnectionPool : IDisposable
     {
         // properties
+        /// <summary>
+        /// Gets the generation of the connection pool.
+        /// </summary>
+        /// <value>
+        /// The generation.
+        /// </value>
+        int Generation { get; }
+
         /// <summary>
         /// Gets the server identifier.
         /// </summary>

src/MongoDB.Driver.Core/Core/Connections/BinaryConnection.cs

@@ -126,6 +126,11 @@ public EndPoint EndPoint
             get { return _endPoint; }
         }
 
+        public int Generation
+        {
+            get => throw new NotSupportedException();
+        }
+
         public bool IsExpired
         {
             get

src/MongoDB.Driver.Core/Core/Connections/IConnection.cs

@@ -57,6 +57,14 @@ public interface IConnection : IDisposable
         /// </value>
         EndPoint EndPoint { get; }
 
+        /// <summary>
+        /// Gets the generation of the pool when this connection was created.
+        /// </summary>
+        /// <value>
+        /// The generation of the pool when this connection was created.
+        /// </value>
+        int Generation { get; }
+
         /// <summary>
         /// Gets a value indicating whether this instance is expired.
         /// </summary>

src/MongoDB.Driver.Core/Core/Connections/IsMasterResult.cs

@@ -364,6 +364,25 @@ public TagSet Tags
             }
         }
 
+        /// <summary>
+        /// Get the TopologyVersion.
+        /// </summary>
+        /// <value>
+        /// Null if TopologyVersion was not included in the isMaster response.
+        /// </value>
+        public TopologyVersion TopologyVersion
+        {
+            get
+            {
+                if (_wrapped.TryGetValue("topologyVersion", out var topologyVersionValue) &&
+                    topologyVersionValue is BsonDocument topologyVersion)
+                {
+                    return Servers.TopologyVersion.FromBsonDocument(topologyVersion);
+                }
+                return null;
+            }
+        }
+
         /// <summary>
         /// Gets the maximum wire version.
         /// </summary>

src/MongoDB.Driver.Core/Core/Servers/IServer.cs

@@ -103,8 +103,16 @@ public interface IClusterableServer : IServer, IDisposable
         /// Invalidates this instance (sets the server type to Unknown and clears the connection pool).
         /// </summary>
         /// <param name="reasonInvalidated">The reason the instance was invalidated.</param>
+        [Obsolete("Use Invalidate with TopologyVersion instead.")]
         void Invalidate(string reasonInvalidated);
 
+        /// <summary>
+        /// Invalidates this instance (sets the server type to Unknown and clears the connection pool).
+        /// </summary>
+        /// <param name="reasonInvalidated">The reason the instance was invalidated.</param>
+        /// <param name="responseTopologyVersion">The topology of the version of the response that invalidated this instance.</param>
+        void Invalidate(string reasonInvalidated, TopologyVersion responseTopologyVersion);
+
         /// <summary>
         /// Requests a heartbeat as soon as possible.
         /// </summary>

src/MongoDB.Driver.Core/Core/Servers/IServerMonitor.cs

@@ -23,6 +23,9 @@ namespace MongoDB.Driver.Core.Servers
     /// <seealso cref="System.IDisposable" />
     internal interface IServerMonitor : IDisposable
     {
+        /// <summary>
+        /// The monitor's view of the server, which could be considered stale relative to the <see cref="Server"/>'s view.
+        /// </summary>
         ServerDescription Description { get; }
 
         /// <summary>
@@ -35,12 +38,6 @@ internal interface IServerMonitor : IDisposable
         /// </summary>
         void Initialize();
 
-        /// <summary>
-        /// Instructs the monitor to refresh its description immediately.
-        /// </summary>
-        /// <param name="reasonInvalidated">The reason the server was invalidated.</param>
-        void Invalidate(string reasonInvalidated);
-
         /// <summary>
         /// Requests a heartbeat as soon as possible.
         /// </summary>