Skip to content

Commit 5caa4ee

Browse files
committed
CSHARP-2126: Change handling of network errors or timeouts during connection handshake
1 parent ec9ac39 commit 5caa4ee

File tree

4 files changed

+201
-28
lines changed

4 files changed

+201
-28
lines changed

src/MongoDB.Driver.Core/Core/Servers/Server.cs

Lines changed: 26 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -144,23 +144,14 @@ public IChannelHandle GetChannel(CancellationToken cancellationToken)
144144
// wanted to cancel their operation. It will be better for the
145145
// collective to complete opening the connection than the throw
146146
// it away.
147-
connection.Open(CancellationToken.None);
147+
148+
connection.Open(CancellationToken.None); // This results in the initial isMaster being sent
148149
return new ServerChannel(this, connection);
149150
}
150151
catch (Exception ex)
151152
{
152-
if (ShouldClearConnectionPool(ex))
153-
{
154-
try
155-
{
156-
_connectionPool.Clear();
157-
}
158-
catch
159-
{
160-
// ignore exceptions
161-
}
153+
HandleBeforeHandshakeCompletesException(ex);
162154

163-
}
164155
connection.Dispose();
165156
throw;
166157
}
@@ -183,17 +174,8 @@ public async Task<IChannelHandle> GetChannelAsync(CancellationToken cancellation
183174
}
184175
catch (Exception ex)
185176
{
186-
if (ShouldClearConnectionPool(ex))
187-
{
188-
try
189-
{
190-
_connectionPool.Clear();
191-
}
192-
catch
193-
{
194-
// ignore exceptions
195-
}
196-
}
177+
HandleBeforeHandshakeCompletesException(ex);
178+
197179
connection.Dispose();
198180
throw;
199181
}
@@ -287,6 +269,21 @@ private void HandleChannelException(IConnection connection, Exception ex)
287269
}
288270
}
289271

272+
private void HandleBeforeHandshakeCompletesException(Exception ex)
273+
{
274+
if (ex is MongoAuthenticationException)
275+
{
276+
_connectionPool.Clear();
277+
return;
278+
}
279+
280+
if (ex is MongoConnectionException connectionException &&
281+
(connectionException.IsNetworkException || connectionException.ContainsSocketTimeoutException))
282+
{
283+
Invalidate($"ChannelException during handshake: {ex}.", clearConnectionPool: true);
284+
}
285+
}
286+
290287
private void Invalidate(string reasonInvalidated, bool clearConnectionPool)
291288
{
292289
if (clearConnectionPool)
@@ -346,11 +343,6 @@ private bool IsRecovering(ServerErrorCode code, string message)
346343
return false;
347344
}
348345

349-
private bool ShouldClearConnectionPool(Exception ex)
350-
{
351-
return ex is MongoAuthenticationException;
352-
}
353-
354346
private bool ShouldClearConnectionPoolForChannelException(Exception ex, SemanticVersion serverVersion)
355347
{
356348
if (ex is MongoNotPrimaryException mongoNotPrimaryException && mongoNotPrimaryException.Code == (int)ServerErrorCode.NotMaster)
@@ -363,6 +355,12 @@ private bool ShouldClearConnectionPoolForChannelException(Exception ex, Semantic
363355

364356
private bool ShouldInvalidateServer(Exception exception)
365357
{
358+
if (exception is MongoConnectionException mongoConnectionException &&
359+
mongoConnectionException.ContainsSocketTimeoutException)
360+
{
361+
return false;
362+
}
363+
366364
if (__invalidatingExceptions.Contains(exception.GetType()))
367365
{
368366
return true;

src/MongoDB.Driver.Core/MongoConnectionException.cs

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
*/
1515

1616
using System;
17+
using System.IO;
18+
using System.Net.Sockets;
1719
#if NET452
1820
using System.Runtime.Serialization;
1921
#endif
@@ -78,6 +80,25 @@ public ConnectionId ConnectionId
7880
get { return _connectionId; }
7981
}
8082

83+
/// <summary>
84+
/// Whether or not this exception contains a socket timeout exception.
85+
/// </summary>
86+
public bool ContainsSocketTimeoutException
87+
{
88+
get
89+
{
90+
for (var exception = InnerException; exception != null; exception = exception.InnerException)
91+
{
92+
if (exception is SocketException socketException &&
93+
socketException.SocketErrorCode == SocketError.TimedOut)
94+
{
95+
return true;
96+
}
97+
}
98+
return false;
99+
}
100+
}
101+
81102
/// <summary>
82103
/// Determines whether the exception is network error or no.
83104
/// </summary>

tests/MongoDB.Driver.Core.TestHelpers/CoreExceptionHelper.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
using System;
1717
using System.IO;
1818
using System.Net;
19+
using System.Net.Sockets;
1920
using MongoDB.Bson;
2021
using MongoDB.Driver.Core.Clusters;
2122
using MongoDB.Driver.Core.Connections;
@@ -117,5 +118,18 @@ public static MongoCommandException CreateMongoWriteConcernException(BsonDocumen
117118

118119
return writeConcernException;
119120
}
121+
122+
public static SocketException CreateSocketException(string errorType)
123+
{
124+
switch (errorType)
125+
{
126+
case "timedout":
127+
return new SocketException((int)SocketError.TimedOut);
128+
case "networkunreachable":
129+
return new SocketException((int)SocketError.NetworkUnreachable);
130+
default:
131+
throw new ArgumentException("Unknown error type.");
132+
}
133+
}
120134
}
121135
}

tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerTests.cs

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
using MongoDB.Bson;
2525
using MongoDB.Bson.IO;
2626
using MongoDB.Bson.Serialization.Serializers;
27+
using MongoDB.Bson.TestHelpers;
2728
using MongoDB.Bson.TestHelpers.XunitExtensions;
2829
using MongoDB.Driver.Core.Bindings;
2930
using MongoDB.Driver.Core.Clusters;
@@ -32,6 +33,7 @@
3233
using MongoDB.Driver.Core.ConnectionPools;
3334
using MongoDB.Driver.Core.Connections;
3435
using MongoDB.Driver.Core.Events;
36+
using MongoDB.Driver.Core.TestHelpers;
3537
using MongoDB.Driver.Core.TestHelpers.XunitExtensions;
3638
using MongoDB.Driver.Core.WireProtocol;
3739
using MongoDB.Driver.Core.WireProtocol.Messages.Encoders;
@@ -266,6 +268,133 @@ public void GetChannel_should_get_a_connection(
266268
channel.Should().NotBeNull();
267269
}
268270

271+
[Theory]
272+
[ParameterAttributeData]
273+
public void GetChannel_should_update_topology_and_clear_connection_pool_on_network_error_or_timeout(
274+
[Values("timedout", "networkunreachable")] string errorType,
275+
[Values(false, true)] bool async)
276+
{
277+
var serverId = new ServerId(_clusterId, _endPoint);
278+
var connectionId = new ConnectionId(serverId);
279+
var innerMostException = CoreExceptionHelper.CreateSocketException(errorType);
280+
281+
var openConnectionException = new MongoConnectionException(connectionId, "Oops", new IOException("Cry", innerMostException));
282+
var mockConnection = new Mock<IConnectionHandle>();
283+
mockConnection.Setup(c => c.Open(It.IsAny<CancellationToken>())).Throws(openConnectionException);
284+
mockConnection.Setup(c => c.OpenAsync(It.IsAny<CancellationToken>())).ThrowsAsync(openConnectionException);
285+
var mockConnectionPool = new Mock<IConnectionPool>();
286+
mockConnectionPool.Setup(p => p.AcquireConnection(It.IsAny<CancellationToken>())).Returns(mockConnection.Object);
287+
mockConnectionPool.Setup(p => p.AcquireConnectionAsync(It.IsAny<CancellationToken>())).ReturnsAsync(mockConnection.Object);
288+
var mockConnectionPoolFactory = new Mock<IConnectionPoolFactory>();
289+
mockConnectionPoolFactory
290+
.Setup(f => f.CreateConnectionPool(It.IsAny<ServerId>(), _endPoint))
291+
.Returns(mockConnectionPool.Object);
292+
var mockMonitorServerDescription = new ServerDescription(serverId, _endPoint);
293+
var mockServerMonitor = new Mock<IServerMonitor>();
294+
mockServerMonitor.SetupGet(m => m.Description).Returns(mockMonitorServerDescription);
295+
mockServerMonitor
296+
.Setup(m => m.Invalidate(It.IsAny<string>()))
297+
.Callback((string reason) => MockMonitorInvalidate(reason));
298+
var mockServerMonitorFactory = new Mock<IServerMonitorFactory>();
299+
mockServerMonitorFactory.Setup(f => f.Create(It.IsAny<ServerId>(), _endPoint)).Returns(mockServerMonitor.Object);
300+
301+
var subject = new Server(_clusterId, _clusterClock, _clusterConnectionMode, _settings, _endPoint, mockConnectionPoolFactory.Object, mockServerMonitorFactory.Object, _capturedEvents);
302+
subject.Initialize();
303+
304+
IChannelHandle channel = null;
305+
Exception exception;
306+
if (async)
307+
{
308+
exception = Record.Exception(() => channel = subject.GetChannelAsync(CancellationToken.None).GetAwaiter().GetResult());
309+
}
310+
else
311+
{
312+
exception = Record.Exception(() => channel = subject.GetChannel(CancellationToken.None));
313+
}
314+
315+
channel.Should().BeNull();
316+
exception.Should().Be(openConnectionException);
317+
subject.Description.Type.Should().Be(ServerType.Unknown);
318+
subject.Description.ReasonChanged.Should().Contain("ChannelException during handshake");
319+
mockServerMonitor.Verify(m => m.Invalidate(It.IsAny<string>()), Times.Once);
320+
mockConnectionPool.Verify(p => p.Clear(), Times.Once);
321+
322+
void MockMonitorInvalidate(string reason)
323+
{
324+
var currentDescription = mockServerMonitor.Object.Description;
325+
mockServerMonitor.SetupGet(m => m.Description).Returns(currentDescription.With(reason));
326+
}
327+
}
328+
329+
[Theory]
330+
[InlineData(nameof(MongoConnectionException), true)]
331+
[InlineData("MongoConnectionExceptionWithSocketTimeout", false)]
332+
public void HandleChannelException_should_update_topology_as_expected_on_network_error_or_timeout(
333+
string errorType, bool shouldUpdateTopology)
334+
{
335+
var serverId = new ServerId(_clusterId, _endPoint);
336+
var connectionId = new ConnectionId(serverId);
337+
Exception innerMostException;
338+
switch (errorType)
339+
{
340+
case "MongoConnectionExceptionWithSocketTimeout":
341+
innerMostException = new SocketException((int)SocketError.TimedOut);
342+
break;
343+
case nameof(MongoConnectionException):
344+
innerMostException = new SocketException((int)SocketError.NetworkUnreachable);
345+
break;
346+
default: throw new ArgumentException("Unknown error type.");
347+
}
348+
349+
var operationUsingChannelException = new MongoConnectionException(connectionId, "Oops", new IOException("Cry", innerMostException));
350+
var mockConnection = new Mock<IConnectionHandle>();
351+
var isMasterResult = new IsMasterResult(new BsonDocument { {"compressors", new BsonArray()}});
352+
// the server version doesn't matter when we're not testing MongoNotPrimaryExceptions, but is needed when
353+
// Server calls ShouldClearConnectionPoolForException
354+
var buildInfoResult = new BuildInfoResult(new BsonDocument { {"version", "4.4.0"} });
355+
mockConnection.SetupGet(c => c.Description)
356+
.Returns(new ConnectionDescription(new ConnectionId(serverId, 0), isMasterResult, buildInfoResult));
357+
var mockConnectionPool = new Mock<IConnectionPool>();
358+
mockConnectionPool.Setup(p => p.AcquireConnection(It.IsAny<CancellationToken>())).Returns(mockConnection.Object);
359+
mockConnectionPool.Setup(p => p.AcquireConnectionAsync(It.IsAny<CancellationToken>())).ReturnsAsync(mockConnection.Object);
360+
var mockConnectionPoolFactory = new Mock<IConnectionPoolFactory>();
361+
mockConnectionPoolFactory
362+
.Setup(f => f.CreateConnectionPool(It.IsAny<ServerId>(), _endPoint))
363+
.Returns(mockConnectionPool.Object);
364+
var mockMonitorServerInitialDescription = new ServerDescription(serverId, _endPoint).With(reasonChanged: "Initial D", type: ServerType.Standalone);
365+
var mockServerMonitor = new Mock<IServerMonitor>();
366+
mockServerMonitor.SetupGet(m => m.Description).Returns(mockMonitorServerInitialDescription);
367+
mockServerMonitor
368+
.Setup(m => m.Invalidate(It.IsAny<string>()))
369+
.Callback((string reason) => MockMonitorInvalidate(reason));
370+
var mockServerMonitorFactory = new Mock<IServerMonitorFactory>();
371+
mockServerMonitorFactory.Setup(f => f.Create(It.IsAny<ServerId>(), _endPoint)).Returns(mockServerMonitor.Object);
372+
var subject = new Server(_clusterId, _clusterClock, _clusterConnectionMode, _settings, _endPoint, mockConnectionPoolFactory.Object, mockServerMonitorFactory.Object, _capturedEvents);
373+
subject.Initialize();
374+
375+
subject.HandleChannelException(mockConnection.Object, operationUsingChannelException);
376+
377+
if (shouldUpdateTopology)
378+
{
379+
mockServerMonitor.Verify(m => m.Invalidate(It.IsAny<string>()), Times.Once);
380+
subject.Description.Type.Should().Be(ServerType.Unknown);
381+
subject.Description.ReasonChanged.Should().Contain("ChannelException");
382+
}
383+
else
384+
{
385+
mockServerMonitor.Verify(m => m.Invalidate(It.IsAny<string>()), Times.Never);
386+
subject.Description.Should().Be(mockMonitorServerInitialDescription);
387+
}
388+
389+
void MockMonitorInvalidate(string reason)
390+
{
391+
var currentDescription = mockServerMonitor.Object.Description;
392+
mockServerMonitor
393+
.SetupGet(m => m.Description)
394+
.Returns(currentDescription.With(reason, type: ServerType.Unknown));
395+
}
396+
}
397+
269398
[Fact]
270399
public void Initialize_should_initialize_the_server()
271400
{
@@ -407,6 +536,7 @@ internal void IsRecovering_should_return_expected_result_for_message(string mess
407536
[InlineData(nameof(MongoNotPrimaryException), true)]
408537
[InlineData(nameof(SocketException), true)]
409538
[InlineData(nameof(TimeoutException), false)]
539+
[InlineData("MongoConnectionExceptionWithSocketTimeout", false)]
410540
[InlineData(nameof(MongoExecutionTimeoutException), false)]
411541
internal void ShouldInvalidateServer_should_return_expected_result_for_exceptionType(string exceptionTypeName, bool expectedResult)
412542
{
@@ -426,6 +556,11 @@ internal void ShouldInvalidateServer_should_return_expected_result_for_exception
426556
case nameof(MongoNodeIsRecoveringException): exception = new MongoNodeIsRecoveringException(connectionId, command, commandResult); break;
427557
case nameof(MongoNotPrimaryException): exception = new MongoNotPrimaryException(connectionId, command, commandResult); break;
428558
case nameof(SocketException): exception = new SocketException(); break;
559+
case "MongoConnectionExceptionWithSocketTimeout":
560+
var innermostException = new SocketException((int)SocketError.TimedOut);
561+
var innerException = new IOException("Execute Order 66", innermostException);
562+
exception = new MongoConnectionException(connectionId, "Yes, Lord Sidious", innerException);
563+
break;
429564
case nameof(TimeoutException): exception = new TimeoutException(); break;
430565
case nameof(MongoExecutionTimeoutException): exception = new MongoExecutionTimeoutException(connectionId, "message"); break;
431566
default: throw new Exception($"Invalid exceptionTypeName: {exceptionTypeName}.");
@@ -595,6 +730,11 @@ public void Command_should_update_the_session_and_cluster_cluster_times()
595730

596731
internal static class ServerReflector
597732
{
733+
public static void HandleChannelException(this Server server, IConnection connection, Exception ex)
734+
{
735+
Reflector.Invoke(server, nameof(HandleChannelException), connection, ex);
736+
}
737+
598738
public static bool IsNotMaster(this Server server, ServerErrorCode code, string message)
599739
{
600740
var methodInfo = typeof(Server).GetMethod(nameof(IsNotMaster), BindingFlags.NonPublic | BindingFlags.Instance);

0 commit comments

Comments
 (0)