CSHARP-2872: Expand use of error labels for RetryableWrites.

Author: DmitryLukyanov

src/MongoDB.Driver.Core/Core/Misc/Feature.cs

@@ -74,9 +74,11 @@ public class Feature
         private static readonly Feature __partialIndexes = new Feature("PartialIndexes", new SemanticVersion(3, 2, 0));
         private static readonly ReadConcernFeature __readConcern = new ReadConcernFeature("ReadConcern", new SemanticVersion(3, 2, 0));
         private static readonly Feature __retryableReads = new Feature("RetryableReads", new SemanticVersion(3, 6, 0));
+        private static readonly Feature __retryableWrites = new Feature("RetryableWrites", new SemanticVersion(3, 6, 0));
         private static readonly Feature __scramSha1Authentication = new Feature("ScramSha1Authentication", new SemanticVersion(3, 0, 0));
         private static readonly Feature __scramSha256Authentication = new Feature("ScramSha256Authentication", new SemanticVersion(4, 0, 0, ""));
         private static readonly Feature __serverExtractsUsernameFromX509Certificate = new Feature("ServerExtractsUsernameFromX509Certificate", new SemanticVersion(3, 3, 12));
+        private static readonly Feature __serverReturnsRetryableWriteErrorLabel = new Feature("ServerReturnsRetryableWriteErrorLabel", new SemanticVersion(4, 3, 0));
         private static readonly Feature __shardedTransactions = new Feature("ShardedTransactions", new SemanticVersion(4, 1, 6));
         private static readonly Feature __tailableCursor = new Feature("TailableCursor", new SemanticVersion(3, 2, 0));
         private static readonly Feature __transactions = new Feature("Transactions", new SemanticVersion(4, 0, 0));
@@ -340,6 +342,11 @@ public class Feature
         /// </summary>
         public static Feature RetryableReads => __retryableReads;
 
+        /// <summary>
+        /// Gets the retryable writes feature.
+        /// </summary>
+        public static Feature RetryableWrites => __retryableWrites;
+
         /// <summary>
         /// Gets the scram sha1 authentication feature.
         /// </summary>
@@ -355,6 +362,11 @@ public class Feature
         /// </summary>
         public static Feature ServerExtractsUsernameFromX509Certificate => __serverExtractsUsernameFromX509Certificate;
 
+        /// <summary>
+        /// Gets the server returns retryable writeError label feature.
+        /// </summary>
+        public static Feature ServerReturnsRetryableWriteErrorLabel => __serverReturnsRetryableWriteErrorLabel;
+
         /// <summary>
         /// Gets the sharded transactions feature.
         /// </summary>

src/MongoDB.Driver.Core/Core/Operations/RetryabilityHelper.cs

@@ -16,11 +16,15 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using MongoDB.Bson;
 
 namespace MongoDB.Driver.Core.Operations
 {
     internal static class RetryabilityHelper
     {
+        // private constants
+        private const string RetryableWriteErrorLabel = "RetryableWriteError";
+
         // private static fields
         private static readonly HashSet<ServerErrorCode> __notResumableChangeStreamErrorCodes;
         private static readonly HashSet<string> __notResumableChangeStreamErrorLabels;
@@ -45,7 +49,6 @@ static RetryabilityHelper()
                 typeof(MongoCursorNotFoundException)
             };
 
-
             __retryableReadExceptions = new HashSet<Type>(resumableAndRetryableExceptions);
 
             __retryableWriteExceptions = new HashSet<Type>(resumableAndRetryableExceptions);
@@ -62,7 +65,7 @@ static RetryabilityHelper()
 
             __retryableWriteErrorCodes = new HashSet<ServerErrorCode>(resumableAndRetryableErrorCodes)
             {
-                ServerErrorCode.WriteConcernFailed
+                ServerErrorCode.ExceededTimeLimit
             };
 
             __notResumableChangeStreamErrorCodes = new HashSet<ServerErrorCode>()
@@ -79,6 +82,14 @@ static RetryabilityHelper()
         }
 
         // public static methods
+        public static void AddRetryableWriteErrorLabelIfRequired(MongoException exception)
+        {
+            if (ShouldRetryableWriteExceptionLabelBeAdded(exception))
+            {
+                exception.AddErrorLabel(RetryableWriteErrorLabel);
+            }
+        }
+
         public static bool IsResumableChangeStreamException(Exception exception)
         {
             var commandException = exception as MongoCommandException;
@@ -117,6 +128,12 @@ public static bool IsRetryableReadException(Exception exception)
         }
 
         public static bool IsRetryableWriteException(Exception exception)
+        {
+            return exception is MongoException mongoException ? mongoException.HasErrorLabel(RetryableWriteErrorLabel) : false;
+        }
+
+        // private static methods
+        private static bool ShouldRetryableWriteExceptionLabelBeAdded(Exception exception)
         {
             if (__retryableWriteExceptions.Contains(exception.GetType()))
             {
@@ -153,15 +170,9 @@ public static bool IsRetryableWriteException(Exception exception)
                         case ServerErrorCode.HostUnreachable:
                         case ServerErrorCode.NetworkTimeout:
                         case ServerErrorCode.SocketException:
+                        case ServerErrorCode.ExceededTimeLimit:
                             return true;
                     }
-
-                    var message = writeConcernError.GetValue("errmsg", null)?.AsString;
-                    if (message.IndexOf("not master", StringComparison.OrdinalIgnoreCase) != -1 ||
-                        message.IndexOf("node is recovering", StringComparison.OrdinalIgnoreCase) != -1)
-                    {
-                        return true;
-                    }
                 }
             }
 

src/MongoDB.Driver.Core/Core/WireProtocol/CommandUsingCommandMessageWireProtocol.cs

@@ -122,10 +122,8 @@ public TCommandResult Execute(IConnection connection, CancellationToken cancella
             }
             catch (Exception exception)
             {
-                if (exception is MongoException mongoException && ShouldAddTransientTransactionError(mongoException))
-                {
-                    mongoException.AddErrorLabel("TransientTransactionError");
-                }
+                AddErrorLabelIfRequired(exception, connection.Description.ServerVersion);
+
                 TransactionHelper.UnpinServerIfNeededOnCommandException(_session, exception);
                 throw;
             }
@@ -164,16 +162,32 @@ public async Task<TCommandResult> ExecuteAsync(IConnection connection, Cancellat
             }
             catch (Exception exception)
             {
-                if (exception is MongoException mongoException && ShouldAddTransientTransactionError(mongoException))
-                {
-                    mongoException.AddErrorLabel("TransientTransactionError");
-                }
+                AddErrorLabelIfRequired(exception, connection.Description.ServerVersion);
+
                 TransactionHelper.UnpinServerIfNeededOnCommandException(_session, exception);
                 throw;
             }
         }
 
         // private methods
+        private void AddErrorLabelIfRequired(Exception exception, SemanticVersion serverVersion)
+        {
+            if (exception is MongoException mongoException)
+            {
+                if (ShouldAddTransientTransactionError(mongoException))
+                {
+                    mongoException.AddErrorLabel("TransientTransactionError");
+                }
+
+                if ((exception is MongoConnectionException) || // network error
+                    (Feature.RetryableWrites.IsSupported(serverVersion) &&
+                    !Feature.ServerReturnsRetryableWriteErrorLabel.IsSupported(serverVersion)))
+                {
+                    RetryabilityHelper.AddRetryableWriteErrorLabelIfRequired(mongoException);
+                }
+            }
+        }
+
         private CommandResponseMessage AutoDecryptFieldsIfNecessary(CommandResponseMessage encryptedResponseMessage, CancellationToken cancellationToken)
         {
             if (_documentFieldDecryptor == null)

src/MongoDB.Driver.Core/MongoCommandException.cs

@@ -31,7 +31,12 @@ namespace MongoDB.Driver
     public class MongoCommandException : MongoServerException
     {
         #region static
-        private static void AddErrorLabelsFromCommandResult(MongoCommandException exception, BsonDocument result)
+        /// <summary>
+        /// Adds error labels from a command result document into the top-level label list.
+        /// </summary>
+        /// <param name="exception">The exception.</param>
+        /// <param name="result">The result document.</param>
+        protected static void AddErrorLabelsFromCommandResult(MongoCommandException exception, BsonDocument result)
         {
             // note: make a best effort to extract the error labels from the result, but never throw an exception
             if (result != null)

src/MongoDB.Driver.Core/MongoWriteConcernException.cs

@@ -13,15 +13,12 @@
 * limitations under the License.
 */
 
-using System;
 #if NET452
+using System;
 using System.Runtime.Serialization;
 #endif
-using MongoDB.Bson;
-using MongoDB.Bson.Serialization;
 using MongoDB.Driver.Core.Connections;
 using MongoDB.Driver.Core.Misc;
-using MongoDB.Driver.Core.Operations;
 
 namespace MongoDB.Driver
 {
@@ -33,6 +30,21 @@ namespace MongoDB.Driver
 #endif
     public class MongoWriteConcernException : MongoCommandException
     {
+        #region static
+        private static void AddErrorLabelsFromWriteConcernResult(MongoWriteConcernException exception, WriteConcernResult writeConcernResult)
+        {
+            // note: make a best effort to extract the error labels from the writeConcernResult, but never throw an exception
+            if (writeConcernResult != null && writeConcernResult.Response != null)
+            {
+                if (writeConcernResult.Response.TryGetValue("writeConcernError", out var writeConcernError) &&
+                    writeConcernError.IsBsonDocument)
+                {
+                    AddErrorLabelsFromCommandResult(exception, writeConcernError.AsBsonDocument);
+                }
+            }
+        }
+        #endregion
+
         // fields
         private readonly WriteConcernResult _writeConcernResult;
 
@@ -47,6 +59,8 @@ public MongoWriteConcernException(ConnectionId connectionId, string message, Wri
             : base(connectionId, message, null, writeConcernResult.Response)
         {
             _writeConcernResult = Ensure.IsNotNull(writeConcernResult, nameof(writeConcernResult));
+
+            AddErrorLabelsFromWriteConcernResult(this, _writeConcernResult);
         }
 
 #if NET452
@@ -59,6 +73,8 @@ public MongoWriteConcernException(SerializationInfo info, StreamingContext conte
             : base(info, context)
         {
             _writeConcernResult = (WriteConcernResult)info.GetValue("_writeConcernResult", typeof(WriteConcernResult));
+
+            AddErrorLabelsFromWriteConcernResult(this, _writeConcernResult);
         }
 #endif
 

src/MongoDB.Driver.Core/ServerErrorCode.cs

@@ -22,7 +22,7 @@ internal enum ServerErrorCode
         CappedPositionLost = 136,
         CursorKilled = 237,
         ElectionInProgress = 216,
-        ExceededTimeLimit = 50,
+        ExceededTimeLimit = 262,
         HostNotFound = 7,
         HostUnreachable = 6,
         Interrupted = 11601,
@@ -38,6 +38,7 @@ internal enum ServerErrorCode
         SocketException = 9001,
         UnknownReplWriteConcern = 79,
         UnsatisfiableWriteConcern = 100,
-        WriteConcernFailed = 64
+        WriteConcernFailed = 64,
+        MaxTimeMSExpired = 50
     }
 }

tests/MongoDB.Driver.Core.TestHelpers/CoreExceptionHelper.cs

@@ -92,5 +92,22 @@ public static MongoCommandException CreateMongoCommandException(int code = 1, st
 
             return commandException;
         }
+
+        public static MongoCommandException CreateMongoWriteConcernException(BsonDocument writeConcernResultDocument, string label = null)
+        {
+            var clusterId = new ClusterId(1);
+            var endPoint = new DnsEndPoint("localhost", 27017);
+            var serverId = new ServerId(clusterId, endPoint);
+            var connectionId = new ConnectionId(serverId);
+            var message = "Fake MongoWriteConcernException";
+            var writeConcernResult = new WriteConcernResult(writeConcernResultDocument);
+            var writeConcernException = new MongoWriteConcernException(connectionId, message, writeConcernResult);
+            if (label != null)
+            {
+                writeConcernException.AddErrorLabel(label);
+            }
+
+            return writeConcernException;
+        }
     }
 }