Implement Auto Retry of Socket and Retryable Exceptions

Author: mythz

src/ServiceStack.Redis/Generic/RedisTypedClient.cs

@@ -479,7 +479,7 @@ internal int ReadMultiDataResultCount()
         }
         public void FlushSendBuffer()
         {
-            client.FlushSendBuffer();
+            client.FlushAndResetSendBuffer();
         }
         public void ResetSendBuffer()
         {

src/ServiceStack.Redis/Pipeline/RedisAllPurposePipeline.cs

@@ -34,7 +34,7 @@ protected virtual void Init()
         public void Flush()
         {
             // flush send buffers
-            RedisClient.FlushSendBuffer();
+            RedisClient.FlushAndResetSendBuffer();
 
             //receive expected results
             foreach (var queuedCommand in QueuedCommands)

src/ServiceStack.Redis/Pipeline/RedisPipelineCommand.cs

@@ -38,7 +38,7 @@ public bool ReadAllAsIntsHaveSuccess()
 
         public void Flush()
         {
-            client.FlushSendBuffer();
+            client.FlushAndResetSendBuffer();
         }
     }
 }

src/ServiceStack.Redis/RedisConfig.cs

@@ -22,7 +22,9 @@ public class RedisConfig
         public static int DefaultConnectTimeout = 0;
         public static int DefaultSendTimeout = -1;
         public static int DefaultReceiveTimeout = -1;
+        public static int DefaultRetryTimeout = 3 * 1000;
         public static int DefaultIdleTimeOutSecs = 240;
+        public static int BackOffMultiplier = 10;
 
         public static int BufferLength = 1450;
 

src/ServiceStack.Redis/RedisEndpoint.cs

@@ -16,6 +16,7 @@ public RedisEndpoint()
             ConnectTimeout = RedisConfig.DefaultConnectTimeout;
             SendTimeout = RedisConfig.DefaultSendTimeout;
             ReceiveTimeout = RedisConfig.DefaultReceiveTimeout;
+            RetryTimeout = RedisConfig.DefaultRetryTimeout;
             IdleTimeOutSecs = RedisConfig.DefaultIdleTimeOutSecs;
         }
 
@@ -34,6 +35,7 @@ public RedisEndpoint(string host, int port, string password = null, long db = Re
         public int ConnectTimeout { get; set; }
         public int SendTimeout { get; set; }
         public int ReceiveTimeout { get; set; }
+        public int RetryTimeout { get; set; }
         public int IdleTimeOutSecs { get; set; }
         public long Db { get; set; }
         public string Client { get; set; }
@@ -61,6 +63,8 @@ public override string ToString()
                 args.Add("SendTimeout=" + SendTimeout);
             if (ReceiveTimeout != RedisConfig.DefaultReceiveTimeout)
                 args.Add("ReceiveTimeout=" + ReceiveTimeout);
+            if (RetryTimeout != RedisConfig.DefaultRetryTimeout)
+                args.Add("RetryTimeout=" + RetryTimeout);
             if (IdleTimeOutSecs != RedisConfig.DefaultIdleTimeOutSecs)
                 args.Add("IdleTimeOutSecs=" + IdleTimeOutSecs);
             if (NamespacePrefix != null)
@@ -80,6 +84,7 @@ protected bool Equals(RedisEndpoint other)
                 && ConnectTimeout == other.ConnectTimeout 
                 && SendTimeout == other.SendTimeout 
                 && ReceiveTimeout == other.ReceiveTimeout 
+                && RetryTimeout == other.RetryTimeout
                 && IdleTimeOutSecs == other.IdleTimeOutSecs 
                 && Db == other.Db 
                 && string.Equals(Client, other.Client) 
@@ -105,6 +110,7 @@ public override int GetHashCode()
                 hashCode = (hashCode * 397) ^ ConnectTimeout;
                 hashCode = (hashCode * 397) ^ SendTimeout;
                 hashCode = (hashCode * 397) ^ ReceiveTimeout;
+                hashCode = (hashCode * 397) ^ RetryTimeout;
                 hashCode = (hashCode * 397) ^ IdleTimeOutSecs;
                 hashCode = (hashCode * 397) ^ Db.GetHashCode();
                 hashCode = (hashCode * 397) ^ (Client != null ? Client.GetHashCode() : 0);

src/ServiceStack.Redis/RedisExtensions.cs

@@ -93,6 +93,9 @@ public static RedisEndpoint ToRedisEndpoint(this string connectionString, int? d
                         case "receivetimeout":
                             endpoint.ReceiveTimeout = int.Parse(value);
                             break;
+                        case "retrytimeout":
+                            endpoint.RetryTimeout = int.Parse(value);
+                            break;
                         case "idletimeout":
                         case "idletimeoutsecs":
                             endpoint.IdleTimeOutSecs = int.Parse(value);

src/ServiceStack.Redis/RedisNativeClient.cs

@@ -104,7 +104,12 @@ internal bool Active
         /// </summary>
         public string NamespacePrefix { get; set; }
         public int ConnectTimeout { get; set; }
-        public int RetryTimeout { get; set; }
+        private TimeSpan retryTimeout;
+        public int RetryTimeout
+        {
+            get { return (int) retryTimeout.TotalMilliseconds; }
+            set { retryTimeout = TimeSpan.FromMilliseconds(value); }
+        }
         public int RetryCount { get; set; }
         public int SendTimeout { get; set; }
         public int ReceiveTimeout { get; set; }
@@ -168,6 +173,7 @@ private void Init(RedisEndpoint config)
             ConnectTimeout = config.ConnectTimeout;
             SendTimeout = config.SendTimeout;
             ReceiveTimeout = config.ReceiveTimeout;
+            RetryTimeout = config.RetryTimeout;
             Password = config.Password;
             NamespacePrefix = config.NamespacePrefix;
             Client = config.Client;
@@ -771,12 +777,12 @@ public void BgSave()
 
         public void Shutdown()
         {
-            SendCommand(Commands.Shutdown);
+            SendWithoutRead(Commands.Shutdown);
         }
 
         public void ShutdownNoSave()
         {
-            SendCommand(Commands.Shutdown, Commands.NoSave);
+            SendWithoutRead(Commands.Shutdown, Commands.NoSave);
         }
 
         public void BgRewriteAof()
@@ -786,7 +792,7 @@ public void BgRewriteAof()
 
         public void Quit()
         {
-            SendCommand(Commands.Quit);
+            SendWithoutRead(Commands.Quit);
         }
 
         public void FlushDb()
@@ -924,8 +930,7 @@ internal void Multi()
             //make sure socket is connected. Otherwise, fetch of server info will interfere
             //with pipeline
             AssertConnectedSocket();
-            if (!SendCommand(Commands.Multi))
-                throw CreateConnectionError();
+            SendWithoutRead(Commands.Multi);
         }
 
         /// <summary>
@@ -934,9 +939,7 @@ internal void Multi()
         /// <returns>Number of results</returns>
         internal void Exec()
         {
-            if (!SendCommand(Commands.Exec))
-                throw CreateConnectionError();
-
+            SendWithoutRead(Commands.Exec);
         }
 
         internal void Discard()