Merge pull request #245 from qiniu/fix/retry-upload

Fix retry on form upload and part upload

Author: bachue

CHANGELOG.md

@@ -1,3 +1,9 @@
+**Next Release Date**
+
+Next Version
+
+修复：表单直传、分片上传配置 MaxRetryTimes 无效（v7.4.0 引入）
+
 **2022-10-26**
 
 v8.3.0

src/Qiniu/Storage/Config.cs

@@ -40,7 +40,8 @@ public class Config
         /// <summary>
         /// 重试请求次数
         /// </summary>
-        public int MaxRetryTimes = 3;
+        /// 默认值应与 <see cref="PutExtra.MaxRetryTimes"/> 一致
+        public int MaxRetryTimes { set; get; } = 3;
 
         /// <summary>
         /// 获取资源管理域名

src/Qiniu/Storage/FormUploader.cs

@@ -81,6 +81,7 @@ public HttpResult UploadStream(Stream stream, string key, string token, PutExtra
             if (putExtra == null)
             {
                 putExtra = new PutExtra();
+                putExtra.MaxRetryTimes = config.MaxRetryTimes;
             }
             if (string.IsNullOrEmpty(putExtra.MimeType )) {
                 putExtra.MimeType = "application/octet-stream";
@@ -182,17 +183,8 @@ public HttpResult UploadStream(Stream stream, string key, string token, PutExtra
                     ms.Write(partData2, 0, partData2.Length);
                     ms.Write(partData3, 0, partData3.Length);
 
-                    //get upload host
-                    string ak = UpToken.GetAccessKeyFromUpToken(token);
-                    string bucket = UpToken.GetBucketFromUpToken(token);
-                    if (ak == null || bucket == null)
-                    {
-                        return HttpResult.InvalidToken;
-                    }
-
-                    string uploadHost = this.config.UpHost(ak, bucket);
                     putExtra.ProgressHandler(stream.Length / 5, stream.Length);
-                    result = httpManager.PostMultipart(uploadHost, ms.ToArray(), boundary, null);
+                    result = PostFormWithRetry(token, ms.ToArray(), boundary, putExtra);
                     putExtra.ProgressHandler(stream.Length, stream.Length);
                     if (result.Code == (int)HttpCode.OK)
                     {
@@ -265,6 +257,31 @@ public static UploadControllerAction DefaultUploadController()
         {
             return UploadControllerAction.Activated;
         }
-    }
 
-}
+        private HttpResult PostFormWithRetry(string token, byte[] data, string boundary, PutExtra putExtra)
+        {
+            //get upload host
+            string ak = UpToken.GetAccessKeyFromUpToken(token);
+            string bucket = UpToken.GetBucketFromUpToken(token);
+            if (ak == null || bucket == null)
+            {
+                return HttpResult.InvalidToken;
+            }
+
+            string uploadHost = this.config.UpHost(ak, bucket);
+            HttpResult result = httpManager.PostMultipart(uploadHost, data, boundary, null);
+
+            int retryTimes = 0;
+            while (
+                retryTimes < putExtra.MaxRetryTimes &&
+                UploadUtil.ShouldRetry(result.Code, result.RefCode)
+            )
+            {
+                result = httpManager.PostMultipart(uploadHost, data, boundary, null);
+                retryTimes += 1;
+            }
+
+            return result;
+        }
+    }
+}

src/Qiniu/Storage/PutExtra.cs

@@ -31,7 +31,8 @@ public class PutExtra
         /// <summary>
         /// 最大重试次数
         /// </summary>
-        public int MaxRetryTimes { set; get; }
+        /// 默认值应与 <see cref="Config.MaxRetryTimes"/> 一致
+        public int MaxRetryTimes { set; get; } = 3;
 
         /// <summary>
         /// 块并发上传的线程数量

src/Qiniu/Storage/ResumableUploader.cs

@@ -84,6 +84,7 @@ public HttpResult UploadStream(Stream stream, string key, string upToken, PutExt
             if (putExtra == null)
             {
                 putExtra = new PutExtra();
+                putExtra.MaxRetryTimes = config.MaxRetryTimes;
             }
             if (putExtra.ProgressHandler == null)
             {
@@ -588,7 +589,7 @@ private void processMakeBlocks(Dictionary<long, byte[]> blockDataDict, string up
                 byte[] blockData = blockDataDict[blockIndex];
                 ResumeBlocker resumeBlocker = new ResumeBlocker(doneEvent, blockData, blockIndex, upToken, putExtra,
                     resumeInfo, blockMakeResults, progressLock, uploadedBytesDict, fileSize, encodedObjectName);
-                ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlock), resumeBlocker);
+                ThreadPool.QueueUserWorkItem(new WaitCallback(this.MakeBlockWithRetry), resumeBlocker);
             }
 
             try
@@ -602,11 +603,37 @@ private void processMakeBlocks(Dictionary<long, byte[]> blockDataDict, string up
             }
         }
 
+        private void MakeBlockWithRetry(object resumeBlockerObj)
+        {
+            ResumeBlocker resumeBlocker = (ResumeBlocker)resumeBlockerObj;
+            ManualResetEvent doneEvent = resumeBlocker.DoneEvent;
+            Dictionary<long, HttpResult> blockMakeResults = resumeBlocker.BlockMakeResults;
+            long blockIndex = resumeBlocker.BlockIndex;
+            PutExtra putExtra = resumeBlocker.PutExtra;
+
+            HttpResult result = MakeBlock(resumeBlockerObj);
+
+
+            int retryTimes = 0;
+            while (
+                retryTimes < putExtra.MaxRetryTimes &&
+                UploadUtil.ShouldRetry(result.Code, result.RefCode)
+            )
+            {
+                result = MakeBlock(resumeBlockerObj);
+                retryTimes += 1;
+            }
+
+            //return the http result
+            blockMakeResults.Add(blockIndex, result);
+            doneEvent.Set();
+        }
+
         /// <summary>
         /// 创建块(携带首片数据),v1检查CRC32,v2检查md5
         /// </summary>
         /// <param name="resumeBlockerObj">创建分片上传的块请求</param>
-        private void MakeBlock(object resumeBlockerObj)
+        private HttpResult MakeBlock(object resumeBlockerObj)
         {
             ResumeBlocker resumeBlocker = (ResumeBlocker)resumeBlockerObj;
             ManualResetEvent doneEvent = resumeBlocker.DoneEvent;
@@ -632,7 +659,7 @@ private void MakeBlock(object resumeBlockerObj)
                     result.RefText += string.Format("[{0}] [ResumableUpload] Info: upload task is aborted, mkblk {1}\n",
                         DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), blockIndex);
                     blockMakeResults.Add(blockIndex, result);
-                    return;
+                    return result;
                 }
                 else
                 {
@@ -658,7 +685,7 @@ private void MakeBlock(object resumeBlockerObj)
                 {
                     result = HttpResult.InvalidToken;
                     doneEvent.Set();
-                    return;
+                    return result;
                 }
 
                 string uploadHost = this.config.UpHost(ak, bucket);
@@ -793,9 +820,7 @@ private void MakeBlock(object resumeBlockerObj)
                 }
             }
 
-            //return the http result
-            blockMakeResults.Add(blockIndex, result);
-            doneEvent.Set();
+            return result;
         }
 
         /// <summary>

src/Qiniu/Storage/UploadUtil.cs

@@ -0,0 +1,44 @@
+using Qiniu.Http;
+
+namespace Qiniu.Storage
+{
+    public static class UploadUtil
+    {
+        public static bool ShouldRetry(int code, int refCode)
+        {
+            if (code == (int) HttpCode.OK)
+            {
+                return false;
+            }
+
+            // allow list
+            if (
+                refCode == (int)HttpCode.USER_UNDEF ||
+                refCode == (int)HttpCode.USER_NEED_RETRY
+            )
+            {
+                return true;
+            }
+            
+            // block list
+            int codeSeries = code / 100;
+
+            if (codeSeries == 4 && code != (int)HttpCode.CRC32_CHECK_FAILEd)
+            {
+                return false;
+            }
+
+            if (
+                code == (int)HttpCode.FILE_NOT_EXIST ||
+                code == (int)HttpCode.FILE_EXISTS ||
+                code == (int)HttpCode.CALLBACK_FAILED
+            )
+            {
+                return false;
+            }
+            
+            // others need retry
+            return true;
+        }
+    }
+}

src/QiniuTests/Storage/UploadUtilTests.cs

@@ -0,0 +1,45 @@
+using NUnit.Framework;
+using System.Collections;
+using Qiniu.Tests;
+
+namespace Qiniu.Storage.Tests
+{
+    public class UploadUtilTestCases
+    {
+        public static IEnumerable ShouldRetryTestCases
+        {
+            get
+            {
+                // 200 always false
+                yield return new TestCaseData(200, 200).Returns(false);
+                
+                // some refCode should retry
+                yield return new TestCaseData(0, 3).Returns(true);
+                yield return new TestCaseData(0, 0).Returns(true);
+                
+                // 4xx shouldn't retry but 406
+                yield return new TestCaseData(406, 406).Returns(true);
+                yield return new TestCaseData(400, 400).Returns(false);
+                
+                // some code shouldn't retry
+                yield return new TestCaseData(612, 612).Returns(false);
+                yield return new TestCaseData(614, 614).Returns(false);
+                yield return new TestCaseData(579, 579).Returns(false);
+                
+                // any others should retry
+                yield return new TestCaseData(500, 500).Returns(true);
+                yield return new TestCaseData(502, 502).Returns(true);
+            }
+        }
+    }
+
+    [TestFixture]
+    public class UploadUtilTests : TestEnv
+    {
+        [TestCaseSource(typeof(UploadUtilTestCases), nameof(UploadUtilTestCases.ShouldRetryTestCases))]
+        public bool ShouldRetryTest(int code, int refCode)
+        {
+            return UploadUtil.ShouldRetry(code, refCode);
+        }
+    }
+}