Skip to content

Commit c1869fe

Browse files
committed
1. add multi-thread block upload for resume upload
2. disable the bput request for resume upload
1 parent 6323765 commit c1869fe

File tree

10 files changed

+467
-544
lines changed

10 files changed

+467
-544
lines changed

src/Qiniu.sln

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio 14
4-
VisualStudioVersion = 14.0.25123.0
3+
# Visual Studio 15
4+
VisualStudioVersion = 15.0.27004.2002
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu", "Qiniu\Qiniu.csproj", "{2F5B0328-DE8B-4B53-A500-3077E340A51B}"
77
EndProject
@@ -10,29 +10,40 @@ EndProject
1010
Global
1111
GlobalSection(SolutionConfigurationPlatforms) = preSolution
1212
Debug|Any CPU = Debug|Any CPU
13+
Debug|x64 = Debug|x64
1314
Debug|x86 = Debug|x86
1415
Release|Any CPU = Release|Any CPU
16+
Release|x64 = Release|x64
1517
Release|x86 = Release|x86
1618
EndGlobalSection
1719
GlobalSection(ProjectConfigurationPlatforms) = postSolution
18-
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
19-
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|Any CPU.Build.0 = Debug|Any CPU
20+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|Any CPU.ActiveCfg = Debug|x86
21+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|x64.ActiveCfg = Debug|x64
22+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|x64.Build.0 = Debug|x64
2023
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|x86.ActiveCfg = Debug|x86
2124
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Debug|x86.Build.0 = Debug|x86
22-
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|Any CPU.ActiveCfg = Release|Any CPU
23-
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|Any CPU.Build.0 = Release|Any CPU
25+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|Any CPU.ActiveCfg = Release|x86
26+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|x64.ActiveCfg = Release|x64
27+
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|x64.Build.0 = Release|x64
2428
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|x86.ActiveCfg = Release|x86
2529
{2F5B0328-DE8B-4B53-A500-3077E340A51B}.Release|x86.Build.0 = Release|x86
2630
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
2731
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
32+
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|x64.ActiveCfg = Debug|x64
33+
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|x64.Build.0 = Debug|x64
2834
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|x86.ActiveCfg = Debug|Any CPU
2935
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Debug|x86.Build.0 = Debug|Any CPU
3036
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
3137
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|Any CPU.Build.0 = Release|Any CPU
38+
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|x64.ActiveCfg = Release|x64
39+
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|x64.Build.0 = Release|x64
3240
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|x86.ActiveCfg = Release|Any CPU
3341
{E8CB1665-53F7-46A5-9AFD-B85AD08262D0}.Release|x86.Build.0 = Release|Any CPU
3442
EndGlobalSection
3543
GlobalSection(SolutionProperties) = preSolution
3644
HideSolutionNode = FALSE
3745
EndGlobalSection
46+
GlobalSection(ExtensibilityGlobals) = postSolution
47+
SolutionGuid = {0358A72B-AB04-47A5-8AC2-BCDE2447D2A4}
48+
EndGlobalSection
3849
EndGlobal

src/Qiniu/Qiniu.csproj

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,29 +13,6 @@
1313
<FileAlignment>512</FileAlignment>
1414
<TargetFrameworkProfile />
1515
</PropertyGroup>
16-
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
17-
<DebugSymbols>true</DebugSymbols>
18-
<DebugType>full</DebugType>
19-
<Optimize>false</Optimize>
20-
<BaseIntermediateOutputPath>..\..\bin\net40\obj\</BaseIntermediateOutputPath>
21-
<OutputPath>..\..\bin\</OutputPath>
22-
<DefineConstants>DEBUG;TRACE;Net40</DefineConstants>
23-
<ErrorReport>prompt</ErrorReport>
24-
<WarningLevel>4</WarningLevel>
25-
<Prefer32Bit>false</Prefer32Bit>
26-
<DocumentationFile>..\..\bin\Qiniu.xml</DocumentationFile>
27-
</PropertyGroup>
28-
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
29-
<DebugType>pdbonly</DebugType>
30-
<Optimize>true</Optimize>
31-
<BaseIntermediateOutputPath>..\..\bin\net40\obj\</BaseIntermediateOutputPath>
32-
<OutputPath>..\..\bin\</OutputPath>
33-
<DefineConstants>TRACE;Net40</DefineConstants>
34-
<ErrorReport>prompt</ErrorReport>
35-
<WarningLevel>4</WarningLevel>
36-
<Prefer32Bit>false</Prefer32Bit>
37-
<DocumentationFile>..\..\bin\Qiniu.xml</DocumentationFile>
38-
</PropertyGroup>
3916
<PropertyGroup>
4017
<SignAssembly>false</SignAssembly>
4118
</PropertyGroup>
@@ -62,12 +39,32 @@
6239
<ErrorReport>prompt</ErrorReport>
6340
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
6441
</PropertyGroup>
42+
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
43+
<DebugSymbols>true</DebugSymbols>
44+
<OutputPath>bin\x64\Debug\</OutputPath>
45+
<DefineConstants>DEBUG;TRACE;Net40</DefineConstants>
46+
<DocumentationFile>..\..\bin\net40\Qiniu.xml</DocumentationFile>
47+
<DebugType>full</DebugType>
48+
<PlatformTarget>x64</PlatformTarget>
49+
<ErrorReport>prompt</ErrorReport>
50+
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
51+
</PropertyGroup>
52+
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x64'">
53+
<OutputPath>bin\x64\Release\</OutputPath>
54+
<DefineConstants>TRACE;Net40</DefineConstants>
55+
<DocumentationFile>..\..\bin\net40\Qiniu.xml</DocumentationFile>
56+
<Optimize>true</Optimize>
57+
<DebugType>pdbonly</DebugType>
58+
<PlatformTarget>x64</PlatformTarget>
59+
<ErrorReport>prompt</ErrorReport>
60+
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
61+
</PropertyGroup>
6562
<ItemGroup>
6663
<Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
6764
<HintPath>..\packages\Newtonsoft.Json.10.0.3\lib\net40\Newtonsoft.Json.dll</HintPath>
6865
</Reference>
69-
<Reference Include="nunit.framework, Version=3.7.1.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
70-
<HintPath>..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll</HintPath>
66+
<Reference Include="nunit.framework, Version=3.8.1.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
67+
<HintPath>..\packages\NUnit.3.8.1\lib\net40\nunit.framework.dll</HintPath>
7168
</Reference>
7269
<Reference Include="System" />
7370
<Reference Include="System.Core" />
@@ -91,6 +88,7 @@
9188
<Compile Include="Storage\PrefopResult.cs" />
9289
<Compile Include="Storage\PutExtra.cs" />
9390
<Compile Include="Storage\QiniuException.cs" />
91+
<Compile Include="Storage\ResumeBlocker.cs" />
9492
<Compile Include="Storage\Zone.cs" />
9593
<Compile Include="Storage\ZoneHelper.cs" />
9694
<Compile Include="Storage\ZoneInfo.cs" />

src/Qiniu/Storage/FormUploader.cs

Lines changed: 110 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -101,148 +101,139 @@ public HttpResult UploadStream(Stream stream, string key, string token, PutExtra
101101

102102
HttpResult result = new HttpResult();
103103

104-
try
104+
using (stream)
105105
{
106-
string boundary = HttpManager.CreateFormDataBoundary();
107-
StringBuilder bodyBuilder = new StringBuilder();
108-
bodyBuilder.AppendLine("--" + boundary);
109-
110-
if (key != null)
106+
try
111107
{
112-
//write key when it is not null
113-
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"key\"");
114-
bodyBuilder.AppendLine();
115-
bodyBuilder.AppendLine(key);
108+
string boundary = HttpManager.CreateFormDataBoundary();
109+
StringBuilder bodyBuilder = new StringBuilder();
116110
bodyBuilder.AppendLine("--" + boundary);
117-
}
118111

119-
//write token
120-
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"token\"");
121-
bodyBuilder.AppendLine();
122-
bodyBuilder.AppendLine(token);
123-
bodyBuilder.AppendLine("--" + boundary);
112+
if (key != null)
113+
{
114+
//write key when it is not null
115+
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"key\"");
116+
bodyBuilder.AppendLine();
117+
bodyBuilder.AppendLine(key);
118+
bodyBuilder.AppendLine("--" + boundary);
119+
}
124120

125-
//write extra params
126-
if (putExtra.Params != null && putExtra.Params.Count > 0)
127-
{
128-
foreach (var p in putExtra.Params)
121+
//write token
122+
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"token\"");
123+
bodyBuilder.AppendLine();
124+
bodyBuilder.AppendLine(token);
125+
bodyBuilder.AppendLine("--" + boundary);
126+
127+
//write extra params
128+
if (putExtra.Params != null && putExtra.Params.Count > 0)
129129
{
130-
if (p.Key.StartsWith("x:"))
130+
foreach (var p in putExtra.Params)
131131
{
132-
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"{0}\"", p.Key);
133-
bodyBuilder.AppendLine();
134-
bodyBuilder.AppendLine();
135-
bodyBuilder.AppendLine(p.Value);
136-
bodyBuilder.AppendLine("--" + boundary);
132+
if (p.Key.StartsWith("x:"))
133+
{
134+
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"{0}\"", p.Key);
135+
bodyBuilder.AppendLine();
136+
bodyBuilder.AppendLine();
137+
bodyBuilder.AppendLine(p.Value);
138+
bodyBuilder.AppendLine("--" + boundary);
139+
}
137140
}
138141
}
139-
}
140142

141-
//prepare data buffer
142-
int bufferSize = 1024 * 1024;
143-
byte[] buffer = new byte[bufferSize];
144-
int bytesRead = 0;
145-
putExtra.ProgressHandler(0, stream.Length);
146-
MemoryStream dataMS = new MemoryStream();
147-
while ((bytesRead = stream.Read(buffer, 0, bufferSize)) != 0)
148-
{
149-
dataMS.Write(buffer, 0, bytesRead);
150-
}
151-
152-
//write crc32
153-
uint crc32 = CRC32.CheckSumBytes(dataMS.ToArray());
154-
//write key when it is not null
155-
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"crc32\"");
156-
bodyBuilder.AppendLine();
157-
bodyBuilder.AppendLine(crc32.ToString());
158-
bodyBuilder.AppendLine("--" + boundary);
143+
//prepare data buffer
144+
int bufferSize = 1024 * 1024;
145+
byte[] buffer = new byte[bufferSize];
146+
int bytesRead = 0;
147+
putExtra.ProgressHandler(0, stream.Length);
148+
MemoryStream dataMS = new MemoryStream();
149+
while ((bytesRead = stream.Read(buffer, 0, bufferSize)) != 0)
150+
{
151+
dataMS.Write(buffer, 0, bytesRead);
152+
}
159153

160-
//write fname
161-
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"file\"; filename=\"{0}\"", fname);
162-
bodyBuilder.AppendLine();
154+
//write crc32
155+
uint crc32 = CRC32.CheckSumBytes(dataMS.ToArray());
156+
//write key when it is not null
157+
bodyBuilder.AppendLine("Content-Disposition: form-data; name=\"crc32\"");
158+
bodyBuilder.AppendLine();
159+
bodyBuilder.AppendLine(crc32.ToString());
160+
bodyBuilder.AppendLine("--" + boundary);
163161

164-
//write mime type
165-
bodyBuilder.AppendFormat("Content-Type: {0}", putExtra.MimeType);
166-
bodyBuilder.AppendLine();
167-
bodyBuilder.AppendLine();
162+
//write fname
163+
bodyBuilder.AppendFormat("Content-Disposition: form-data; name=\"file\"; filename=\"{0}\"", fname);
164+
bodyBuilder.AppendLine();
168165

169-
//write file data
170-
StringBuilder bodyEnd = new StringBuilder();
171-
bodyEnd.AppendLine();
172-
bodyEnd.AppendLine("--" + boundary + "--");
166+
//write mime type
167+
bodyBuilder.AppendFormat("Content-Type: {0}", putExtra.MimeType);
168+
bodyBuilder.AppendLine();
169+
bodyBuilder.AppendLine();
173170

174-
byte[] partData1 = Encoding.UTF8.GetBytes(bodyBuilder.ToString());
175-
byte[] partData2 = dataMS.ToArray();
176-
byte[] partData3 = Encoding.UTF8.GetBytes(bodyEnd.ToString());
171+
//write file data
172+
StringBuilder bodyEnd = new StringBuilder();
173+
bodyEnd.AppendLine();
174+
bodyEnd.AppendLine("--" + boundary + "--");
177175

178-
MemoryStream ms = new MemoryStream();
179-
ms.Write(partData1, 0, partData1.Length);
180-
ms.Write(partData2, 0, partData2.Length);
181-
ms.Write(partData3, 0, partData3.Length);
176+
byte[] partData1 = Encoding.UTF8.GetBytes(bodyBuilder.ToString());
177+
byte[] partData2 = dataMS.ToArray();
178+
byte[] partData3 = Encoding.UTF8.GetBytes(bodyEnd.ToString());
182179

183-
//get upload host
184-
string ak = UpToken.GetAccessKeyFromUpToken(token);
185-
string bucket = UpToken.GetBucketFromUpToken(token);
186-
if (ak == null || bucket == null)
187-
{
188-
return HttpResult.InvalidToken;
189-
}
180+
MemoryStream ms = new MemoryStream();
181+
ms.Write(partData1, 0, partData1.Length);
182+
ms.Write(partData2, 0, partData2.Length);
183+
ms.Write(partData3, 0, partData3.Length);
190184

191-
string uploadHost = this.config.UpHost(ak, bucket);
192-
putExtra.ProgressHandler(stream.Length / 5, stream.Length);
193-
result = httpManager.PostMultipart(uploadHost, ms.ToArray(), boundary, null);
194-
putExtra.ProgressHandler(stream.Length, stream.Length);
195-
if (result.Code == (int)HttpCode.OK)
196-
{
197-
result.RefText += string.Format("[{0}] [FormUpload] Uploaded: #STREAM# ==> \"{1}\"\n",
198-
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), key);
199-
}
200-
else
201-
{
202-
result.RefText += string.Format("[{0}] [FormUpload] Failed: code = {1}, text = {2}\n",
203-
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Code, result.Text);
204-
}
185+
//get upload host
186+
string ak = UpToken.GetAccessKeyFromUpToken(token);
187+
string bucket = UpToken.GetBucketFromUpToken(token);
188+
if (ak == null || bucket == null)
189+
{
190+
return HttpResult.InvalidToken;
191+
}
205192

206-
//close memory stream
207-
ms.Close();
208-
dataMS.Close();
209-
}
210-
catch (Exception ex)
211-
{
212-
StringBuilder sb = new StringBuilder();
213-
sb.AppendFormat("[{0}] [FormUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
214-
Exception e = ex;
215-
while (e != null)
216-
{
217-
sb.Append(e.Message + " ");
218-
e = e.InnerException;
219-
}
220-
sb.AppendLine();
193+
string uploadHost = this.config.UpHost(ak, bucket);
194+
putExtra.ProgressHandler(stream.Length / 5, stream.Length);
195+
result = httpManager.PostMultipart(uploadHost, ms.ToArray(), boundary, null);
196+
putExtra.ProgressHandler(stream.Length, stream.Length);
197+
if (result.Code == (int)HttpCode.OK)
198+
{
199+
result.RefText += string.Format("[{0}] [FormUpload] Uploaded: #STREAM# ==> \"{1}\"\n",
200+
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), key);
201+
}
202+
else
203+
{
204+
result.RefText += string.Format("[{0}] [FormUpload] Failed: code = {1}, text = {2}\n",
205+
DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), result.Code, result.Text);
206+
}
221207

222-
if (ex is QiniuException)
223-
{
224-
QiniuException qex = (QiniuException)ex;
225-
result.Code = qex.HttpResult.Code;
226-
result.RefCode = qex.HttpResult.Code;
227-
result.Text = qex.HttpResult.Text;
228-
result.RefText += sb.ToString();
208+
//close memory stream
209+
ms.Close();
210+
dataMS.Close();
229211
}
230-
else
231-
{
232-
result.RefCode = (int)HttpCode.USER_UNDEF;
233-
result.RefText += sb.ToString();
234-
}
235-
}
236-
finally
237-
{
238-
if (stream != null)
212+
catch (Exception ex)
239213
{
240-
try
214+
StringBuilder sb = new StringBuilder();
215+
sb.AppendFormat("[{0}] [FormUpload] Error: ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"));
216+
Exception e = ex;
217+
while (e != null)
218+
{
219+
sb.Append(e.Message + " ");
220+
e = e.InnerException;
221+
}
222+
sb.AppendLine();
223+
224+
if (ex is QiniuException)
225+
{
226+
QiniuException qex = (QiniuException)ex;
227+
result.Code = qex.HttpResult.Code;
228+
result.RefCode = qex.HttpResult.Code;
229+
result.Text = qex.HttpResult.Text;
230+
result.RefText += sb.ToString();
231+
}
232+
else
241233
{
242-
stream.Close();
243-
stream.Dispose();
234+
result.RefCode = (int)HttpCode.USER_UNDEF;
235+
result.RefText += sb.ToString();
244236
}
245-
catch (Exception) { }
246237
}
247238
}
248239

src/Qiniu/Storage/PutExtra.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,10 @@ public class PutExtra
3232
/// 最大重试次数
3333
/// </summary>
3434
public int MaxRetryTimes { set; get; }
35+
36+
/// <summary>
37+
/// 块并发上传的线程数量
38+
/// </summary>
39+
public int BlockUploadThreads { set; get; }
3540
}
3641
}

0 commit comments

Comments
 (0)