Skip to content
This repository was archived by the owner on Dec 5, 2024. It is now read-only.

Commit f71de4e

Browse files
shanashana
committed
Use a local webserver to serve files for testing. Fix partial downloads.
1 parent e61d48c commit f71de4e

File tree

12 files changed

+564
-184
lines changed

12 files changed

+564
-184
lines changed

‎GitHub.Unity.sln

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TaskSystem", "src\tests\Tas
2727
EndProject
2828
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestApp", "src\tests\TestApp\TestApp.csproj", "{08B87D2A-8CF1-4211-B7AA-5209F00F72F8}"
2929
EndProject
30+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestWebServer", "src\tests\TestWebServer\TestWebServer.csproj", "{3DD3451C-30FA-4294-A3A9-1E080342F867}"
31+
EndProject
3032
Global
3133
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3234
Debug|Any CPU = Debug|Any CPU
@@ -94,6 +96,12 @@ Global
9496
{08B87D2A-8CF1-4211-B7AA-5209F00F72F8}.dev|Any CPU.Build.0 = Debug|Any CPU
9597
{08B87D2A-8CF1-4211-B7AA-5209F00F72F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
9698
{08B87D2A-8CF1-4211-B7AA-5209F00F72F8}.Release|Any CPU.Build.0 = Release|Any CPU
99+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
100+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.Debug|Any CPU.Build.0 = Debug|Any CPU
101+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.dev|Any CPU.ActiveCfg = Debug|Any CPU
102+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.dev|Any CPU.Build.0 = Debug|Any CPU
103+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.Release|Any CPU.ActiveCfg = Release|Any CPU
104+
{3DD3451C-30FA-4294-A3A9-1E080342F867}.Release|Any CPU.Build.0 = Release|Any CPU
97105
EndGlobalSection
98106
GlobalSection(SolutionProperties) = preSolution
99107
HideSolutionNode = FALSE
@@ -106,5 +114,6 @@ Global
106114
{66A1D219-F61D-4AE4-9BD7-AAEB97276FFF} = {D17F1B4C-42DC-4E78-BCEF-9F239A084C4D}
107115
{1A382F40-FD9E-43E1-89C1-320073F35CE9} = {D17F1B4C-42DC-4E78-BCEF-9F239A084C4D}
108116
{08B87D2A-8CF1-4211-B7AA-5209F00F72F8} = {D17F1B4C-42DC-4E78-BCEF-9F239A084C4D}
117+
{3DD3451C-30FA-4294-A3A9-1E080342F867} = {D17F1B4C-42DC-4E78-BCEF-9F239A084C4D}
109118
EndGlobalSection
110119
EndGlobal

‎src/GitHub.Api/GitHub.Api.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@
157157
<Compile Include="Git\Tasks\GitLfsInstallTask.cs" />
158158
<Compile Include="Git\Tasks\GitRemoteChangeTask.cs" />
159159
<Compile Include="Git\Tasks\GitRemoveFromIndexTask.cs" />
160+
<Compile Include="IO\Utils.cs" />
160161
<Compile Include="Threading\IMainThreadSynchronizationContext.cs" />
161162
<Compile Include="Installer\IZipHelper.cs" />
162163
<Compile Include="Installer\ZipHelper.cs" />

‎src/GitHub.Api/IO/Utils.cs

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
using System;
2+
using System.Diagnostics;
3+
using System.IO;
4+
using System.Net;
5+
6+
namespace GitHub.Unity
7+
{
8+
public static class Utils
9+
{
10+
public static bool Copy(Stream source, Stream destination,
11+
long totalSize = 0,
12+
int chunkSize = 8192,
13+
Func<long, long, bool> progress = null,
14+
int progressUpdateRate = 100)
15+
{
16+
byte[] buffer = new byte[chunkSize];
17+
int bytesRead = 0;
18+
long totalRead = 0;
19+
float averageSpeed = -1f;
20+
float lastSpeed = 0f;
21+
float smoothing = 0.005f;
22+
long readLastSecond = 0;
23+
long timeToFinish = 0;
24+
Stopwatch watch = null;
25+
bool success = true;
26+
27+
bool trackProgress = totalSize > 0 && progress != null;
28+
if (trackProgress)
29+
watch = new Stopwatch();
30+
31+
do
32+
{
33+
if (trackProgress)
34+
watch.Start();
35+
36+
bytesRead = source.Read(buffer, 0, totalRead + chunkSize > totalSize ? (int)(totalSize - totalRead) : chunkSize);
37+
38+
if (trackProgress)
39+
watch.Stop();
40+
41+
totalRead += bytesRead;
42+
43+
if (bytesRead > 0)
44+
{
45+
destination.Write(buffer, 0, bytesRead);
46+
if (trackProgress)
47+
{
48+
readLastSecond += bytesRead;
49+
if (watch.ElapsedMilliseconds >= progressUpdateRate || totalRead == totalSize || bytesRead == 0)
50+
{
51+
watch.Reset();
52+
if (bytesRead == 0) // we've reached the end
53+
totalSize = totalRead;
54+
55+
lastSpeed = readLastSecond;
56+
readLastSecond = 0;
57+
averageSpeed = averageSpeed < 0f
58+
? lastSpeed
59+
: smoothing * lastSpeed + (1f - smoothing) * averageSpeed;
60+
timeToFinish = Math.Max(1L,
61+
(long)((totalSize - totalRead) / (averageSpeed / progressUpdateRate)));
62+
63+
Logging.Debug($"totalRead: {totalRead} of {totalSize}");
64+
success = progress(totalRead, timeToFinish);
65+
if (!success)
66+
break;
67+
}
68+
}
69+
else // we still need to call the callback if it's there, so we can abort if needed
70+
{
71+
success = progress?.Invoke(totalRead, timeToFinish) ?? true;
72+
if (!success)
73+
break;
74+
}
75+
}
76+
} while (bytesRead > 0 && (totalSize == 0 || totalSize > totalRead));
77+
78+
if (totalRead > 0)
79+
destination.Flush();
80+
81+
return success;
82+
}
83+
84+
public static bool Download(ILogging logger, UriString url,
85+
Stream destinationStream,
86+
Func<long, long, bool> onProgress)
87+
{
88+
long bytes = destinationStream.Length;
89+
90+
var expectingResume = bytes > 0;
91+
92+
var webRequest = (HttpWebRequest)WebRequest.Create(url);
93+
94+
if (expectingResume)
95+
{
96+
// classlib for 3.5 doesn't take long overloads...
97+
webRequest.AddRange((int)bytes);
98+
}
99+
100+
webRequest.Method = "GET";
101+
webRequest.Timeout = 5000;
102+
103+
if (expectingResume)
104+
logger.Trace($"Resuming download of {url}");
105+
else
106+
logger.Trace($"Downloading {url}");
107+
108+
using (var webResponse = (HttpWebResponse) webRequest.GetResponseWithoutException())
109+
{
110+
var httpStatusCode = webResponse.StatusCode;
111+
logger.Trace($"Downloading {url} StatusCode:{(int)webResponse.StatusCode}");
112+
113+
if (expectingResume && httpStatusCode == HttpStatusCode.RequestedRangeNotSatisfiable)
114+
{
115+
onProgress(bytes, bytes);
116+
return true;
117+
}
118+
119+
if (!(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.PartialContent))
120+
{
121+
return false;
122+
}
123+
124+
if (expectingResume && httpStatusCode == HttpStatusCode.OK)
125+
{
126+
expectingResume = false;
127+
destinationStream.Seek(0, SeekOrigin.Begin);
128+
}
129+
130+
var responseLength = webResponse.ContentLength;
131+
if (expectingResume)
132+
{
133+
if (!onProgress(bytes, bytes + responseLength))
134+
return false;
135+
}
136+
137+
using (var responseStream = webResponse.GetResponseStream())
138+
{
139+
return Copy(responseStream, destinationStream, responseLength,
140+
progress: (totalRead, timeToFinish) => {
141+
return onProgress(totalRead, responseLength);
142+
});
143+
}
144+
}
145+
}
146+
}
147+
}

‎src/GitHub.Api/Tasks/DownloadTask.cs

Lines changed: 2 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,140 +1,11 @@
11
using System;
2-
using System.Diagnostics;
32
using System.IO;
43
using System.Net;
54
using System.Text;
65
using System.Threading;
76

87
namespace GitHub.Unity
98
{
10-
public static class Utils
11-
{
12-
public static bool Copy(Stream source, Stream destination, int chunkSize)
13-
{
14-
return Copy(source, destination, chunkSize, 0, null, 1000);
15-
}
16-
17-
public static bool Copy(Stream source, Stream destination, int chunkSize, long totalSize,
18-
Func<long, long, bool> progress, int progressUpdateRate)
19-
{
20-
byte[] buffer = new byte[chunkSize];
21-
int bytesRead = 0;
22-
long totalRead = 0;
23-
float averageSpeed = -1f;
24-
float lastSpeed = 0f;
25-
float smoothing = 0.005f;
26-
long readLastSecond = 0;
27-
long timeToFinish = 0;
28-
Stopwatch watch = null;
29-
bool success = true;
30-
31-
bool trackProgress = totalSize > 0 && progress != null;
32-
if (trackProgress)
33-
watch = new Stopwatch();
34-
35-
do
36-
{
37-
if (trackProgress)
38-
watch.Start();
39-
40-
bytesRead = source.Read(buffer, 0, chunkSize);
41-
42-
if (trackProgress)
43-
watch.Stop();
44-
45-
totalRead += bytesRead;
46-
47-
if (bytesRead > 0)
48-
{
49-
destination.Write(buffer, 0, bytesRead);
50-
if (trackProgress)
51-
{
52-
readLastSecond += bytesRead;
53-
if (watch.ElapsedMilliseconds >= progressUpdateRate || totalRead == totalSize)
54-
{
55-
watch.Reset();
56-
lastSpeed = readLastSecond;
57-
readLastSecond = 0;
58-
averageSpeed = averageSpeed < 0f
59-
? lastSpeed
60-
: smoothing * lastSpeed + (1f - smoothing) * averageSpeed;
61-
timeToFinish = Math.Max(1L,
62-
(long)((totalSize - totalRead) / (averageSpeed / progressUpdateRate)));
63-
64-
Logging.Debug($"totalRead: {totalRead} of {totalSize}");
65-
success = progress(totalRead, timeToFinish);
66-
if (!success)
67-
break;
68-
}
69-
}
70-
}
71-
} while (bytesRead > 0);
72-
73-
if (totalRead > 0)
74-
destination.Flush();
75-
76-
return success;
77-
}
78-
79-
public static bool Download(ILogging logger, UriString url,
80-
Stream destinationStream,
81-
Func<long, long, bool> onProgress)
82-
{
83-
long bytes = destinationStream.Length;
84-
85-
var expectingResume = bytes > 0;
86-
87-
var webRequest = (HttpWebRequest)WebRequest.Create(url);
88-
89-
if (expectingResume)
90-
{
91-
// classlib for 3.5 doesn't take long overloads...
92-
webRequest.AddRange((int)bytes);
93-
}
94-
95-
webRequest.Method = "GET";
96-
webRequest.Timeout = 3000;
97-
98-
if (expectingResume)
99-
logger.Trace($"Resuming download of {url}");
100-
else
101-
logger.Trace($"Downloading {url}");
102-
103-
using (var webResponse = (HttpWebResponse) webRequest.GetResponseWithoutException())
104-
{
105-
var httpStatusCode = webResponse.StatusCode;
106-
logger.Trace($"Downloading {url} StatusCode:{(int)webResponse.StatusCode}");
107-
108-
if (expectingResume && httpStatusCode == HttpStatusCode.RequestedRangeNotSatisfiable)
109-
{
110-
onProgress(bytes, bytes);
111-
return true;
112-
}
113-
114-
if (!(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.PartialContent))
115-
{
116-
return false;
117-
}
118-
119-
var responseLength = webResponse.ContentLength;
120-
if (expectingResume)
121-
{
122-
if (!onProgress(bytes, bytes + responseLength))
123-
return false;
124-
}
125-
126-
using (var responseStream = webResponse.GetResponseStream())
127-
{
128-
return Copy(responseStream, destinationStream, 8192, responseLength,
129-
(totalRead, timeToFinish) => {
130-
return onProgress(totalRead, responseLength);
131-
}
132-
, 100);
133-
}
134-
}
135-
}
136-
}
137-
1389
public static class WebRequestExtensions
13910
{
14011
public static WebResponse GetResponseWithoutException(this WebRequest request)
@@ -158,8 +29,6 @@ public static WebResponse GetResponseWithoutException(this WebRequest request)
15829
class DownloadTask : TaskBase<string>
15930
{
16031
protected readonly IFileSystem fileSystem;
161-
private long bytes;
162-
private bool restarted;
16332

16433
public DownloadTask(CancellationToken token,
16534
IFileSystem fileSystem, UriString url,
@@ -305,27 +174,8 @@ public DownloadTextTask(CancellationToken token,
305174

306175
protected override string RunDownload(bool success)
307176
{
308-
string result = null;
309-
310-
RaiseOnStart();
311-
312-
try
313-
{
314-
result = base.RunDownload(success);
315-
result = fileSystem.ReadAllText(result, Encoding.UTF8);
316-
}
317-
catch (Exception ex)
318-
{
319-
Errors = ex.Message;
320-
if (!RaiseFaultHandlers(ex))
321-
throw;
322-
}
323-
finally
324-
{
325-
RaiseOnEnd(result);
326-
}
327-
328-
return result;
177+
var result = base.RunDownload(success);
178+
return fileSystem.ReadAllText(result, Encoding.UTF8);
329179
}
330180
}
331181
}

‎src/tests/IntegrationTests/BaseIntegrationTest.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,15 @@ protected void InitializeEnvironment(NPath repoPath,
3636
}
3737

3838
[TestFixtureSetUp]
39-
public void TestFixtureSetUp()
39+
public virtual void TestFixtureSetUp()
4040
{
4141
Logger = Logging.GetLogger(GetType());
4242
Factory = new TestUtils.SubstituteFactory();
4343
GitHub.Unity.Guard.InUnitTestRunner = true;
4444
}
4545

4646
[TestFixtureTearDown]
47-
public void TestFixtureTearDown()
47+
public virtual void TestFixtureTearDown()
4848
{
4949
}
5050

0 commit comments

Comments
 (0)