Skip to content
This repository was archived by the owner on Dec 5, 2024. It is now read-only.

Commit 7873cd2

Browse files
shanashana
committed
Make downloading parallel and fix concurrent threading
Make the downloads in pairs (file and md5) in a downloader class that can run them all parallel. I kept getting deadlocks and it's something to do with how the concurrent scheduler is set up, so replaced it with a scheduler that just fires a thread per task and does nothing smart about it.
1 parent d89ee2c commit 7873cd2

File tree

15 files changed

+636
-345
lines changed

15 files changed

+636
-345
lines changed

‎src/GitHub.Api/GitHub.Api.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
<Compile Include="Helpers\Validation.cs" />
119119
<Compile Include="Installer\GitInstaller.cs" />
120120
<Compile Include="Installer\UnzipTask.cs" />
121+
<Compile Include="Managers\Downloader.cs" />
121122
<Compile Include="OutputProcessors\GitAheadBehindStatusOutputProcessor.cs" />
122123
<Compile Include="OutputProcessors\LfsVersionOutputProcessor.cs" />
123124
<Compile Include="OutputProcessors\VersionOutputProcessor.cs" />

‎src/GitHub.Api/IO/NiceIO.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,6 +1074,11 @@ public static NPath Resolve(this NPath path)
10741074

10751075
return new NPath(Mono.Unix.UnixPath.GetCompleteRealPath(path.ToString()));
10761076
}
1077+
1078+
public static string CalculateMD5(this NPath path)
1079+
{
1080+
return NPath.FileSystem.CalculateFileMD5(path);
1081+
}
10771082
}
10781083

10791084
public enum SlashMode

‎src/GitHub.Api/IO/Utils.cs‎

Lines changed: 4 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -82,68 +82,11 @@ public static bool Copy(Stream source, Stream destination,
8282

8383
return success;
8484
}
85-
86-
public static bool Download(ILogging logger, UriString url,
87-
Stream destinationStream,
88-
Func<long, long, bool> onProgress)
85+
public static bool VerifyFileIntegrity(NPath file, NPath md5file)
8986
{
90-
long bytes = destinationStream.Length;
91-
92-
var expectingResume = bytes > 0;
93-
94-
var webRequest = (HttpWebRequest)WebRequest.Create(url);
95-
96-
if (expectingResume)
97-
{
98-
// classlib for 3.5 doesn't take long overloads...
99-
webRequest.AddRange((int)bytes);
100-
}
101-
102-
webRequest.Method = "GET";
103-
webRequest.Timeout = ApplicationConfiguration.WebTimeout;
104-
105-
if (expectingResume)
106-
logger.Trace($"Resuming download of {url}");
107-
else
108-
logger.Trace($"Downloading {url}");
109-
110-
using (var webResponse = (HttpWebResponse) webRequest.GetResponseWithoutException())
111-
{
112-
var httpStatusCode = webResponse.StatusCode;
113-
logger.Trace($"Downloading {url} StatusCode:{(int)webResponse.StatusCode}");
114-
115-
if (expectingResume && httpStatusCode == HttpStatusCode.RequestedRangeNotSatisfiable)
116-
{
117-
onProgress(bytes, bytes);
118-
return true;
119-
}
120-
121-
if (!(httpStatusCode == HttpStatusCode.OK || httpStatusCode == HttpStatusCode.PartialContent))
122-
{
123-
return false;
124-
}
125-
126-
if (expectingResume && httpStatusCode == HttpStatusCode.OK)
127-
{
128-
expectingResume = false;
129-
destinationStream.Seek(0, SeekOrigin.Begin);
130-
}
131-
132-
var responseLength = webResponse.ContentLength;
133-
if (expectingResume)
134-
{
135-
if (!onProgress(bytes, bytes + responseLength))
136-
return false;
137-
}
138-
139-
using (var responseStream = webResponse.GetResponseStream())
140-
{
141-
return Copy(responseStream, destinationStream, responseLength,
142-
progress: (totalRead, timeToFinish) => {
143-
return onProgress(totalRead, responseLength);
144-
});
145-
}
146-
}
87+
var expected = md5file.ReadAllText();
88+
var actual = file.CalculateMD5();
89+
return expected.Equals(actual, StringComparison.InvariantCultureIgnoreCase);
14790
}
14891
}
14992
}

‎src/GitHub.Api/Installer/GitInstaller.cs‎

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Threading;
3+
using System.Threading.Tasks;
34
using GitHub.Logging;
45

56
namespace GitHub.Unity
@@ -116,22 +117,27 @@ public void SetupGitIfNeeded(ActionTask<NPath> onSuccess, ITask onFailure)
116117
return;
117118
}
118119

119-
var task = new FuncTask<NPath>(cancellationToken, () =>
120+
var isGitExtractedTask = new FuncTask<NPath>(cancellationToken, () =>
120121
{
121122
if (!IsGitExtracted())
122-
{
123-
Logger.Trace("SetupGitIfNeeded: Skipped");
124-
throw new Exception();
125-
}
123+
return null;
124+
Logger.Trace("SetupGitIfNeeded: Skipped");
126125
return installDetails.GitExecutablePath;
127126
});
128-
var extractTask = ExtractPortableGit();
129-
extractTask.Then(onSuccess, TaskRunOptions.OnSuccess, taskIsTopOfChain: true);
130-
extractTask.Then(onFailure, TaskRunOptions.OnFailure, taskIsTopOfChain: true);
127+
isGitExtractedTask.OnEnd += (t, res, _, __) =>
128+
{
129+
if (res == null)
130+
{
131+
var extractTask = ExtractPortableGit();
132+
extractTask.Then(onSuccess, TaskRunOptions.OnSuccess, taskIsTopOfChain: true);
133+
extractTask.Then(onFailure, TaskRunOptions.OnFailure, taskIsTopOfChain: true);
134+
t.Then(extractTask);
135+
}
136+
else
137+
t.Then(onSuccess);
138+
};
131139

132-
task.Then(onSuccess, TaskRunOptions.OnSuccess, taskIsTopOfChain: true);
133-
task.Then(extractTask, TaskRunOptions.OnFailure, taskIsTopOfChain: true);
134-
task.Start();
140+
isGitExtractedTask.Start();
135141
}
136142

137143
private FuncTask<NPath> ExtractPortableGit()
@@ -157,6 +163,7 @@ private FuncTask<NPath> CreateUnzipTasks(NPath gitExtractPath, NPath gitLfsExtra
157163
environment.FileSystem, GitInstallDetails.GitExtractedMD5);
158164
var unzipGitLfsTask = new UnzipTask(cancellationToken, gitLfsArchivePath, gitLfsExtractPath, sharpZipLibHelper,
159165
environment.FileSystem, GitInstallDetails.GitLfsExtractedMD5);
166+
160167
var moveGitTask = new FuncTask<NPath>(cancellationToken, () => MoveGitAndLfs(gitExtractPath, gitLfsExtractPath, tempZipExtractPath));
161168
return unzipGitTask
162169
.Then(unzipGitLfsTask)
@@ -191,24 +198,12 @@ private ITask CreateDownloadTask()
191198
gitArchiveFilePath = installDetails.PluginDataPath.Combine("git.zip");
192199
gitLfsArchivePath = installDetails.PluginDataPath.Combine("git-lfs.zip");
193200

194-
var downloadGitMd5Task = new DownloadTextTask(TaskManager.Instance.Token, environment.FileSystem,
195-
installDetails.GitZipMd5Url, installDetails.PluginDataPath);
196-
197-
var downloadGitTask = new DownloadTask(TaskManager.Instance.Token, environment.FileSystem,
198-
installDetails.GitZipUrl, installDetails.PluginDataPath);
199-
200-
var downloadGitLfsMd5Task = new DownloadTextTask(TaskManager.Instance.Token, environment.FileSystem,
201-
installDetails.GitLfsZipMd5Url, installDetails.PluginDataPath);
201+
var downloader = new Downloader();
202202

203-
var downloadGitLfsTask = new DownloadTask(TaskManager.Instance.Token, environment.FileSystem,
204-
installDetails.GitLfsZipUrl, installDetails.PluginDataPath);
203+
downloader.QueueDownload(installDetails.GitZipUrl, installDetails.GitZipMd5Url, installDetails.PluginDataPath);
204+
downloader.QueueDownload(installDetails.GitLfsZipUrl, installDetails.GitLfsZipMd5Url, installDetails.PluginDataPath);
205205

206-
return
207-
downloadGitMd5Task.Then((b, s) => { downloadGitTask.ValidationHash = s; })
208-
.Then(downloadGitTask)
209-
.Then(downloadGitLfsMd5Task)
210-
.Then((b, s) => { downloadGitLfsTask.ValidationHash = s; })
211-
.Then(downloadGitLfsTask);
206+
return downloader;
212207
}
213208

214209
private bool IsGitExtracted()

‎src/GitHub.Api/Installer/UnzipTask.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ protected virtual void RunUnzip(bool success)
7878
if (expectedMD5 != null)
7979
{
8080
var calculatedMD5 = fileSystem.CalculateFolderMD5(extractedPath);
81-
success = !calculatedMD5.Equals(expectedMD5, StringComparison.InvariantCultureIgnoreCase);
81+
success = calculatedMD5.Equals(expectedMD5, StringComparison.InvariantCultureIgnoreCase);
8282
if (!success)
8383
{
8484
extractedPath.DeleteIfExists();
@@ -100,7 +100,7 @@ protected virtual void RunUnzip(bool success)
100100
if (!success)
101101
{
102102
Token.ThrowIfCancellationRequested();
103-
throw new UnzipException("Error downloading file", exception);
103+
throw new UnzipException("Error unzipping file", exception);
104104
}
105105
}
106106
protected int RetryCount { get; }

0 commit comments

Comments
 (0)