Skip to content

Commit 16ef0b3

Browse files
sanjuyadav24Sanju Yadav
sanjuyadav24
and
Sanju Yadav
authored
Moved vso-task-lib download to runtime (#5277)
* Moved vso-task-lib download to runtime * Moved vso-task-lib download to runtime --------- Co-authored-by: Sanju Yadav <[email protected]>
1 parent 80dd37c commit 16ef0b3

File tree

3 files changed

+145
-2
lines changed

3 files changed

+145
-2
lines changed

‎src/Agent.Worker/Handlers/NodeHandler.cs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,23 @@ public async Task RunAsync()
107107

108108
if (!PlatformUtil.RunningOnWindows && !AgentKnobs.IgnoreVSTSTaskLib.GetValue(ExecutionContext).AsBoolean())
109109
{
110+
Dictionary<string, string> telemetryData = new Dictionary<string, string>
111+
{
112+
{ "JobId", ExecutionContext.Variables.System_JobId.ToString()},
113+
{ "PlanId", ExecutionContext.Variables.Get(Constants.Variables.System.PlanId)},
114+
{ "AgentName", ExecutionContext.Variables.Get(Constants.Variables.Agent.Name)},
115+
{ "MachineName", ExecutionContext.Variables.Get(Constants.Variables.Agent.MachineName)},
116+
{ "AgentVersion", ExecutionContext.Variables.Get(Constants.Variables.Agent.Version)},
117+
{ "IsSelfHosted", ExecutionContext.Variables.Get(Constants.Variables.Agent.IsSelfHosted)},
118+
{ "IsAzureVM", ExecutionContext.Variables.Get(Constants.Variables.System.IsAzureVM)},
119+
{ "IsDockerContainer", ExecutionContext.Variables.Get(Constants.Variables.System.IsDockerContainer)},
120+
{ "VsoTaskLibUsed", "true" },
121+
{ "Platform", PlatformUtil.HostOS.ToString() }
122+
};
123+
ExecutionContext.PublishTaskRunnerTelemetry(telemetryData);
124+
125+
await VsoTaskLibManager.DownloadVsoTaskLibAsync(ExecutionContext);
126+
110127
// Ensure compat vso-task-lib exist at the root of _work folder
111128
// This will make vsts-agent work against 2015 RTM/QU1 TFS, since tasks in those version doesn't package with task lib
112129
// Put the 0.5.5 version vso-task-lib into the root of _work/node_modules folder, so tasks are able to find those lib.

‎src/Agent.Worker/VsoTaskLibManager.cs

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
using Microsoft.VisualStudio.Services.Agent.Util;
2+
using System;
3+
using System.IO;
4+
using System.IO.Compression;
5+
using System.Net.Http;
6+
using System.Threading;
7+
using System.Threading.Tasks;
8+
using Agent.Sdk;
9+
using Agent.Sdk.Knob;
10+
11+
namespace Microsoft.VisualStudio.Services.Agent.Worker
12+
{
13+
public static class VsoTaskLibManager
14+
{
15+
/// <summary>
16+
/// Downloads and installs vso-task-lib at runtime if not already present
17+
/// </summary>
18+
/// <param name="executionContext">The execution context</param>
19+
/// <returns>Task representing the async operation</returns>
20+
public static async Task DownloadVsoTaskLibAsync(IExecutionContext executionContext)
21+
{
22+
ArgUtil.NotNull(executionContext, nameof(executionContext));
23+
string externalsPath = Path.Combine(executionContext.GetVariableValueOrDefault("Agent.HomeDirectory"), Constants.Path.ExternalsDirectory);
24+
ArgUtil.NotNull(externalsPath, nameof(externalsPath));
25+
26+
string vsoTaskLibExternalsPath = Path.Combine(externalsPath, "vso-task-lib");
27+
var retryOptions = new RetryOptions() { CurrentCount = 0, Limit = 3 };
28+
29+
if (!Directory.Exists(vsoTaskLibExternalsPath))
30+
{
31+
const string vsoTaskLibDownloadUrl = "https://vstsagenttools.blob.core.windows.net/tools/vso-task-lib/0.5.5/vso-task-lib.tar.gz";
32+
string tempVsoTaskLibDirectory = Path.Combine(externalsPath, "vso-task-lib_download_temp");
33+
34+
await DownloadAsync(executionContext, vsoTaskLibDownloadUrl, tempVsoTaskLibDirectory, vsoTaskLibExternalsPath, retryOptions);
35+
}
36+
else
37+
{
38+
executionContext.Debug($"vso-task-lib download already exists at {vsoTaskLibExternalsPath}.");
39+
}
40+
}
41+
42+
public static async Task DownloadAsync(IExecutionContext executionContext, string blobUrl, string tempDirectory, string extractPath, IRetryOptions retryOptions)
43+
{
44+
Directory.CreateDirectory(tempDirectory);
45+
Directory.CreateDirectory(extractPath);
46+
string downloadPath = Path.ChangeExtension(Path.Combine(tempDirectory, "download"), ".tar.gz");
47+
string toolName = new DirectoryInfo(extractPath).Name;
48+
49+
const int timeout = 180;
50+
const int bufferSize = 4096;
51+
const int retryDelay = 10000;
52+
53+
using var downloadCts = new CancellationTokenSource(TimeSpan.FromSeconds(timeout));
54+
using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(downloadCts.Token, executionContext.CancellationToken);
55+
var cancellationToken = linkedTokenSource.Token;
56+
57+
using var handler = executionContext.GetHostContext().CreateHttpClientHandler();
58+
using var httpClient = new HttpClient(handler);
59+
60+
for (; retryOptions.CurrentCount < retryOptions.Limit; retryOptions.CurrentCount++)
61+
{
62+
try
63+
{
64+
executionContext.Debug($"Downloading {toolName} (attempt {retryOptions.CurrentCount + 1}/{retryOptions.Limit}).");
65+
using var stream = await httpClient.GetStreamAsync(blobUrl, cancellationToken);
66+
using var fs = new FileStream(downloadPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize, true);
67+
await stream.CopyToAsync(fs, cancellationToken);
68+
executionContext.Debug($"Finished downloading {toolName}.");
69+
await fs.FlushAsync(cancellationToken);
70+
ExtractTarGz(downloadPath, extractPath, executionContext, toolName);
71+
executionContext.Debug($"{toolName} has been extracted and cleaned up");
72+
break;
73+
}
74+
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
75+
{
76+
executionContext.Debug($"{toolName} download has been cancelled.");
77+
throw;
78+
}
79+
catch (Exception ex)
80+
{
81+
if (retryOptions.CurrentCount + 1 == retryOptions.Limit)
82+
{
83+
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None);
84+
executionContext.Error($"Retry limit for {toolName} download has been exceeded.");
85+
executionContext.Error(ex);
86+
return;
87+
}
88+
executionContext.Debug($"Failed to download {toolName}: {ex.Message}");
89+
executionContext.Debug($"Retry {toolName} download in 10 seconds.");
90+
await Task.Delay(retryDelay, cancellationToken);
91+
}
92+
}
93+
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None);
94+
executionContext.Debug($"{toolName} download directory has been cleaned up.");
95+
}
96+
97+
/// <summary>
98+
/// Extracts a .tar.gz file to the specified directory using the tar command.
99+
/// </summary>
100+
private static void ExtractTarGz(string tarGzPath, string extractPath, IExecutionContext executionContext, string toolName)
101+
{
102+
Directory.CreateDirectory(extractPath);
103+
executionContext.Debug($"Extracting {toolName} using tar...");
104+
using (var process = new System.Diagnostics.Process
105+
{
106+
StartInfo = new System.Diagnostics.ProcessStartInfo
107+
{
108+
FileName = "tar",
109+
Arguments = $"-xzf \"{tarGzPath}\" -C \"{extractPath}\"",
110+
RedirectStandardOutput = true,
111+
RedirectStandardError = true,
112+
UseShellExecute = false,
113+
CreateNoWindow = true,
114+
}
115+
})
116+
{
117+
process.Start();
118+
process.WaitForExit();
119+
if (process.ExitCode != 0)
120+
{
121+
var error = process.StandardError.ReadToEnd();
122+
executionContext.Error($"tar extraction failed: {error}");
123+
throw new Exception($"tar extraction failed: {error}");
124+
}
125+
}
126+
}
127+
}
128+
}

‎src/Misc/externals.sh

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,6 @@ elif [[ "$PACKAGERUNTIME" == "win-arm64" || "$PACKAGERUNTIME" == "win-arm32" ]];
235235
else
236236
# Download external tools for Linux and OSX.
237237

238-
acquireExternalTool "$CONTAINER_URL/vso-task-lib/0.5.5/vso-task-lib.tar.gz" vso-task-lib
239-
240238
if [[ "$PACKAGERUNTIME" == "osx-arm64" ]]; then
241239
ARCH="darwin-x64"
242240
if [[ "$INCLUDE_NODE6" == "true" ]]; then

0 commit comments

Comments
 (0)