Add mechanism to download Git on Initialize step (#4850)

* Add GitManager

* Add Git 2.42.0.2 handling

* Set default version as 2.39.4

* Handle error on extracting

* Add debug message

* Move consts

* Set defaultVersion as property

* Update unit test

* Change Directory to File check

* Set Git only on windows platform

---------

Co-authored-by: Kirill Ivlev <[email protected]>

Author: ivanduplenskikh

src/Agent.Plugins/GitCliManager.cs

@@ -87,7 +87,22 @@ public bool EnsureGitLFSVersion(Version requiredVersion, bool throwOnNotMatch)
             string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value;
             ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir));
 
-            string gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe");
+            string gitPath = null;
+
+            if (AgentKnobs.UseGit2_39_4.GetValue(context).AsBoolean())
+            {
+                gitPath = Path.Combine(agentHomeDir, "externals", "git-2.39.4", "cmd", $"git.exe");
+            }
+            else if (AgentKnobs.UseGit2_42_0_2.GetValue(context).AsBoolean())
+            {
+                gitPath = Path.Combine(agentHomeDir, "externals", "git-2.42.0.2", "cmd", $"git.exe");
+            }
+
+            if (gitPath is null || !File.Exists(gitPath))
+            {
+                context.Debug("gitPath is null or does not exist. Falling back to default git path.");
+                gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe");
+            }
 
             string gitLfsPath;
 

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -107,6 +107,20 @@ public class AgentKnobs
             new EnvironmentKnobSource("system.prefergitfrompath"),
             new BuiltInDefaultKnobSource("false"));
 
+        public static readonly Knob UseGit2_39_4 = new Knob(
+            nameof(UseGit2_39_4),
+            "If true, Git v2.39.4 will be used instead of the default version.",
+            new RuntimeKnobSource("USE_GIT_2_39_4"),
+            new EnvironmentKnobSource("USE_GIT_2_39_4"),
+            new BuiltInDefaultKnobSource("false"));
+
+        public static readonly Knob UseGit2_42_0_2 = new Knob(
+            nameof(UseGit2_42_0_2),
+            "If true, Git v2.42.0.2 will be used instead of the default version.",
+            new RuntimeKnobSource("USE_GIT_2_42_0_2"),
+            new EnvironmentKnobSource("USE_GIT_2_42_0_2"),
+            new BuiltInDefaultKnobSource("false"));
+
         public static readonly Knob DisableGitPrompt = new Knob(
             nameof(DisableGitPrompt),
             "If true, git will not prompt on the terminal (e.g., when asking for HTTP authentication).",

src/Agent.Worker/GitManager.cs

@@ -0,0 +1,126 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Agent.Sdk;
+using Microsoft.VisualStudio.Services.Agent.Util;
+
+namespace Microsoft.VisualStudio.Services.Agent.Worker
+{
+    [ServiceLocator(Default = typeof(GitManager))]
+    public interface IGitManager : IAgentService
+    {
+        Task DownloadAsync(IExecutionContext executionContext, string version = GitManager.defaultGitVersion);
+    }
+
+    public class GitManager : AgentService, IGitManager
+    {
+        private const int timeout = 180;
+        private const int defaultFileStreamBufferSize = 4096;
+        private const int retryDelay = 10000;
+        private const int retryLimit = 3;
+
+        public const string defaultGitVersion = "2.39.4";
+
+        public async Task DownloadAsync(IExecutionContext executionContext, string version = defaultGitVersion)
+        {
+            Trace.Entering();
+            ArgUtil.NotNull(executionContext, nameof(executionContext));
+            ArgUtil.NotNullOrEmpty(version, nameof(version));
+
+            Uri gitUrl = GitStore.GetDownloadUrl(version);
+            var gitFileName = gitUrl.Segments[^1];
+            var externalsFolder = HostContext.GetDirectory(WellKnownDirectory.Externals);
+            var gitExternalsPath = Path.Combine(externalsFolder, $"git-{version}");
+            var gitPath = Path.Combine(gitExternalsPath, gitFileName);
+
+            if (File.Exists(gitPath))
+            {
+                executionContext.Debug($"Git instance {gitFileName} already exists.");
+                return;
+            }
+
+            var tempDirectory = Path.Combine(externalsFolder, "git_download_temp");
+            Directory.CreateDirectory(tempDirectory);
+            var downloadGitPath = Path.ChangeExtension(Path.Combine(tempDirectory, gitFileName), ".completed");
+
+            if (File.Exists(downloadGitPath))
+            {
+                executionContext.Debug($"Git intance {version} already downloaded.");
+                return;
+            }
+
+            Trace.Info($@"Git zip file will be downloaded and saved as ""{downloadGitPath}""");
+
+            int retryCount = 0;
+
+            while (true)
+            {
+                using CancellationTokenSource downloadToken = new(TimeSpan.FromSeconds(timeout));
+                using var downloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(downloadToken.Token, executionContext.CancellationToken);
+
+                try
+                {
+                    using HttpClient client = new();
+                    using Stream stream = await client.GetStreamAsync(gitUrl);
+                    using FileStream fs = new(downloadGitPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: defaultFileStreamBufferSize, useAsync: true);
+
+                    await stream.CopyToAsync(fs);
+                    Trace.Info("Finished Git downloading.");
+                    await fs.FlushAsync(downloadCancellation.Token);
+                    fs.Close();
+                    break;
+                }
+                catch (OperationCanceledException) when (executionContext.CancellationToken.IsCancellationRequested)
+                {
+                    Trace.Info($"Git download has been cancelled.");
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    retryCount++;
+                    Trace.Info("Failed to download Git");
+                    Trace.Error(ex);
+
+                    if (retryCount > retryLimit)
+                    {
+                        Trace.Info($"Retry limit to download Git has been reached.");
+                        break;
+                    }
+                    else
+                    {
+                        Trace.Info("Retry Git download in 10 seconds.");
+                        await Task.Delay(retryDelay, executionContext.CancellationToken);
+                    }
+                }
+            }
+
+            try
+            {
+                ZipFile.ExtractToDirectory(downloadGitPath, gitExternalsPath);
+                File.WriteAllText(downloadGitPath, DateTime.UtcNow.ToString());
+                Trace.Info("Git has been extracted and cleaned up");
+            }
+            catch (Exception ex)
+            {
+                Trace.Error(ex);
+            }
+        }
+    }
+
+    internal class GitStore
+    {
+        private static readonly string baseUrl = "https://vstsagenttools.blob.core.windows.net/tools/mingit";
+        private static readonly string bit = PlatformUtil.BuiltOnX86 ? "32" : "64";
+        internal static Uri GetDownloadUrl(string version = GitManager.defaultGitVersion)
+        {
+            return new Uri($"{baseUrl}/{version}/MinGit-{version}-{bit}-bit.zip");
+        }
+    }
+}

src/Agent.Worker/JobExtension.cs

@@ -239,7 +239,7 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
                     context.Output("Finished checking job knob settings.");
 
                     // Ensure that we send git telemetry before potential path env changes during the pipeline execution
-                    var isSelfHosted =  StringUtil.ConvertToBoolean(jobContext.Variables.Get(Constants.Variables.Agent.IsSelfHosted));
+                    var isSelfHosted = StringUtil.ConvertToBoolean(jobContext.Variables.Get(Constants.Variables.Agent.IsSelfHosted));
                     if (PlatformUtil.RunningOnWindows && isSelfHosted)
                     {
                         try
@@ -276,6 +276,24 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
                             prepareStep.Condition = ExpressionManager.Succeeded;
                             preJobSteps.Add(prepareStep);
                         }
+
+                        string gitVersion = null;
+
+                        if (AgentKnobs.UseGit2_39_4.GetValue(jobContext).AsBoolean())
+                        {
+                            gitVersion = "2.39.4";
+                        }
+                        else if (AgentKnobs.UseGit2_42_0_2.GetValue(jobContext).AsBoolean())
+                        {
+                            gitVersion = "2.42.0.2";
+                        }
+
+                        if (gitVersion is not null)
+                        {
+                            context.Debug($"Downloading Git v{gitVersion}");
+                            var gitManager = HostContext.GetService<IGitManager>();
+                            await gitManager.DownloadAsync(context, gitVersion);
+                        }
                     }
 
                     // build up 3 lists of steps, pre-job, job, post-job

src/Test/L0/Plugin/TestGitCliManager/TestGitCliManagerL0.cs

@@ -2,7 +2,6 @@
 using Microsoft.VisualStudio.Services.Agent.Util;
 using Moq;
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;

src/Test/L0/Worker/GitManagerL0.cs

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.IO;
+using System.Threading;
+using Microsoft.VisualStudio.Services.Agent.Worker;
+using Moq;
+using Xunit;
+
+namespace Microsoft.VisualStudio.Services.Agent.Tests.Worker
+{
+    public sealed class GitManagerL0
+    {
+        [Fact]
+        [Trait("Level", "L0")]
+        [Trait("Category", "Worker")]
+        [Trait("SkipOn", "darwin")]
+        [Trait("SkipOn", "linux")]
+        public async void DownloadAsync()
+        {
+            using var tokenSource = new CancellationTokenSource();
+            using var hostContext = new TestHostContext(this);
+            GitManager gitManager = new();
+            gitManager.Initialize(hostContext);
+            var executionContext = new Mock<IExecutionContext>();
+            executionContext.Setup(x => x.CancellationToken).Returns(tokenSource.Token);
+            await gitManager.DownloadAsync(executionContext.Object);
+
+            var externalsPath = hostContext.GetDirectory(WellKnownDirectory.Externals);
+
+            Assert.True(Directory.Exists(Path.Combine(externalsPath, "git-2.39.4")));
+            Assert.True(File.Exists(Path.Combine(externalsPath, "git-2.39.4", "cmd", "git.exe")));
+        }
+    }
+}