C#: Add retry logic to file (nuget.exe, dotnet-install.sh) downloads

tamasvajk · tamasvajk · commit 8b6c293b5cf1 · 2024-08-19T12:19:51.000+02:00
diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
@@ -7,6 +7,7 @@
 using Microsoft.Build.Construction;
 using Semmle.Util;
 using Semmle.Autobuild.Shared;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.CSharp.Tests
 {
@@ -203,7 +204,7 @@ public void CreateDirectory(string path)
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName)
+        public void DownloadFile(string address, string fileName, ILogger logger)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
diff --git a/csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs
@@ -7,6 +7,7 @@
 using Microsoft.Build.Construction;
 using System.Xml;
 using System.IO;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Cpp.Tests
 {
@@ -189,7 +190,7 @@ public void CreateDirectory(string path)
                 throw new ArgumentException($"Missing CreateDirectory, {path}");
         }
 
-        public void DownloadFile(string address, string fileName)
+        public void DownloadFile(string address, string fileName, ILogger logger)
         {
             if (!DownloadFiles.Contains((address, fileName)))
                 throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs
@@ -157,7 +157,8 @@ private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAut
             BuildScript.DownloadFile(
                 FileUtils.NugetExeUrl,
                 path,
-                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
+                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),
+                builder.Logger)
             &
             BuildScript.Create(_ =>
             {
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -248,7 +248,8 @@ BuildScript GetInstall(string pwsh) =>
                     var downloadDotNetInstallSh = BuildScript.DownloadFile(
                         "https://dot.net/v1/dotnet-install.sh",
                         dotnetInstallPath,
-                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"),
+                        logger);
 
                     var chmod = new CommandBuilder(actions).
                         RunCommand("chmod").
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs
@@ -145,7 +145,7 @@ private string DownloadNugetExe(string sourceDir)
 
             Directory.CreateDirectory(directory);
             logger.LogInfo("Attempting to download nuget.exe");
-            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
+            FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);
             logger.LogInfo($"Downloaded nuget.exe to {nuget}");
             return nuget;
         }
diff --git a/csharp/extractor/Semmle.Util/BuildActions.cs b/csharp/extractor/Semmle.Util/BuildActions.cs
@@ -6,7 +6,7 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Xml;
-using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -165,7 +165,7 @@ public interface IBuildActions
         /// <summary>
         /// Downloads the resource with the specified URI to a local file.
         /// </summary>
-        void DownloadFile(string address, string fileName);
+        void DownloadFile(string address, string fileName, ILogger logger);
 
         /// <summary>
         /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
@@ -280,8 +280,8 @@ XmlDocument IBuildActions.LoadXml(string filename)
 
         public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
 
-        public void DownloadFile(string address, string fileName) =>
-            FileUtils.DownloadFile(address, fileName);
+        public void DownloadFile(string address, string fileName, ILogger logger) =>
+            FileUtils.DownloadFile(address, fileName, logger);
 
         public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
 
diff --git a/csharp/extractor/Semmle.Util/BuildScript.cs b/csharp/extractor/Semmle.Util/BuildScript.cs
@@ -1,6 +1,7 @@
 ﻿using System;
 using System.Collections.Generic;
 using System.IO;
+using Semmle.Util.Logging;
 
 namespace Semmle.Util
 {
@@ -275,14 +276,14 @@ public static BuildScript Create(Func<IBuildActions, int> func) =>
         /// <summary>
         /// Creates a build script that downloads the specified file.
         /// </summary>
-        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
+        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>
             Create(actions =>
             {
                 if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
                     actions.CreateDirectory(dir);
                 try
                 {
-                    actions.DownloadFile(address, fileName);
+                    actions.DownloadFile(address, fileName, logger);
                     return 0;
                 }
                 catch (Exception e)
diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs
@@ -4,6 +4,7 @@
 using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using Semmle.Util.Logging;
 
@@ -99,19 +100,49 @@ public static string ComputeFileHash(string filePath)
             return hex.ToString();
         }
 
-        private static async Task DownloadFileAsync(string address, string filename)
+        private static async Task DownloadFileAsync(string address, string filename, HttpClient httpClient, CancellationToken token)
         {
-            using var httpClient = new HttpClient();
-            using var contentStream = await httpClient.GetStreamAsync(address);
+            using var contentStream = await httpClient.GetStreamAsync(address, token);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
-            await contentStream.CopyToAsync(stream);
+            await contentStream.CopyToAsync(stream, CancellationToken.None);
+        }
+
+        private static void DownloadFileWithRetry(string address, string fileName, int tryCount, int timeoutMilliSeconds, ILogger logger)
+        {
+            logger.LogDebug($"Downloading {address} to {fileName}.");
+            using HttpClient client = new();
+
+            for (var i = 0; i < tryCount; i++)
+            {
+                logger.LogDebug($"Attempt {i + 1} of {tryCount}. Timeout: {timeoutMilliSeconds} ms.");
+                using var cts = new CancellationTokenSource();
+                cts.CancelAfter(timeoutMilliSeconds);
+                try
+                {
+                    DownloadFileAsync(address, fileName, client, cts.Token).GetAwaiter().GetResult();
+                    logger.LogDebug($"Downloaded {address} to {fileName}.");
+                    return;
+                }
+                catch (Exception exc)
+                {
+                    logger.LogDebug($"Failed to download {address} to {fileName}. Exception: {exc.Message}");
+                    timeoutMilliSeconds *= 2;
+
+                    if (i == tryCount - 1)
+                    {
+                        logger.LogDebug($"Failed to download {address} to {fileName} after {tryCount} attempts.");
+                        // Rethrowing the last exception
+                        throw;
+                    }
+                }
+            }
         }
 
         /// <summary>
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
-        public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
+        public static void DownloadFile(string address, string fileName, ILogger logger) =>
+           DownloadFileWithRetry(address, fileName, tryCount: 3, timeoutMilliSeconds: 10000, logger);
 
         public static string ConvertPathToSafeRelativePath(string path)
         {

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@`
`7`	`7`	`using Microsoft.Build.Construction;`
`8`	`8`	`using Semmle.Util;`
`9`	`9`	`using Semmle.Autobuild.Shared;`
	`10`	`+using Semmle.Util.Logging;`
`10`	`11`
`11`	`12`	`namespace Semmle.Autobuild.CSharp.Tests`
`12`	`13`	`{`
`@@ -203,7 +204,7 @@ public void CreateDirectory(string path)`
`203`	`204`	`throw new ArgumentException($"Missing CreateDirectory, {path}");`
`204`	`205`	`}`
`205`	`206`
`206`		`- public void DownloadFile(string address, string fileName)`
	`207`	`+ public void DownloadFile(string address, string fileName, ILogger logger)`
`207`	`208`	`{`
`208`	`209`	`if (!DownloadFiles.Contains((address, fileName)))`
`209`	`210`	`throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");`
Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@`
`7`	`7`	`using Microsoft.Build.Construction;`
`8`	`8`	`using System.Xml;`
`9`	`9`	`using System.IO;`
	`10`	`+using Semmle.Util.Logging;`
`10`	`11`
`11`	`12`	`namespace Semmle.Autobuild.Cpp.Tests`
`12`	`13`	`{`
`@@ -189,7 +190,7 @@ public void CreateDirectory(string path)`
`189`	`190`	`throw new ArgumentException($"Missing CreateDirectory, {path}");`
`190`	`191`	`}`
`191`	`192`
`192`		`- public void DownloadFile(string address, string fileName)`
	`193`	`+ public void DownloadFile(string address, string fileName, ILogger logger)`
`193`	`194`	`{`
`194`	`195`	`if (!DownloadFiles.Contains((address, fileName)))`
`195`	`196`	`throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");`
Original file line number	Diff line number	Diff line change
`@@ -157,7 +157,8 @@ private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAut`
`157`	`157`	`BuildScript.DownloadFile(`
`158`	`158`	`FileUtils.NugetExeUrl,`
`159`	`159`	`path,`
`160`		`- e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))`
	`160`	`+ e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"),`
	`161`	`+ builder.Logger)`
`161`	`162`	`&`
`162`	`163`	`BuildScript.Create(_ =>`
`163`	`164`	`{`
Original file line number	Diff line number	Diff line change
`@@ -145,7 +145,7 @@ private string DownloadNugetExe(string sourceDir)`
`145`	`145`
`146`	`146`	`Directory.CreateDirectory(directory);`
`147`	`147`	`logger.LogInfo("Attempting to download nuget.exe");`
`148`		`- FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);`
	`148`	`+ FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget, logger);`
`149`	`149`	`logger.LogInfo($"Downloaded nuget.exe to {nuget}");`
`150`	`150`	`return nuget;`
`151`	`151`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,7 @@`
`1`	`1`	`using System;`
`2`	`2`	`using System.Collections.Generic;`
`3`	`3`	`using System.IO;`
	`4`	`+using Semmle.Util.Logging;`
`4`	`5`
`5`	`6`	`namespace Semmle.Util`
`6`	`7`	`{`
`@@ -275,14 +276,14 @@ public static BuildScript Create(Func<IBuildActions, int> func) =>`
`275`	`276`	`/// <summary>`
`276`	`277`	`/// Creates a build script that downloads the specified file.`
`277`	`278`	`/// </summary>`
`278`		`- public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>`
	`279`	`+ public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback, ILogger logger) =>`
`279`	`280`	`Create(actions =>`
`280`	`281`	`{`
`281`	`282`	`if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))`
`282`	`283`	`actions.CreateDirectory(dir);`
`283`	`284`	`try`
`284`	`285`	`{`
`285`		`- actions.DownloadFile(address, fileName);`
	`286`	`+ actions.DownloadFile(address, fileName, logger);`
`286`	`287`	`return 0;`
`287`	`288`	`}`
`288`	`289`	`catch (Exception e)`