Skip to content

Commit 9aa85f2

Browse files
tamasvajktamasvajk
committed
C#: Validate all nuget feeds to respond in reasonable time
1 parent e426398 commit 9aa85f2

File tree

8 files changed

+208
-39
lines changed

8 files changed

+208
-39
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

Lines changed: 163 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Linq;
5+
using System.Net.Http;
56
using System.Text.RegularExpressions;
7+
using System.Threading;
68
using System.Threading.Tasks;
79
using Semmle.Util;
810

@@ -14,6 +16,13 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
1416
{
1517
try
1618
{
19+
var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
20+
if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
21+
{
22+
DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
23+
return;
24+
}
25+
1726
using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
1827
{
1928
var count = nuget.InstallPackages();
@@ -139,7 +148,7 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
139148
CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
140149
}
141150

142-
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
151+
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
143152
{
144153
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
145154
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -172,30 +181,9 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
172181
}
173182

174183
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
175-
176-
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
177-
string? nugetConfig = null;
178-
if (nugetConfigs.Length > 1)
179-
{
180-
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
181-
nugetConfig = allFiles
182-
.SelectRootFiles(sourceDir)
183-
.SelectFileNamesByName("nuget.config")
184-
.FirstOrDefault();
185-
if (nugetConfig == null)
186-
{
187-
logger.LogInfo("Could not find a top-level nuget.config file.");
188-
}
189-
}
190-
else
191-
{
192-
nugetConfig = nugetConfigs.FirstOrDefault();
193-
}
194-
195-
if (nugetConfig != null)
196-
{
197-
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
198-
}
184+
var nugetConfig = withNugetConfig
185+
? GetNugetConfig(allFiles)
186+
: null;
199187

200188
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
201189

@@ -221,6 +209,37 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
221209
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
222210
}
223211

212+
private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
213+
214+
private string? GetNugetConfig(List<FileInfo> allFiles)
215+
{
216+
var nugetConfigs = GetAllNugetConfigs(allFiles);
217+
string? nugetConfig;
218+
if (nugetConfigs.Length > 1)
219+
{
220+
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
221+
nugetConfig = allFiles
222+
.SelectRootFiles(sourceDir)
223+
.SelectFileNamesByName("nuget.config")
224+
.FirstOrDefault();
225+
if (nugetConfig == null)
226+
{
227+
logger.LogInfo("Could not find a top-level nuget.config file.");
228+
}
229+
}
230+
else
231+
{
232+
nugetConfig = nugetConfigs.FirstOrDefault();
233+
}
234+
235+
if (nugetConfig != null)
236+
{
237+
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
238+
}
239+
240+
return nugetConfig;
241+
}
242+
224243
private void LogAllUnusedPackages(DependencyContainer dependencies)
225244
{
226245
var allPackageDirectories = GetAllPackageDirectories();
@@ -279,9 +298,6 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
279298
.Select(d => Path.GetFileName(d).ToLowerInvariant());
280299
}
281300

282-
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
283-
private static partial Regex TargetFramework();
284-
285301
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
286302
{
287303
logger.LogInfo($"Restoring package {package}...");
@@ -358,7 +374,126 @@ private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
358374
}
359375
}
360376

377+
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
378+
{
379+
using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
380+
var buffer = new byte[1024];
381+
int bytesRead;
382+
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
383+
{
384+
// do nothing
385+
}
386+
}
387+
388+
private bool IsFeedReachable(string feed)
389+
{
390+
using HttpClient client = new();
391+
var timeoutSeconds = 1;
392+
var tryCount = 4;
393+
394+
for (var i = 0; i < tryCount; i++)
395+
{
396+
using var cts = new CancellationTokenSource();
397+
cts.CancelAfter(timeoutSeconds * 1000);
398+
try
399+
{
400+
ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
401+
return true;
402+
}
403+
catch (Exception exc)
404+
{
405+
if (exc is TaskCanceledException tce &&
406+
tce.CancellationToken == cts.Token &&
407+
cts.Token.IsCancellationRequested)
408+
{
409+
logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutSeconds} seconds.");
410+
timeoutSeconds *= 2;
411+
continue;
412+
}
413+
414+
// We're only interested in timeouts.
415+
logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
416+
return true;
417+
}
418+
}
419+
420+
logger.LogError($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
421+
return false;
422+
}
423+
424+
private bool CheckFeeds(List<FileInfo> allFiles)
425+
{
426+
logger.LogInfo("Checking Nuget feeds...");
427+
var feeds = GetAllFeeds(allFiles);
428+
429+
var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
430+
?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)
431+
.ToHashSet() ?? [];
432+
433+
if (excludedFeeds.Count > 0)
434+
{
435+
logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds)}");
436+
}
437+
438+
var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
439+
if (!allFeedsReachable)
440+
{
441+
diagnosticsWriter.AddEntry(new DiagnosticMessage(
442+
Language.CSharp,
443+
"buildless/unreachable-feed",
444+
"Found unreachable Nuget feed in C# analysis with build-mode 'none'",
445+
visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
446+
markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
447+
severity: DiagnosticMessage.TspSeverity.Warning
448+
));
449+
}
450+
CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
451+
return allFeedsReachable;
452+
}
453+
454+
private IEnumerable<string> GetFeeds(string nugetConfig)
455+
{
456+
logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
457+
var results = dotnet.GetNugetFeeds(nugetConfig);
458+
var regex = EnabledNugetFeed();
459+
foreach (var result in results)
460+
{
461+
var match = regex.Match(result);
462+
if (!match.Success)
463+
{
464+
logger.LogError($"Failed to parse feed from '{result}'");
465+
continue;
466+
}
467+
468+
var url = match.Groups[1].Value;
469+
if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
470+
!url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
471+
{
472+
logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
473+
continue;
474+
}
475+
476+
yield return url;
477+
}
478+
}
479+
480+
private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
481+
{
482+
var nugetConfigs = GetAllNugetConfigs(allFiles);
483+
var feeds = nugetConfigs
484+
.SelectMany(nf => GetFeeds(nf))
485+
.Where(str => !string.IsNullOrWhiteSpace(str))
486+
.ToHashSet();
487+
return feeds;
488+
}
489+
490+
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
491+
private static partial Regex TargetFramework();
492+
361493
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
362494
private static partial Regex LegacyNugetPackage();
495+
496+
[GeneratedRegex(@"^E (.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
497+
private static partial Regex EnabledNugetFeed();
363498
}
364499
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public sealed partial class DependencyManager : IDisposable
2020
{
2121
private readonly AssemblyCache assemblyCache;
2222
private readonly ILogger logger;
23+
private readonly IDiagnosticsWriter diagnosticsWriter;
2324

2425
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
2526
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -52,6 +53,9 @@ public DependencyManager(string srcDir, ILogger logger)
5253
var startTime = DateTime.Now;
5354

5455
this.logger = logger;
56+
this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
57+
Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir) ?? "",
58+
$"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
5559
this.sourceDir = new DirectoryInfo(srcDir);
5660

5761
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -177,8 +181,7 @@ private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
177181
var frameworkLocations = new HashSet<string>();
178182

179183
var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
180-
var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
181-
_ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
184+
var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
182185
if (!string.IsNullOrWhiteSpace(frameworkReferences))
183186
{
184187
RemoveFrameworkNugetPackages(dllPaths);
@@ -740,6 +743,8 @@ public void Dispose()
740743
{
741744
Dispose(tempWorkingDirectory, "temporary working");
742745
}
746+
747+
diagnosticsWriter?.Dispose();
743748
}
744749
}
745750
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
1616
public partial class DotNet : IDotNet
1717
{
1818
private readonly IDotNetCliInvoker dotnetCliInvoker;
19+
private readonly ILogger logger;
1920
private readonly TemporaryDirectory? tempWorkingDirectory;
2021

2122
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
2223
{
2324
this.tempWorkingDirectory = tempWorkingDirectory;
2425
this.dotnetCliInvoker = dotnetCliInvoker;
26+
this.logger = logger;
2527
Info();
2628
}
2729

@@ -89,17 +91,18 @@ public bool AddPackage(string folder, string package)
8991
return dotnetCliInvoker.RunCommand(args);
9092
}
9193

92-
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
94+
public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
9395

94-
public IList<string> GetListedSdks() => GetListed("--list-sdks");
96+
public IList<string> GetListedSdks() => GetResultList("--list-sdks");
9597

96-
private IList<string> GetListed(string args)
98+
private IList<string> GetResultList(string args)
9799
{
98-
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
100+
if (dotnetCliInvoker.RunCommand(args, out var results))
99101
{
100-
return artifacts;
102+
return results;
101103
}
102-
return new List<string>();
104+
logger.LogWarning($"Running 'dotnet {args}' failed.");
105+
return [];
103106
}
104107

105108
public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ public bool Exec(string execArgs)
108111
return dotnetCliInvoker.RunCommand(args);
109112
}
110113

114+
public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
115+
111116
// The version number should be kept in sync with the version .NET version used for building the application.
112117
public const string LatestDotNetSdkVersion = "8.0.101";
113118

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,20 @@ internal class EnvironmentVariableNames
1616
/// Controls whether to use framework dependencies from subfolders.
1717
/// </summary>
1818
public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
19+
20+
/// <summary>
21+
/// Controls whether to check the responsiveness of NuGet feeds.
22+
/// </summary>
23+
public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
24+
25+
/// <summary>
26+
/// Specifies the NuGet feeds to exclude from the responsiveness check.
27+
/// </summary>
28+
public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_EXCLUDED_FROM_CHECK";
29+
30+
/// <summary>
31+
/// Specifies the location of the diagnostic directory.
32+
/// </summary>
33+
public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
1934
}
2035
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public interface IDotNet
1313
IList<string> GetListedRuntimes();
1414
IList<string> GetListedSdks();
1515
bool Exec(string execArgs);
16+
IList<string> GetNugetFeeds(string nugetConfig);
1617
}
1718

1819
public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

‎csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
2626
public IList<string> GetListedSdks() => sdks;
2727

2828
public bool Exec(string execArgs) => true;
29+
30+
public IList<string> GetNugetFeeds(string nugetConfig) => [];
2931
}
3032

3133
public class RuntimeTests

‎csharp/extractor/Semmle.Util/EnvironmentVariables.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,12 @@ public static int GetDefaultNumberOfThreads()
2727
}
2828
return threads;
2929
}
30+
31+
public static bool GetBoolean(string name)
32+
{
33+
var env = Environment.GetEnvironmentVariable(name);
34+
var _ = bool.TryParse(env, out var value);
35+
return value;
36+
}
3037
}
3138
}

‎csharp/extractor/Semmle.Util/FileUtils.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,7 @@ public static string ComputeFileHash(string filePath)
102102
private static async Task DownloadFileAsync(string address, string filename)
103103
{
104104
using var httpClient = new HttpClient();
105-
using var request = new HttpRequestMessage(HttpMethod.Get, address);
106-
using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
105+
using var contentStream = await httpClient.GetStreamAsync(address);
107106
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
108107
await contentStream.CopyToAsync(stream);
109108
}
@@ -112,7 +111,7 @@ private static async Task DownloadFileAsync(string address, string filename)
112111
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
113112
/// </summary>
114113
public static void DownloadFile(string address, string fileName) =>
115-
DownloadFileAsync(address, fileName).Wait();
114+
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
116115

117116
public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
118117
{

0 commit comments

Comments
 (0)