Skip to content

Commit 8d0856f

Browse files
committed
Fix failing test
1 parent 0f7fc90 commit 8d0856f

File tree

4 files changed

+68
-22
lines changed

4 files changed

+68
-22
lines changed

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,16 @@ private List<string> GetReachableFallbackNugetFeeds()
9898
fallbackFeeds.Add(PublicNugetFeed);
9999
}
100100

101-
logger.LogInfo("Checking fallback Nuget feed reachability");
102-
var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed)).ToList();
101+
logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
102+
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: true);
103+
var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)).ToList();
103104
if (reachableFallbackFeeds.Count == 0)
104105
{
105-
logger.LogWarning("No fallback Nuget feeds are reachable. Skipping fallback Nuget package restoration.");
106+
logger.LogWarning("No fallback Nuget feeds are reachable.");
107+
}
108+
else
109+
{
110+
logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
106111
}
107112

108113
return reachableFallbackFeeds;
@@ -183,11 +188,15 @@ private void DownloadMissingPackagesFromSpecificFeeds(List<FileInfo> allNonBinar
183188
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
184189
if (reachableFallbackFeeds.Count > 0)
185190
{
186-
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false, fallbackNugetFeeds: reachableFallbackFeeds);
191+
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfigFromSrc: false, fallbackNugetFeeds: reachableFallbackFeeds);
192+
}
193+
else
194+
{
195+
logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback Nuget feeds are reachable.");
187196
}
188197
}
189198

190-
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true, IEnumerable<string>? fallbackNugetFeeds = null)
199+
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfigFromSrc = true, IEnumerable<string>? fallbackNugetFeeds = null)
191200
{
192201
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
193202
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -221,7 +230,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLo
221230

222231
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
223232
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig"));
224-
var nugetConfig = withNugetConfig
233+
var nugetConfig = withNugetConfigFromSrc
225234
? GetNugetConfig(allFiles)
226235
: CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
227236

@@ -232,7 +241,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLo
232241

233242
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
234243
{
235-
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
244+
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource, tryWithoutNugetConfig: withNugetConfigFromSrc);
236245
if (!success)
237246
{
238247
return;
@@ -254,6 +263,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLo
254263
if (fallbackNugetFeeds is null)
255264
{
256265
// We're not overriding the inherited Nuget feeds
266+
logger.LogInfo("No fallback Nuget feeds provided. Not creating a fallback nuget.config file.");
257267
return null;
258268
}
259269

@@ -365,7 +375,7 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
365375
.Select(d => Path.GetFileName(d).ToLowerInvariant());
366376
}
367377

368-
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
378+
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
369379
{
370380
logger.LogInfo($"Restoring package {package}...");
371381
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
@@ -389,7 +399,7 @@ private bool TryRestorePackageManually(string package, string? nugetConfig, Pack
389399
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
390400
if (!res.Success)
391401
{
392-
if (res.HasNugetPackageSourceError && nugetConfig is not null)
402+
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
393403
{
394404
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
395405
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
@@ -452,16 +462,10 @@ private static async Task ExecuteGetRequest(string address, HttpClient httpClien
452462
}
453463
}
454464

455-
private bool IsFeedReachable(string feed)
465+
private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
456466
{
457467
logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
458468
using HttpClient client = new();
459-
int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
460-
? timeoutMilliSeconds
461-
: 1000;
462-
int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
463-
? tryCount
464-
: 4;
465469

466470
for (var i = 0; i < tryCount; i++)
467471
{
@@ -470,6 +474,7 @@ private bool IsFeedReachable(string feed)
470474
try
471475
{
472476
ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
477+
logger.LogInfo($"Querying Nuget feed '{feed}' succeeded.");
473478
return true;
474479
}
475480
catch (Exception exc)
@@ -478,21 +483,41 @@ private bool IsFeedReachable(string feed)
478483
tce.CancellationToken == cts.Token &&
479484
cts.Token.IsCancellationRequested)
480485
{
481-
logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
486+
logger.LogInfo($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
482487
timeoutMilliSeconds *= 2;
483488
continue;
484489
}
485490

486491
// We're only interested in timeouts.
487-
logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
488-
return true;
492+
var start = allowExceptions ? "Considering" : "Not considering";
493+
logger.LogInfo($"Querying Nuget feed '{feed}' failed in a timely manner. {start} the feed for use. The reason for the failure: {exc.Message}");
494+
return allowExceptions;
489495
}
490496
}
491497

492498
logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
493499
return false;
494500
}
495501

502+
private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
503+
{
504+
int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds)
505+
? timeoutMilliSeconds
506+
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
507+
? timeoutMilliSeconds
508+
: 1000;
509+
logger.LogDebug($"Initial timeout for Nuget feed reachability check is {timeoutMilliSeconds}ms.");
510+
511+
int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount)
512+
? tryCount
513+
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
514+
? tryCount
515+
: 4;
516+
logger.LogDebug($"Number of tries for Nuget feed reachability check is {tryCount}.");
517+
518+
return (timeoutMilliSeconds, tryCount);
519+
}
520+
496521
private bool CheckFeeds(List<FileInfo> allFiles)
497522
{
498523
logger.LogInfo("Checking Nuget feeds...");
@@ -507,7 +532,9 @@ private bool CheckFeeds(List<FileInfo> allFiles)
507532
logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
508533
}
509534

510-
var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
535+
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
536+
537+
var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
511538
if (!allFeedsReachable)
512539
{
513540
logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
339339
{
340340
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
341341

342-
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
342+
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
343343
{
344344
runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
345345
}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,22 @@ internal class EnvironmentVariableNames
3333
public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
3434

3535
/// <summary>
36-
/// Specifies how many requests to make to the NuGet feed to check its responsiveness.
36+
/// Specifies the timeout (as an integer) in milliseconds for the initial check of fallback NuGet feeds responsiveness. The value is then doubled for each subsequent check.
37+
/// This is primarily used in testing.
38+
/// </summary>
39+
internal const string NugetFeedResponsivenessInitialTimeoutForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT";
40+
41+
/// <summary>
42+
/// Specifies how many requests to make to the NuGet feeds to check their responsiveness.
3743
/// </summary>
3844
public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
3945

46+
/// <summary>
47+
/// Specifies how many requests to make to the fallback NuGet feeds to check their responsiveness.
48+
/// This is primarily used in testing.
49+
/// </summary>
50+
internal const string NugetFeedResponsivenessRequestCountForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT";
51+
4052
/// <summary>
4153
/// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
4254
/// The default value is `https://api.nuget.org/v3/index.json`.

csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/test.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,12 @@
66
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT"] = "1" # 1ms, the GET request should fail with such short timeout
77
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"] = "1" # Limit the count of checks to 1
88
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED"] = "https://abc.de:8000/packages/" # Exclude this feed from check
9+
10+
# Making sure the reachability test of `nuget.org` succeeds:
11+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT"] = "1000"
12+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT"] = "5"
13+
# The second feed is ignored in the fallback restore, because of network issues:
14+
os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"] = "https://api.nuget.org/v3/index.json https://abc.def:8000/packages/"
15+
916
run_codeql_database_create([], lang="csharp", extra_args=["--build-mode=none"])
1017
check_diagnostics()

0 commit comments

Comments
 (0)