Skip to content

Commit 5cf7112

Browse files
tamasvajktamasvajk
authored
Merge pull request github#16541 from tamasvajk/buildless/use-nuget-config-fallback
C#: Use nuget feeds from nuget.config in fallback restore
2 parents 4567b17 + 182325d commit 5cf7112

File tree

15 files changed

+179
-9
lines changed

15 files changed

+179
-9
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ internal class EnvironmentVariableNames
6060
/// </summary>
6161
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
6262

63+
/// <summary>
64+
/// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
65+
/// </summary>
66+
public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
67+
6368
/// <summary>
6469
/// Specifies the path to the nuget executable to be used for package restoration.
6570
/// </summary>

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,14 @@ public HashSet<AssemblyLookupLocation> Restore()
9898
logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
9999
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
100100

101+
HashSet<string>? explicitFeeds = null;
102+
101103
try
102104
{
103-
if (checkNugetFeedResponsiveness && !CheckFeeds())
105+
if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
104106
{
105107
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
106-
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
108+
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
107109
return unresponsiveMissingPackageLocation is null
108110
? []
109111
: [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ public HashSet<AssemblyLookupLocation> Restore()
163165
LogAllUnusedPackages(dependencies);
164166

165167
var missingPackageLocation = checkNugetFeedResponsiveness
166-
? DownloadMissingPackagesFromSpecificFeeds()
168+
? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
167169
: DownloadMissingPackages();
168170

169171
if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ public HashSet<AssemblyLookupLocation> Restore()
173175
return assemblyLookupLocations;
174176
}
175177

176-
private List<string> GetReachableFallbackNugetFeeds()
178+
private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
177179
{
178180
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
179181
if (fallbackFeeds.Count == 0)
180182
{
181183
fallbackFeeds.Add(PublicNugetOrgFeed);
182-
logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
184+
logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
185+
186+
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
187+
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
188+
189+
if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
190+
{
191+
// There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
192+
// But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
193+
fallbackFeeds.UnionWith(feedsFromNugetConfigs);
194+
logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
195+
}
183196
}
184197

185198
logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ private List<string> GetReachableFallbackNugetFeeds()
194207
logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
195208
}
196209

210+
compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
211+
197212
return reachableFallbackFeeds;
198213
}
199214

@@ -272,9 +287,9 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
272287
compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
273288
}
274289

275-
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
290+
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
276291
{
277-
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
292+
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
278293
if (reachableFallbackFeeds.Count > 0)
279294
{
280295
return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount,
623638
return (timeoutMilliSeconds, tryCount);
624639
}
625640

626-
private bool CheckFeeds()
641+
private bool CheckFeeds(out HashSet<string> explicitFeeds)
627642
{
628643
logger.LogInfo("Checking Nuget feeds...");
629-
var (explicitFeeds, allFeeds) = GetAllFeeds();
644+
(explicitFeeds, var allFeeds) = GetAllFeeds();
630645

631646
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
632647
.ToHashSet() ?? [];

‎csharp/ql/integration-tests/all-platforms/standalone_resx/CompilationInfo.expected

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
| Failed solution restore with package source error | 0.0 |
44
| NuGet feed responsiveness checked | 1.0 |
55
| Project files on filesystem | 1.0 |
6+
| Reachable fallback Nuget feed count | 1.0 |
67
| Resource extraction enabled | 1.0 |
78
| Restored .NET framework variants | 1.0 |
89
| Restored projects through solution files | 0.0 |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/CompilationInfo.expected

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
| Fallback nuget restore | 1.0 |
55
| NuGet feed responsiveness checked | 1.0 |
66
| Project files on filesystem | 1.0 |
7+
| Reachable fallback Nuget feed count | 1.0 |
78
| Resolved assembly conflicts | 7.0 |
89
| Resource extraction enabled | 0.0 |
910
| Restored .NET framework variants | 0.0 |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/CompilationInfo.expected

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
| Inherited Nuget feed count | 1.0 |
44
| NuGet feed responsiveness checked | 1.0 |
55
| Project files on filesystem | 1.0 |
6+
| Reachable fallback Nuget feed count | 1.0 |
67
| Resolved assembly conflicts | 7.0 |
78
| Resource extraction enabled | 0.0 |
89
| Restored .NET framework variants | 0.0 |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.expected

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/Assemblies.ql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import csharp
2+
3+
private string getPath(Assembly a) {
4+
not a.getCompilation().getOutputAssembly() = a and
5+
exists(string s | s = a.getFile().getAbsolutePath() |
6+
result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
7+
)
8+
}
9+
10+
from Assembly a
11+
select getPath(a)

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.expected

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
| All Nuget feeds reachable | 0.0 |
2+
| Fallback nuget restore | 1.0 |
3+
| NuGet feed responsiveness checked | 1.0 |
4+
| Project files on filesystem | 1.0 |
5+
| Reachable fallback Nuget feed count | 2.0 |
6+
| Resolved assembly conflicts | 7.0 |
7+
| Resource extraction enabled | 0.0 |
8+
| Restored .NET framework variants | 0.0 |
9+
| Solution files on filesystem | 1.0 |
10+
| Source files generated | 0.0 |
11+
| Source files on filesystem | 1.0 |
12+
| Successfully ran fallback nuget restore | 1.0 |
13+
| Unresolved references | 0.0 |
14+
| UseWPF set | 0.0 |
15+
| UseWindowsForms set | 0.0 |
16+
| WebView extraction enabled | 1.0 |

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/CompilationInfo.ql

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import csharp
2+
import semmle.code.csharp.commons.Diagnostics
3+
4+
query predicate compilationInfo(string key, float value) {
5+
key != "Resolved references" and
6+
not key.matches("Compiler diagnostic count for%") and
7+
exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
8+
key = infoKey and
9+
value = infoValue.toFloat()
10+
or
11+
not exists(infoValue.toFloat()) and
12+
key = infoKey + ": " + infoValue and
13+
value = 1
14+
)
15+
}

‎csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"markdownMessage": "C# analysis with build-mode 'none' completed.",
3+
"severity": "unknown",
4+
"source": {
5+
"extractorName": "csharp",
6+
"id": "csharp/autobuilder/buildless/complete",
7+
"name": "C# analysis with build-mode 'none' completed"
8+
},
9+
"visibility": {
10+
"cliSummaryTable": true,
11+
"statusPage": false,
12+
"telemetry": true
13+
}
14+
}
15+
{
16+
"markdownMessage": "C# with build-mode set to 'none'. This means that all C# source in the working directory will be scanned, with build tools, such as Nuget and Dotnet CLIs, only contributing information about external dependencies.",
17+
"severity": "note",
18+
"source": {
19+
"extractorName": "csharp",
20+
"id": "csharp/autobuilder/buildless/mode-active",
21+
"name": "C# with build-mode set to 'none'"
22+
},
23+
"visibility": {
24+
"cliSummaryTable": true,
25+
"statusPage": true,
26+
"telemetry": true
27+
}
28+
}
29+
{
30+
"markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
31+
"severity": "warning",
32+
"source": {
33+
"extractorName": "csharp",
34+
"id": "csharp/autobuilder/buildless/unreachable-feed",
35+
"name": "Found unreachable Nuget feed in C# analysis with build-mode 'none'"
36+
},
37+
"visibility": {
38+
"cliSummaryTable": true,
39+
"statusPage": true,
40+
"telemetry": true
41+
}
42+
}

0 commit comments

Comments
 (0)