Skip to content

Commit e372ca9

Browse files
jjonesczjjonescz
committed
Allow downloading stable compiler versions
1 parent 3403aa1 commit e372ca9

File tree

5 files changed

+187
-29
lines changed

5 files changed

+187
-29
lines changed

‎src/Shared/Util.cs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@ namespace DotNetLab;
88

99
public static class Util
1010
{
11+
extension(AsyncEnumerable)
12+
{
13+
public static IAsyncEnumerable<T> Create<T>(T item)
14+
{
15+
return AsyncEnumerable.Repeat(item, 1);
16+
}
17+
}
18+
1119
extension<T>(IEnumerable<T> collection)
1220
{
1321
public IList<T> AsList()
@@ -102,6 +110,16 @@ public static async IAsyncEnumerable<T> Concat<T>(this IAsyncEnumerable<T> a, IE
102110
/// </summary>
103111
public static R EnsureSync() => default;
104112

113+
public static async Task<T?> FirstOrNullAsync<T>(this IAsyncEnumerable<T> source) where T : struct
114+
{
115+
await foreach (var item in source)
116+
{
117+
return item;
118+
}
119+
120+
return null;
121+
}
122+
105123
public static void ForEach<T>(this IEnumerable<T> source, Action<T> action)
106124
{
107125
foreach (var item in source)
@@ -159,6 +177,14 @@ public static string JoinToString<T>(this IEnumerable<T> source, string separato
159177
return string.Join(separator, source.Select(x => $"{quote}{x}{quote}"));
160178
}
161179

180+
public static async IAsyncEnumerable<TResult> SelectAsync<T, TResult>(this IAsyncEnumerable<T> source, Func<T, Task<TResult>> selector)
181+
{
182+
await foreach (var item in source)
183+
{
184+
yield return await selector(item);
185+
}
186+
}
187+
162188
public static async Task<IEnumerable<TResult>> SelectAsync<T, TResult>(this IEnumerable<T> source, Func<T, Task<TResult>> selector)
163189
{
164190
var results = new List<TResult>(source.TryGetNonEnumeratedCount(out var count) ? count : 0);
@@ -189,6 +215,40 @@ public static async Task<ImmutableArray<TResult>> SelectAsArrayAsync<T, TResult>
189215
return results.DrainToImmutable();
190216
}
191217

218+
public static async IAsyncEnumerable<TResult> SelectManyAsync<T, TCollection, TResult>(this IAsyncEnumerable<T> source, Func<T, Task<IEnumerable<TCollection>>> selector, Func<T, TCollection, TResult> resultSelector)
219+
{
220+
await foreach (var item in source)
221+
{
222+
foreach (var subitem in await selector(item))
223+
{
224+
yield return resultSelector(item, subitem);
225+
}
226+
}
227+
}
228+
229+
public static async Task<IEnumerable<TResult>> SelectManyAsync<T, TResult>(this IEnumerable<T> source, Func<T, Task<IEnumerable<TResult>>> selector)
230+
{
231+
var results = new List<TResult>();
232+
foreach (var item in source)
233+
{
234+
results.AddRange(await selector(item));
235+
}
236+
return results;
237+
}
238+
239+
public static async Task<IEnumerable<TResult>> SelectManyAsync<T, TCollection, TResult>(this IEnumerable<T> source, Func<T, Task<IEnumerable<TCollection>>> selector, Func<T, TCollection, TResult> resultSelector)
240+
{
241+
var results = new List<TResult>();
242+
foreach (var item in source)
243+
{
244+
foreach (var subitem in await selector(item))
245+
{
246+
results.AddRange(resultSelector(item, subitem));
247+
}
248+
}
249+
return results;
250+
}
251+
192252
public static IEnumerable<TResult> SelectNonNull<T, TResult>(this IEnumerable<T> source, Func<T, TResult?> selector)
193253
{
194254
foreach (var item in source)

‎src/Worker/Lab/NuGetDownloader.cs‎

Lines changed: 78 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
using Microsoft.Extensions.Logging;
2+
using Microsoft.Extensions.Options;
13
using NuGet.Common;
24
using NuGet.Packaging;
35
using NuGet.Protocol;
@@ -64,47 +66,63 @@ internal sealed class NuGetDownloaderPlugin(
6466

6567
internal sealed class NuGetDownloader : ICompilerDependencyResolver
6668
{
67-
private readonly SourceRepository repository;
6869
private readonly SourceCacheContext cacheContext;
69-
private readonly AsyncLazy<FindPackageByIdResource> findPackageById;
70+
private readonly ImmutableArray<AsyncLazy<FindPackageByIdResource>> findPackageByIds;
7071

71-
public NuGetDownloader()
72+
public NuGetDownloader(
73+
IOptions<NuGetDownloaderOptions> options,
74+
ILogger<NuGetDownloader> logger)
7275
{
76+
Options = options.Value;
77+
Logger = logger;
7378
ImmutableArray<Lazy<INuGetResourceProvider>> providers =
7479
[
7580
new(() => new RegistrationResourceV3Provider()),
7681
new(() => new DependencyInfoResourceV3Provider()),
77-
new(() => new CustomHttpHandlerResourceV3Provider()),
82+
new(() => new CustomHttpHandlerResourceV3Provider(this)),
7883
new(() => new HttpSourceResourceProvider()),
7984
new(() => new ServiceIndexResourceV3Provider()),
8085
new(() => new RemoteV3FindPackageByIdResourceProvider()),
8186
];
82-
repository = Repository.CreateSource(
87+
IEnumerable<string> sourceUrls =
88+
[
89+
"https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json",
90+
"https://api.nuget.org/v3/index.json",
91+
];
92+
var repositories = sourceUrls.Select(url => Repository.CreateSource(
8393
providers,
84-
"https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json");
94+
url));
8595
cacheContext = new SourceCacheContext();
86-
findPackageById = new(() => repository.GetResourceAsync<FindPackageByIdResource>());
96+
findPackageByIds = repositories.SelectAsArray(repository =>
97+
new AsyncLazy<FindPackageByIdResource>(() => repository.GetResourceAsync<FindPackageByIdResource>()));
8798
}
8899

100+
public NuGetDownloaderOptions Options { get; }
101+
public ILogger<NuGetDownloader> Logger { get; }
102+
89103
public async Task<CompilerDependency?> TryResolveCompilerAsync(
90104
CompilerInfo info,
91105
CompilerVersionSpecifier specifier,
92106
BuildConfiguration configuration)
93107
{
94-
NuGetVersion version;
108+
(FindPackageByIdResource? findPackageById, NuGetVersion version) result;
95109
if (specifier is CompilerVersionSpecifier.NuGetLatest)
96110
{
97-
var versions = await (await findPackageById).GetAllVersionsAsync(
98-
info.PackageId,
99-
cacheContext,
100-
NullLogger.Instance,
101-
CancellationToken.None);
102-
version = versions.FirstOrDefault() ??
111+
var versions = findPackageByIds.ToAsyncEnumerable()
112+
.SelectAsync(static async lazy => await lazy)
113+
.SelectManyAsync(findPackageById =>
114+
findPackageById.GetAllVersionsAsync(
115+
info.PackageId,
116+
cacheContext,
117+
NullLogger.Instance,
118+
CancellationToken.None),
119+
(findPackageById, version) => (findPackageById, version));
120+
result = await versions.FirstOrNullAsync() ??
103121
throw new InvalidOperationException($"Package '{info.PackageId}' not found.");
104122
}
105123
else if (specifier is CompilerVersionSpecifier.NuGet nuGetSpecifier)
106124
{
107-
version = nuGetSpecifier.Version;
125+
result = (null, nuGetSpecifier.Version);
108126
}
109127
else
110128
{
@@ -113,14 +131,27 @@ public NuGetDownloader()
113131

114132
var package = new NuGetDownloadablePackage(specifier, info.PackageFolder, async () =>
115133
{
134+
var (findPackageById, version) = result;
135+
136+
var finders = findPackageById != null
137+
? AsyncEnumerable.Create(findPackageById)
138+
: findPackageByIds.ToAsyncEnumerable().SelectAsync(async lazy => await lazy);
139+
116140
var stream = new MemoryStream();
117-
var success = await (await findPackageById).CopyNupkgToStreamAsync(
118-
info.PackageId,
119-
version,
120-
stream,
121-
cacheContext,
122-
NullLogger.Instance,
123-
CancellationToken.None);
141+
var success = await finders.AnyAsync(async (findPackageById, cancellationToken) =>
142+
{
143+
try
144+
{
145+
return await findPackageById.CopyNupkgToStreamAsync(
146+
info.PackageId,
147+
version,
148+
stream,
149+
cacheContext,
150+
NullLogger.Instance,
151+
cancellationToken);
152+
}
153+
catch (Newtonsoft.Json.JsonReaderException) { return false; }
154+
});
124155

125156
if (!success)
126157
{
@@ -183,21 +214,24 @@ public async Task<ImmutableArray<LoadedAssembly>> GetAssembliesAsync()
183214

184215
internal sealed class CustomHttpHandlerResourceV3Provider : ResourceProvider
185216
{
186-
public CustomHttpHandlerResourceV3Provider()
217+
private readonly NuGetDownloader nuGetDownloader;
218+
219+
public CustomHttpHandlerResourceV3Provider(NuGetDownloader nuGetDownloader)
187220
: base(typeof(HttpHandlerResource), nameof(CustomHttpHandlerResourceV3Provider))
188221
{
222+
this.nuGetDownloader = nuGetDownloader;
189223
}
190224

191225
public override Task<Tuple<bool, INuGetResource?>> TryCreate(SourceRepository source, CancellationToken token)
192226
{
193227
return Task.FromResult(TryCreate(source));
194228
}
195229

196-
private static Tuple<bool, INuGetResource?> TryCreate(SourceRepository source)
230+
private Tuple<bool, INuGetResource?> TryCreate(SourceRepository source)
197231
{
198232
if (source.PackageSource.IsHttp)
199233
{
200-
var clientHandler = new CorsClientHandler();
234+
var clientHandler = new CorsClientHandler(nuGetDownloader);
201235
var messageHandler = new ServerWarningLogHandler(clientHandler);
202236
return new(true, new HttpHandlerResourceV3(clientHandler, messageHandler));
203237
}
@@ -206,15 +240,31 @@ public CustomHttpHandlerResourceV3Provider()
206240
}
207241
}
208242

209-
internal sealed class CorsClientHandler : HttpClientHandler
243+
internal sealed class CorsClientHandler(NuGetDownloader nuGetDownloader) : HttpClientHandler
210244
{
211-
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
245+
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
212246
{
213247
if (request.RequestUri?.AbsolutePath.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) == true)
214248
{
215249
request.RequestUri = request.RequestUri.WithCorsProxy();
216250
}
217251

218-
return base.SendAsync(request, cancellationToken);
252+
var response = await base.SendAsync(request, cancellationToken);
253+
254+
if (nuGetDownloader.Options.LogRequests)
255+
{
256+
nuGetDownloader.Logger.LogDebug(
257+
"Sent: {Method} {Uri}, Received: {Status}",
258+
request.Method,
259+
request.RequestUri,
260+
response.StatusCode);
261+
}
262+
263+
return response;
219264
}
220265
}
266+
267+
internal sealed class NuGetDownloaderOptions
268+
{
269+
public bool LogRequests { get; set; }
270+
}

‎src/Worker/WorkerServices.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ public static IServiceProvider CreateTest(
2424
{
2525
options.AssembliesAreAlwaysInDllFormat = true;
2626
});
27+
services.Configure<NuGetDownloaderOptions>(static options =>
28+
{
29+
options.LogRequests = true;
30+
});
2731
configureServices?.Invoke(services);
2832
});
2933
}
@@ -54,7 +58,7 @@ private static IServiceProvider Create(
5458
services.AddScoped<AssemblyDownloader>();
5559
services.AddScoped<CompilerProxy>();
5660
services.AddScoped<DependencyRegistry>();
57-
services.AddScoped<Lazy<NuGetDownloader>>();
61+
services.AddScoped(sp => new Lazy<NuGetDownloader>(() => ActivatorUtilities.CreateInstance<NuGetDownloader>(sp)));
5862
services.AddScoped<SdkDownloader>();
5963
services.AddScoped<CompilerDependencyProvider>();
6064
services.AddScoped<BuiltInCompilerProvider>();

‎test/UnitTests/Utils/TestLoggerProvider.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using Microsoft.Extensions.Logging;
2+
3+
namespace DotNetLab;
4+
5+
internal sealed class TestLoggerProvider(ITestOutputHelper output) : ILoggerProvider
6+
{
7+
public ILogger CreateLogger(string categoryName) => new Logger(output);
8+
9+
public void Dispose() { }
10+
11+
private sealed class Logger(ITestOutputHelper output) : ILogger
12+
{
13+
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
14+
15+
public bool IsEnabled(LogLevel logLevel) => true;
16+
17+
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
18+
{
19+
output.WriteLine(formatter(state, exception));
20+
}
21+
}
22+
}

‎test/UnitTests/Utils/TestUtil.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using Microsoft.Extensions.Logging;
3+
4+
namespace DotNetLab;
5+
6+
internal static class TestUtil
7+
{
8+
extension(WorkerServices)
9+
{
10+
public static IServiceProvider CreateTest(
11+
ITestOutputHelper output,
12+
HttpMessageHandler? httpMessageHandler = null,
13+
Action<ServiceCollection>? configureServices = null)
14+
{
15+
return WorkerServices.CreateTest(httpMessageHandler, (services) =>
16+
{
17+
services.AddSingleton<ILoggerProvider>(new TestLoggerProvider(output));
18+
configureServices?.Invoke(services);
19+
});
20+
}
21+
}
22+
}

0 commit comments

Comments
 (0)