Skip to content

Commit 8ff297d

Browse files
soninarensoninaren
authored
[pack] Selective extension loading for bundles (#4980)
1 parent 331c901 commit 8ff297d

25 files changed

+828
-410
lines changed

‎src/WebJobs.Script.WebHost/Configuration/ScriptApplicationHostOptionsSetup.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,15 @@ public class ScriptApplicationHostOptionsSetup : IConfigureNamedOptions<ScriptAp
1616
private readonly IConfiguration _configuration;
1717
private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
1818
private readonly IDisposable _standbyOptionsOnChangeSubscription;
19+
private readonly IServiceProvider _serviceProvider;
1920

2021
public ScriptApplicationHostOptionsSetup(IConfiguration configuration, IOptionsMonitor<StandbyOptions> standbyOptions,
21-
IOptionsMonitorCache<ScriptApplicationHostOptions> cache)
22+
IOptionsMonitorCache<ScriptApplicationHostOptions> cache, IServiceProvider serviceProvider)
2223
{
2324
_cache = cache ?? throw new ArgumentNullException(nameof(cache));
2425
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
2526
_standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
26-
27+
_serviceProvider = serviceProvider;
2728
// If standby options change, invalidate this options cache.
2829
_standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => _cache.Clear());
2930
}
@@ -40,6 +41,7 @@ public void Configure(string name, ScriptApplicationHostOptions options)
4041

4142
// Indicate that a WebHost is hosting the ScriptHost
4243
options.HasParentScope = true;
44+
options.RootServiceProvider = _serviceProvider;
4345

4446
// During assignment, we need a way to get the non-placeholder ScriptPath
4547
// while we are still in PlaceholderMode. This is a way for us to request it from the

‎src/WebJobs.Script.WebHost/FileMonitoringService.cs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Collections.Immutable;
67
using System.Globalization;
78
using System.IO;
89
using System.Linq;
@@ -30,6 +31,7 @@ public class FileMonitoringService : IHostedService, IDisposable
3031
private readonly IList<IDisposable> _eventSubscriptions = new List<IDisposable>();
3132
private readonly Func<Task> _restart;
3233
private readonly Action _shutdown;
34+
private readonly ImmutableArray<string> _rootDirectorySnapshot;
3335
private AutoRecoveringFileSystemWatcher _debugModeFileWatcher;
3436
private AutoRecoveringFileSystemWatcher _diagnosticModeFileWatcher;
3537
private FileWatcherEventSource _fileEventSource;
@@ -54,6 +56,23 @@ public FileMonitoringService(IOptions<ScriptJobHostOptions> scriptOptions, ILogg
5456

5557
_shutdown = Shutdown;
5658
_shutdown = _shutdown.Debounce(milliseconds: 500);
59+
_rootDirectorySnapshot = GetDirectorySnapshot();
60+
}
61+
62+
internal ImmutableArray<string> GetDirectorySnapshot()
63+
{
64+
if (_scriptOptions.RootScriptPath != null)
65+
{
66+
try
67+
{
68+
return Directory.EnumerateDirectories(_scriptOptions.RootScriptPath).ToImmutableArray();
69+
}
70+
catch (DirectoryNotFoundException)
71+
{
72+
_logger.LogInformation($"Unable to get directory snapshot. No directory present at {_scriptOptions.RootScriptPath}");
73+
}
74+
}
75+
return ImmutableArray<string>.Empty;
5776
}
5877

5978
public Task StartAsync(CancellationToken cancellationToken)
@@ -162,7 +181,7 @@ private void OnFileChanged(FileSystemEventArgs e)
162181
changeDescription = "File";
163182
}
164183
else if ((e.ChangeType == WatcherChangeTypes.Deleted || Directory.Exists(e.FullPath))
165-
&& !_scriptOptions.RootScriptDirectorySnapshot.SequenceEqual(Directory.EnumerateDirectories(_scriptOptions.RootScriptPath)))
184+
&& !_rootDirectorySnapshot.SequenceEqual(Directory.EnumerateDirectories(_scriptOptions.RootScriptPath)))
166185
{
167186
// Check directory snapshot only if "Deleted" change or if directory changed
168187
changeDescription = "Directory";

‎src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,12 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
4343
private readonly IScriptWebHostEnvironment _webHostEnvironment;
4444
private readonly IEnvironment _environment;
4545
private readonly HostNameProvider _hostNameProvider;
46+
private readonly IFunctionMetadataProvider _functionMetadataProvider;
4647
private readonly SemaphoreSlim _syncSemaphore = new SemaphoreSlim(1, 1);
4748

4849
private CloudBlockBlob _hashBlob;
4950

50-
public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILogger<FunctionsSyncManager> logger, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider)
51+
public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILogger<FunctionsSyncManager> logger, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider, IFunctionMetadataProvider functionMetadataProvider)
5152
{
5253
_applicationHostOptions = applicationHostOptions;
5354
_logger = logger;
@@ -59,6 +60,7 @@ public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostId
5960
_webHostEnvironment = webHostEnvironment;
6061
_environment = environment;
6162
_hostNameProvider = hostNameProvider;
63+
_functionMetadataProvider = functionMetadataProvider;
6264
}
6365

6466
internal bool ArmCacheEnabled
@@ -247,7 +249,7 @@ internal async Task<CloudBlockBlob> GetHashBlobAsync()
247249
public async Task<string> GetSyncTriggersPayload()
248250
{
249251
var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
250-
var functionsMetadata = WebFunctionsManager.GetFunctionsMetadata(hostOptions, _workerConfigs, _logger);
252+
var functionsMetadata = _functionMetadataProvider.GetFunctionMetadata();
251253

252254
// trigger information used by the ScaleController
253255
var triggers = await GetFunctionTriggers(functionsMetadata, hostOptions);

‎src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ public class WebFunctionsManager : IWebFunctionsManager
2828
private readonly ISecretManagerProvider _secretManagerProvider;
2929
private readonly IFunctionsSyncManager _functionsSyncManager;
3030
private readonly HostNameProvider _hostNameProvider;
31+
private readonly IFunctionMetadataProvider _functionMetadataProvider;
3132

32-
public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILoggerFactory loggerFactory, HttpClient client, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider)
33+
public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILoggerFactory loggerFactory, HttpClient client, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataProvider functionMetadataProvider)
3334
{
3435
_applicationHostOptions = applicationHostOptions;
3536
_logger = loggerFactory?.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
@@ -38,12 +39,13 @@ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applica
3839
_secretManagerProvider = secretManagerProvider;
3940
_functionsSyncManager = functionsSyncManager;
4041
_hostNameProvider = hostNameProvider;
42+
_functionMetadataProvider = functionMetadataProvider;
4143
}
4244

4345
public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies)
4446
{
4547
var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
46-
var functionsMetadata = GetFunctionsMetadata(hostOptions, _workerConfigs, _logger, includeProxies);
48+
var functionsMetadata = GetFunctionsMetadata(hostOptions, _logger, includeProxies);
4749

4850
return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider);
4951
}
@@ -57,10 +59,9 @@ internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMet
5759
return await tasks.WhenAll();
5860
}
5961

60-
internal static IEnumerable<FunctionMetadata> GetFunctionsMetadata(ScriptJobHostOptions hostOptions, IEnumerable<WorkerConfig> workerConfigs, ILogger logger, bool includeProxies = false)
62+
internal IEnumerable<FunctionMetadata> GetFunctionsMetadata(ScriptJobHostOptions hostOptions, ILogger logger, bool includeProxies = false)
6163
{
62-
var functionDirectories = FileUtility.EnumerateDirectories(hostOptions.RootScriptPath);
63-
IEnumerable<FunctionMetadata> functionsMetadata = FunctionMetadataManager.ReadFunctionsMetadata(functionDirectories, null, workerConfigs, logger, fileSystem: FileUtility.Instance);
64+
IEnumerable<FunctionMetadata> functionsMetadata = _functionMetadataProvider.GetFunctionMetadata();
6465

6566
if (includeProxies)
6667
{
@@ -162,10 +163,8 @@ await functionMetadata
162163
/// <returns>(success, FunctionMetadataResponse)</returns>
163164
public async Task<(bool, FunctionMetadataResponse)> TryGetFunction(string name, HttpRequest request)
164165
{
165-
// TODO: DI (FACAVAL) Follow up with ahmels - Since loading of function metadata is no longer tied to the script host, we
166-
// should be able to inject an IFunctionMetadataManager here and bypass this step.
167166
var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
168-
var functionMetadata = FunctionMetadataManager.ReadFunctionMetadata(Path.Combine(hostOptions.RootScriptPath, name), null, _workerConfigs, new Dictionary<string, ICollection<string>>(), fileSystem: FileUtility.Instance);
167+
var functionMetadata = _functionMetadataProvider.GetFunctionMetadata().First(metadata => metadata.Name == name);
169168
if (functionMetadata != null)
170169
{
171170
string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);

‎src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
101101

102102
// Management services
103103
services.AddSingleton<IFunctionsSyncManager, FunctionsSyncManager>();
104+
services.AddSingleton<IFunctionMetadataProvider, FunctionMetadataProvider>();
104105
services.AddSingleton<IWebFunctionsManager, WebFunctionsManager>();
105106
services.AddSingleton<IInstanceManager, InstanceManager>();
106107
services.AddSingleton(_ => new HttpClient());

‎src/WebJobs.Script/Config/ScriptApplicationHostOptions.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

4+
using System;
5+
46
namespace Microsoft.Azure.WebJobs.Script
57
{
68
public class ScriptApplicationHostOptions
@@ -25,5 +27,7 @@ public class ScriptApplicationHostOptions
2527
/// a set of common services will not be registered as they are supplied from the parent WebHost.
2628
/// </summary>
2729
public bool HasParentScope { get; set; }
30+
31+
public IServiceProvider RootServiceProvider { get; set; }
2832
}
2933
}

‎src/WebJobs.Script/Config/ScriptJobHostOptions.cs‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,30 +35,6 @@ public string RootScriptPath
3535
}
3636
}
3737

38-
public ImmutableArray<string> RootScriptDirectorySnapshot
39-
{
40-
get
41-
{
42-
if (_rootScriptPath != null && _directorySnapshot.IsDefaultOrEmpty)
43-
{
44-
// take a startup time function directory snapshot so we can detect function additions/removals
45-
// we'll also use this snapshot when reading function metadata as part of startup
46-
// taking this snapshot once and reusing at various points during initialization allows us to
47-
// minimize disk operations
48-
try
49-
{
50-
_directorySnapshot = Directory.EnumerateDirectories(_rootScriptPath).ToImmutableArray();
51-
}
52-
catch (DirectoryNotFoundException)
53-
{
54-
_directorySnapshot = ImmutableArray<string>.Empty;
55-
}
56-
}
57-
58-
return _directorySnapshot;
59-
}
60-
}
61-
6238
/// <summary>
6339
/// Gets the current ScriptHost instance id.
6440
/// </summary>

‎src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs‎

Lines changed: 54 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,16 @@ public class ScriptStartupTypeLocator : IWebJobsStartupTypeLocator
2929
private readonly string _rootScriptPath;
3030
private readonly ILogger _logger;
3131
private readonly IExtensionBundleManager _extensionBundleManager;
32+
private readonly IFunctionMetadataProvider _functionMetadataProvider;
3233

3334
private static string[] _builtinExtensionAssemblies = GetBuiltinExtensionAssemblies();
3435

35-
public ScriptStartupTypeLocator(string rootScriptPath, ILogger<ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager)
36+
public ScriptStartupTypeLocator(string rootScriptPath, ILogger<ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager, IFunctionMetadataProvider functionMetadataProvider)
3637
{
3738
_rootScriptPath = rootScriptPath ?? throw new ArgumentNullException(nameof(rootScriptPath));
3839
_extensionBundleManager = extensionBundleManager ?? throw new ArgumentNullException(nameof(extensionBundleManager));
3940
_logger = logger;
41+
_functionMetadataProvider = functionMetadataProvider;
4042
}
4143

4244
private static string[] GetBuiltinExtensionAssemblies()
@@ -83,58 +85,64 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
8385

8486
var startupTypes = new List<Type>();
8587

86-
foreach (var item in extensionItems)
87-
{
88-
string startupExtensionName = item.Name ?? item.TypeName;
89-
_logger.ScriptStartUpLoadingStartUpExtension(startupExtensionName);
90-
91-
// load the Type for each startup extension into the function assembly load context
92-
Type extensionType = Type.GetType(item.TypeName,
93-
assemblyName =>
94-
{
95-
if (_builtinExtensionAssemblies.Contains(assemblyName.Name, StringComparer.OrdinalIgnoreCase))
96-
{
97-
_logger.ScriptStartUpBelongExtension(item.TypeName);
98-
return null;
99-
}
88+
var functionBindings = _functionMetadataProvider.GetFunctionMetadata(forceRefresh: true).SelectMany(f => f.Bindings.Select(b => b.Type));
10089

101-
string path = item.HintPath;
102-
if (string.IsNullOrEmpty(path))
103-
{
104-
path = assemblyName.Name + ".dll";
105-
}
90+
var bundleConfigured = _extensionBundleManager.IsExtensionBundleConfigured();
91+
foreach (var extensionItem in extensionItems)
92+
{
93+
if (!bundleConfigured
94+
|| extensionItem.Bindings.Count == 0
95+
|| extensionItem.Bindings.Intersect(functionBindings, StringComparer.OrdinalIgnoreCase).Any())
96+
{
97+
string startupExtensionName = extensionItem.Name ?? extensionItem.TypeName;
98+
_logger.ScriptStartUpLoadingStartUpExtension(startupExtensionName);
10699

107-
var hintUri = new Uri(path, UriKind.RelativeOrAbsolute);
108-
if (!hintUri.IsAbsoluteUri)
100+
// load the Type for each startup extension into the function assembly load context
101+
Type extensionType = Type.GetType(extensionItem.TypeName,
102+
assemblyName =>
109103
{
110-
path = Path.Combine(binPath, path);
111-
}
104+
if (_builtinExtensionAssemblies.Contains(assemblyName.Name, StringComparer.OrdinalIgnoreCase))
105+
{
106+
_logger.ScriptStartUpBelongExtension(extensionItem.TypeName);
107+
return null;
108+
}
109+
110+
string path = extensionItem.HintPath;
111+
if (string.IsNullOrEmpty(path))
112+
{
113+
path = assemblyName.Name + ".dll";
114+
}
115+
116+
var hintUri = new Uri(path, UriKind.RelativeOrAbsolute);
117+
if (!hintUri.IsAbsoluteUri)
118+
{
119+
path = Path.Combine(binPath, path);
120+
}
121+
122+
if (File.Exists(path))
123+
{
124+
return FunctionAssemblyLoadContext.Shared.LoadFromAssemblyPath(path, true);
125+
}
112126

113-
if (File.Exists(path))
127+
return null;
128+
},
129+
(assembly, typeName, ignoreCase) =>
114130
{
115-
return FunctionAssemblyLoadContext.Shared.LoadFromAssemblyPath(path, true);
116-
}
117-
118-
return null;
119-
},
120-
(assembly, typeName, ignoreCase) =>
131+
_logger.ScriptStartUpLoadedExtension(startupExtensionName, assembly.GetName().Version.ToString());
132+
return assembly?.GetType(typeName, false, ignoreCase);
133+
}, false, true);
134+
if (extensionType == null)
121135
{
122-
_logger.ScriptStartUpLoadedExtension(startupExtensionName, assembly.GetName().Version.ToString());
123-
return assembly?.GetType(typeName, false, ignoreCase);
124-
}, false, true);
125-
126-
if (extensionType == null)
127-
{
128-
_logger.ScriptStartUpUnableToLoadExtension(startupExtensionName, item.TypeName);
129-
continue;
130-
}
131-
if (!typeof(IWebJobsStartup).IsAssignableFrom(extensionType))
132-
{
133-
_logger.ScriptStartUpTypeIsNotValid(item.TypeName, nameof(IWebJobsStartup));
134-
continue;
136+
_logger.ScriptStartUpUnableToLoadExtension(startupExtensionName, extensionItem.TypeName);
137+
continue;
138+
}
139+
if (!typeof(IWebJobsStartup).IsAssignableFrom(extensionType))
140+
{
141+
_logger.ScriptStartUpTypeIsNotValid(extensionItem.TypeName, nameof(IWebJobsStartup));
142+
continue;
143+
}
144+
startupTypes.Add(extensionType);
135145
}
136-
137-
startupTypes.Add(extensionType);
138146
}
139147

140148
return startupTypes;

0 commit comments

Comments
 (0)