Skip to content

Commit 0bf733e

Browse files
mathewcmathewc
committed
Add feature flag for dynamic extension loading (#1878)
1 parent def69c2 commit 0bf733e

File tree

8 files changed

+319
-256
lines changed

8 files changed

+319
-256
lines changed

‎src/WebJobs.Script/Binding/BuiltinExtensionBindingProvider.cs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Reflection;
7+
using Microsoft.Azure.WebJobs.Host;
8+
using Microsoft.Azure.WebJobs.Script.Extensibility;
9+
using Newtonsoft.Json.Linq;
10+
11+
namespace Microsoft.Azure.WebJobs.Script.Binding
12+
{
13+
/// <summary>
14+
/// We have a backwards compat requirement to whitelist #r references to certain "builtin" dlls.
15+
/// Hook into #r resolution pipeline and apply the whitelist.
16+
/// </summary>
17+
internal class BuiltinExtensionBindingProvider : ScriptBindingProvider
18+
{
19+
// For backwards compat, we support a #r directly to these assemblies.
20+
private static HashSet<string> _assemblyWhitelist = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
21+
{
22+
{ "Twilio.Api" },
23+
{ "Microsoft.Azure.WebJobs.Extensions.Twilio" },
24+
{ "Microsoft.Azure.NotificationHubs" },
25+
{ "Microsoft.WindowsAzure.Mobile" },
26+
{ "Microsoft.Azure.WebJobs.Extensions.MobileApps" },
27+
{ "Microsoft.Azure.WebJobs.Extensions.NotificationHubs" },
28+
{ "Microsoft.WindowsAzure.Mobile" },
29+
{ "Microsoft.Azure.WebJobs.Extensions.MobileApps" },
30+
{ "Microsoft.Azure.Documents.Client" },
31+
{ "Microsoft.Azure.WebJobs.Extensions.DocumentDB" },
32+
{ "Microsoft.Azure.ApiHub.Sdk" },
33+
{ "Microsoft.Azure.WebJobs.Extensions.ApiHub" },
34+
{ "Microsoft.ServiceBus" },
35+
{ "Sendgrid" },
36+
};
37+
38+
public BuiltinExtensionBindingProvider(JobHostConfiguration config, JObject hostMetadata, TraceWriter traceWriter)
39+
: base(config, hostMetadata, traceWriter)
40+
{
41+
}
42+
43+
public override bool TryCreate(ScriptBindingContext context, out ScriptBinding binding)
44+
{
45+
binding = null;
46+
return false;
47+
}
48+
49+
public override bool TryResolveAssembly(string assemblyName, out Assembly assembly)
50+
{
51+
if (_assemblyWhitelist.Contains(assemblyName))
52+
{
53+
assembly = Assembly.Load(assemblyName);
54+
return true;
55+
}
56+
57+
return base.TryResolveAssembly(assemblyName, out assembly);
58+
}
59+
}
60+
}

‎src/WebJobs.Script/Binding/ExtensionLoader.cs‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Reflection;
9+
using Microsoft.Azure.WebJobs.Host;
10+
using Microsoft.Azure.WebJobs.Host.Config;
11+
using Microsoft.Azure.WebJobs.Script.Config;
12+
using Microsoft.Azure.WebJobs.Script.Description;
13+
using Microsoft.Extensions.Logging;
14+
15+
namespace Microsoft.Azure.WebJobs.Script.Binding
16+
{
17+
internal class ExtensionLoader
18+
{
19+
private readonly TraceWriter _traceWriter;
20+
private readonly ILogger _startupLogger;
21+
private readonly ScriptHostConfiguration _config;
22+
private readonly bool _dynamicExtensionLoadingEnabled;
23+
24+
// The set of "built in" binding types. These are types that are directly accesible without
25+
// needing an explicit load gesture.
26+
private static IReadOnlyDictionary<string, string> _builtinBindingTypes = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
27+
{
28+
{ "bot", "Microsoft.Azure.WebJobs.Extensions.BotFramework.Config.BotFrameworkConfiguration, Microsoft.Azure.WebJobs.Extensions.BotFramework" },
29+
{ "sendgrid", "Microsoft.Azure.WebJobs.Extensions.SendGrid.SendGridConfiguration, Microsoft.Azure.WebJobs.Extensions.SendGrid" },
30+
{ "eventGridTrigger", "Microsoft.Azure.WebJobs.Extensions.EventGrid.EventGridExtensionConfig, Microsoft.Azure.WebJobs.Extensions.EventGrid" }
31+
};
32+
33+
// The set of "built in" binding types that are built on the older/obsolete ScriptBindingProvider APIs
34+
// Over time these will migrate to the above set as they are reworked.
35+
private static IReadOnlyDictionary<string, string> _builtinScriptBindingTypes = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
36+
{
37+
{ "twilioSms", "Microsoft.Azure.WebJobs.Script.Binding.TwilioScriptBindingProvider" },
38+
{ "notificationHub", "Microsoft.Azure.WebJobs.Script.Binding.NotificationHubScriptBindingProvider" },
39+
{ "cosmosDBTrigger", "Microsoft.Azure.WebJobs.Script.Binding.DocumentDBScriptBindingProvider" },
40+
{ "documentDB", "Microsoft.Azure.WebJobs.Script.Binding.DocumentDBScriptBindingProvider" },
41+
{ "mobileTable", "Microsoft.Azure.WebJobs.Script.Binding.MobileAppsScriptBindingProvider" },
42+
{ "apiHubFileTrigger", "Microsoft.Azure.WebJobs.Script.Binding.ApiHubScriptBindingProvider" },
43+
{ "apiHubFile", "Microsoft.Azure.WebJobs.Script.Binding.ApiHubScriptBindingProvider" },
44+
{ "apiHubTable", "Microsoft.Azure.WebJobs.Script.Binding.ApiHubScriptBindingProvider" },
45+
{ "serviceBusTrigger", "Microsoft.Azure.WebJobs.Script.Binding.ServiceBusScriptBindingProvider" },
46+
{ "serviceBus", "Microsoft.Azure.WebJobs.Script.Binding.ServiceBusScriptBindingProvider" },
47+
{ "eventHubTrigger", "Microsoft.Azure.WebJobs.Script.Binding.ServiceBusScriptBindingProvider" },
48+
{ "eventHub", "Microsoft.Azure.WebJobs.Script.Binding.ServiceBusScriptBindingProvider" },
49+
};
50+
51+
public ExtensionLoader(ScriptHostConfiguration config, TraceWriter traceWriter, ILogger startupLogger)
52+
{
53+
_config = config;
54+
_traceWriter = traceWriter;
55+
_startupLogger = startupLogger;
56+
_dynamicExtensionLoadingEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagsEnableDynamicExtensionLoading);
57+
}
58+
59+
/// <summary>
60+
/// Get the ScriptBindingProviderType for a given binding type.
61+
/// </summary>
62+
/// <param name="bindingTypeName">The Type name of the binding.</param>
63+
/// <returns>The binding provider Type, or null.</returns>
64+
public static Type GetScriptBindingProvider(string bindingTypeName)
65+
{
66+
string assemblyQualifiedTypeName;
67+
if (_builtinScriptBindingTypes.TryGetValue(bindingTypeName, out assemblyQualifiedTypeName))
68+
{
69+
var type = Type.GetType(assemblyQualifiedTypeName);
70+
return type;
71+
}
72+
return null;
73+
}
74+
75+
public void LoadCustomExtensions()
76+
{
77+
string extensionsPath = _config.RootExtensionsPath;
78+
if (!string.IsNullOrWhiteSpace(extensionsPath))
79+
{
80+
foreach (var dir in Directory.EnumerateDirectories(extensionsPath))
81+
{
82+
LoadCustomExtensions(dir);
83+
}
84+
}
85+
}
86+
87+
private void LoadCustomExtensions(string extensionsPath)
88+
{
89+
foreach (var path in Directory.EnumerateFiles(extensionsPath, "*.dll"))
90+
{
91+
// We don't want to load and reflect over every dll.
92+
// By convention, restrict to based on filenames.
93+
var filename = Path.GetFileName(path);
94+
if (!filename.ToLowerInvariant().Contains("extension"))
95+
{
96+
continue;
97+
}
98+
99+
try
100+
{
101+
// See GetNugetPackagesPath() for details
102+
// Script runtime is already setup with assembly resolution hooks, so use LoadFrom
103+
Assembly assembly = Assembly.LoadFrom(path);
104+
LoadExtensions(assembly, path);
105+
}
106+
catch (Exception e)
107+
{
108+
string msg = $"Failed to load custom extension from '{path}'.";
109+
_traceWriter.Error(msg, e);
110+
_startupLogger.LogError(0, e, msg);
111+
}
112+
}
113+
}
114+
115+
// Load extensions that are directly referenced by the user types.
116+
public void LoadDirectlyReferencedExtensions(IEnumerable<Type> userTypes)
117+
{
118+
var possibleExtensionAssemblies = UserTypeScanner.GetPossibleExtensionAssemblies(userTypes);
119+
120+
foreach (var kv in possibleExtensionAssemblies)
121+
{
122+
var assembly = kv.Key;
123+
var locationHint = kv.Value;
124+
LoadExtensions(assembly, locationHint);
125+
}
126+
}
127+
128+
private void LoadExtensions(Assembly assembly, string locationHint)
129+
{
130+
foreach (var type in assembly.ExportedTypes)
131+
{
132+
if (!typeof(IExtensionConfigProvider).IsAssignableFrom(type))
133+
{
134+
continue;
135+
}
136+
137+
if (IsExtensionLoaded(type))
138+
{
139+
continue;
140+
}
141+
142+
var extensionConfigProvider = (IExtensionConfigProvider)Activator.CreateInstance(type);
143+
LoadExtension(extensionConfigProvider, locationHint);
144+
}
145+
}
146+
147+
private bool IsExtensionLoaded(Type type)
148+
{
149+
var registry = _config.HostConfig.GetService<IExtensionRegistry>();
150+
var extensions = registry.GetExtensions<IExtensionConfigProvider>();
151+
foreach (var extension in extensions)
152+
{
153+
var loadedExtentionType = extension.GetType();
154+
if (loadedExtentionType == type)
155+
{
156+
return true;
157+
}
158+
}
159+
return false;
160+
}
161+
162+
private void LoadExtension(IExtensionConfigProvider extensionConfigProvider, string locationHint = null)
163+
{
164+
var extensionConfigProviderType = extensionConfigProvider.GetType();
165+
string extensionName = extensionConfigProviderType.Name;
166+
167+
string msg = null;
168+
if (!string.IsNullOrEmpty(locationHint))
169+
{
170+
msg = $"Loaded binding extension '{extensionName}' from '{locationHint}'";
171+
}
172+
else
173+
{
174+
msg = $"Loaded custom extension '{extensionName}'";
175+
}
176+
177+
_traceWriter.Info(msg);
178+
_startupLogger.LogInformation(msg);
179+
_config.HostConfig.AddExtension(extensionConfigProvider);
180+
}
181+
182+
public void LoadBuiltinExtensions(IEnumerable<string> bindingTypes)
183+
{
184+
foreach (var bindingType in bindingTypes)
185+
{
186+
string assemblyQualifiedTypeName;
187+
if (_builtinBindingTypes.TryGetValue(bindingType, out assemblyQualifiedTypeName))
188+
{
189+
Type typeExtension = Type.GetType(assemblyQualifiedTypeName);
190+
if (typeExtension == null)
191+
{
192+
string errorMsg = $"Can't find binding provider '{assemblyQualifiedTypeName}' for '{bindingType}'";
193+
_traceWriter.Error(errorMsg);
194+
_startupLogger?.LogError(errorMsg);
195+
}
196+
else
197+
{
198+
IExtensionConfigProvider extension = (IExtensionConfigProvider)Activator.CreateInstance(typeExtension);
199+
LoadExtension(extension);
200+
}
201+
}
202+
}
203+
}
204+
205+
public IEnumerable<string> DiscoverBindingTypes(IEnumerable<FunctionMetadata> functions)
206+
{
207+
if (_dynamicExtensionLoadingEnabled)
208+
{
209+
// only load binding extensions that are actually used
210+
var bindingTypes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
211+
foreach (var function in functions)
212+
{
213+
foreach (var binding in function.InputBindings.Concat(function.OutputBindings))
214+
{
215+
bindingTypes.Add(binding.Type);
216+
}
217+
}
218+
219+
return bindingTypes;
220+
}
221+
else
222+
{
223+
// if not enabled, load all binding extensions
224+
return _builtinBindingTypes.Keys.Concat(_builtinScriptBindingTypes.Keys).ToArray();
225+
}
226+
}
227+
}
228+
}

‎src/WebJobs.Script/Binding/GeneralScriptBindingProvider.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313

1414
namespace Microsoft.Azure.WebJobs.Script.Binding
1515
{
16-
// This single binder can service all SDK extensions by leveraging the SDK metadata provider.
16+
/// <summary>
17+
/// This general purpose binder can service all SDK extensions by leveraging the SDK metadata provider.
18+
/// This should eventually replace all other ScriptBindingProviders.
19+
/// </summary>
1720
internal class GeneralScriptBindingProvider : ScriptBindingProvider
1821
{
1922
private IJobHostMetadataProvider _metadataProvider;

‎src/WebJobs.Script/GlobalSuppressions.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,4 +228,5 @@
228228
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Description.FunctionLogger.#.ctor(Microsoft.Azure.WebJobs.Script.ScriptHost,System.String,System.String)")]
229229
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Dir", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Description.FunctionLogger.#.ctor(Microsoft.Azure.WebJobs.Script.ScriptHost,System.String,System.String)")]
230230
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Dir", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.IFunctionTraceWriterFactory.#Create(System.String,System.String)")]
231-
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.ScriptHost.#Create(Microsoft.Azure.WebJobs.Script.IScriptHostEnvironment,Microsoft.Azure.WebJobs.Script.Eventing.IScriptEventManager,Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.Description.ProxyClientExecutor)")]
231+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2000:Dispose objects before losing scope", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.ScriptHost.#Create(Microsoft.Azure.WebJobs.Script.IScriptHostEnvironment,Microsoft.Azure.WebJobs.Script.Eventing.IScriptEventManager,Microsoft.Azure.WebJobs.Script.ScriptHostConfiguration,Microsoft.Azure.WebJobs.Script.Config.ScriptSettingsManager,Microsoft.Azure.WebJobs.Script.Description.ProxyClientExecutor)")]
232+
[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Reliability", "CA2001:AvoidCallingProblematicMethods", MessageId = "System.Reflection.Assembly.LoadFrom", Scope = "member", Target = "Microsoft.Azure.WebJobs.Script.Binding.ExtensionLoader.#LoadCustomExtensions(System.String)")]

0 commit comments

Comments
 (0)