Skip to content

Commit a2d9bc4

Browse files
brettsambrettsam
committed
Merge branch 'dev' into v3.x
2 parents dcb71fd + 2d6c40c commit a2d9bc4

File tree

74 files changed

+1805
-525
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+1805
-525
lines changed

‎WebJobs.Script.sln‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger", "HttpTrigger"
297297
sample\PowerShell\HttpTrigger\run.ps1 = sample\PowerShell\HttpTrigger\run.ps1
298298
EndProjectSection
299299
EndProject
300+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Python", "Python", "{0AE3CE25-4CD9-4769-AE58-399FC59CF70F}"
301+
ProjectSection(SolutionItems) = preProject
302+
sample\Python\host.json = sample\Python\host.json
303+
EndProjectSection
304+
EndProject
305+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger", "HttpTrigger", "{BA45A727-34B7-484F-9B93-B1755AF09A2A}"
306+
ProjectSection(SolutionItems) = preProject
307+
sample\Python\HttpTrigger\__init__.py = sample\Python\HttpTrigger\__init__.py
308+
sample\Python\HttpTrigger\function.json = sample\Python\HttpTrigger\function.json
309+
EndProjectSection
310+
EndProject
300311
Global
301312
GlobalSection(SharedMSBuildProjectFiles) = preSolution
302313
test\WebJobs.Script.Tests.Shared\WebJobs.Script.Tests.Shared.projitems*{35c9ccb7-d8b6-4161-bb0d-bcfa7c6dcffb}*SharedItemsImports = 13
@@ -386,6 +397,8 @@ Global
386397
{120DC6B3-B4B0-4CA6-A1B6-13177EAE325B} = {908055D8-096D-49BA-850D-C96F676C1561}
387398
{FA3EB27D-D1C1-4AE0-A928-CF3882D929CD} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
388399
{EA8288BA-CB4D-4B9C-ADF8-F4B7C41466EF} = {FA3EB27D-D1C1-4AE0-A928-CF3882D929CD}
400+
{0AE3CE25-4CD9-4769-AE58-399FC59CF70F} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
401+
{BA45A727-34B7-484F-9B93-B1755AF09A2A} = {0AE3CE25-4CD9-4769-AE58-399FC59CF70F}
389402
EndGlobalSection
390403
GlobalSection(ExtensibilityGlobals) = postSolution
391404
SolutionGuid = {85400884-5FFD-4C27-A571-58CB3C8CAAC5}

‎sample/CSharp/BlobTrigger/run.csx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
#r "Microsoft.WindowsAzure.Storage"
1+
#r "Microsoft.Azure.Storage"
22

3-
using Microsoft.WindowsAzure.Storage.Blob;
3+
using Microsoft.Azure.Storage.Blob;
44

55
public static async Task Run(CloudBlockBlob blob, CloudBlockBlob output, TraceWriter log)
66
{

‎src/WebJobs.Script.WebHost/ContainerManagement/LinuxContainerInitializationHostService.cs‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
using System;
55
using System.Threading;
66
using System.Threading.Tasks;
7+
using Microsoft.Azure.Storage.Blob;
8+
using Microsoft.Azure.Storage.RetryPolicies;
79
using Microsoft.Azure.WebJobs.Script.WebHost.Management;
810
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
911
using Microsoft.Extensions.Hosting;
1012
using Microsoft.Extensions.Logging;
11-
using Microsoft.WindowsAzure.Storage.Blob;
12-
using Microsoft.WindowsAzure.Storage.RetryPolicies;
1313
using Newtonsoft.Json;
1414

1515
namespace Microsoft.Azure.WebJobs.Script.WebHost.ContainerManagement
@@ -19,28 +19,50 @@ public class LinuxContainerInitializationHostService : IHostedService
1919
private readonly IEnvironment _environment;
2020
private readonly IInstanceManager _instanceManager;
2121
private readonly ILogger _logger;
22+
private readonly StartupContextProvider _startupContextProvider;
2223
private CancellationToken _cancellationToken;
2324

24-
public LinuxContainerInitializationHostService(IEnvironment environment, IInstanceManager instanceManager, ILogger<LinuxContainerInitializationHostService> logger)
25+
public LinuxContainerInitializationHostService(IEnvironment environment, IInstanceManager instanceManager, ILogger<LinuxContainerInitializationHostService> logger, StartupContextProvider startupContextProvider)
2526
{
2627
_environment = environment;
2728
_instanceManager = instanceManager;
2829
_logger = logger;
30+
_startupContextProvider = startupContextProvider;
2931
}
3032

3133
public async Task StartAsync(CancellationToken cancellationToken)
3234
{
3335
_logger.LogInformation("Initializing LinuxContainerInitializationService.");
3436
_cancellationToken = cancellationToken;
3537

36-
// The service should be registered in IsLinuxContainerEnvironment only. But do additional check here.
38+
// The service should be registered in Linux Consumption only, but do additional check here.
3739
if (_environment.IsLinuxConsumption())
3840
{
39-
await ApplyContextIfPresent();
41+
await ApplyStartContextIfPresent();
4042
}
4143
}
4244

43-
private async Task ApplyContextIfPresent()
45+
private async Task ApplyStartContextIfPresent()
46+
{
47+
var startContext = await GetStartContextOrNullAsync();
48+
49+
if (!string.IsNullOrEmpty(startContext))
50+
{
51+
_logger.LogInformation("Applying host context");
52+
53+
var encryptedAssignmentContext = JsonConvert.DeserializeObject<EncryptedHostAssignmentContext>(startContext);
54+
var assignmentContext = _startupContextProvider.SetContext(encryptedAssignmentContext);
55+
56+
bool success = _instanceManager.StartAssignment(assignmentContext, false);
57+
_logger.LogInformation($"StartAssignment invoked (Success={success})");
58+
}
59+
else
60+
{
61+
_logger.LogInformation("No host context specified. Waiting for host assignment");
62+
}
63+
}
64+
65+
private async Task<string> GetStartContextOrNullAsync()
4466
{
4567
var startContext = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ContainerStartContext);
4668

@@ -61,26 +83,7 @@ private async Task ApplyContextIfPresent()
6183
_logger.LogInformation("Host context specified via CONTAINER_START_CONTEXT");
6284
}
6385

64-
if (!string.IsNullOrEmpty(startContext))
65-
{
66-
_logger.LogInformation("Applying host context");
67-
68-
var encryptedAssignmentContext = JsonConvert.DeserializeObject<EncryptedHostAssignmentContext>(startContext);
69-
var containerKey = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ContainerEncryptionKey);
70-
var assignmentContext = encryptedAssignmentContext.Decrypt(containerKey);
71-
if (_instanceManager.StartAssignment(assignmentContext, false))
72-
{
73-
_logger.LogInformation("Start assign HostAssignmentContext success");
74-
}
75-
else
76-
{
77-
_logger.LogError("Start assign HostAssignmentContext failed");
78-
}
79-
}
80-
else
81-
{
82-
_logger.LogInformation("No host context specified. Waiting for host assignment");
83-
}
86+
return startContext;
8487
}
8588

8689
private async Task<string> GetAssignmentContextFromSasUri(string sasUri)

‎src/WebJobs.Script.WebHost/Controllers/FunctionsController.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ public async Task<IActionResult> Delete(string name)
167167
return NotFound();
168168
}
169169

170-
(var deleted, var error) = _functionsManager.TryDeleteFunction(function);
170+
(var deleted, var error) = await _functionsManager.TryDeleteFunction(function);
171171

172172
if (deleted)
173173
{

‎src/WebJobs.Script.WebHost/Controllers/InstanceController.cs‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,14 @@ public class InstanceController : Controller
2222
private readonly IEnvironment _environment;
2323
private readonly IInstanceManager _instanceManager;
2424
private readonly ILogger _logger;
25+
private readonly StartupContextProvider _startupContextProvider;
2526

26-
public InstanceController(IEnvironment environment, IInstanceManager instanceManager, ILoggerFactory loggerFactory)
27+
public InstanceController(IEnvironment environment, IInstanceManager instanceManager, ILoggerFactory loggerFactory, StartupContextProvider startupContextProvider)
2728
{
2829
_environment = environment;
2930
_instanceManager = instanceManager;
3031
_logger = loggerFactory.CreateLogger<InstanceController>();
32+
_startupContextProvider = startupContextProvider;
3133
}
3234

3335
[HttpPost]
@@ -36,30 +38,36 @@ public InstanceController(IEnvironment environment, IInstanceManager instanceMan
3638
public async Task<IActionResult> Assign([FromBody] EncryptedHostAssignmentContext encryptedAssignmentContext)
3739
{
3840
_logger.LogDebug($"Starting container assignment for host : {Request?.Host}. ContextLength is: {encryptedAssignmentContext.EncryptedContext?.Length}");
39-
var containerKey = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ContainerEncryptionKey);
40-
var assignmentContext = encryptedAssignmentContext.IsWarmup
41-
? null
42-
: encryptedAssignmentContext.Decrypt(containerKey);
4341

44-
// before starting the assignment we want to perform as much
45-
// up front validation on the context as possible
46-
string error = await _instanceManager.ValidateContext(assignmentContext, encryptedAssignmentContext.IsWarmup);
47-
if (error != null)
42+
bool succeeded = false;
43+
if (!encryptedAssignmentContext.IsWarmup)
4844
{
49-
return StatusCode(StatusCodes.Status400BadRequest, error);
50-
}
45+
var assignmentContext = _startupContextProvider.SetContext(encryptedAssignmentContext);
46+
47+
// before starting the assignment we want to perform as much
48+
// up front validation on the context as possible
49+
string error = await _instanceManager.ValidateContext(assignmentContext, encryptedAssignmentContext.IsWarmup);
50+
if (error != null)
51+
{
52+
return StatusCode(StatusCodes.Status400BadRequest, error);
53+
}
54+
55+
// Wait for Sidecar specialization to complete before returning ok.
56+
// This shouldn't take too long so ok to do this sequentially.
57+
error = await _instanceManager.SpecializeMSISidecar(assignmentContext, encryptedAssignmentContext.IsWarmup);
58+
if (error != null)
59+
{
60+
return StatusCode(StatusCodes.Status500InternalServerError, error);
61+
}
5162

52-
// Wait for Sidecar specialization to complete before returning ok.
53-
// This shouldn't take too long so ok to do this sequentially.
54-
error = await _instanceManager.SpecializeMSISidecar(assignmentContext, encryptedAssignmentContext.IsWarmup);
55-
if (error != null)
63+
succeeded = _instanceManager.StartAssignment(assignmentContext, encryptedAssignmentContext.IsWarmup);
64+
}
65+
else
5666
{
57-
return StatusCode(StatusCodes.Status500InternalServerError, error);
67+
succeeded = true;
5868
}
5969

60-
var result = _instanceManager.StartAssignment(assignmentContext, encryptedAssignmentContext.IsWarmup);
61-
62-
return result || encryptedAssignmentContext.IsWarmup
70+
return succeeded
6371
? Accepted()
6472
: StatusCode(StatusCodes.Status409Conflict, "Instance already assigned");
6573
}

‎src/WebJobs.Script.WebHost/Diagnostics/FunctionInstanceLogger.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@
66
using System.Linq;
77
using System.Threading;
88
using System.Threading.Tasks;
9+
using Microsoft.Azure.Cosmos.Table;
910
using Microsoft.Azure.WebJobs.Host.Executors;
1011
using Microsoft.Azure.WebJobs.Host.Loggers;
1112
using Microsoft.Azure.WebJobs.Logging;
1213
using Microsoft.Azure.WebJobs.Script.Description;
1314
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1415
using Microsoft.Extensions.Configuration;
1516
using Microsoft.Extensions.Logging;
16-
using Microsoft.WindowsAzure.Storage;
1717

1818
namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
1919
{

‎src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,18 @@ public static async Task<JObject> ToFunctionTrigger(this FunctionMetadata functi
7777
// Read function.json as a JObject
7878
var functionConfig = await GetFunctionConfig(functionMetadataFilePath);
7979

80-
// Find the trigger and add functionName to it
81-
foreach (JObject binding in (JArray)functionConfig["bindings"])
80+
if (functionConfig.TryGetValue("bindings", out JToken value) &&
81+
value is JArray)
8282
{
83-
var type = (string)binding["type"];
84-
if (type.EndsWith("Trigger", StringComparison.OrdinalIgnoreCase))
83+
// Find the trigger and add functionName to it
84+
foreach (JObject binding in (JArray)value)
8585
{
86-
binding.Add("functionName", functionMetadata.Name);
87-
return binding;
86+
var type = (string)binding["type"];
87+
if (type != null && type.EndsWith("Trigger", StringComparison.OrdinalIgnoreCase))
88+
{
89+
binding.Add("functionName", functionMetadata.Name);
90+
return binding;
91+
}
8892
}
8993
}
9094

‎src/WebJobs.Script.WebHost/FileMonitoringService.cs‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
2323
{
2424
public class FileMonitoringService : IHostedService, IDisposable
2525
{
26+
private const int _delayedWatchersDelaySeconds = 5;
2627
private readonly ScriptJobHostOptions _scriptOptions;
2728
private readonly IScriptEventManager _eventManager;
2829
private readonly IApplicationLifetime _applicationLifetime;
@@ -101,16 +102,6 @@ public Task StopAsync(CancellationToken cancellationToken)
101102
/// </summary>
102103
private void InitializeFileWatchers()
103104
{
104-
_debugModeFileWatcher = new AutoRecoveringFileSystemWatcher(_hostLogPath, ScriptConstants.DebugSentinelFileName,
105-
includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);
106-
_debugModeFileWatcher.Changed += OnDebugModeFileChanged;
107-
_logger.LogDebug("Debug file watch initialized.");
108-
109-
_diagnosticModeFileWatcher = new AutoRecoveringFileSystemWatcher(_hostLogPath, ScriptConstants.DiagnosticSentinelFileName,
110-
includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);
111-
_diagnosticModeFileWatcher.Changed += OnDiagnosticModeFileChanged;
112-
_logger.LogDebug("Diagnostic file watch initialized.");
113-
114105
if (_scriptOptions.FileWatchingEnabled)
115106
{
116107
_fileEventSource = new FileWatcherEventSource(_eventManager, EventSources.ScriptFiles, _scriptOptions.RootScriptPath);
@@ -126,10 +117,46 @@ private void InitializeFileWatchers()
126117
.Subscribe((msg) => ScheduleRestartAsync(false)
127118
.ContinueWith(t => _logger.LogCritical(t.Exception.Message),
128119
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously)));
120+
121+
// Delay starting up for logging and debug file watchers to avoid long start up times
122+
Utility.ExecuteAfterDelay(InitializeNonBlockingFileWatchers, TimeSpan.FromSeconds(_delayedWatchersDelaySeconds));
123+
}
124+
125+
/// <summary>
126+
/// Initializes the file and directory monitoring that does not need to happen as part of a Host startup
127+
/// These watchers can be started after a delay to avoid startup performance issue
128+
/// </summary>
129+
private void InitializeNonBlockingFileWatchers()
130+
{
131+
if (_watchersStopped)
132+
{
133+
return;
134+
}
135+
136+
lock (_stopWatchersLock)
137+
{
138+
if (!_watchersStopped)
139+
{
140+
_debugModeFileWatcher = new AutoRecoveringFileSystemWatcher(_hostLogPath, ScriptConstants.DebugSentinelFileName,
141+
includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);
142+
_debugModeFileWatcher.Changed += OnDebugModeFileChanged;
143+
_logger.LogDebug("Debug file watch initialized.");
144+
145+
_diagnosticModeFileWatcher = new AutoRecoveringFileSystemWatcher(_hostLogPath, ScriptConstants.DiagnosticSentinelFileName,
146+
includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);
147+
_diagnosticModeFileWatcher.Changed += OnDiagnosticModeFileChanged;
148+
_logger.LogDebug("Diagnostic file watch initialized.");
149+
}
150+
}
129151
}
130152

131153
private void StopFileWatchers()
132154
{
155+
if (_watchersStopped)
156+
{
157+
return;
158+
}
159+
133160
lock (_stopWatchersLock)
134161
{
135162
if (_watchersStopped)

‎src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,17 @@
1111
using System.Text.RegularExpressions;
1212
using System.Threading;
1313
using System.Threading.Tasks;
14+
using Microsoft.Azure.Storage;
15+
using Microsoft.Azure.Storage.Blob;
1416
using Microsoft.Azure.WebJobs.Host.Executors;
1517
using Microsoft.Azure.WebJobs.Script.Description;
1618
using Microsoft.Azure.WebJobs.Script.Models;
1719
using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
20+
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
1821
using Microsoft.Azure.WebJobs.Script.WebHost.Security;
1922
using Microsoft.Extensions.Configuration;
2023
using Microsoft.Extensions.Logging;
2124
using Microsoft.Extensions.Options;
22-
using Microsoft.WindowsAzure.Storage;
23-
using Microsoft.WindowsAzure.Storage.Blob;
2425
using Newtonsoft.Json;
2526
using Newtonsoft.Json.Linq;
2627

@@ -301,31 +302,32 @@ public async Task<SyncTriggersPayload> GetSyncTriggersPayload()
301302
string.Compare(secretsStorageType, "files", StringComparison.OrdinalIgnoreCase) == 0 ||
302303
string.Compare(secretsStorageType, "blob", StringComparison.OrdinalIgnoreCase) == 0)
303304
{
304-
JObject secrets = new JObject();
305-
result.Add("secrets", secrets);
305+
var functionAppSecrets = new FunctionAppSecrets();
306306

307307
// add host secrets
308308
var hostSecretsInfo = await _secretManagerProvider.Current.GetHostSecretsAsync();
309-
var hostSecrets = new JObject();
310-
hostSecrets.Add("master", hostSecretsInfo.MasterKey);
311-
hostSecrets.Add("function", JObject.FromObject(hostSecretsInfo.FunctionKeys));
312-
hostSecrets.Add("system", JObject.FromObject(hostSecretsInfo.SystemKeys));
313-
secrets.Add("host", hostSecrets);
309+
functionAppSecrets.Host = new FunctionAppSecrets.HostSecrets
310+
{
311+
Master = hostSecretsInfo.MasterKey,
312+
Function = hostSecretsInfo.FunctionKeys,
313+
System = hostSecretsInfo.SystemKeys
314+
};
314315

315316
// add function secrets
316-
var functionSecrets = new JArray();
317-
var httpFunctions = functionsMetadata.Where(p => !p.IsProxy && p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name);
318-
foreach (var functionName in httpFunctions)
317+
var httpFunctions = functionsMetadata.Where(p => !p.IsProxy && p.InputBindings.Any(q => q.IsTrigger && string.Compare(q.Type, "httptrigger", StringComparison.OrdinalIgnoreCase) == 0)).Select(p => p.Name).ToArray();
318+
functionAppSecrets.Function = new FunctionAppSecrets.FunctionSecrets[httpFunctions.Length];
319+
for (int i = 0; i < httpFunctions.Length; i++)
319320
{
320-
var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(functionName);
321-
var currElement = new JObject()
321+
var currFunctionName = httpFunctions[i];
322+
var currSecrets = await _secretManagerProvider.Current.GetFunctionSecretsAsync(currFunctionName);
323+
functionAppSecrets.Function[i] = new FunctionAppSecrets.FunctionSecrets
322324
{
323-
{ "name", functionName },
324-
{ "secrets", JObject.FromObject(currSecrets) }
325+
Name = currFunctionName,
326+
Secrets = currSecrets
325327
};
326-
functionSecrets.Add(currElement);
327328
}
328-
secrets.Add("function", functionSecrets);
329+
330+
result.Add("secrets", JObject.FromObject(functionAppSecrets));
329331
}
330332
else
331333
{

0 commit comments

Comments
 (0)