Skip to content

Commit 021e7ae

Browse files
mathewcmathewc
committed
Implementing Linux Container specialization
1 parent 034b812 commit 021e7ae

File tree

11 files changed

+229
-41
lines changed

11 files changed

+229
-41
lines changed

‎src/WebJobs.Script.WebHost/Management/InstanceManager.cs‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.IO.Compression;
88
using System.Net.Http;
99
using System.Threading.Tasks;
10+
using Microsoft.Azure.WebJobs.Script.Config;
1011
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
1112
using Microsoft.Extensions.Logging;
1213

@@ -19,17 +20,24 @@ public class InstanceManager : IInstanceManager
1920

2021
private readonly WebHostSettings _webHostSettings;
2122
private readonly ILogger _logger;
23+
private readonly ScriptSettingsManager _settingsManager;
2224
private readonly HttpClient _client;
2325

24-
public InstanceManager(WebHostSettings webHostSettings, ILoggerFactory loggerFactory, HttpClient client)
26+
public InstanceManager(ScriptSettingsManager settingsManager, WebHostSettings webHostSettings, ILoggerFactory loggerFactory, HttpClient client)
2527
{
28+
_settingsManager = settingsManager;
2629
_webHostSettings = webHostSettings;
2730
_logger = loggerFactory.CreateLogger(nameof(InstanceManager));
2831
_client = client;
2932
}
3033

3134
public bool StartAssignment(HostAssignmentContext assignmentContext)
3235
{
36+
if (!WebScriptHostManager.InStandbyMode)
37+
{
38+
return false;
39+
}
40+
3341
if (_assignmentContext == null)
3442
{
3543
lock (_assignmentLock)
@@ -57,11 +65,10 @@ private async Task Specialize(HostAssignmentContext assignmentContext)
5765
{
5866
try
5967
{
60-
// download zip
6168
var zip = assignmentContext.ZipUrl;
62-
6369
if (!string.IsNullOrEmpty(zip))
6470
{
71+
// download zip and extract
6572
var filePath = Path.GetTempFileName();
6673

6774
await DownloadAsync(new Uri(zip), filePath);
@@ -75,7 +82,9 @@ private async Task Specialize(HostAssignmentContext assignmentContext)
7582
assignmentContext.ApplyAppSettings();
7683
}
7784

78-
// TODO: specialize
85+
// set flags which will trigger specialization
86+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
87+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
7988
}
8089
catch (Exception ex)
8190
{

‎src/WebJobs.Script.WebHost/Models/EncryptedHostAssignmentContext.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,20 @@ public class EncryptedHostAssignmentContext
1212
[JsonProperty("encryptedContext")]
1313
public string EncryptedContext { get; set; }
1414

15+
public static EncryptedHostAssignmentContext Create(HostAssignmentContext context, string key)
16+
{
17+
string json = JsonConvert.SerializeObject(context);
18+
var encryptionKey = Convert.FromBase64String(key);
19+
string encrypted = SimpleWebTokenHelper.Encrypt(json, encryptionKey);
20+
21+
return new EncryptedHostAssignmentContext { EncryptedContext = encrypted };
22+
}
23+
1524
public HostAssignmentContext Decrypt(string key)
1625
{
1726
var encryptionKey = Convert.FromBase64String(key);
1827
var decrypted = SimpleWebTokenHelper.Decrypt(encryptionKey, EncryptedContext);
28+
1929
return JsonConvert.DeserializeObject<HostAssignmentContext>(decrypted);
2030
}
2131
}

‎src/WebJobs.Script.WebHost/Security/SimpleWebTokenHelper.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ public static class SimpleWebTokenHelper
2121
/// <returns>a SWT signed by this app</returns>
2222
public static string CreateToken(DateTime validUntil) => Encrypt($"exp={validUntil.Ticks}");
2323

24-
internal static string Encrypt(string value)
24+
internal static string Encrypt(string value, byte[] key = null)
2525
{
26-
using (var aes = new AesManaged { Key = GetWebSiteAuthEncryptionKey() })
26+
key = key ?? GetWebSiteAuthEncryptionKey();
27+
28+
using (var aes = new AesManaged { Key = key })
2729
{
2830
// IV is always generated for the key every time
2931
aes.GenerateIV();

‎src/WebJobs.Script.WebHost/StandbyManager.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,8 @@ public static async Task<HttpResponseMessage> WarmUp(HttpRequest request, WebScr
3636

3737
public static bool IsWarmUpRequest(HttpRequest request)
3838
{
39-
return ScriptSettingsManager.Instance.IsAppServiceEnvironment &&
40-
WebScriptHostManager.InStandbyMode &&
41-
request.IsAntaresInternalRequest() &&
39+
return WebScriptHostManager.InStandbyMode &&
40+
((ScriptSettingsManager.Instance.IsAppServiceEnvironment && request.IsAntaresInternalRequest()) || ScriptSettingsManager.Instance.IsLinuxContainerEnvironment) &&
4241
(request.Path.StartsWithSegments(new PathString($"/api/{WarmUpFunctionName}")) ||
4342
request.Path.StartsWithSegments(new PathString($"/api/{WarmUpAlternateRoute}")));
4443
}

‎src/WebJobs.Script.WebHost/Startup.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System;
55
using Microsoft.AspNetCore.Builder;
66
using Microsoft.AspNetCore.Hosting;
7+
using Microsoft.Azure.WebJobs.Script.Config;
78
using Microsoft.Extensions.Configuration;
89
using Microsoft.Extensions.DependencyInjection;
910
using Microsoft.Extensions.Logging;
@@ -15,6 +16,12 @@ public class Startup
1516
public Startup(IConfiguration configuration)
1617
{
1718
Configuration = configuration;
19+
20+
if (ScriptSettingsManager.Instance.IsLinuxContainerEnvironment)
21+
{
22+
// Linux containers always start out in placeholder mode
23+
ScriptSettingsManager.Instance.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
24+
}
1825
}
1926

2027
public IConfiguration Configuration { get; }

‎test/WebJobs.Script.Tests.Integration/Host/StandbyManagerTests.cs‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,18 @@
77
using System.Linq;
88
using System.Net;
99
using System.Net.Http;
10+
using System.Text;
1011
using System.Threading.Tasks;
1112
using System.Web.Http;
1213
using Microsoft.AspNetCore.TestHost;
1314
using Microsoft.Azure.WebJobs.Script.Config;
1415
using Microsoft.Azure.WebJobs.Script.WebHost;
16+
using Microsoft.Azure.WebJobs.Script.WebHost.Authentication;
17+
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
1518
using Microsoft.Extensions.DependencyInjection;
1619
using Microsoft.Extensions.Logging;
1720
using Microsoft.WebJobs.Script.Tests;
21+
using Newtonsoft.Json;
1822
using Xunit;
1923

2024
namespace Microsoft.Azure.WebJobs.Script.Tests
@@ -42,6 +46,10 @@ public void IsWarmUpRequest_ReturnsExpectedValue()
4246
};
4347
using (var env = new TestScopedEnvironmentVariable(vars))
4448
{
49+
// in this test we're forcing a transition from non-placeholder mode to placeholder mode
50+
// which can't happen in the wild, so we force a reset here
51+
WebScriptHostManager.ResetStandbyMode();
52+
4553
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
4654
Assert.False(StandbyManager.IsWarmUpRequest(request));
4755

@@ -58,6 +66,24 @@ public void IsWarmUpRequest_ReturnsExpectedValue()
5866
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/foo");
5967
Assert.False(StandbyManager.IsWarmUpRequest(request));
6068
}
69+
70+
vars = new Dictionary<string, string>
71+
{
72+
{ EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0" },
73+
{ EnvironmentSettingNames.AzureWebsiteInstanceId, null }
74+
};
75+
using (var env = new TestScopedEnvironmentVariable(vars))
76+
{
77+
WebScriptHostManager.ResetStandbyMode();
78+
79+
_settingsManager.SetSetting(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "1");
80+
Assert.False(StandbyManager.IsWarmUpRequest(request));
81+
82+
request = HttpTestHelpers.CreateHttpRequest("POST", "http://azure.com/api/warmup");
83+
_settingsManager.SetSetting(EnvironmentSettingNames.ContainerName, "TestContainer");
84+
Assert.True(_settingsManager.IsLinuxContainerEnvironment);
85+
Assert.True(StandbyManager.IsWarmUpRequest(request));
86+
}
6187
}
6288

6389
[Fact]
@@ -161,6 +187,123 @@ await TestHelpers.Await(() =>
161187
}
162188
}
163189

190+
[Fact]
191+
public async Task StandbyMode_EndToEnd_LinuxContainer()
192+
{
193+
byte[] bytes = TestHelpers.GenerateKeyBytes();
194+
var encryptionKey = Convert.ToBase64String(bytes);
195+
196+
var vars = new Dictionary<string, string>
197+
{
198+
{ EnvironmentSettingNames.ContainerName, "TestContainer" },
199+
{ EnvironmentSettingNames.ContainerEncryptionKey, encryptionKey },
200+
{ EnvironmentSettingNames.AzureWebsiteContainerReady, null },
201+
{ "AzureWebEncryptionKey", "0F75CA46E7EBDD39E4CA6B074D1F9A5972B849A55F91A248" }
202+
};
203+
using (var env = new TestScopedEnvironmentVariable(vars))
204+
{
205+
var httpConfig = new HttpConfiguration();
206+
207+
var testRootPath = Path.Combine(Path.GetTempPath(), "StandbyModeTest_Linux");
208+
await FileUtility.DeleteDirectoryAsync(testRootPath, true);
209+
210+
var loggerProvider = new TestLoggerProvider();
211+
var loggerProviderFactory = new TestLoggerProviderFactory(loggerProvider);
212+
var webHostSettings = new WebHostSettings
213+
{
214+
IsSelfHost = true,
215+
LogPath = Path.Combine(testRootPath, "Logs"),
216+
SecretsPath = Path.Combine(testRootPath, "Secrets"),
217+
ScriptPath = Path.Combine(testRootPath, "WWWRoot")
218+
};
219+
220+
var loggerFactory = new LoggerFactory();
221+
loggerFactory.AddProvider(loggerProvider);
222+
223+
var webHostBuilder = Program.CreateWebHostBuilder()
224+
.ConfigureServices(c =>
225+
{
226+
c.AddSingleton(webHostSettings)
227+
.AddSingleton<ILoggerProviderFactory>(loggerProviderFactory)
228+
.AddSingleton<ILoggerFactory>(loggerFactory);
229+
});
230+
231+
var httpServer = new TestServer(webHostBuilder);
232+
var httpClient = httpServer.CreateClient();
233+
httpClient.BaseAddress = new Uri("https://localhost/");
234+
235+
TestHelpers.WaitForWebHost(httpClient);
236+
237+
var traces = loggerProvider.GetAllLogMessages().ToArray();
238+
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Starting Host (HostId=placeholder-host")));
239+
Assert.NotNull(traces.Single(p => p.FormattedMessage.StartsWith("Host is in standby mode")));
240+
241+
// issue warmup request and verify
242+
var request = new HttpRequestMessage(HttpMethod.Get, "api/warmup");
243+
var response = await httpClient.SendAsync(request);
244+
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
245+
string responseBody = await response.Content.ReadAsStringAsync();
246+
Assert.Equal("WarmUp complete.", responseBody);
247+
248+
// issue warmup request with restart and verify
249+
request = new HttpRequestMessage(HttpMethod.Get, "api/warmup?restart=1");
250+
response = await httpClient.SendAsync(request);
251+
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
252+
responseBody = await response.Content.ReadAsStringAsync();
253+
Assert.Equal("WarmUp complete.", responseBody);
254+
255+
// Now specialize the host by invoking assign
256+
var secretManager = httpServer.Host.Services.GetService<ISecretManager>();
257+
var masterKey = (await secretManager.GetHostSecretsAsync()).MasterKey;
258+
string uri = "admin/instance/assign";
259+
request = new HttpRequestMessage(HttpMethod.Post, uri);
260+
var environment = new Dictionary<string, string>();
261+
var assignmentContext = new HostAssignmentContext
262+
{
263+
SiteId = 1234,
264+
SiteName = "TestSite",
265+
Environment = environment
266+
};
267+
var encryptedAssignmentContext = EncryptedHostAssignmentContext.Create(assignmentContext, encryptionKey);
268+
string json = JsonConvert.SerializeObject(encryptedAssignmentContext);
269+
request.Content = new StringContent(json, Encoding.UTF8, "application/json");
270+
request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, masterKey);
271+
response = await httpClient.SendAsync(request);
272+
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
273+
274+
// give time for the specialization to happen
275+
string[] logLines = null;
276+
await TestHelpers.Await(() =>
277+
{
278+
// wait for the trace indicating that the host has been specialized
279+
logLines = loggerProvider.GetAllLogMessages().Where(p => p.FormattedMessage != null).Select(p => p.FormattedMessage).ToArray();
280+
return logLines.Contains("Generating 0 job function(s)");
281+
}, userMessageCallback: () => string.Join(Environment.NewLine, loggerProvider.GetAllLogMessages().Select(p => $"[{p.Timestamp.ToString("HH:mm:ss.fff")}] {p.FormattedMessage}")));
282+
283+
httpServer.Dispose();
284+
httpClient.Dispose();
285+
286+
await Task.Delay(2000);
287+
288+
var hostConfig = WebHostResolver.CreateScriptHostConfiguration(webHostSettings, true);
289+
var expectedHostId = hostConfig.HostConfig.HostId;
290+
291+
// verify the rest of the expected logs
292+
string text = string.Join(Environment.NewLine, logLines);
293+
Assert.True(logLines.Count(p => p.Contains("Stopping Host")) >= 1);
294+
Assert.Equal(1, logLines.Count(p => p.Contains("Creating StandbyMode placeholder function directory")));
295+
Assert.Equal(1, logLines.Count(p => p.Contains("StandbyMode placeholder function directory created")));
296+
Assert.Equal(2, logLines.Count(p => p.Contains("Starting Host (HostId=placeholder-host")));
297+
Assert.Equal(2, logLines.Count(p => p.Contains("Host is in standby mode")));
298+
Assert.Equal(2, logLines.Count(p => p.Contains("Executed 'Functions.WarmUp' (Succeeded")));
299+
Assert.Equal(1, logLines.Count(p => p.Contains("Starting host specialization")));
300+
Assert.Equal(1, logLines.Count(p => p.Contains($"Starting Host (HostId={expectedHostId}")));
301+
Assert.Contains("Generating 0 job function(s)", logLines);
302+
303+
WebScriptHostManager.ResetStandbyMode();
304+
}
305+
}
306+
164307
public void Dispose()
165308
{
166309
}

‎test/WebJobs.Script.Tests.Shared/TestHelpers.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Linq;
99
using System.Net;
1010
using System.Net.Http;
11+
using System.Security.Cryptography;
1112
using System.Text;
1213
using System.Threading.Tasks;
1314
using Microsoft.WindowsAzure.Storage.Blob;
@@ -31,6 +32,20 @@ public static string FunctionsTestDirectory
3132
}
3233
}
3334

35+
public static byte[] GenerateKeyBytes()
36+
{
37+
using (var aes = new AesManaged())
38+
{
39+
aes.GenerateKey();
40+
return aes.Key;
41+
}
42+
}
43+
44+
public static string GenerateKeyHexString(byte[] key = null)
45+
{
46+
return BitConverter.ToString(key ?? GenerateKeyBytes()).Replace("-", string.Empty);
47+
}
48+
3449
public static string NewRandomString(int length = 10)
3550
{
3651
return new string(

‎test/WebJobs.Script.Tests/Helpers/SimpleWebTokenTests.cs‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ public void EncryptShouldThrowIdNoEncryptionKeyDefined()
3131
[InlineData("value")]
3232
public void EncryptShouldGenerateDecryptableValues(string valueToEncrypt)
3333
{
34-
var key = GenerateBytesKey();
35-
var stringKey = GenerateKeyHexString(key);
34+
var key = TestHelpers.GenerateKeyBytes();
35+
var stringKey = TestHelpers.GenerateKeyHexString(key);
3636
Environment.SetEnvironmentVariable("WEBSITE_AUTH_ENCRYPTION_KEY", stringKey);
3737

3838
var encrypted = SimpleWebTokenHelper.Encrypt(valueToEncrypt);
@@ -45,8 +45,8 @@ public void EncryptShouldGenerateDecryptableValues(string valueToEncrypt)
4545
[Fact]
4646
public void CreateTokenShouldCreateAValidToken()
4747
{
48-
var key = GenerateBytesKey();
49-
var stringKey = GenerateKeyHexString(key);
48+
var key = TestHelpers.GenerateKeyBytes();
49+
var stringKey = TestHelpers.GenerateKeyHexString(key);
5050
var timeStamp = DateTime.UtcNow;
5151
Environment.SetEnvironmentVariable("WEBSITE_AUTH_ENCRYPTION_KEY", stringKey);
5252

@@ -56,20 +56,6 @@ public void CreateTokenShouldCreateAValidToken()
5656
Assert.Equal($"exp={timeStamp.Ticks}", decrypted);
5757
}
5858

59-
public static byte[] GenerateBytesKey()
60-
{
61-
using (var aes = new AesManaged())
62-
{
63-
aes.GenerateKey();
64-
return aes.Key;
65-
}
66-
}
67-
68-
public static string GenerateKeyHexString(byte[] key = null)
69-
{
70-
return BitConverter.ToString(key ?? GenerateBytesKey()).Replace("-", string.Empty);
71-
}
72-
7359
public void Dispose()
7460
{
7561
// Clean up

0 commit comments

Comments
 (0)