Skip to content

Commit e870425

Browse files
khkh-mskhkh-ms
authored
Support RFP=1 in Linux Consumption for V3.x. (#8488)
* lin con rfp=1 * minor change * fixed an issue * logs for testings * test without metrics * change in getpackage type * cleanup. * Unit test fix. * Added unit tests * Removed the code witch is not needed. * Fix after testing. * Addressed the review comments. # Conflicts: # src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs # test/WebJobs.Script.Tests.Integration/Management/PackageDownloadHandlerTests.cs # test/WebJobs.Script.Tests.Integration/Management/RunFromPackageHandlerTests.cs * Passing FileSystem object. # Conflicts: # test/WebJobs.Script.Tests.Integration/Management/RunFromPackageHandlerTests.cs * Added unit tests # Conflicts: # test/WebJobs.Script.Tests.Integration/Management/PackageDownloadHandlerTests.cs * typos. * Addressed the review comments. * Fixed the merge issue.s * merge fix * Feedback # Conflicts: # src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs # test/WebJobs.Script.Tests.Integration/Management/PackageDownloadHandlerTests.cs # test/WebJobs.Script.Tests.Integration/Management/RunFromPackageHandlerTests.cs * rfp=1 is now Squashfs * minor fix for v3.
1 parent 493ee56 commit e870425

File tree

12 files changed

+361
-54
lines changed

12 files changed

+361
-54
lines changed

‎src/WebJobs.Script.WebHost/Management/InstanceManager.cs‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -314,7 +314,7 @@ private async Task ApplyContext(HostAssignmentContext assignmentContext)
314314
var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder);
315315
RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
316316

317-
if (_environment.SupportsAzureFileShareMount())
317+
if (_environment.SupportsAzureFileShareMount() || pkgContext.IsRunFromLocalPackage())
318318
{
319319
var azureFilesMounted = false;
320320
if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
@@ -334,13 +334,30 @@ private async Task ApplyContext(HostAssignmentContext assignmentContext)
334334
_logger.LogWarning("App is configured to use both Run-From-Package and AzureFiles. Run-From-Package will take precedence");
335335
}
336336
var blobContextApplied =
337-
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath,
337+
await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath,
338338
azureFilesMounted, false);
339339

340340
if (!blobContextApplied && azureFilesMounted)
341341
{
342-
_logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyBlobPackageContext)}. Attempting to use local disk instead");
343-
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
342+
_logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyRunFromPackageContext)}. Attempting to use local disk instead");
343+
await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, false);
344+
}
345+
}
346+
else if (pkgContext.IsRunFromLocalPackage())
347+
{
348+
if (!azureFilesMounted)
349+
{
350+
const string mountErrorMessage = "App Run-From-Package is set as '1'. AzureFiles is needed but is not configured.";
351+
_logger.LogWarning(mountErrorMessage);
352+
throw new Exception(mountErrorMessage);
353+
}
354+
355+
var blobContextApplied =
356+
await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, azureFilesMounted);
357+
358+
if (!blobContextApplied)
359+
{
360+
_logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyRunFromPackageContext)}.");
344361
}
345362
}
346363
else
@@ -352,7 +369,7 @@ await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptP
352369
{
353370
if (pkgContext.IsRunFromPackage(options, _logger))
354371
{
355-
await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
372+
await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, false);
356373
}
357374
else if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
358375
{

‎src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IPackageDownloadHandler.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) .NET Foundation. All rights reserved.
22
// Licensed under the MIT License. See License.txt in the project root for license information.
33

4+
using System.IO.Abstractions;
45
using System.Threading.Tasks;
56
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
67

‎src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IRunFromPackageHandler.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public interface IRunFromPackageHandler
1010
{
1111
Task<bool> MountAzureFileShare(HostAssignmentContext assignmentContext);
1212

13-
Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted,
13+
Task<bool> ApplyRunFromPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted,
1414
bool throwOnFailure = true);
1515
}
1616
}

‎src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33

44
using System;
55
using System.IO;
6+
using System.IO.Abstractions;
67
using System.Net;
78
using System.Net.Http;
89
using System.Net.Http.Headers;
910
using System.Threading;
1011
using System.Threading.Tasks;
12+
using DryIoc;
1113
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1214
using Microsoft.Azure.WebJobs.Script.WebHost.Models;
1315
using Microsoft.Extensions.Logging;
@@ -25,20 +27,34 @@ public class PackageDownloadHandler : IPackageDownloadHandler
2527
private readonly IBashCommandHandler _bashCommandHandler;
2628
private readonly ILogger<PackageDownloadHandler> _logger;
2729
private readonly IMetricsLogger _metricsLogger;
30+
private readonly IEnvironment _environment;
31+
private readonly IFileSystem _fileSystem;
2832

2933
public PackageDownloadHandler(HttpClient httpClient, IManagedIdentityTokenProvider managedIdentityTokenProvider,
30-
IBashCommandHandler bashCommandHandler, ILogger<PackageDownloadHandler> logger,
34+
IBashCommandHandler bashCommandHandler, IEnvironment environment, IFileSystem fileSystem, ILogger<PackageDownloadHandler> logger,
3135
IMetricsLogger metricsLogger)
3236
{
3337
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
3438
_managedIdentityTokenProvider = managedIdentityTokenProvider ?? throw new ArgumentNullException(nameof(managedIdentityTokenProvider));
3539
_bashCommandHandler = bashCommandHandler ?? throw new ArgumentNullException(nameof(bashCommandHandler));
40+
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
41+
_fileSystem = fileSystem ?? FileUtility.Instance;
3642
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
3743
_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
3844
}
3945

46+
/// <summary>
47+
/// Download the package from blob storage or fileshare.
48+
/// </summary>
49+
/// <param name="pkgContext">Package Context.</param>
50+
/// <returns>Path of the downloaded package.</returns>
4051
public async Task<string> Download(RunFromPackageContext pkgContext)
4152
{
53+
if (pkgContext.IsRunFromLocalPackage())
54+
{
55+
return CopyPackageFile(pkgContext);
56+
}
57+
4258
if (Utility.TryCleanUrl(pkgContext.Url, out var cleanedUrl))
4359
{
4460
_logger.LogDebug("Downloading app contents from '{cleanedUrl}'", cleanedUrl);
@@ -214,5 +230,59 @@ private async Task<bool> IsResourceAccessibleWithoutAuthorization(string resourc
214230
return false;
215231
}
216232
}
233+
234+
private string CopyPackageFile(RunFromPackageContext pkgContext)
235+
{
236+
var packageFolderPath = _environment.GetSitePackagesPath();
237+
Action<string> copyPackageFileFailed = (message) =>
238+
{
239+
_logger.LogWarning(message);
240+
throw new InvalidOperationException(message);
241+
};
242+
243+
if (!_fileSystem.Directory.Exists(packageFolderPath))
244+
{
245+
copyPackageFileFailed($"{ScriptConstants.SitePackagesFolderName} folder doesn't exist at {packageFolderPath}.");
246+
}
247+
248+
var packageNameTxtPath = _environment.GetSitePackageNameTxtPath();
249+
if (!_fileSystem.File.Exists(packageNameTxtPath))
250+
{
251+
copyPackageFileFailed($"{ScriptConstants.SitePackageNameTxtFileName} doesn't exist at {packageNameTxtPath}.");
252+
}
253+
254+
var packageFileName = _fileSystem.File.ReadAllText(packageNameTxtPath);
255+
256+
if (string.IsNullOrEmpty(packageFileName))
257+
{
258+
copyPackageFileFailed($"{ScriptConstants.SitePackageNameTxtFileName} is empty at {packageNameTxtPath}.");
259+
}
260+
261+
var packageFilePath = _fileSystem.Path.Combine(packageFolderPath, packageFileName);
262+
if (!_fileSystem.File.Exists(packageFilePath))
263+
{
264+
copyPackageFileFailed($"{packageFileName} doesn't exist at {packageFilePath}.");
265+
}
266+
267+
var tmpPath = _fileSystem.Path.GetTempPath();
268+
var fileName = _fileSystem.Path.GetFileName(packageFileName);
269+
var filePath = _fileSystem.Path.Combine(tmpPath, fileName);
270+
271+
var copyMetricName = pkgContext.IsWarmUpRequest
272+
? MetricEventNames.LinuxContainerSpecializationZipMountCopyWarmup
273+
: MetricEventNames.LinuxContainerSpecializationZipMountCopy;
274+
275+
using (_metricsLogger.LatencyEvent(copyMetricName))
276+
{
277+
_fileSystem.File.Copy(packageFilePath, filePath, true);
278+
279+
var fileInfo = _fileSystem.FileInfo.FromFileName(filePath);
280+
_logger.LogInformation($"Downloaded Package size is {fileInfo.Length}");
281+
}
282+
283+
_logger.LogInformation($"{nameof(CopyPackageFile)} was successful. {packageFileName} was copied from {packageFilePath} to {filePath}.");
284+
285+
return filePath;
286+
}
217287
}
218288
}

‎src/WebJobs.Script.WebHost/Management/LinuxSpecialization/RunFromPackageHandler.cs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public RunFromPackageHandler(IEnvironment environment, IMeshServiceClient meshSe
3838
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
3939
}
4040

41-
public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted, bool throwOnFailure = true)
41+
public async Task<bool> ApplyRunFromPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted, bool throwOnFailure = true)
4242
{
4343
try
4444
{
@@ -51,8 +51,10 @@ public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext
5151
EnvironmentSettingNames.DefaultLocalSitePackagesPath)
5252
: string.Empty;
5353

54-
// download zip and extract
55-
var filePath = await _packageDownloadHandler.Download(pkgContext);
54+
// download zip
55+
string filePath = await _packageDownloadHandler.Download(pkgContext);
56+
57+
// extract zip
5658
await UnpackPackage(filePath, targetPath, pkgContext, localSitePackagesPath);
5759

5860
string bundlePath = Path.Combine(targetPath, "worker-bundle");
@@ -65,7 +67,7 @@ public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext
6567
}
6668
catch (Exception e)
6769
{
68-
_logger.LogDebug(e, nameof(ApplyBlobPackageContext));
70+
_logger.LogDebug(e, nameof(ApplyRunFromPackageContext));
6971
if (throwOnFailure)
7072
{
7173
throw;
@@ -129,7 +131,7 @@ private async Task UnpackPackage(string filePath, string scriptPath, RunFromPack
129131
private CodePackageType GetPackageType(string filePath, RunFromPackageContext pkgContext)
130132
{
131133
// cloud build always builds squashfs
132-
if (pkgContext.IsScmRunFromPackage())
134+
if (pkgContext.IsScmRunFromPackage() || pkgContext.IsRunFromLocalPackage())
133135
{
134136
return CodePackageType.Squashfs;
135137
}

‎src/WebJobs.Script.WebHost/Models/RunFromPackageContext.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,22 @@ public bool IsScmRunFromPackage()
2929
return string.Equals(EnvironmentVariableName, EnvironmentSettingNames.ScmRunFromPackage, StringComparison.OrdinalIgnoreCase);
3030
}
3131

32+
public bool IsWebsiteRunFromPackage()
33+
{
34+
return string.Equals(EnvironmentVariableName, EnvironmentSettingNames.AzureWebsiteRunFromPackage, StringComparison.OrdinalIgnoreCase) ||
35+
string.Equals(EnvironmentVariableName, EnvironmentSettingNames.AzureWebsiteAltZipDeployment, StringComparison.OrdinalIgnoreCase);
36+
}
37+
3238
public bool IsRunFromPackage(ScriptApplicationHostOptions options, ILogger logger)
3339
{
3440
return (IsScmRunFromPackage() && ScmRunFromPackageBlobExists(options, logger)) || (!IsScmRunFromPackage() && !string.IsNullOrEmpty(Url) && Url != "1");
3541
}
3642

43+
public bool IsRunFromLocalPackage()
44+
{
45+
return IsWebsiteRunFromPackage() && string.Equals(Url, "1", StringComparison.OrdinalIgnoreCase);
46+
}
47+
3748
private bool ScmRunFromPackageBlobExists(ScriptApplicationHostOptions options, ILogger logger)
3849
{
3950
var blobExists = options.IsScmRunFromPackage;

‎src/WebJobs.Script/Diagnostics/MetricEventNames.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ public static class MetricEventNames
7979
public const string LinuxContainerSpecializationZipDownloadWarmup = "linux.container.specialization.zip.download.warmup";
8080
public const string LinuxContainerSpecializationZipWrite = "linux.container.specialization.zip.write";
8181
public const string LinuxContainerSpecializationZipWriteWarmup = "linux.container.specialization.zip.write.warmup";
82+
public const string LinuxContainerSpecializationZipMountCopy = "linux.container.specialization.zip.mountcopy";
83+
public const string LinuxContainerSpecializationZipMountCopyWarmup = "linux.container.specialization.zip.mountcopy.warmup";
8284
public const string LinuxContainerSpecializationZipHead = "linux.container.specialization.zip.head";
8385
public const string LinuxContainerSpecializationZipHeadWarmup = "linux.container.specialization.zip.head.warmup";
8486
public const string LinuxContainerSpecializationFuseMount = "linux.container.specialization.mount";

‎src/WebJobs.Script/Environment/EnvironmentExtensions.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,21 @@ public static bool AzureFilesAppSettingsExist(this IEnvironment environment)
103103
!string.IsNullOrEmpty(environment.GetEnvironmentVariable(AzureFilesContentShare));
104104
}
105105

106+
public static string GetAzureWebsiteHomePath(this IEnvironment environment)
107+
{
108+
return environment.GetEnvironmentVariable(AzureWebsiteHomePath);
109+
}
110+
111+
public static string GetSitePackagesPath(this IEnvironment environment)
112+
{
113+
return Path.Combine(environment.GetAzureWebsiteHomePath(), ScriptConstants.DataFolderName, ScriptConstants.SitePackagesFolderName);
114+
}
115+
116+
public static string GetSitePackageNameTxtPath(this IEnvironment environment)
117+
{
118+
return Path.Combine(environment.GetSitePackagesPath(), ScriptConstants.SitePackageNameTxtFileName);
119+
}
120+
106121
public static bool IsCoreTools(this IEnvironment environment)
107122
{
108123
return !string.IsNullOrEmpty(environment.GetEnvironmentVariable(CoreToolsEnvironment));

‎src/WebJobs.Script/ScriptConstants.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,10 @@ public static class ScriptConstants
194194
public const string HelpLinkKey = "MS_HelpLink";
195195
public const string ErrorCodeKey = "MS_ErrorCode";
196196

197+
public const string DataFolderName = "data";
198+
public const string SitePackagesFolderName = "SitePackages";
199+
public const string SitePackageNameTxtFileName = "packagename.txt";
200+
197201
public static readonly ImmutableArray<string> HttpMethods = ImmutableArray.Create("get", "post", "delete", "head", "patch", "put", "options");
198202
public static readonly ImmutableArray<string> AssemblyFileTypes = ImmutableArray.Create(".dll", ".exe");
199203
public static readonly string HostUserAgent = $"azure-functions-host/{ScriptHost.Version}";

0 commit comments

Comments
 (0)