Merge branch 'dev' into jviau/abort-channel

Author: jviau

release_notes.md

@@ -5,3 +5,5 @@
 -->
 - Memory allocation optimizations in `ScriptStartupTypeLocator.GetExtensionsStartupTypesAsync` (#11012)
 - Fix invocation timeout when incoming request contains "x-ms-invocation-id" header (#10980)
+- Warn if .azurefunctions folder does not exist (#10967)
+- Memory allocation & CPU optimizations in `GrpcMessageExtensionUtilities.ConvertFromHttpMessageToExpando` (#11054)

src/WebJobs.Script.Grpc/MessageExtensions/GrpcMessageExtensionUtilities.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Dynamic;
-using System.Linq;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
 using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
@@ -13,30 +12,58 @@ namespace Microsoft.Azure.WebJobs.Script.Grpc
 {
     internal static class GrpcMessageExtensionUtilities
     {
-        public static object ConvertFromHttpMessageToExpando(RpcHttp inputMessage)
+        private static readonly object BoxedTrue = true;
+        private static readonly object BoxedFalse = false;
+        private static readonly IReadOnlyDictionary<string, object> EmptyHeaders = new Dictionary<string, object>();
+
+        public static ExpandoObject ConvertFromHttpMessageToExpando(RpcHttp inputMessage)
         {
-            if (inputMessage == null)
+            if (inputMessage is null)
             {
                 return null;
             }
 
-            dynamic expando = new ExpandoObject();
-            expando.method = inputMessage.Method;
-            expando.query = inputMessage.Query as IDictionary<string, string>;
-            expando.statusCode = inputMessage.StatusCode;
-            expando.headers = inputMessage.Headers.ToDictionary(p => p.Key, p => (object)p.Value);
-            expando.enableContentNegotiation = inputMessage.EnableContentNegotiation;
+            var expando = new ExpandoObject();
+            IDictionary<string, object> dict = expando;
+
+            dict["method"] = inputMessage.Method;
+            dict["query"] = inputMessage.Query;
+            dict["statusCode"] = inputMessage.StatusCode;
+            dict["enableContentNegotiation"] = inputMessage.EnableContentNegotiation ? BoxedTrue : BoxedFalse;
 
-            expando.cookies = new List<Tuple<string, string, CookieOptions>>();
-            foreach (RpcHttpCookie cookie in inputMessage.Cookies)
+            if (inputMessage.Headers is { Count: > 0 })
+            {
+                var headerDict = new Dictionary<string, object>(inputMessage.Headers.Count);
+                foreach (var kvp in inputMessage.Headers)
+                {
+                    headerDict[kvp.Key] = kvp.Value;
+                }
+                dict["headers"] = headerDict;
+            }
+            else
             {
-                expando.cookies.Add(RpcHttpCookieConverter(cookie));
+                dict["headers"] = EmptyHeaders;
             }
 
-            if (inputMessage.Body != null)
+            if (inputMessage.Cookies is { Count: > 0 })
             {
-                expando.body = inputMessage.Body.ToObject();
+                var cookiesList = new List<Tuple<string, string, CookieOptions>>(inputMessage.Cookies.Count);
+                foreach (var cookie in inputMessage.Cookies)
+                {
+                    cookiesList.Add(RpcHttpCookieConverter(cookie));
+                }
+                dict["cookies"] = cookiesList;
             }
+            else
+            {
+                dict["cookies"] = Array.Empty<Tuple<string, string, CookieOptions>>();
+            }
+
+            if (inputMessage.Body is not null)
+            {
+                dict["body"] = inputMessage.Body.ToObject();
+            }
+
             return expando;
         }
 
@@ -80,27 +107,17 @@ public static Tuple<string, string, CookieOptions> RpcHttpCookieConverter(RpcHtt
 
         internal static void UpdateWorkerMetadata(this WorkerMetadata workerMetadata, RpcWorkerConfig workerConfig)
         {
-            workerMetadata.RuntimeName = string.IsNullOrEmpty(workerMetadata.RuntimeName)
-                                            ? workerConfig.Description.Language : workerMetadata.RuntimeName;
-            workerMetadata.RuntimeVersion = string.IsNullOrEmpty(workerMetadata.RuntimeVersion)
-                                            ? workerConfig.Description.DefaultRuntimeVersion : workerMetadata.RuntimeVersion;
+            workerMetadata.RuntimeName ??= workerConfig.Description.Language;
+            workerMetadata.RuntimeVersion ??= workerConfig.Description.DefaultRuntimeVersion;
         }
 
-        private static SameSiteMode RpcSameSiteEnumConverter(RpcHttpCookie.Types.SameSite sameSite)
+        private static SameSiteMode RpcSameSiteEnumConverter(RpcHttpCookie.Types.SameSite sameSite) => sameSite switch
         {
-            switch (sameSite)
-            {
-                case RpcHttpCookie.Types.SameSite.Strict:
-                    return SameSiteMode.Strict;
-                case RpcHttpCookie.Types.SameSite.Lax:
-                    return SameSiteMode.Lax;
-                case RpcHttpCookie.Types.SameSite.None:
-                    return SameSiteMode.Unspecified;
-                case RpcHttpCookie.Types.SameSite.ExplicitNone:
-                    return SameSiteMode.None;
-                default:
-                    return SameSiteMode.Unspecified;
-            }
-        }
+            RpcHttpCookie.Types.SameSite.Strict => SameSiteMode.Strict,
+            RpcHttpCookie.Types.SameSite.Lax => SameSiteMode.Lax,
+            RpcHttpCookie.Types.SameSite.None => SameSiteMode.Unspecified,
+            RpcHttpCookie.Types.SameSite.ExplicitNone => SameSiteMode.None,
+            _ => SameSiteMode.Unspecified
+        };
     }
-}
+}

src/WebJobs.Script.WebHost/DependencyInjection/DependencyValidator/DependencyValidator.cs

@@ -9,6 +9,7 @@
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.FileProvisioning;
+using Microsoft.Azure.WebJobs.Script.Host;
 using Microsoft.Azure.WebJobs.Script.Scale;
 using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Workers;
@@ -48,6 +49,7 @@ private static ExpectedDependencyBuilder CreateExpectedDependencies()
                 .Expect<WorkerConsoleLogService>()
                 .Expect<FunctionInvocationDispatcherShutdownManager>()
                 .Expect<WorkerConcurrencyManager>()
+                .Optional<FunctionAppValidationService>() // Conditionally registered.
                 .Optional<FuncAppFileProvisioningService>() // Used by powershell.
                 .Optional<JobHostService>() // Missing when host is offline.
                 .Optional<FunctionsSyncService>() // Conditionally registered.

src/WebJobs.Script/DependencyInjection/ScriptStartupTypeLocator.cs

@@ -110,7 +110,7 @@ public async Task<IEnumerable<Type>> GetExtensionsStartupTypesAsync()
                 }
             }
 
-            bool isDotnetIsolatedApp = IsDotnetIsolatedApp(functionMetadataCollection);
+            bool isDotnetIsolatedApp = Utility.IsDotnetIsolatedApp(_environment, functionMetadataCollection);
             bool isDotnetApp = isPrecompiledFunctionApp || isDotnetIsolatedApp;
             var isLogicApp = _environment.IsLogicApp();
 
@@ -348,12 +348,6 @@ void CollectError(Type extensionType, Version minimumVersion, ExtensionStartupTy
             }
         }
 
-        private bool IsDotnetIsolatedApp(IEnumerable<FunctionMetadata> functions)
-        {
-            string workerRuntime = Utility.GetWorkerRuntime(functions, _environment);
-            return workerRuntime?.Equals(RpcWorkerConstants.DotNetIsolatedLanguageWorkerName, StringComparison.OrdinalIgnoreCase) ?? false;
-        }
-
         private ExtensionRequirementsInfo GetExtensionRequirementsInfo()
         {
             ExtensionRequirementsInfo requirementsInfo = _extensionRequirementOptions.Value.Bundles != null || _extensionRequirementOptions.Value.Extensions != null

src/WebJobs.Script/Diagnostics/Extensions/LoggerExtension.cs

@@ -184,14 +184,24 @@ internal static class LoggerExtension
             "Referenced bundle {bundleId} of version {bundleVersion} does not meet the required minimum version of {minimumVersion}. Update your extension bundle reference in host.json to reference {minimumVersion2} or later.");
 
         private static readonly Action<ILogger, string, Exception> _hostJsonZipDeploymentIssue =
-    LoggerMessage.Define<string>(LogLevel.Error,
-        new EventId(338, nameof(HostJsonZipDeploymentIssue)),
-        "No functions were found. A valid host.json file wasn't found in the package root. However, one was located at: {hostJsonFilesPath}. This state indicates that your deployment package was created incorrectly. For deployment package requirements, see https://aka.ms/deployment-zip-push.");
+            LoggerMessage.Define<string>(LogLevel.Error,
+            new EventId(338, nameof(HostJsonZipDeploymentIssue)),
+            "No functions were found. A valid host.json file wasn't found in the package root. However, one was located at: {hostJsonFilesPath}. This state indicates that your deployment package was created incorrectly. For deployment package requirements, see https://aka.ms/deployment-zip-push.");
 
         private static readonly Action<ILogger, Exception> _noHostJsonFile =
-    LoggerMessage.Define(LogLevel.Information,
-        new EventId(339, nameof(NoHostJsonFile)),
-        "No functions were found. This can occur before you deploy code to your function app or when the host.json file is missing from the most recent deployment. Make sure that your deployment package includes the host.json file in the root of the package. For deployment package requirements, see https://aka.ms/functions-deployment-technologies.");
+            LoggerMessage.Define(LogLevel.Information,
+            new EventId(339, nameof(NoHostJsonFile)),
+            "No functions were found. This can occur before you deploy code to your function app or when the host.json file is missing from the most recent deployment. Make sure that your deployment package includes the host.json file in the root of the package. For deployment package requirements, see https://aka.ms/functions-deployment-technologies.");
+
+        private static readonly Action<ILogger, string, Exception> _missingAzureFunctionsFolder =
+            LoggerMessage.Define<string>(LogLevel.Warning,
+            new EventId(340, nameof(MissingAzureFunctionsFolder)),
+            "Could not find the .azurefunctions folder in the deployed artifacts of a .NET isolated function app. Make sure that your deployment package includes the .azurefunctions folder at the root of the package. For deployment package requirements, see https://aka.ms/functions-deployment-technologies. If this is not intended to be a .NET isolated app, please ensure that the {functionWorkerRuntime} app setting is configured correctly.");
+
+        private static readonly Action<ILogger, string, string, Exception> _incorrectAzureFunctionsFolderPath =
+            LoggerMessage.Define<string, string>(LogLevel.Warning,
+            new EventId(341, nameof(IncorrectAzureFunctionsFolderPath)),
+            "Could not find the .azurefunctions folder in the deployed artifacts of a .NET isolated function app. However, it is found to be located at: {path}. Make sure that your deployment package includes the .azurefunctions folder at the root of the package. For deployment package requirements, see https://aka.ms/functions-deployment-technologies. If this is not intended to be a .NET isolated app, please ensure that the {functionWorkerRuntime} app setting is configured correctly.");
 
         private static readonly Action<ILogger, string, Exception> _publishingMetrics =
             LoggerMessage.Define<string>(LogLevel.Debug, new EventId(338, nameof(PublishingMetrics)), "{metrics}");
@@ -387,5 +397,15 @@ public static void NoHostJsonFile(this ILogger logger)
         {
             _noHostJsonFile(logger, null);
         }
+
+        public static void MissingAzureFunctionsFolder(this ILogger logger)
+        {
+            _missingAzureFunctionsFolder(logger, EnvironmentSettingNames.FunctionWorkerRuntime, null);
+        }
+
+        public static void IncorrectAzureFunctionsFolderPath(this ILogger logger, string path)
+        {
+            _incorrectAzureFunctionsFolderPath(logger, path, EnvironmentSettingNames.FunctionWorkerRuntime, null);
+        }
     }
 }

src/WebJobs.Script/Host/FunctionAppValidationService.cs

@@ -0,0 +1,79 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Azure.WebJobs.Script.Host
+{
+    /// <summary>
+    /// A background service responsible for validating function app payload.
+    /// </summary>
+    internal sealed class FunctionAppValidationService : BackgroundService
+    {
+        private readonly IEnvironment _environment;
+        private readonly ILogger<FunctionAppValidationService> _logger;
+        private readonly IOptions<ScriptJobHostOptions> _scriptOptions;
+
+        public FunctionAppValidationService(
+            ILogger<FunctionAppValidationService> logger,
+            IOptions<ScriptJobHostOptions> scriptOptions,
+            IEnvironment environment)
+        {
+            _scriptOptions = scriptOptions ?? throw new ArgumentNullException(nameof(scriptOptions));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+            _environment = environment ?? throw new ArgumentNullException(nameof(environment));
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken cancellationToken)
+        {
+            if (!_scriptOptions.Value.IsStandbyConfiguration)
+            {
+                // Adding a delay to ensure that this validation does not impact the cold start performance
+                Utility.ExecuteAfterColdStartDelay(_environment, Validate, cancellationToken);
+            }
+
+            await Task.CompletedTask;
+        }
+
+        private void Validate()
+        {
+            try
+            {
+                string azureFunctionsDirPath = Path.Combine(_scriptOptions.Value.RootScriptPath, ScriptConstants.AzureFunctionsSystemDirectoryName);
+
+                if (_scriptOptions.Value.RootScriptPath is not null &&
+                    !_scriptOptions.Value.IsDefaultHostConfig &&
+                    Utility.IsDotnetIsolatedApp(environment: _environment) &&
+                    !Directory.Exists(azureFunctionsDirPath))
+                {
+                    // Search for the .azurefunctions directory within nested directories to verify scenarios where it isn't located at the root. This situation occurs when a function app has been improperly zipped.
+                    IEnumerable<string> azureFunctionsDirectories = Directory.GetDirectories(_scriptOptions.Value.RootScriptPath, ScriptConstants.AzureFunctionsSystemDirectoryName, SearchOption.AllDirectories)
+                        .Where(dir => !dir.Equals(azureFunctionsDirPath, StringComparison.OrdinalIgnoreCase));
+
+                    if (azureFunctionsDirectories.Any())
+                    {
+                        string azureFunctionsDirectoriesPath = string.Join(", ", azureFunctionsDirectories).Replace(_scriptOptions.Value.RootScriptPath, string.Empty);
+                        _logger.IncorrectAzureFunctionsFolderPath(azureFunctionsDirectoriesPath);
+                    }
+                    else
+                    {
+                        _logger.MissingAzureFunctionsFolder();
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogTrace("Unable to validate deployed function app payload", ex);
+            }
+        }
+    }
+}

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -31,6 +31,7 @@
 using Microsoft.Azure.WebJobs.Script.Extensibility;
 using Microsoft.Azure.WebJobs.Script.ExtensionBundle;
 using Microsoft.Azure.WebJobs.Script.FileProvisioning;
+using Microsoft.Azure.WebJobs.Script.Host;
 using Microsoft.Azure.WebJobs.Script.Http;
 using Microsoft.Azure.WebJobs.Script.ManagedDependencies;
 using Microsoft.Azure.WebJobs.Script.Scale;
@@ -182,6 +183,11 @@ public static IHostBuilder AddScriptHostCore(this IHostBuilder builder, ScriptAp
 
             builder.ConfigureServices((context, services) =>
             {
+                if (!SystemEnvironment.Instance.IsPlaceholderModeEnabled())
+                {
+                    services.AddHostedService<FunctionAppValidationService>();
+                }
+
                 services.AddSingleton<ExternalConfigurationStartupValidator>();
                 services.AddSingleton<IHostedService>(s =>
                 {

src/WebJobs.Script/Utility.cs

@@ -671,6 +671,12 @@ internal static bool IsSingleLanguage(IEnumerable<FunctionMetadata> functions, s
             return ContainsFunctionWithWorkerRuntime(filteredFunctions, workerRuntime);
         }
 
+        internal static bool IsDotnetIsolatedApp(IEnvironment environment, IEnumerable<FunctionMetadata> functions = null)
+        {
+            string workerRuntime = GetWorkerRuntime(functions, environment);
+            return workerRuntime?.Equals(RpcWorkerConstants.DotNetIsolatedLanguageWorkerName, StringComparison.OrdinalIgnoreCase) ?? false;
+        }
+
         internal static string GetWorkerRuntime(IEnumerable<FunctionMetadata> functions, IEnvironment environment = null)
         {
             if (environment != null)

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SpecializationE2ETests.cs

@@ -873,6 +873,46 @@ public async Task ResponseCompressionWorksAfterSpecialization(string acceptEncod
             Assert.Equal(expectedContentEncodingResponseHeaderValue, value?.First());
         }
 
+        [Fact]
+        public async Task Specialization_DotnetIsolatedApp_MissingAzureFunctionsDir_Logs()
+        {
+            Guid guid = Guid.NewGuid();
+            string path = "test-path" + guid.ToString();
+
+            if (!Directory.Exists(path))
+            {
+                Directory.CreateDirectory(path);
+            }
+
+            string json = "{\r\n  \"version\": \"2.0\",\r\n  \"isDefaultHostConfig\": false\r\n}";
+            File.WriteAllText(Path.Combine(path, "host.json"), json);
+
+            var builder = InitializeDotNetIsolatedPlaceholderBuilder(path);
+
+            using var testServer = new TestServer(builder);
+
+            var standbyManager = testServer.Services.GetService<IStandbyManager>();
+            Assert.NotNull(standbyManager);
+
+            _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteContainerReady, "1");
+            _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "dotnet-isolated");
+            SystemEnvironment.Instance.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
+            _environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsitePlaceholderMode, "0");
+
+            await standbyManager.SpecializeHostAsync();
+
+            // Assert: Verify that the host has specialized
+            var scriptHostManager = testServer.Services.GetService<IScriptHostManager>();
+            Assert.NotNull(scriptHostManager);
+            Assert.Equal(ScriptHostState.Running, scriptHostManager.State);
+
+            await TestHelpers.Await(() =>
+            {
+                int completed = _loggerProvider.GetAllLogMessages().Count(p => p.FormattedMessage.Contains("Could not find the .azurefunctions folder in the deployed artifacts of a .NET isolated function app."));
+                return completed > 0;
+            });
+        }
+
         [Fact]
         public async Task DotNetIsolated_PlaceholderHit_WithProxies()
         {