Increase default grpc max message size (#2651)

* Increase default grpc max message size to 32MB for dynamic sku, 128MB for dedicated

Author: pragnagopa

schemas/json/host.json

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Grpc.Core;
@@ -15,9 +16,13 @@ public class GrpcServer : IRpcServer, IDisposable
         private Server _server;
         private bool _disposed = false;
 
-        public GrpcServer(FunctionRpc.FunctionRpcBase serviceImpl)
+        public GrpcServer(FunctionRpc.FunctionRpcBase serviceImpl, int grpcMaxMessageLength)
         {
-            _server = new Server
+            ChannelOption maxReceiveMessageLength = new ChannelOption(ChannelOptions.MaxReceiveMessageLength, grpcMaxMessageLength);
+            ChannelOption maxSendMessageLength = new ChannelOption(ChannelOptions.MaxSendMessageLength, grpcMaxMessageLength);
+            ChannelOption[] grpcChannelOptions = { maxReceiveMessageLength, maxSendMessageLength };
+
+            _server = new Server(grpcChannelOptions)
             {
                 Services = { FunctionRpc.BindService(serviceImpl) },
                 Ports = { new ServerPort("127.0.0.1", ServerPort.PickUnused, ServerCredentials.Insecure) }

src/WebJobs.Script.Grpc/Server/GrpcServer.cs

@@ -86,6 +86,11 @@ public ScriptHostConfiguration()
         /// </summary>
         public TimeSpan? FunctionTimeout { get; set; }
 
+        /// <summary>
+        /// Gets or sets a value for grpc_max_message_length.
+        /// </summary>
+        public int MaxMessageLengthBytes { get; set; }
+
         /// <summary>
         /// Gets or sets a value indicating whether the host is running
         /// outside of the normal Azure hosting environment. E.g. when running

src/WebJobs.Script/Config/ScriptHostConfiguration.cs

@@ -72,6 +72,8 @@ public class ScriptHost : JobHost
         private static readonly Regex FunctionNameValidationRegex = new Regex(@"^[a-z][a-z0-9_\-]{0,127}$(?<!^host$)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
         private static readonly Regex ProxyNameValidationRegex = new Regex(@"[^a-zA-Z0-9_-]", RegexOptions.Compiled | RegexOptions.IgnoreCase);
         public static readonly string Version = GetAssemblyFileVersion(typeof(ScriptHost).Assembly);
+        internal static readonly int DefaultMaxMessageLengthBytesDynamicSku = 32 * 1024 * 1024;
+        internal static readonly int DefaultMaxMessageLengthBytes = 128 * 1024 * 1024;
         private ScriptSettingsManager _settingsManager;
         private bool _shutdownScheduled;
         private ILogger _startupLogger;
@@ -498,16 +500,22 @@ internal void InitializeFunctionDescriptors(Collection<FunctionMetadata> functio
             _descriptorProviders = new List<FunctionDescriptorProvider>();
             if (string.IsNullOrEmpty(language))
             {
+                _startupLogger.LogTrace("Adding all the Function descriptors.");
                 _descriptorProviders.Add(new DotNetFunctionDescriptorProvider(this, ScriptConfig));
                 _descriptorProviders.Add(new WorkerFunctionDescriptorProvider(this, ScriptConfig, _functionDispatcher));
             }
-            else if (language.Equals(ScriptConstants.DotNetLanguageWorkerName, StringComparison.OrdinalIgnoreCase))
-            {
-                _descriptorProviders.Add(new DotNetFunctionDescriptorProvider(this, ScriptConfig));
-            }
             else
             {
-                _descriptorProviders.Add(new WorkerFunctionDescriptorProvider(this, ScriptConfig, _functionDispatcher));
+                _startupLogger.LogTrace($"Adding Function descriptor for language {language}.");
+                switch (language.ToLower())
+                {
+                    case ScriptConstants.DotNetLanguageWorkerName:
+                        _descriptorProviders.Add(new DotNetFunctionDescriptorProvider(this, ScriptConfig));
+                        break;
+                    default:
+                        _descriptorProviders.Add(new WorkerFunctionDescriptorProvider(this, ScriptConfig, _functionDispatcher));
+                        break;
+                }
             }
 
             Collection<FunctionDescriptor> functions;
@@ -645,7 +653,7 @@ private JObject ApplyHostConfiguration()
             string sanitizedJson = SanitizeHostJson(hostConfigObject);
             string readFileMessage = $"Host configuration file read:{Environment.NewLine}{sanitizedJson}";
 
-            ApplyConfiguration(hostConfigObject, ScriptConfig);
+            ApplyConfiguration(hostConfigObject, ScriptConfig, _startupLogger);
 
             if (_settingsManager.FileSystemIsReadOnly)
             {
@@ -698,7 +706,7 @@ private JObject ApplyHostConfiguration()
         private void InitializeWorkers(string language)
         {
             var serverImpl = new FunctionRpcService(EventManager);
-            var server = new GrpcServer(serverImpl);
+            var server = new GrpcServer(serverImpl, ScriptConfig.MaxMessageLengthBytes);
 
             // TODO: async initialization of script host - hook into startasync method?
             server.StartAsync().GetAwaiter().GetResult();
@@ -733,7 +741,7 @@ private void InitializeWorkers(string language)
                 // TODO: We still have some hard coded languages, so we need to handle them. Remove this switch once we've moved away from that.
                 switch (language.ToLower())
                 {
-                    case ScriptConstants.NodeLanguageWrokerName:
+                    case ScriptConstants.NodeLanguageWorkerName:
                         providers.Add(new NodeWorkerProvider());
                         break;
                     case ScriptConstants.JavaLanguageWrokerName:
@@ -1405,11 +1413,6 @@ internal Collection<FunctionDescriptor> GetFunctionDescriptors(IEnumerable<Funct
                         ValidateFunction(descriptor, httpFunctions);
                         functionDescriptors.Add(descriptor);
                     }
-                    else
-                    {
-                        string functionLanguage = _settingsManager.Configuration[ScriptConstants.FunctionWorkerRuntimeSettingName];
-                        throw new ArgumentException($"Could not find a valid provider. {ScriptConstants.FunctionWorkerRuntimeSettingName} Appsetting is set to {functionLanguage}. Check that you have the correct language provider enabled and installed");
-                    }
                 }
                 catch (Exception ex)
                 {
@@ -1491,7 +1494,7 @@ internal static bool HttpRoutesConflict(HttpTriggerAttribute httpTrigger, HttpTr
             return httpTrigger.Methods.Intersect(otherHttpTrigger.Methods).Any();
         }
 
-        internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration scriptConfig)
+        internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration scriptConfig, ILogger logger = null)
         {
             var hostConfig = scriptConfig.HostConfig;
 
@@ -1600,6 +1603,7 @@ internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration
                 }
             }
 
+            value = null;
             if (config.TryGetValue("functionTimeout", out value))
             {
                 TimeSpan requestedTimeout = TimeSpan.Parse((string)value, CultureInfo.InvariantCulture);
@@ -1620,10 +1624,44 @@ internal static void ApplyConfiguration(JObject config, ScriptHostConfiguration
             }
             scriptConfig.HostConfig.FunctionTimeout = ScriptHost.CreateTimeoutConfiguration(scriptConfig);
 
+            ApplyLanguageWorkerConfig(config, scriptConfig, logger);
             ApplyLoggerConfig(config, scriptConfig);
             ApplyApplicationInsightsConfig(config, scriptConfig);
         }
 
+        private static void ApplyLanguageWorkerConfig(JObject config, ScriptHostConfiguration scriptConfig, ILogger logger)
+        {
+            JToken value = null;
+            JObject languageWorkerSection = (JObject)config["languageWorker"];
+            int requestedGrpcMaxMessageLength = ScriptSettingsManager.Instance.IsDynamicSku ? DefaultMaxMessageLengthBytesDynamicSku : DefaultMaxMessageLengthBytes;
+            if (languageWorkerSection != null)
+            {
+                if (languageWorkerSection.TryGetValue("maxMessageLength", out value))
+                {
+                    int valueInBytes = int.Parse((string)value) * 1024 * 1024;
+                    if (ScriptSettingsManager.Instance.IsDynamicSku)
+                    {
+                        string message = $"Cannot set {nameof(scriptConfig.MaxMessageLengthBytes)} on Consumption plan. Default MaxMessageLength: {DefaultMaxMessageLengthBytesDynamicSku} will be used";
+                        logger?.LogWarning(message);
+                    }
+                    else
+                    {
+                        if (valueInBytes < 0 || valueInBytes > 2000 * 1024 * 1024)
+                        {
+                            // Current grpc max message limits
+                            string message = $"MaxMessageLength must be between 4MB and 2000MB.Default MaxMessageLength: {DefaultMaxMessageLengthBytes} will be used";
+                            logger?.LogWarning(message);
+                        }
+                        else
+                        {
+                            requestedGrpcMaxMessageLength = valueInBytes;
+                        }
+                    }
+                }
+            }
+            scriptConfig.MaxMessageLengthBytes = requestedGrpcMaxMessageLength;
+        }
+
         internal static void ApplyLoggerConfig(JObject configJson, ScriptHostConfiguration scriptConfig)
         {
             scriptConfig.LogFilter = new LogCategoryFilter();

src/WebJobs.Script/Host/ScriptHost.cs

@@ -48,7 +48,7 @@ public static List<IWorkerProvider> ReadWorkerProviderFromConfig(ScriptHostConfi
 
             if (!string.IsNullOrEmpty(language))
             {
-                logger.LogInformation($"Reading Worker config for the lanuage: {language}");
+                logger.LogInformation($"Reading Worker config for the language: {language}");
                 string languageWorkerDirectory = Path.Combine(workerDirPath, language);
                 var provider = GetProviderFromConfig(languageWorkerDirectory, logger);
                 if (provider != null)

src/WebJobs.Script/Rpc/Configuration/GenericWorkerProvider.cs

@@ -14,11 +14,11 @@ namespace Microsoft.Azure.WebJobs.Script.Rpc
 {
     internal class NodeWorkerProvider : IWorkerProvider
     {
-        private string pathToWorkerDir = WorkerProviderHelper.BuildWorkerDirectoryPath(ScriptConstants.NodeLanguageWrokerName);
+        private string pathToWorkerDir = WorkerProviderHelper.BuildWorkerDirectoryPath(ScriptConstants.NodeLanguageWorkerName);
 
         public WorkerDescription GetDescription() => new WorkerDescription
         {
-            Language = ScriptConstants.NodeLanguageWrokerName,
+            Language = ScriptConstants.NodeLanguageWorkerName,
             Extension = ".js",
             DefaultExecutablePath = "node",
             DefaultWorkerPath = Path.Combine("dist", "src", "nodejsWorker.js"),

src/WebJobs.Script/Rpc/Configuration/NodeWorkerProvider.cs

@@ -122,6 +122,7 @@ internal void StartWorker()
             var workerContext = new WorkerCreateContext()
             {
                 RequestId = Guid.NewGuid().ToString(),
+                MaxMessageLength = _scriptConfig.MaxMessageLengthBytes,
                 WorkerId = _workerId,
                 Arguments = _workerConfig.Arguments,
                 WorkingDirectory = _scriptConfig.RootScriptPath,

src/WebJobs.Script/Rpc/LanguageWorkerChannel.cs

@@ -32,8 +32,8 @@ public string GetArguments(WorkerCreateContext context)
             var argumentsBuilder = context.Arguments.ExecutableArguments.Aggregate(new StringBuilder(), MergeArguments);
             argumentsBuilder.AppendFormat(" \"{0}\"", context.Arguments.WorkerPath);
             context.Arguments.WorkerArguments.Aggregate(argumentsBuilder, MergeArguments);
-            argumentsBuilder.AppendFormat(" --host {0} --port {1} --workerId {2} --requestId {3}",
-                context.ServerUri.Host, context.ServerUri.Port, context.WorkerId, context.RequestId);
+            argumentsBuilder.AppendFormat(" --host {0} --port {1} --workerId {2} --requestId {3} --grpcMaxMessageLength {4}",
+                context.ServerUri.Host, context.ServerUri.Port, context.WorkerId, context.RequestId, context.MaxMessageLength);
             return argumentsBuilder.ToString();
         }
     }

src/WebJobs.Script/Rpc/ProcessManagement/DefaultWorkerProcessFactory.cs

@@ -18,5 +18,7 @@ internal class WorkerCreateContext
         public string RequestId { get; set; }
 
         public string WorkingDirectory { get; set; }
+
+        public int MaxMessageLength { get; set; }
     }
 }

src/WebJobs.Script/Rpc/ProcessManagement/WorkerCreateContext.cs

@@ -70,7 +70,7 @@ public static class ScriptConstants
         public const string ColdStartEventName = "ColdStart";
         public const string FunctionWorkerRuntimeSettingName = "FUNCTIONS_WORKER_RUNTIME";
         public const string DotNetLanguageWorkerName = "dotnet";
-        public const string NodeLanguageWrokerName = "node";
+        public const string NodeLanguageWorkerName = "node";
         public const string JavaLanguageWrokerName = "java";
         public const string DefaultWorkersDirectoryName = "workers";