Worker config updates to make path to jave.exe configurable (#3210)

pragnagopa · web-flow · commit 395b0c9929e8 · 2018-08-01T12:36:02.000-07:00
* Add support to configure Java.exe path
diff --git a/src/WebJobs.Script.Grpc/Abstractions/WorkerDescription.cs b/src/WebJobs.Script.Grpc/Abstractions/WorkerDescription.cs
@@ -12,25 +12,25 @@ public class WorkerDescription
         /// <summary>
         /// Gets or sets the name of the supported language. This is the same name as the IConfiguration section for the worker.
         /// </summary>
-        [JsonProperty(PropertyName = "language", Required = Required.Always)]
+        [JsonProperty(PropertyName = "language")]
         public string Language { get; set; }
 
         /// <summary>
         /// Gets or sets the supported file extension type. Functions are registered with workers based on extension.
         /// </summary>
-        [JsonProperty(PropertyName = "extension", Required = Required.Always)]
+        [JsonProperty(PropertyName = "extension")]
         public string Extension { get; set; }
 
         /// <summary>
         /// Gets or sets the default executable path.
         /// </summary>
-        [JsonProperty(PropertyName = "defaultExecutablePath", Required = Required.Always)]
+        [JsonProperty(PropertyName = "defaultExecutablePath")]
         public string DefaultExecutablePath { get; set; }
 
         /// <summary>
         /// Gets or sets the default path to the worker (relative to the bin/workers/{language} directory)
         /// </summary>
-        [JsonProperty(PropertyName = "defaultWorkerPath", Required = Required.Always)]
+        [JsonProperty(PropertyName = "defaultWorkerPath")]
         public string DefaultWorkerPath { get; set; }
 
         /// <summary>
diff --git a/src/WebJobs.Script/Rpc/Configuration/WorkerConfigFactory.cs b/src/WebJobs.Script/Rpc/Configuration/WorkerConfigFactory.cs
@@ -50,7 +50,7 @@ public IEnumerable<WorkerConfig> GetConfigs(IEnumerable<IWorkerProvider> provide
 
                 if (description.Language.Equals(LanguageWorkerConstants.JavaLanguageWorkerName))
                 {
-                    arguments.ExecutablePath = GetExecutablePathForJava();
+                    arguments.ExecutablePath = GetExecutablePathForJava(description.DefaultExecutablePath);
                 }
 
                 if (provider.TryConfigureArguments(arguments, _config, _logger))
@@ -117,6 +117,7 @@ internal void AddProvider(string workerDir, ILogger logger)
         {
             try
             {
+                Dictionary<string, WorkerDescription> descriptionProfiles = new Dictionary<string, WorkerDescription>();
                 string workerConfigPath = Path.Combine(workerDir, LanguageWorkerConstants.WorkerConfigFileName);
                 if (!File.Exists(workerConfigPath))
                 {
@@ -130,11 +131,15 @@ internal void AddProvider(string workerDir, ILogger logger)
                 workerDescription.WorkerDirectory = workerDir;
                 var languageSection = _config.GetSection($"{LanguageWorkerConstants.LanguageWorkersSectionName}:{workerDescription.Language}");
                 workerDescription.Arguments = workerDescription.Arguments ?? new List<string>();
-                var argumentsSection = languageSection.GetSection($"{LanguageWorkerConstants.WorkerDescriptionArguments}");
-                if (argumentsSection.Value != null)
+
+                descriptionProfiles = GetWorkerDescriptionProfiles(workerConfig);
+                if (ScriptSettingsManager.Instance.IsAppServiceEnvironment)
                 {
-                    workerDescription.Arguments.AddRange(Regex.Split(argumentsSection.Value, @"\s+"));
+                    //Overwrite default Description with AppServiceEnv profile
+                    workerDescription = GetWorkerDescriptionFromProfiles(LanguageWorkerConstants.WorkerDescriptionAppServiceEnvProfileName, descriptionProfiles, workerDescription);
                 }
+                GetDefaultExecutablePathFromAppSettings(workerDescription, languageSection);
+                AddArgumentsFromAppSettings(workerDescription, languageSection);
                 if (File.Exists(workerDescription.GetWorkerPath()))
                 {
                     logger.LogTrace($"Will load worker provider for language: {workerDescription.Language}");
@@ -151,23 +156,64 @@ internal void AddProvider(string workerDir, ILogger logger)
             }
         }
 
-        internal string GetExecutablePathForJava()
+        private static Dictionary<string, WorkerDescription> GetWorkerDescriptionProfiles(JObject workerConfig)
+        {
+            Dictionary<string, WorkerDescription> descriptionProfiles = new Dictionary<string, WorkerDescription>();
+            var profiles = workerConfig.Property("profiles")?.Value.ToObject<JObject>();
+            if (profiles != null)
+            {
+                foreach (var profile in profiles)
+                {
+                    string name = profile.Key;
+                    JToken value = profile.Value;
+                    WorkerDescription description = profile.Value.ToObject<WorkerDescription>();
+                    descriptionProfiles.Add(name, description);
+                }
+            }
+            return descriptionProfiles;
+        }
+
+        private static WorkerDescription GetWorkerDescriptionFromProfiles(string key, Dictionary<string, WorkerDescription> descriptionProfiles, WorkerDescription defaultWorkerDescription)
+        {
+            WorkerDescription profileDescription = null;
+            if (descriptionProfiles.TryGetValue(key, out profileDescription))
+            {
+                profileDescription.Arguments = profileDescription.Arguments?.Count > 0 ? profileDescription.Arguments : defaultWorkerDescription.Arguments;
+                profileDescription.DefaultExecutablePath = string.IsNullOrEmpty(profileDescription.DefaultExecutablePath) ? defaultWorkerDescription.DefaultExecutablePath : profileDescription.DefaultExecutablePath;
+                profileDescription.DefaultWorkerPath = string.IsNullOrEmpty(profileDescription.DefaultWorkerPath) ? defaultWorkerDescription.DefaultWorkerPath : profileDescription.DefaultWorkerPath;
+                profileDescription.Extension = string.IsNullOrEmpty(profileDescription.Extension) ? defaultWorkerDescription.Extension : profileDescription.Extension;
+                profileDescription.Language = string.IsNullOrEmpty(profileDescription.Language) ? defaultWorkerDescription.Language : profileDescription.Language;
+                profileDescription.WorkerDirectory = string.IsNullOrEmpty(profileDescription.WorkerDirectory) ? defaultWorkerDescription.WorkerDirectory : profileDescription.WorkerDirectory;
+                return profileDescription;
+            }
+            return defaultWorkerDescription;
+        }
+
+        private static void GetDefaultExecutablePathFromAppSettings(WorkerDescription workerDescription, IConfigurationSection languageSection)
+        {
+            var defaultExecutablePath = languageSection.GetSection($"{LanguageWorkerConstants.WorkerDescriptionDefaultExecutablePath}");
+            workerDescription.DefaultExecutablePath = defaultExecutablePath.Value != null ? defaultExecutablePath.Value : workerDescription.DefaultExecutablePath;
+        }
+
+        private static void AddArgumentsFromAppSettings(WorkerDescription workerDescription, IConfigurationSection languageSection)
+        {
+            var argumentsSection = languageSection.GetSection($"{LanguageWorkerConstants.WorkerDescriptionArguments}");
+            if (argumentsSection.Value != null)
+            {
+                workerDescription.Arguments.AddRange(Regex.Split(argumentsSection.Value, @"\s+"));
+            }
+        }
+
+        internal string GetExecutablePathForJava(string defaultExecutablePath)
         {
                 string javaHome = ScriptSettingsManager.Instance.GetSetting("JAVA_HOME");
                 if (string.IsNullOrEmpty(javaHome))
                 {
-                    return LanguageWorkerConstants.JavaLanguageWorkerName;
+                    return defaultExecutablePath;
                 }
                 else
                 {
-                    if (ScriptSettingsManager.Instance.IsAppServiceEnvironment)
-                    {
-                        return Path.GetFullPath(Path.Combine(javaHome, "..", LanguageWorkerConstants.AppServiceEnvJavaVersion, "bin", LanguageWorkerConstants.JavaLanguageWorkerName));
-                    }
-                    else
-                    {
-                        return Path.GetFullPath(Path.Combine(javaHome, "bin", LanguageWorkerConstants.JavaLanguageWorkerName));
-                    }
+                    return Path.GetFullPath(Path.Combine(javaHome, "bin", defaultExecutablePath));
                 }
         }
     }
diff --git a/src/WebJobs.Script/Rpc/LanguageWorkerChannel.cs b/src/WebJobs.Script/Rpc/LanguageWorkerChannel.cs
@@ -337,14 +337,18 @@ internal void StartProcess(string workerId, Process process)
                 // TODO: per language stdout/err parser?
                 if (e.Data != null)
                 {
-                    if (e.Data.IndexOf("warn", StringComparison.OrdinalIgnoreCase) > 0)
+                    if (e.Data.IndexOf("warn", StringComparison.OrdinalIgnoreCase) > -1)
                     {
                         _logger.LogWarning(e.Data);
                     }
-                    else
+                    else if (e.Data.IndexOf("error", StringComparison.OrdinalIgnoreCase) > -1)
                     {
                         _logger.LogError(e.Data);
                     }
+                    else
+                    {
+                        _logger.LogInformation(e.Data);
+                    }
                 }
             };
             process.OutputDataReceived += (sender, e) =>
@@ -371,9 +375,9 @@ internal void StartProcess(string workerId, Process process)
                     HandleWorkerError(new Exception("Worker process is not attached"));
                 }
             };
-            _logger.LogInformation($"Starting {process.StartInfo.FileName} language worker process with Arguments={process.StartInfo.Arguments}");
+            _logger.LogInformation($"Starting language worker process: {process.StartInfo.FileName} {process.StartInfo.Arguments}");
             process.Start();
-            _logger.LogInformation($"{process.StartInfo.FileName} process with Id={process.Id} started");
+            _logger.LogInformation($"{process.StartInfo.FileName} process with Id: {process.Id} started");
             process.BeginErrorReadLine();
             process.BeginOutputReadLine();
         }
diff --git a/src/WebJobs.Script/Rpc/LanguageWorkerConstants.cs b/src/WebJobs.Script/Rpc/LanguageWorkerConstants.cs
@@ -8,6 +8,7 @@ public static class LanguageWorkerConstants
         public const string FunctionWorkerRuntimeSettingName = "FUNCTIONS_WORKER_RUNTIME";
         public const string DotNetLanguageWorkerName = "dotnet";
         public const string NodeLanguageWorkerName = "node";
+        public const string JavaLanguageWorkerName = "java";
         public const string WorkerConfigFileName = "worker.config.json";
         public const string DefaultWorkersDirectoryName = "workers";
 
@@ -24,8 +25,8 @@ public static class LanguageWorkerConstants
         public const string WorkerDescription = "Description";
         public const string WorkerDescriptionArguments = "arguments";
 
-        // Java
-        public const string AppServiceEnvJavaVersion = "zulu8.23.0.3-jdk8.0.144-win_x64";
-        public const string JavaLanguageWorkerName = "java";
+        // Profiles
+        public const string WorkerDescriptionProfiles = "profiles";
+        public const string WorkerDescriptionAppServiceEnvProfileName = "AppServiceEnvironment";
     }
 }
diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj
@@ -31,7 +31,7 @@
       <NoWarn>NU1701</NoWarn>
     </PackageReference>
     <PackageReference Include="Microsoft.Azure.AppService.Proxy.Client" Version="2.0.5350001-beta-fc119b98" />
-    <PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="1.1.0-beta8" />
+    <PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="1.1.0-beta9" />
     <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="1.0.0-beta3" />
     <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.0-beta7-11351" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions" Version="3.0.0-beta7-10629" />
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests.cs
@@ -201,7 +201,7 @@ public async Task HttpTrigger_Get_Succeeds()
             await InvokeHttpTrigger("HttpTrigger");
         }
 
-        [Fact(Skip = "Investigate test failure")]
+        [Fact]
         public async Task HttpTrigger_Java_Get_Succeeds()
         {
             await InvokeHttpTrigger("HttpTrigger-Java");
diff --git a/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj b/test/WebJobs.Script.Tests.Integration/WebJobs.Script.Tests.Integration.csproj
@@ -28,7 +28,7 @@
     <PackageReference Include="Microsoft.AspNetCore.TestHost" Version="2.1.1" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="1.9.1" />
     <PackageReference Include="Microsoft.Azure.EventHubs" Version="1.0.3" />
-    <PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="1.1.0-beta8" />
+    <PackageReference Include="Microsoft.Azure.Functions.JavaWorker" Version="1.1.0-beta9" />
     <PackageReference Include="Microsoft.Azure.Functions.NodeJsWorker" Version="1.0.0-beta3" />
     <PackageReference Include="Microsoft.Azure.Mobile.Client" Version="4.0.2" />
     <PackageReference Include="Microsoft.Azure.ServiceBus" Version="2.0.0" />
diff --git a/test/WebJobs.Script.Tests/Rpc/GenericWorkerProviderTests.cs b/test/WebJobs.Script.Tests/Rpc/GenericWorkerProviderTests.cs
@@ -172,7 +172,54 @@ public void ReadWorkerProviderFromConfig_InvalidWorker()
             Assert.Empty(providers);
         }
 
-        private IEnumerable<IWorkerProvider> TestReadWorkerProviderFromConfig(IEnumerable<TestLanguageWorkerConfig> configs, ILogger testLogger, string language = null, Dictionary<string, string> keyValuePairs = null)
+        [Fact]
+        public void ReadWorkerProviderFromConfig_AddAppSvcProfile_ReturnsAppServiceEnvDescription()
+        {
+            var expectedArguments = new string[] { "-v", "verbose" };
+            var configs = new List<TestLanguageWorkerConfig>() { MakeTestConfig(testLanguage, expectedArguments, false, LanguageWorkerConstants.WorkerDescriptionAppServiceEnvProfileName) };
+            var testLogger = new TestLogger(testLanguage);
+
+            // Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
+            IEnumerable<IWorkerProvider> providers = TestReadWorkerProviderFromConfig(configs, testLogger, null, null, true);
+            Assert.Single(providers);
+
+            IWorkerProvider worker = providers.FirstOrDefault();
+            Assert.Equal("myFooPath", worker.GetDescription().DefaultExecutablePath);
+        }
+
+        [Fact]
+        public void ReadWorkerProviderFromConfig_AddProfile_ReturnsDefaultDescription()
+        {
+            var expectedArguments = new string[] { "-v", "verbose" };
+            var configs = new List<TestLanguageWorkerConfig>() { MakeTestConfig(testLanguage, expectedArguments, false, "TestProfile") };
+            var testLogger = new TestLogger(testLanguage);
+
+            // Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
+            IEnumerable<IWorkerProvider> providers = TestReadWorkerProviderFromConfig(configs, testLogger);
+            Assert.Single(providers);
+
+            IWorkerProvider worker = providers.FirstOrDefault();
+            Assert.Equal("foopath", worker.GetDescription().DefaultExecutablePath);
+        }
+
+        [Fact]
+        public void ReadWorkerProviderFromConfig_OverrideDefaultExePath()
+        {
+            var configs = new List<TestLanguageWorkerConfig>() { MakeTestConfig(testLanguage, new string[0], false, LanguageWorkerConstants.WorkerDescriptionAppServiceEnvProfileName) };
+            var testLogger = new TestLogger(testLanguage);
+            var testExePath = "./mySrc/myIndex";
+            Dictionary<string, string> keyValuePairs = new Dictionary<string, string>
+            {
+                [$"{LanguageWorkerConstants.LanguageWorkersSectionName}:{testLanguage}:{LanguageWorkerConstants.WorkerDescriptionDefaultExecutablePath}"] = testExePath
+            };
+            var providers = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), null, keyValuePairs, true);
+            Assert.Single(providers);
+
+            IWorkerProvider worker = providers.FirstOrDefault();
+            Assert.Equal(testExePath, worker.GetDescription().DefaultExecutablePath);
+        }
+
+        private IEnumerable<IWorkerProvider> TestReadWorkerProviderFromConfig(IEnumerable<TestLanguageWorkerConfig> configs, ILogger testLogger, string language = null, Dictionary<string, string> keyValuePairs = null, bool appSvcEnv = false)
         {
             var workerPathSection = $"{LanguageWorkerConstants.LanguageWorkersSectionName}:{LanguageWorkerConstants.WorkersDirectorySectionName}";
             try
@@ -187,7 +234,17 @@ private IEnumerable<IWorkerProvider> TestReadWorkerProviderFromConfig(IEnumerabl
                 var scriptHostConfig = new ScriptHostConfiguration();
                 var scriptSettingsManager = new ScriptSettingsManager(config);
                 var configFactory = new WorkerConfigFactory(config, testLogger);
-
+                if (appSvcEnv)
+                {
+                    var testEnvVariables = new Dictionary<string, string>
+                    {
+                        { EnvironmentSettingNames.AzureWebsiteInstanceId, "123" },
+                    };
+                    using (var variables = new TestScopedSettings(scriptSettingsManager, testEnvVariables))
+                    {
+                        return configFactory.GetWorkerProviders(testLogger, scriptSettingsManager, language: language);
+                    }
+                }
                 return configFactory.GetWorkerProviders(testLogger, scriptSettingsManager, language: language);
             }
             finally
@@ -240,17 +297,17 @@ private static IConfigurationRoot TestConfigBuilder(string workerPathSection, Di
             return config;
         }
 
-        private static TestLanguageWorkerConfig MakeTestConfig(string language, string[] arguments, bool invalid = false)
+        private static TestLanguageWorkerConfig MakeTestConfig(string language, string[] arguments, bool invalid = false, string addAppSvcProfile = "")
         {
-            string json = GetTestWorkerConfig(language, arguments, invalid).ToString();
+            string json = GetTestWorkerConfig(language, arguments, invalid, addAppSvcProfile).ToString();
             return new TestLanguageWorkerConfig()
             {
                 Json = json,
                 Language = language,
             };
         }
 
-        private static JObject GetTestWorkerConfig(string language, string[] arguments, bool invalid)
+        private static JObject GetTestWorkerConfig(string language, string[] arguments, bool invalid, string profileName)
         {
             var description = new WorkerDescription()
             {
@@ -263,6 +320,19 @@ private static JObject GetTestWorkerConfig(string language, string[] arguments,
 
             JObject config = new JObject();
             config["Description"] = JObject.FromObject(description);
+
+            if (!string.IsNullOrEmpty(profileName))
+            {
+                var appSvcDescription = new WorkerDescription()
+                {
+                    DefaultExecutablePath = "myFooPath",
+                };
+
+                JObject profiles = new JObject();
+                profiles[profileName] = JObject.FromObject(appSvcDescription);
+                config[LanguageWorkerConstants.WorkerDescriptionProfiles] = profiles;
+            }
+
             if (invalid)
             {
                 config["Description"] = "invalidWorkerConfig";
diff --git a/test/WebJobs.Script.Tests/Rpc/WorkerConfigFactoryTests.cs b/test/WebJobs.Script.Tests/Rpc/WorkerConfigFactoryTests.cs

Original file line number	Diff line number	Diff line change
`@@ -337,14 +337,18 @@ internal void StartProcess(string workerId, Process process)`
`337`	`337`	`// TODO: per language stdout/err parser?`
`338`	`338`	`if (e.Data != null)`
`339`	`339`	`{`
`340`		`- if (e.Data.IndexOf("warn", StringComparison.OrdinalIgnoreCase) > 0)`
	`340`	`+ if (e.Data.IndexOf("warn", StringComparison.OrdinalIgnoreCase) > -1)`
`341`	`341`	`{`
`342`	`342`	`_logger.LogWarning(e.Data);`
`343`	`343`	`}`
`344`		`- else`
	`344`	`+ else if (e.Data.IndexOf("error", StringComparison.OrdinalIgnoreCase) > -1)`
`345`	`345`	`{`
`346`	`346`	`_logger.LogError(e.Data);`
`347`	`347`	`}`
	`348`	`+ else`
	`349`	`+ {`
	`350`	`+ _logger.LogInformation(e.Data);`
	`351`	`+ }`
`348`	`352`	`}`
`349`	`353`	`};`
`350`	`354`	`process.OutputDataReceived += (sender, e) =>`
`@@ -371,9 +375,9 @@ internal void StartProcess(string workerId, Process process)`
`371`	`375`	`HandleWorkerError(new Exception("Worker process is not attached"));`
`372`	`376`	`}`
`373`	`377`	`};`
`374`		`- _logger.LogInformation($"Starting {process.StartInfo.FileName} language worker process with Arguments={process.StartInfo.Arguments}");`
	`378`	`+ _logger.LogInformation($"Starting language worker process: {process.StartInfo.FileName} {process.StartInfo.Arguments}");`
`375`	`379`	`process.Start();`
`376`		`- _logger.LogInformation($"{process.StartInfo.FileName} process with Id={process.Id} started");`
	`380`	`+ _logger.LogInformation($"{process.StartInfo.FileName} process with Id: {process.Id} started");`
`377`	`381`	`process.BeginErrorReadLine();`
`378`	`382`	`process.BeginOutputReadLine();`
`379`	`383`	`}`
Original file line number	Diff line number	Diff line change
`@@ -201,7 +201,7 @@ public async Task HttpTrigger_Get_Succeeds()`
`201`	`201`	`await InvokeHttpTrigger("HttpTrigger");`
`202`	`202`	`}`
`203`	`203`
`204`		`- [Fact(Skip = "Investigate test failure")]`
	`204`	`+ [Fact]`
`205`	`205`	`public async Task HttpTrigger_Java_Get_Succeeds()`
`206`	`206`	`{`
`207`	`207`	`await InvokeHttpTrigger("HttpTrigger-Java");`