removing test filter; fixing issues in failing tests

Author: brettsam

appveyor.yml

@@ -40,10 +40,8 @@ test_script:
 
     $success = $success -and $?
 
-    dotnet test .\test\WebJobs.Script.Tests.Integration\ -v q --no-build --filter "(FullyQualifiedName~Tests.NodeEndToEndTests|FullyQualifiedName~NodeContentTests|FullyQualifiedName~Tests.SecretsRepositoryTests|FullyQualifiedName~StandbyModeTests)"
-
+    dotnet test .\test\WebJobs.Script.Tests.Integration\ -v q --no-build
+    
     $success = $success -and $?
 
-    # Skipping integration tests for initial build # dotnet test .\test\WebJobs.Script.Tests.Integration\ -v q --no-build
-
     if (-not $success) { exit 1 }

test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsEndToEndTestsBase.cs

@@ -217,6 +217,8 @@ public async Task Validate_HostLogs()
             // slightly out-of-order or on different threads
             TraceTelemetry[] traces = null;
 
+            int expectedCount = 11;
+
             await TestHelpers.Await(() =>
             {
                 traces = _fixture.Channel.Telemetries
@@ -225,22 +227,25 @@ await TestHelpers.Await(() =>
                     .OrderBy(t => t.Message)
                     .ToArray();
 
-                return traces.Length >= 9;
+                // When these two messages are logged, we know we've completed initialization.
+                return traces
+                .Where(t => t.Message.Contains("Host lock lease acquired by instance ID") || t.Message.Contains("Job host started"))
+                .Count() == 2;
             });
 
-            Assert.True(traces.Length == 9, $"Expected 9 messages, but found {traces.Length}. Actual logs:{Environment.NewLine}{string.Join(Environment.NewLine, traces.Select(t => t.Message))}");
+            Assert.True(traces.Length == expectedCount, $"Expected {expectedCount} messages, but found {traces.Length}. Actual logs:{Environment.NewLine}{string.Join(Environment.NewLine, traces.Select(t => t.Message))}");
 
             ValidateTrace(traces[0], "A function whitelist has been specified", LogCategories.Startup);
             ValidateTrace(traces[1], "Found the following functions:\r\n", LogCategories.Startup);
             ValidateTrace(traces[2], "Generating 2 job function(s)", LogCategories.Startup);
             ValidateTrace(traces[3], "Host configuration file read:", LogCategories.Startup);
             ValidateTrace(traces[4], "Host id explicitly set", LogCategories.Startup, expectedLevel: SeverityLevel.Warning);
-            ValidateTrace(traces[5], "Host lock lease acquired by instance ID", ScriptConstants.LogCategoryHostGeneral);
-            ValidateTrace(traces[6], "Job host started", LogCategories.Startup);
-            ValidateTrace(traces[7], "Reading host configuration file", LogCategories.Startup);
-            ValidateTrace(traces[8], "Starting Host (HostId=function-tests-", ScriptConstants.LogCategoryHostGeneral);
-
-            await Task.CompletedTask;
+            ValidateTrace(traces[5], "Host initialized (", LogCategories.Startup);
+            ValidateTrace(traces[6], "Host lock lease acquired by instance ID", ScriptConstants.LogCategoryHostGeneral);
+            ValidateTrace(traces[7], "Host started (", LogCategories.Startup);
+            ValidateTrace(traces[8], "Job host started", LogCategories.Startup);
+            ValidateTrace(traces[9], "Reading host configuration file", LogCategories.Startup);
+            ValidateTrace(traces[10], "Starting Host (HostId=function-tests-", ScriptConstants.LogCategoryHostGeneral);
         }
 
         private static void ValidateMetric(MetricTelemetry telemetry, string expectedOperationId, string expectedOperationName)

test/WebJobs.Script.Tests.Integration/ApplicationInsights/ApplicationInsightsTestFixture.cs

@@ -2,12 +2,14 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Net.Http;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.WebHost;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.WebJobs.Script.Tests;
@@ -33,7 +35,19 @@ public ApplicationInsightsTestFixture(string scriptRoot, string testId)
                 .UseStartup<Startup>()
                 .ConfigureServices(services =>
                 {
-                    ScriptSettingsManager.Instance.ApplicationInsightsInstrumentationKey = TestChannelLoggerProviderFactory.ApplicationInsightsKey;
+                    var settingsManager = new ScriptSettingsManager();
+                    settingsManager.SetConfigurationFactory(() =>
+                    {
+                        return new ConfigurationBuilder()
+                            .AddInMemoryCollection(new Dictionary<string, string>
+                            {
+                                [EnvironmentSettingNames.AppInsightsInstrumentationKey] = TestChannelLoggerProviderFactory.ApplicationInsightsKey
+                            })
+                            .AddEnvironmentVariables()
+                            .Build();
+                    });
+
+                    services.Replace(new ServiceDescriptor(typeof(ScriptSettingsManager), settingsManager));
                     services.Replace(new ServiceDescriptor(typeof(WebHostSettings), HostSettings));
                     services.Replace(new ServiceDescriptor(typeof(ILoggerProviderFactory), new TestChannelLoggerProviderFactory(Channel)));
                     services.Replace(new ServiceDescriptor(typeof(ISecretManager), new TestSecretManager()));
@@ -76,6 +90,7 @@ public void Dispose()
         {
             _testServer?.Dispose();
             HttpClient?.Dispose();
+            ScriptSettingsManager.Instance.Reset();
         }
     }
 }

test/WebJobs.Script.Tests.Integration/Controllers/ControllerScenarioTestFixture.cs

@@ -4,16 +4,12 @@
 using System.IO;
 using System.Net.Http;
 using System.Web.Http;
-using Autofac;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.Azure.WebJobs.Script.Config;
-using Microsoft.Azure.WebJobs.Script.Tests.Integration;
 using Microsoft.Azure.WebJobs.Script.WebHost;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging.Console;
+using Microsoft.Extensions.DependencyInjection;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.Controllers
 {
@@ -28,7 +24,7 @@ public ControllerScenarioTestFixture()
         }
 
         public ControllerScenarioTestFixture(bool isAuthDisabled)
-        {
+        {            
             _config = new HttpConfiguration();
             _settingsManager = ScriptSettingsManager.Instance;
 

test/WebJobs.Script.Tests.Integration/Host/ScriptHostManagerTests.cs

@@ -71,19 +71,19 @@ string GetErrorTraces()
                             userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
 
                         // Wait for initial execution.
-                        TestHelpers.Await(() =>
+                        TestHelpers.Await(async () =>
                          {
-                             bool exists = blob1.ExistsAsync().GetAwaiter().GetResult();
+                             bool exists = await blob1.ExistsAsync();
                              return exists;
                          }, timeout: 10 * 1000, userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
 
                         // This changes the bindings so that we now write to blob2
                         var blob2 = UpdateOutputName("first", "testblob", fixture).Result;
 
                         // wait for newly executed
-                        TestHelpers.Await(() =>
+                        TestHelpers.Await(async () =>
                          {
-                             bool exists = blob2.ExistsAsync().GetAwaiter().GetResult();
+                             bool exists = await blob2.ExistsAsync();
                              return exists;
                          }, timeout: 30 * 1000, userMessageCallback: GetErrorTraces).GetAwaiter().GetResult();
 
@@ -128,7 +128,7 @@ string GetErrorTraces()
         }
 
         [Fact]
-        public void RenameFunctionAndRestart()
+        public async Task RenameFunctionAndRestart()
         {
             var oldDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/TimerTrigger");
             var newDirectory = Path.Combine(Directory.GetCurrentDirectory(), "TestScripts/Node/MovedTrigger");
@@ -142,6 +142,7 @@ public void RenameFunctionAndRestart()
             };
 
             var blob = fixture.TestOutputContainer.GetBlockBlobReference("testblob");
+            await blob.DeleteIfExistsAsync();
             var mockEnvironment = new Mock<IScriptHostEnvironment>();
 
             using (var eventManager = new ScriptEventManager())
@@ -167,12 +168,12 @@ public void RenameFunctionAndRestart()
                         userMessageCallback: () => "Host did not start in time.").GetAwaiter().GetResult();
 
                         // Wait for initial execution.
-                        TestHelpers.Await(() =>
-                    {
-                        bool exists = blob.ExistsAsync().GetAwaiter().GetResult();
-                        return exists;
-                    }, timeout: 10 * 1000,
-                    userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'TimerTrigger' in time.").GetAwaiter().GetResult();
+                        TestHelpers.Await(async () =>
+                        {
+                            bool exists = await blob.ExistsAsync();
+                            return exists;
+                        }, timeout: 10 * 1000,
+                        userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'TimerTrigger' in time.").GetAwaiter().GetResult();
 
                         // find __dirname from blob
                         string text;
@@ -189,15 +190,15 @@ public void RenameFunctionAndRestart()
 
                         resetEvent.Wait(TimeSpan.FromSeconds(10));
 
-                        blob.DeleteIfExistsAsync();
+                        blob.DeleteIfExistsAsync().GetAwaiter().GetResult();
 
                         // wait for newly executed
-                        TestHelpers.Await(() =>
-                    {
-                        bool exists = blob.ExistsAsync().GetAwaiter().GetResult();
-                        return exists;
-                    }, timeout: 30 * 1000,
-                    userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'MovedTrigger' in time.").GetAwaiter().GetResult();
+                        TestHelpers.Await(async () =>
+                        {
+                            bool exists = await blob.ExistsAsync();
+                            return exists;
+                        }, timeout: 30 * 1000,
+                        userMessageCallback: () => $"Blob '{blob.Uri}' was not created by 'MovedTrigger' in time.").GetAwaiter().GetResult();
 
                         using (var stream = new MemoryStream())
                         {

test/WebJobs.Script.Tests.Integration/Host/StandbyManagerTests.cs

@@ -20,7 +20,7 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
-    public class StandbyManagerTests
+    public class StandbyManagerTests : IDisposable
     {
         private readonly ScriptSettingsManager _settingsManager;
 
@@ -154,5 +154,10 @@ await TestHelpers.Await(() =>
                 WebScriptHostManager.ResetStandbyMode();
             }
         }
+
+        public void Dispose()
+        {
+            ScriptSettingsManager.Instance.Reset();
+        }
     }
 }