Isolating StandbyMode tests

mathewc · mathewc · commit 0e91a02a3e31 · 2018-06-07T12:54:50.000-07:00
diff --git a/run-tests.ps1 b/run-tests.ps1
@@ -33,14 +33,16 @@ $tests = @(
   @{project ="WebJobs.Script.Tests"; description="Unit Tests"},
   @{project ="WebJobs.Script.Scaling.Tests"; description="Scaling Tests"},
   @{project ="WebJobs.Script.Tests.Integration"; description="Non-E2E integration tests"; filter ="Category!=E2E"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="C# end to end tests"; filter ="E2E=CSharpEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Node end to end tests"; filter ="E2E=NodeEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Direct load end to end tests"; filter ="E2E=DirectLoadEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="F# end to end tests"; filter ="E2E=FSharpEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Language worker end to end tests"; filter ="E2E=LanguageWorkerSelectionEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Node script host end to end tests"; filter ="E2E=NodeScriptHostTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Raw assembly end to end tests"; filter ="E2E=RawAssemblyEndToEndTests"},
-  @{project ="WebJobs.Script.Tests.Integration"; description="Samples end to end tests"; filter ="E2E=SamplesEndToEndTests"}
+  @{project ="WebJobs.Script.Tests.Integration"; description="C# end to end tests"; filter ="Group=CSharpEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Node end to end tests"; filter ="Group=NodeEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Direct load end to end tests"; filter ="Group=DirectLoadEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="F# end to end tests"; filter ="Group=FSharpEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Language worker end to end tests"; filter ="Group=LanguageWorkerSelectionEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Node script host end to end tests"; filter ="Group=NodeScriptHostTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Raw assembly end to end tests"; filter ="Group=RawAssemblyEndToEndTests"},
+  @{project ="WebJobs.Script.Tests.Integration"; description="Samples end to end tests"; filter ="Group=SamplesEndToEndTests"}
+  @{project ="WebJobs.Script.Tests.Integration"; description="Standby mode end to end tests"; filter ="Group=StandbyModeEndToEndTests"}
+  @{project ="WebJobs.Script.Tests.Integration"; description="InstanceManager end to end tests"; filter ="Group=InstanceManagerTests"}
 )
 
 $success = $true
diff --git a/test/WebJobs.Script.Tests.Integration/Host/StandbyManagerTests.cs b/test/WebJobs.Script.Tests.Integration/Host/StandbyManagerTests.cs
@@ -28,6 +28,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, TestTraits.StandbyModeTests)]
     public class StandbyManagerTests : IDisposable
     {
         private readonly ScriptSettingsManager _settingsManager;
diff --git a/test/WebJobs.Script.Tests.Integration/Host/StandbyModeTests.cs b/test/WebJobs.Script.Tests.Integration/Host/StandbyModeTests.cs
@@ -18,6 +18,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, TestTraits.StandbyModeTests)]
     public class StandbyModeTests : IDisposable
     {
         private readonly WebHostResolver _webHostResolver;
diff --git a/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/DirectLoadEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/DirectLoadEndToEndTests.cs
@@ -12,11 +12,12 @@
 using Microsoft.Extensions.Logging;
 using System;
 using Microsoft.Azure.WebJobs.Host;
+using Microsoft.WebJobs.Script.Tests;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(DirectLoadEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(DirectLoadEndToEndTests))]
     public class DirectLoadEndToEndTests : IClassFixture<DirectLoadEndToEndTests.TestFixture>
     {
         TestFixture Fixture;
diff --git a/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/LanguageWorkerSelectionEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/LanguageWorkerSelectionEndToEndTests.cs
@@ -12,8 +12,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.ScriptHostEndToEnd
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(LanguageWorkerSelectionEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(LanguageWorkerSelectionEndToEndTests))]
     public class LanguageWorkerSelectionEndToEndTests
     {
         [Theory]
diff --git a/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/NodeScriptHostTests.cs b/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/NodeScriptHostTests.cs
@@ -16,8 +16,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(NodeScriptHostTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(NodeScriptHostTests))]
     public class NodeScriptHostTests : IClassFixture<NodeScriptHostTests.TestFixture>
     {
         private TestFixture Fixture;
diff --git a/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/RawAssemblyEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/ScriptHostEndToEnd/RawAssemblyEndToEndTests.cs
@@ -13,12 +13,13 @@
 using Microsoft.Azure.WebJobs.Script.Tests.Integration.Properties;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.WebJobs.Script.Tests;
 using Xunit;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(RawAssemblyEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(RawAssemblyEndToEndTests))]
     public class RawAssemblyEndToEndTests : IClassFixture<RawAssemblyEndToEndTests.TestFixture>
     {
         private TestFixture _fixture;
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/CSharpEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/CSharpEndToEndTests.cs
@@ -10,13 +10,14 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.WebJobs.Script.Tests;
 using Newtonsoft.Json.Linq;
 using Xunit;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(CSharpEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(CSharpEndToEndTests))]
     public class CSharpEndToEndTests : EndToEndTestsBase<CSharpEndToEndTests.TestFixture>
     {
         public CSharpEndToEndTests(TestFixture fixture) : base(fixture)
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/FSharpEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/FSharpEndToEndTests.cs
@@ -6,12 +6,13 @@
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.WebJobs.Script.Tests;
 using Xunit;
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(FSharpEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(FSharpEndToEndTests))]
     public class FSharpEndToEndTests : EndToEndTestsBase<FSharpEndToEndTests.TestFixture>
     {
         public FSharpEndToEndTests(TestFixture fixture) : base(fixture)
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/NodeEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/NodeEndToEndTests.cs
@@ -15,6 +15,7 @@
 using Microsoft.Azure.WebJobs.Logging;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Extensions.Logging;
+using Microsoft.WebJobs.Script.Tests;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Microsoft.WindowsAzure.Storage.Queue;
 using Newtonsoft.Json;
@@ -23,8 +24,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(NodeEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(NodeEndToEndTests))]
     public class NodeEndToEndTests : EndToEndTestsBase<NodeEndToEndTests.TestFixture>
     {
         public NodeEndToEndTests(TestFixture fixture) : base(fixture)
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests.cs
@@ -16,6 +16,7 @@
 using Microsoft.Azure.WebJobs.Script.WebHost;
 using Microsoft.Azure.WebJobs.Script.WebHost.Authentication;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
+using Microsoft.WebJobs.Script.Tests;
 using Microsoft.WindowsAzure.Storage.Blob;
 using Microsoft.WindowsAzure.Storage.Table;
 using Newtonsoft.Json;
@@ -24,8 +25,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.EndToEnd
 {
-    [Trait("Category", "E2E")]
-    [Trait("E2E", nameof(SamplesEndToEndTests))]
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(SamplesEndToEndTests))]
     public class SamplesEndToEndTests : IClassFixture<SamplesEndToEndTests.TestFixture>
     {
         private readonly ScriptSettingsManager _settingsManager;
diff --git a/test/WebJobs.Script.Tests.Shared/TestTraits.cs b/test/WebJobs.Script.Tests.Shared/TestTraits.cs
@@ -0,0 +1,26 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.WebJobs.Script.Tests
+{
+    public static class TestTraits
+    {
+        /// <summary>
+        /// Classifies a category of tests. A category may have multiple groups.
+        /// </summary>
+        public const string Category = "Category";
+
+        /// <summary>
+        /// Defines a group of tests to be run together. Useful for test isolation.
+        /// </summary>
+        public const string Group = "Group";
+
+        public const string EndToEnd = "E2E";
+
+        /// <summary>
+        /// Standby mode tests are special in that they set uni-directional
+        /// static state, and benefit from test isolation.
+        /// </summary>
+        public const string StandbyModeTests = "StandbyModeEndToEndTests";
+    }
+}
diff --git a/test/WebJobs.Script.Tests.Shared/WebJobs.Script.Tests.Shared.projitems b/test/WebJobs.Script.Tests.Shared/WebJobs.Script.Tests.Shared.projitems
@@ -12,6 +12,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)HttpTestHelpers.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)SingleThreadedSynchronizationContext.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)TempDirectory.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)TestTraits.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)TestHandler.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)TestHelpers.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)TestHelpers.Functions.cs" />
diff --git a/test/WebJobs.Script.Tests/Managment/InstanceManagerTests.cs b/test/WebJobs.Script.Tests/Managment/InstanceManagerTests.cs
@@ -17,6 +17,8 @@
 
 namespace Microsoft.Azure.WebJobs.Script.Tests.Managment
 {
+    [Trait(TestTraits.Category, TestTraits.EndToEnd)]
+    [Trait(TestTraits.Group, nameof(InstanceManagerTests))]
     public class InstanceManagerTests : IDisposable
     {
         private readonly TestLoggerProvider _loggerProvider;

Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,8 @@`
`28`	`28`
`29`	`29`	`namespace Microsoft.Azure.WebJobs.Script.Tests`
`30`	`30`	`{`
	`31`	`+ [Trait(TestTraits.Category, TestTraits.EndToEnd)]`
	`32`	`+ [Trait(TestTraits.Group, TestTraits.StandbyModeTests)]`
`31`	`33`	`public class StandbyManagerTests : IDisposable`
`32`	`34`	`{`
`33`	`35`	`private readonly ScriptSettingsManager _settingsManager;`
Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,8 @@`
`18`	`18`
`19`	`19`	`namespace Microsoft.Azure.WebJobs.Script.Tests`
`20`	`20`	`{`
	`21`	`+ [Trait(TestTraits.Category, TestTraits.EndToEnd)]`
	`22`	`+ [Trait(TestTraits.Group, TestTraits.StandbyModeTests)]`
`21`	`23`	`public class StandbyModeTests : IDisposable`
`22`	`24`	`{`
`23`	`25`	`private readonly WebHostResolver _webHostResolver;`
Original file line number	Diff line number	Diff line change
`@@ -12,11 +12,12 @@`
`12`	`12`	`using Microsoft.Extensions.Logging;`
`13`	`13`	`using System;`
`14`	`14`	`using Microsoft.Azure.WebJobs.Host;`
	`15`	`+using Microsoft.WebJobs.Script.Tests;`
`15`	`16`
`16`	`17`	`namespace Microsoft.Azure.WebJobs.Script.Tests`
`17`	`18`	`{`
`18`		`- [Trait("Category", "E2E")]`
`19`		`- [Trait("E2E", nameof(DirectLoadEndToEndTests))]`
	`19`	`+ [Trait(TestTraits.Category, TestTraits.EndToEnd)]`
	`20`	`+ [Trait(TestTraits.Group, nameof(DirectLoadEndToEndTests))]`
`20`	`21`	`public class DirectLoadEndToEndTests : IClassFixture<DirectLoadEndToEndTests.TestFixture>`
`21`	`22`	`{`
`22`	`23`	`TestFixture Fixture;`
Original file line number	Diff line number	Diff line change
`@@ -12,8 +12,8 @@`
`12`	`12`
`13`	`13`	`namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.ScriptHostEndToEnd`
`14`	`14`	`{`
`15`		`- [Trait("Category", "E2E")]`
`16`		`- [Trait("E2E", nameof(LanguageWorkerSelectionEndToEndTests))]`
	`15`	`+ [Trait(TestTraits.Category, TestTraits.EndToEnd)]`
	`16`	`+ [Trait(TestTraits.Group, nameof(LanguageWorkerSelectionEndToEndTests))]`
`17`	`17`	`public class LanguageWorkerSelectionEndToEndTests`
`18`	`18`	`{`
`19`	`19`	`[Theory]`
Original file line number	Diff line number	Diff line change
`@@ -16,8 +16,8 @@`
`16`	`16`
`17`	`17`	`namespace Microsoft.Azure.WebJobs.Script.Tests`
`18`	`18`	`{`
`19`		`- [Trait("Category", "E2E")]`
`20`		`- [Trait("E2E", nameof(NodeScriptHostTests))]`
	`19`	`+ [Trait(TestTraits.Category, TestTraits.EndToEnd)]`
	`20`	`+ [Trait(TestTraits.Group, nameof(NodeScriptHostTests))]`
`21`	`21`	`public class NodeScriptHostTests : IClassFixture<NodeScriptHostTests.TestFixture>`
`22`	`22`	`{`
`23`	`23`	`private TestFixture Fixture;`