Add TestCaseFilter support to VSTest Adapter (#95)

* Spec Filtering

* MSpecTestAdapter() improvements

* VisualStudioTestIdentifier[] to IEnumerable<VisualStudioTestIdentifier>

Author: mikeblakeuk

src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_adapter_runs_tests.cs

@@ -4,6 +4,8 @@
 using Machine.VSTestAdapter.Configuration;
 using Machine.VSTestAdapter.Discovery;
 using Machine.VSTestAdapter.Execution;
+using Machine.VSTestAdapter.Helpers;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 
 namespace Machine.VSTestAdapter.Specs.Configuration
@@ -20,7 +22,8 @@ public class When_adapter_runs_tests : WithFakes
   </MSpec>
 </RunSettings>";
 
-        static MSpecTestAdapter adapter;
+        static MSpecTestAdapterExecutor adapter;
+        static MSpecTestAdapterDiscoverer discoverer;
 
         Establish establish = () =>
         {
@@ -32,24 +35,27 @@ public class When_adapter_runs_tests : WithFakes
                 .WhenToldTo(context => context.RunSettings)
                 .Return(The<IRunSettings>());
 
-            adapter = new MSpecTestAdapter(An<ISpecificationDiscoverer>(), The<ISpecificationExecutor>());
+            discoverer = new MSpecTestAdapterDiscoverer(An<ISpecificationDiscoverer>());
+            adapter = new MSpecTestAdapterExecutor(The<ISpecificationExecutor>(), discoverer, An<ISpecificationFilterProvider>());
         };
 
-        Because of = () =>
-            adapter.RunTests(new[] { "dll" }, The<IRunContext>(), An<IFrameworkHandle>());
+        Because of = () => adapter.RunTests(new[] { new TestCase("a", MSpecTestAdapter.Uri, "dll") }, The<IRunContext>(), An<IFrameworkHandle>());
 
         It should_pick_up_DisableFullTestNameInIDE = () =>
             The<ISpecificationExecutor>()
-                .WasToldTo(d => d.RunAssembly("dll",
+                .WasToldTo(d => d.RunAssemblySpecifications("dll",
+                    Param<VisualStudioTestIdentifier[]>.IsAnything,
                     Param<Settings>.Matches(s => s.DisableFullTestNameInIDE),
                     Param<Uri>.IsAnything,
                     Param<IFrameworkHandle>.IsAnything));
 
         It should_pick_up_DisableFullTestNameInOutput = () =>
             The<ISpecificationExecutor>()
-                .WasToldTo(d => d.RunAssembly("dll",
+                .WasToldTo(d => d.RunAssemblySpecifications("dll",
+                    Param<VisualStudioTestIdentifier[]>.IsAnything,
                     Param<Settings>.Matches(s => s.DisableFullTestNameInOutput),
                     Param<Uri>.IsAnything,
                     Param<IFrameworkHandle>.IsAnything));
+
     }
 }

src/Machine.Specifications.Runner.VisualStudio.Specs/Configuration/When_parsing_configuration_and_mspec_section_is_missing.cs

@@ -3,6 +3,7 @@
 
 namespace Machine.VSTestAdapter.Specs.Configuration
 {
+    [Tags("Tag1","Tag2")]
     [Subject(typeof(Settings), "Configuration")]
     public class When_parsing_configuration_and_mspec_section_is_missing
     {

src/Machine.Specifications.Runner.VisualStudio.Specs/Execution/With_AssemblyExecutionSetup.cs

@@ -11,21 +11,21 @@ namespace Machine.VSTestAdapter.Specs.Execution
 {
     public abstract class With_AssemblyExecutionSetup : WithFakes
     {
-        static ISpecificationExecutor Executor;
+        static MSpecTestAdapter Executor;
         static CompileContext compiler;
         static Assembly assembly;
 
         Establish context = () =>
         {
             compiler = new CompileContext();
-            Executor = new SpecificationExecutor();
+            Executor = new MSpecTestAdapter();
 
             var assemblyPath = compiler.Compile(SampleFixture.Code);
             assembly = Assembly.LoadFile(assemblyPath);
         };
 
         Because of = () =>
-            Executor.RunAssembly(assembly.Location, An<Settings>(), new Uri("bla://executor"), The<IFrameworkHandle>());
+            Executor.RunTests(new[] { assembly.Location }, An<IRunContext>(), The<IFrameworkHandle>());
 
         Cleanup after = () =>
             compiler.Dispose();

src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_discovery.cs

@@ -19,7 +19,7 @@ public class When_there_is_an_unhandled_error_during_test_discovery : WithFakes
                     throw new InvalidOperationException();
                 });
 
-            Adapter = new MSpecTestAdapter(The<ISpecificationDiscoverer>(), An<ISpecificationExecutor>());
+            Adapter = new MSpecTestAdapter(The<ISpecificationDiscoverer>(), An<ISpecificationExecutor>(), An<ISpecificationFilterProvider>());
         };
 
         Because of = () => {

src/Machine.Specifications.Runner.VisualStudio.Specs/When_there_is_an_unhandled_error_during_test_execution.cs

@@ -1,30 +1,35 @@
 using System;
+using System.Collections.Generic;
 using Machine.Fakes;
 using Machine.Specifications;
 using Machine.VSTestAdapter.Configuration;
 using Machine.VSTestAdapter.Discovery;
 using Machine.VSTestAdapter.Execution;
+using Machine.VSTestAdapter.Helpers;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
 
 namespace Machine.VSTestAdapter.Specs
 {
     public class When_there_is_an_unhandled_error_during_test_execution : WithFakes
     {
-        static MSpecTestAdapter Adapter;
+        static MSpecTestAdapterExecutor Adapter;
 
         Establish context = () =>
         {
             The<ISpecificationExecutor>()
-                .WhenToldTo(d => d.RunAssembly(Param<string>.IsAnything, Param<Settings>.IsNotNull,
+                .WhenToldTo(d => d.RunAssemblySpecifications(Param<string>.IsAnything, Param<VisualStudioTestIdentifier[]>.IsAnything, Param<Settings>.IsNotNull,
                     Param<Uri>.IsAnything, Param<IFrameworkHandle>.IsAnything))
                 .Throw(new InvalidOperationException());
 
-            Adapter = new MSpecTestAdapter(An<ISpecificationDiscoverer>(), The<ISpecificationExecutor>());
+            var adapterDiscoverer = new MSpecTestAdapterDiscoverer(An<ISpecificationDiscoverer>());
+            Adapter = new MSpecTestAdapterExecutor(The<ISpecificationExecutor>(), adapterDiscoverer, An<ISpecificationFilterProvider>());
         };
 
-        Because of = () => { Adapter.RunTests(new[] {"bla"}, An<IRunContext>(), The<IFrameworkHandle>()); };
-        
+        Because of = () =>
+            Adapter.RunTests(new[] {new TestCase("a", MSpecTestAdapter.Uri, "dll"), }, An<IRunContext>(), The<IFrameworkHandle>());
+
         It should_send_an_error_notification_to_visual_studio = () =>
         {
             The<IFrameworkHandle>()

src/Machine.Specifications.Runner.VisualStudio/Configuration/Settings.cs

@@ -18,9 +18,17 @@ public static Settings Parse(string xml)
             Settings config = new Settings();
 
             XElement mspecConfig = null;
-            try {
-                mspecConfig = XDocument.Parse(xml).XPathSelectElement("RunSettings/MSpec");
-            } catch { }
+            try
+            {
+                if(!string.IsNullOrEmpty(xml))
+                {
+                    mspecConfig = XDocument.Parse(xml).XPathSelectElement("RunSettings/MSpec");
+                }
+            }
+            catch
+            {
+                // ignored
+            }
 
             if (mspecConfig == null)
                 return config;

src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationExecutor.cs

@@ -1,5 +1,4 @@
-using Microsoft.VisualStudio.TestPlatform.ObjectModel;
-using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 using System;
 using System.Collections.Generic;
 using Machine.VSTestAdapter.Helpers;
@@ -9,8 +8,6 @@ namespace Machine.VSTestAdapter.Execution
 {
     public interface ISpecificationExecutor
     {
-        void RunAssembly(string assemblyPath, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle);
-
-        void RunAssemblySpecifications(string assemblyPath, IEnumerable<VisualStudioTestIdentifier> specifications, Settings settings, Uri adaptorUri, IFrameworkHandle frameworkHandle);
+        void RunAssemblySpecifications(string assemblyPath, IEnumerable<VisualStudioTestIdentifier> specifications, Settings settings, Uri adapterUri, IFrameworkHandle frameworkHandle);
     }
-}
+}

src/Machine.Specifications.Runner.VisualStudio/Execution/ISpecificationFilterProvider.cs

@@ -0,0 +1,11 @@
+using System.Collections.Generic;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+
+namespace Machine.VSTestAdapter.Execution
+{
+    public interface ISpecificationFilterProvider
+    {
+        IEnumerable<TestCase> FilteredTests(IEnumerable<TestCase> testCases, IRunContext runContext, IFrameworkHandle frameworkHandle);
+    }
+}

src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationExecutor.cs

@@ -11,28 +11,10 @@ namespace Machine.VSTestAdapter.Execution
 {
     public class SpecificationExecutor : ISpecificationExecutor
     {
-        public void RunAssembly(string source, Settings settings, Uri executorUri, IFrameworkHandle frameworkHandle)
-        {
-            source = Path.GetFullPath(source);
-
-#if !NETSTANDARD
-            using (var scope = new IsolatedAppDomainExecutionScope<TestExecutor>(source)) {
-                TestExecutor executor = scope.CreateInstance();
-#else
-                TestExecutor executor = new TestExecutor();
-#endif
-
-                VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(source, frameworkHandle, executorUri, settings);
-                executor.RunAllTestsInAssembly(source, listener);
-#if !NETSTANDARD
-           }
-#endif
-        }
-
         public void RunAssemblySpecifications(string assemblyPath,
                                               IEnumerable<VisualStudioTestIdentifier> specifications,
                                               Settings settings,
-                                              Uri executorUri,
+                                              Uri adapterUri,
                                               IFrameworkHandle frameworkHandle)
         {
             assemblyPath = Path.GetFullPath(assemblyPath);
@@ -43,12 +25,12 @@ public void RunAssemblySpecifications(string assemblyPath,
 #else
                 TestExecutor executor = new TestExecutor();
 #endif
-                VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(assemblyPath, frameworkHandle, executorUri, settings);
+                VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(assemblyPath, frameworkHandle, adapterUri, settings);
 
                 executor.RunTestsInAssembly(assemblyPath, specifications, listener);
 #if !NETSTANDARD
            }
 #endif
         }
     }
-}
+}

src/Machine.Specifications.Runner.VisualStudio/Execution/SpecificationFilterProvider.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Machine.Specifications.Model;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+
+namespace Machine.VSTestAdapter.Execution
+{
+    public class SpecificationFilterProvider : ISpecificationFilterProvider
+    {
+        static readonly TestProperty TagProperty = TestProperty.Register(nameof(Tag), nameof(Tag), typeof(string), typeof(TestCase));
+        static readonly TestProperty SubjectProperty = TestProperty.Register(nameof(Subject), nameof(Subject), typeof(string), typeof(TestCase));
+
+        readonly Dictionary<string, TestProperty> testCaseProperties = new Dictionary<string, TestProperty>(StringComparer.OrdinalIgnoreCase)
+        {
+            [TestCaseProperties.FullyQualifiedName.Id] = TestCaseProperties.FullyQualifiedName,
+            [TestCaseProperties.DisplayName.Id] = TestCaseProperties.DisplayName
+        };
+
+        readonly Dictionary<string, TestProperty> traitProperties = new Dictionary<string, TestProperty>(StringComparer.OrdinalIgnoreCase)
+        {
+            [TagProperty.Id] = TagProperty,
+            [SubjectProperty.Id] = SubjectProperty
+        };
+
+        readonly string[] supportedProperties;
+
+        public SpecificationFilterProvider()
+        {
+            supportedProperties = testCaseProperties.Keys
+                .Concat(traitProperties.Keys)
+                .ToArray();
+        }
+
+
+        public IEnumerable<TestCase> FilteredTests(IEnumerable<TestCase> testCases, IRunContext runContext, IFrameworkHandle handle)
+        {
+            var filterExpression = runContext.GetTestCaseFilter(supportedProperties, propertyName =>
+            {
+                if (testCaseProperties.TryGetValue(propertyName, out var testProperty))
+                {
+                    return testProperty;
+                }
+                if (traitProperties.TryGetValue(propertyName, out var traitProperty))
+                {
+                    return traitProperty;
+                }
+                return null;
+            });
+
+            if (filterExpression == null)
+            {
+                return testCases;
+            }
+
+            handle?.SendMessage(TestMessageLevel.Informational, $"Machine Specifications Visual Studio Test Adapter - Filter property set '{filterExpression.TestCaseFilterValue}'");
+
+            var filteredTests = testCases
+                .Where(testCase => filterExpression.MatchTestCase(testCase, propertyName =>
+                    {
+                        var value = GetPropertyValue(propertyName, testCase);
+                        return value;
+                    }));
+
+            return filteredTests;
+        }
+
+        object GetPropertyValue(string propertyName, TestObject testCase)
+        {
+            if (testCaseProperties.TryGetValue(propertyName, out var testProperty))
+            {
+                if (testCase.Properties.Contains(testProperty))
+                {
+                    return testCase.GetPropertyValue(testProperty);
+                }
+            }
+
+            if (traitProperties.TryGetValue(propertyName, out var traitProperty))
+            {
+                var val = TraitContains(testCase, traitProperty.Id);
+
+                if (val.Length == 1)
+                {
+                    return val[0];
+                }
+
+                if (val.Length > 1)
+                {
+                    return val;
+                }
+            }
+
+            return null;
+        }
+
+        static string[] TraitContains(TestObject testCase, string traitName)
+        {
+            return testCase?.Traits?
+                .Where(x => x.Name == traitName)
+                .Select(x => x.Value)
+                .ToArray();
+        }
+    }
+}