Graceful failure for unmanaged assemblies

Author: CharliePoole

package-tests.cake

@@ -609,6 +609,16 @@ AddToBothLists(new PackageTest(1, "AppContextBaseDirectory_NET80")
     }
 });
 
+AddToBothLists(new PackageTest(1, "UnmanagedAssemblyTest")
+{
+    Description = "Attempt to run an unmanaged assembly fails gracefully",
+    Arguments = "../../src/TestData/native-assembly/NativeTests.dll",
+    ExpectedResult = new ExpectedResult("Failed:Invalid")
+    {
+        Assemblies = new ExpectedAssemblyResult[] { new ExpectedAssemblyResult("NativeTests.dll", "net-4.6.2") }
+    }
+});
+
 // NOTE: Tests for NUnit.Engine and NUnit.Agent.Core here are quite limited. At this
 // point, the main purpose they serve is to demonstrate that we are ABLE to  run
 // the tests without using the console runner.

src/NUnitCommon/nunit.extensibility/ExtensionManager.cs

@@ -495,15 +495,16 @@ public void FindExtensionsInAssembly(ExtensionAssembly extensionAssembly)
                 // TODO: This is a remnant of older code. In principle, this should be generalized
                 // to something like "HostVersion". However, this can safely remain until
                 // we separate ExtensionManager into its own assembly.
-                string? versionArg = extensionAttr.GetNamedArgument(nameof(ExtensionAttribute.EngineVersion)) as string;
-                if (versionArg is not null)
-                {
-                    if (new Version(versionArg) > CURRENT_ENGINE_VERSION)
-                    {
-                        log.Warning($"  Ignoring {extensionType.Name}. It requires version {versionArg}.");
-                        continue;
-                    }
-                }
+                // Temporarily removing this check for testing
+                //string? versionArg = extensionAttr.GetNamedArgument(nameof(ExtensionAttribute.EngineVersion)) as string;
+                //if (versionArg is not null)
+                //{
+                //    if (new Version(versionArg) > CURRENT_ENGINE_VERSION)
+                //    {
+                //        log.Warning($"  Ignoring {extensionType.Name}. It requires version {versionArg}.");
+                //        continue;
+                //    }
+                //}
 
                 string? extensionAttrPath = (string?)extensionAttr.GetNamedArgument(nameof(ExtensionAttribute.Path));
                 var node = new ExtensionNode(extensionAssembly, extensionType)

src/NUnitEngine/nunit.engine.api/IRuntimeFrameworkService.cs

@@ -26,16 +26,9 @@ public interface IRuntimeFrameworkService
         /// <summary>
         /// Selects a target runtime framework for a TestPackage based on
         /// the settings in the package and the assemblies themselves.
-        /// The package RuntimeFramework setting may be updated as a
-        /// result and the selected runtime is returned.
-        ///
-        /// Note that if a package has subpackages, the subpackages may run on a different
-        /// framework to the top-level package. In future, this method should
-        /// probably not return a simple string, and instead require runners
-        /// to inspect the test package structure, to find all desired frameworks.
+        /// The package RuntimeFramework setting may be updated as a result.
         /// </summary>
         /// <param name="package">A TestPackage</param>
-        /// <returns>The selected RuntimeFramework</returns>
         void SelectRuntimeFramework(TestPackage package);
     }
 }

src/NUnitEngine/nunit.engine/Runners/MasterTestRunner.cs

@@ -1,15 +1,18 @@
 // Copyright (c) Charlie Poole, Rob Prouse and Contributors. MIT License - see LICENSE.txt
 
+using NUnit.Common;
+using NUnit.Engine.Services;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics;
 using System.Globalization;
+using System.IO;
 using System.Reflection;
+using System.Runtime.Versioning;
 using System.Timers;
 using System.Xml;
-using NUnit.Engine.Services;
-using NUnit.Common;
+using TestCentric.Metadata;
 
 namespace NUnit.Engine.Runners
 {
@@ -263,6 +266,33 @@ internal ITestEngineRunner GetEngineRunner()
                 // each contained assembly, including it's target runtime.
                 foreach (var assemblyPackage in leafPackages)
                     _runtimeService.SelectRuntimeFramework(assemblyPackage);
+#else
+                var package = leafPackages[0];
+                string packageName = package.FullName ?? string.Empty;
+
+                if (File.Exists(packageName))
+                try
+                {
+                    using (var assembly = AssemblyDefinition.ReadAssembly(packageName))
+                    {
+                        string targetVersion = assembly.GetRuntimeVersion().ToString();
+                        string frameworkName = assembly.GetFrameworkName();
+                        bool requiresX86 = assembly.RequiresX86();
+                        bool requiresAssemblyResolver = assembly.HasAttribute("NUnit.Framework.TestAssemblyDirectoryResolveAttribute");
+
+                        package.Settings.Set(SettingDefinitions.ImageRuntimeVersion.WithValue(targetVersion));
+                        package.Settings.Set(SettingDefinitions.ImageTargetFrameworkName.WithValue(frameworkName));
+                        package.Settings.Set(SettingDefinitions.ImageRequiresX86.WithValue(true));
+                        package.Settings.Set(SettingDefinitions.RunAsX86.WithValue(true));
+                        package.Settings.Set(SettingDefinitions.ImageRequiresDefaultAppDomainAssemblyResolver.WithValue(requiresAssemblyResolver));
+                    }
+                }
+                catch (BadImageFormatException)
+                {
+                    // "Unmanaged" is not a valid framework identifier but we handle it upstream
+                    // using UnmanagedCodeTestRunner, which doesn't actually try to run it.
+                    package.Settings.Set(SettingDefinitions.ImageTargetFrameworkName.WithValue("Unmanaged,Version=0.0"));
+                }
 #endif
                 _engineRunner = _testRunnerFactory.MakeTestRunner(TestPackage);
             }

….engine/Runners/NotRunnableTestRunner.cs …engine/Runners/NotRunnableTestRunners.cssrc/NUnitEngine/nunit.engine/Runners/NotRunnableTestRunner.cs renamed to src/NUnitEngine/nunit.engine/Runners/NotRunnableTestRunners.cs src/NUnitEngine/nunit.engine/Runners/NotRunnableTestRunner.cs renamed to src/NUnitEngine/nunit.engine/Runners/NotRunnableTestRunners.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Reflection.Emit;
 
 namespace NUnit.Engine.Runners
 {
@@ -157,11 +158,14 @@ public InvalidAssemblyTestRunner(string assemblyPath, string message)
         }
     }
 
-    public class UnmanagedExecutableTestRunner : InvalidAssemblyTestRunner
+    public class UnmanagedExecutableTestRunner : NotRunnableTestRunner
     {
         public UnmanagedExecutableTestRunner(string assemblyPath)
             : base(assemblyPath, "Unmanaged libraries or applications are not supported")
         {
+            _runstate = "NotRunnable";
+            _result = "Failed";
+            _label = "Invalid";
         }
     }
 

src/NUnitEngine/nunit.engine/Services/RuntimeFrameworkService.cs

@@ -173,6 +173,8 @@ public void SelectRuntimeFramework(TestPackage package)
                         targetVersion = new Version(3, 1);
                         break;
                     case "Unmanaged":
+                        package.Settings.Set(SettingDefinitions.ImageTargetFrameworkName.WithValue("Unmanaged,Version=0.0"));
+                        return;
                     default:
                         throw new NUnitEngineException("Unsupported Target Framework: " + imageTargetFrameworkNameSetting);
                 }

src/NUnitEngine/nunit.engine/Services/TestRunnerFactory.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections;
+using System.Diagnostics;
 using System.IO;
 using NUnit.Common;
 using NUnit.Engine.Runners;
@@ -75,15 +76,13 @@ public ITestEngineRunner MakeTestRunner(TestPackage package)
                 return new InvalidAssemblyTestRunner(assemblyPath, $"Not a valid assembly: {assemblyPath}");
 
             string targetFrameworkName = package.Settings.GetValueOrDefault(SettingDefinitions.ImageTargetFrameworkName);
+            string platform = targetFrameworkName.Split(',')[0];
             if (!string.IsNullOrEmpty(targetFrameworkName))
-            {
-                string platform = targetFrameworkName.Split(',')[0];
                 if (platform == "Silverlight" || platform == ".NETPortable" || platform == ".NETStandard" || platform == ".NETCompactFramework")
                     return new InvalidAssemblyTestRunner(assemblyPath, $"Platform {platform} is not supported");
 
-                if (platform == "Unmanaged")
-                    return new UnmanagedExecutableTestRunner(assemblyPath);
-            }
+            if (platform == "Unmanaged")
+                return new UnmanagedExecutableTestRunner(assemblyPath);
 
             bool skipNonTestAssemblies = package.Settings.GetValueOrDefault(SettingDefinitions.SkipNonTestAssemblies);
             if (skipNonTestAssemblies)