Add failed logic for trx logger when TreatNoTestAsError is set to true (#2758)

* Initial implementation

* changing place for outcome

* Parameter send when the value is True. Tests are added

* making tests only for windows

* removing wrong comment

Author: Sanan07

src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs

@@ -131,20 +131,20 @@ public static int GetMaxCpuCount(RunConfiguration runConfiguration)
             return cpuCount;
         }
         /// <summary>
-        /// Gets the value of FailWhenNoTestsFound parameter from runsettings file
+        /// Gets the value of TreatNoTestsAsError parameter from runsettings file
         /// </summary>
         /// <param name="runSettings">Runsetting string value</param>
-        /// <returns>The value of FailWhenNoTestsFound</returns>
+        /// <returns>The value of TreatNoTestsAsError</returns>
         public static bool GetTreatNoTestsAsError(string runSettings)
         {
-            bool failWhenNoTestFound = false;
+            bool treatNoTestsAsError = false;
 
             if (runSettings != null)
             {
                 try
                 {
                     RunConfiguration runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
-                    failWhenNoTestFound = GetTreatNoTestsAsError(runConfiguration);
+                    treatNoTestsAsError = GetTreatNoTestsAsError(runConfiguration);
                 }
                 catch (SettingsException se)
                 {
@@ -155,7 +155,7 @@ public static bool GetTreatNoTestsAsError(string runSettings)
                 }
             }
 
-            return failWhenNoTestFound;
+            return treatNoTestsAsError;
         }
 
         private static bool GetTreatNoTestsAsError(RunConfiguration runConfiguration)

src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs

@@ -22,7 +22,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client
     using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
     using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
     using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces;
-    using CommonResources = Microsoft.VisualStudio.TestPlatform.Common.Resources.Resources;
+    using CommonResources = Common.Resources.Resources;
 
     /// <summary>
     /// Responsible for managing logger extensions and broadcasting results
@@ -52,6 +52,11 @@ internal class TestLoggerManager : ITestLoggerManager
         /// </summary>
         private string targetFramework;
 
+        /// <summary>
+        /// TreatNoTestsAsError value;
+        /// </summary>
+        private bool treatNoTestsAsError;
+
         /// <summary>
         /// Test Logger Events instance which will be passed to loggers when they are initialized.
         /// </summary>
@@ -145,6 +150,7 @@ public void Initialize(string runSettings)
             // Store test run directory. This runsettings is the final runsettings merging CLI args and runsettings.
             this.testRunDirectory = GetResultsDirectory(runSettings);
             this.targetFramework = GetTargetFramework(runSettings)?.Name;
+            this.treatNoTestsAsError = GetTreatNoTestsAsError(runSettings);
 
             var loggers = XmlRunSettingsUtilities.GetLoggerRunSettings(runSettings);
 
@@ -487,6 +493,16 @@ internal Framework GetTargetFramework(string runSettings)
             return targetFramework;
         }
 
+        /// <summary>
+        /// Get TreatNoTestsAsError value of the test run
+        /// </summary>
+        /// <param name="runSettings"></param>
+        /// <returns></returns>
+        internal bool GetTreatNoTestsAsError(string runSettings)
+        {
+            return RunSettingsUtilities.GetTreatNoTestsAsError(runSettings);
+        }
+
         /// <summary>
         /// Enables sending of events to the loggers which are registered.
         /// </summary>
@@ -648,6 +664,13 @@ private Dictionary<string, string> UpdateLoggerParameters(Dictionary<string, str
             // Add default logger parameters...
             loggerParams[DefaultLoggerParameterNames.TestRunDirectory] = testRunDirectory;
             loggerParams[DefaultLoggerParameterNames.TargetFramework] = targetFramework;
+
+            // Add custom logger parameters
+            if (treatNoTestsAsError)
+            {
+                loggerParams[Constants.TreatNoTestsAsError] = treatNoTestsAsError.ToString();
+            }
+            
             return loggerParams;
         }
 

src/Microsoft.TestPlatform.Extensions.TrxLogger/TrxLogger.cs

@@ -20,11 +20,11 @@ namespace Microsoft.VisualStudio.TestPlatform.Extensions.TrxLogger
     using System.Globalization;
     using System.IO;
     using System.Text;
-    using System.Threading;
     using System.Xml;
+    using ObjectModelConstants = ObjectModel.Constants;
     using TrxLoggerConstants = Microsoft.TestPlatform.Extensions.TrxLogger.Utility.Constants;
     using TrxLoggerObjectModel = Microsoft.TestPlatform.Extensions.TrxLogger.ObjectModel;
-    using TrxLoggerResources = Microsoft.VisualStudio.TestPlatform.Extensions.TrxLogger.Resources.TrxResource;
+    using TrxLoggerResources = Resources.TrxResource;
 
     /// <summary>
     /// Logger for Generating TRX
@@ -65,13 +65,13 @@ internal TrxLogger(IFileHelper fileHelper, TrxFileHelper trxFileHelper)
         // The converter class
         private Converter converter;
 
-        private TrxLoggerObjectModel.TestRun testRun;
-        private ConcurrentDictionary<Guid, TrxLoggerObjectModel.ITestResult> results;
-        private ConcurrentDictionary<Guid, TrxLoggerObjectModel.ITestElement> testElements;
+        private TestRun testRun;
+        private ConcurrentDictionary<Guid, ITestResult> results;
+        private ConcurrentDictionary<Guid, ITestElement> testElements;
         private ConcurrentDictionary<Guid, TestEntry> entries;
 
         // Caching results and inner test entries for constant time lookup for inner parents.
-        private ConcurrentDictionary<Guid, TrxLoggerObjectModel.ITestResult> innerResults;
+        private ConcurrentDictionary<Guid, ITestResult> innerResults;
         private ConcurrentDictionary<Guid, TestEntry> innerTestEntries;
 
         private readonly TrxFileHelper trxFileHelper;
@@ -82,7 +82,7 @@ internal TrxLogger(IFileHelper fileHelper, TrxFileHelper trxFileHelper)
         private StringBuilder runLevelStdOut;
 
         // List of run level errors and warnings generated. These are logged in the Trx in the Results Summary.
-        private List<TrxLoggerObjectModel.RunInfo> runLevelErrorsAndWarnings;
+        private List<RunInfo> runLevelErrorsAndWarnings;
 
         private TrxLoggerObjectModel.TestOutcome testRunOutcome = TrxLoggerObjectModel.TestOutcome.Passed;
 
@@ -230,10 +230,11 @@ internal TrxLoggerObjectModel.TestOutcome TestResultOutcome
         /// </param>
         internal void TestMessageHandler(object sender, TestRunMessageEventArgs e)
         {
-            ValidateArg.NotNull<object>(sender, "sender");
-            ValidateArg.NotNull<TestRunMessageEventArgs>(e, "e");
+            ValidateArg.NotNull(sender, "sender");
+            ValidateArg.NotNull(e, "e");
 
             RunInfo runMessage;
+
             switch (e.Level)
             {
                 case TestMessageLevel.Informational:
@@ -263,7 +264,7 @@ internal void TestMessageHandler(object sender, TestRunMessageEventArgs e)
         /// <param name="e">
         /// The eventArgs.
         /// </param>
-        internal void TestResultHandler(object sender, ObjectModel.Logging.TestResultEventArgs e)
+        internal void TestResultHandler(object sender, TestResultEventArgs e)
         {
             // Create test run
             if (this.testRun == null)
@@ -363,6 +364,8 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
                 this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Completed;
             }
 
+            testRunOutcome = changeTestOutcomeIfNecessary(testRunOutcome);
+
             List<string> errorMessages = new List<string>();
             List<CollectorDataEntry> collectorEntries = this.converter.ToCollectionEntries(e.AttachmentSets, this.testRun, this.testResultsDirPath);
             IList<string> resultFiles = this.converter.ToResultFiles(e.AttachmentSets, this.testRun, this.testResultsDirPath, errorMessages);
@@ -760,6 +763,19 @@ private TestEntry GetTestEntry(Guid executionId)
             return testEntry;
         }
 
+        private TrxLoggerObjectModel.TestOutcome changeTestOutcomeIfNecessary (TrxLoggerObjectModel.TestOutcome outcome)
+        {
+            // If no tests discovered/executed and TreatNoTestsAsError was set to True
+            // We will return ResultSummary as Failed
+            // Note : we only send the value of TreatNoTestsAsError if it is "True"
+            if (totalTests == 0 && parametersDictionary.ContainsKey(ObjectModelConstants.TreatNoTestsAsError))
+            {
+                outcome = TrxLoggerObjectModel.TestOutcome.Failed;
+            }
+
+            return outcome;
+        }
+
         #endregion
     }
 }

src/Microsoft.TestPlatform.ObjectModel/Constants.cs

@@ -115,6 +115,11 @@ public static class Constants
         /// </summary>
         public const string LoggerConfigurationNameLower = "configuration";
 
+        /// <summary>
+        /// Name of TreatNoTestsAsError parameter
+        /// </summary>
+        public const string TreatNoTestsAsError = "TreatNoTestsAsError";
+
         /// <summary>
         /// Name of RunConfiguration settings node in RunSettings.
         /// </summary>

test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs

@@ -131,6 +131,54 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn
             TryRemoveDirectory(testResultsDirectory);
         }
 
+        [TestMethod]
+        [TestCategory("Windows-Review")]
+        [NetFullTargetFrameworkDataSource]
+        public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAndTreatNoTestsAsErrorIsTrue(RunnerInfo runnerInfo)
+        {
+            SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue);
+            var trxFileName = "TrxLogger.trx";
+
+            arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\"");
+
+            // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed
+            arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly");
+            arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true");
+
+            this.InvokeVsTest(arguments);
+
+            var trxLogFilePath = Path.Combine(Directory.GetCurrentDirectory(), "TestResults", trxFileName);
+            string outcomeValue = GetElementAtributeValueFromTrx(trxLogFilePath, "ResultSummary", "outcome");
+
+            Assert.AreEqual("Failed", outcomeValue);
+        }
+
+        [TestMethod]
+        [TestCategory("Windows-Review")]
+        [NetFullTargetFrameworkDataSource]
+        public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAndTreatNoTestsAsErrorIsFalse(RunnerInfo runnerInfo)
+        {
+            SetTestEnvironment(this.testEnvironment, runnerInfo);
+
+            var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue);
+            var trxFileName = "TrxLogger.trx";
+
+            arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\"");
+
+            // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed
+            arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly");
+            arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false");
+
+            this.InvokeVsTest(arguments);
+
+            var trxLogFilePath = Path.Combine(Directory.GetCurrentDirectory(), "TestResults", trxFileName);
+            string outcomeValue = GetElementAtributeValueFromTrx(trxLogFilePath, "ResultSummary", "outcome");
+
+            Assert.AreEqual("Completed", outcomeValue);
+        }
+
         private bool IsValidXml(string xmlFilePath)
         {
             try
@@ -166,5 +214,22 @@ private void IsFileAndContentEqual(string filePath)
                 StringAssert.Contains(filePathContent, str);
             }
         }
+
+        private static string GetElementAtributeValueFromTrx(string trxFileName, string fieldName, string attributeName)
+        {
+            using (FileStream file = File.OpenRead(trxFileName))
+            using (XmlReader reader = XmlReader.Create(file))
+            {
+                while (reader.Read())
+                {
+                    if (reader.Name.Equals(fieldName) && reader.NodeType == XmlNodeType.Element && reader.HasAttributes)
+                    {
+                        return reader.GetAttribute(attributeName);
+                    }
+                }
+            }
+
+            return null;
+        }
     }
 }