Changes to handling test passing and exceptions (#83)

* Pass the test immediately if all test blocks pass and expose TestBlockException get and FinallyBlockExceptions

* Only expose readonly finally exceptions

Author: PandaMagnus

IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/FinallyExecutionTests.cs

@@ -3,6 +3,7 @@
 using System;
 using System.Threading.Tasks;
 using Xunit;
+using Xunit.Sdk;
 
 namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests
 {
@@ -201,5 +202,62 @@ public async Task OnlyTestBlockThrowsExpectedExceptionWhenOverridingDefaultFinal
             // Assert
             Assert.False(tc.Passed, "Test case did not get marked as Failed when we expected it.");
         }
+
+        [Fact]
+        public async Task TestCasePassedIsSetTrueBeforeFinallyBlocksRun()
+        {
+            // Arrange
+            TestCase tc = new TestBuilder()
+                .AddDependencyInstance(true)
+                .AddTestBlock<ExampleTestBlockWithBoolReturn>()
+                .AddFinallyBlock<ExampleBlockCheckingTestSuccess>()
+                .Build();
+
+            // Act
+            await tc.ExecuteAsync();
+
+            // Assert
+            Assert.True(tc.Passed, "Test case did not get marked as Passed when we expected it.");
+        }
+
+        [Fact]
+        public async Task TestCasePassedIsSetTrueEvenIfFinallyBlockFails()
+        {
+            // Arrange
+            TestCase tc = new TestBuilder()
+                .AddDependencyInstance(false)
+                .AddTestBlock<ExampleTestBlockWithBoolReturn>()
+                .AddFinallyBlock<ExampleBlockCheckingTestSuccess>()
+                .Build();
+
+            // Act
+            AggregateException exception = await Assert.ThrowsAsync<AggregateException>(() => tc.ExecuteAsync());
+
+            // Assert
+            Assert.Equal(2, exception.InnerExceptions.Count);
+            Assert.Equal(typeof(DivideByZeroException), exception.InnerException?.GetType());
+            Assert.Equal(typeof(DivideByZeroException), exception.InnerExceptions[0].GetType());
+            Assert.Equal(typeof(TrueException), exception.InnerExceptions[1].GetType());
+            Assert.False(tc.Passed, "Test case was marked as Passed when we did not expected it.");
+        }
+
+        [Fact]
+        public async Task CurrentFinallyBlockExceptionCountIncrementsWithEveryFailure()
+        {
+            // Arrange
+            TestCase tc = new TestBuilder()
+                .AddFinallyBlock<ExampleFinallyBlock>(false)
+                .AddFinallyBlock<ExampleFinallyBlock>(false)
+                .Build();
+            tc.ThrowOnFinallyBlockException = false;
+
+            // Act
+            await tc.ExecuteAsync();
+
+            // Assert
+            var test = tc.CurrentFinallyBlockExceptions;
+            Assert.Equal(2, tc.CurrentFinallyBlockExceptions.Count);
+            Assert.True(tc.Passed, "Test case did not get marked as Passed when we expected it.");
+        }
     }
 }

IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/SimulatorClasses.cs

@@ -134,4 +134,12 @@ public async Task Execute()
             await Task.Delay(1);
         }
     }
+
+    public class ExampleBlockCheckingTestSuccess : TestBlock
+    {
+        public void Execute(TestCase tc)
+        {
+            Assert.True(tc.Passed);
+        }
+    }
 }

IntelliTect.TestTools.TestFramework/TestCase.cs

@@ -37,22 +37,37 @@ public TestCase(string testCaseName, string testMethodName, int testCaseId, ISer
         /// If a finally block fails and this property is true, the test case is still considered passed internally, but most unit test frameworks will mark the test failed.
         /// </summary>
         public bool ThrowOnFinallyBlockException { get; set; } = true;
+        /// <summary>
+        /// If any test block throws an exception, it will be stored here. Finally block exceptions are stored separately in FinallyBlockExceptions. If this is not null at the end of the test case execution, the test case is considered failed.
+        /// </summary>
+        public Exception? TestBlockException { get; private set; }
+        /// <summary>
+        /// Gets the list of exceptions that were thrown during the execution of finally blocks.
+        /// </summary>
+        /// <remarks>This collection contains all exceptions that occurred in finally blocks, allowing
+        /// callers to inspect or handle them after execution. The list is empty if no exceptions were thrown in any
+        /// finally block.</remarks>
+        //public readonly List<Exception> FinallBlockExceptions { get; } = [];
+        public IReadOnlyList<Exception> CurrentFinallyBlockExceptions => FinallyBlockExceptions;
+        /// <summary>
+        /// Did the test case pass? This is determined by whether any test block threw an exception.
+        /// </summary>
+        /// <remarks>
+        /// Finally block exceptions do not cause the test case to be marked as failed, but they are still captured and can cause the test case to throw after execution if ThrowOnFinallyBlockException is true.
+        /// </remarks>
+        public bool Passed { get; set; }
 
         // May make sense to make some of the below public if it's needed for debugging.
         // If so, definitely need to change them to internal or private sets.
 
-        internal List<Block> TestBlocks { get; set; } = new();
-        internal List<Block> FinallyBlocks { get; set; } = new();
+        internal List<Block> TestBlocks { get; set; } = [];
+        internal List<Block> FinallyBlocks { get; set; } = [];
         internal bool HasLogger { get; set; } = true;
 
         private ITestCaseLogger? Log { get; set; }
         private IServiceCollection ServiceCollection { get; }
-        private Dictionary<Type, object> BlockOutput { get; } = new();
-        private Exception? TestBlockException { get; set; }
-        private List<Exception> FinallyBlockExceptions { get; } = new();
-
-        // Has this test case passed? Will only be true if every regular test block succeeds.
-        public bool Passed { get; set; }
+        private Dictionary<Type, object> BlockOutput { get; } = [];
+        private List<Exception> FinallyBlockExceptions { get; } = [];
 
         /// <summary>
         /// Legacy method signature. Executes the test case. NOTE: Prefer to use ExecuteAsync, even if you have no awaitable test blocks.
@@ -101,6 +116,16 @@ public async Task ExecuteAsync()
                     }
                 }
 
+                if (TestBlockException is null)
+                {
+                    Passed = true;
+                    Log?.Info("Test case finished successfully.");
+                }
+                else
+                {
+                    Log?.Critical($"Test case failed and will continue to executing Finally blocks. Exception: {TestBlockException}");
+                }
+
                 foreach (var fb in FinallyBlocks)
                 {
                     if (Log is not null) Log.CurrentTestBlock = fb.Type.ToString();
@@ -116,20 +141,6 @@ public async Task ExecuteAsync()
                         Log?.Critical($"Finally block failed: {FinallyBlockExceptions.LastOrDefault()}");
                     }
                 }
-
-                if (TestBlockException is null)
-                {
-                    // Note: This likely needs to be moved up above the finally blocks.
-                    // If a test case "passes" i.e. finishes all of its test blocks, we probably need to know that
-                    //  in the finally blocks.
-                    // Need to think about how to handle "passed" state if a finally block fails.
-                    Passed = true;
-                    Log?.Info("Test case finished successfully.");
-                }
-                else
-                {
-                    Log?.Critical($"Test case failed: {TestBlockException}");
-                }
             }
 
             services.Dispose();