diff --git a/Directory.Packages.props b/Directory.Packages.props index 9c97b932e9..5210ddf284 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -13,6 +13,7 @@ + diff --git a/TUnit.Example.FsCheck.TestProject/PropertyTests.cs b/TUnit.Example.FsCheck.TestProject/PropertyTests.cs new file mode 100644 index 0000000000..27193b2b26 --- /dev/null +++ b/TUnit.Example.FsCheck.TestProject/PropertyTests.cs @@ -0,0 +1,91 @@ +using FsCheck; +using FsCheck.Fluent; +using TUnit.FsCheck; + +namespace TUnit.Example.FsCheck.TestProject; + +public class PropertyTests +{ + [Test, FsCheckProperty] + public bool ReverseReverseIsOriginal(int[] array) + { + var reversed = array.AsEnumerable().Reverse().Reverse().ToArray(); + return array.SequenceEqual(reversed); + } + + [Test, FsCheckProperty] + public bool AbsoluteValueIsNonNegative(int value) + { + return Math.Abs((long)value) >= 0; + } + + [Test, FsCheckProperty] + public bool StringConcatenationLength(string a, string b) + { + if (a == null || b == null) + { + return true; // Skip null cases + } + + return (a + b).Length == a.Length + b.Length; + } + + [Test, FsCheckProperty(MaxTest = 50)] + public bool ListConcatenationPreservesElements(int[] first, int[] second) + { + var combined = first.Concat(second).ToArray(); + return combined.Length == first.Length + second.Length; + } + + [Test, FsCheckProperty] + public void AdditionIsCommutative(int a, int b) + { + var result1 = a + b; + var result2 = b + a; + + if (result1 != result2) + { + throw new InvalidOperationException($"Addition is not commutative: {a} + {b} = {result1}, {b} + {a} = {result2}"); + } + } + + [Test, FsCheckProperty] + public async Task AsyncPropertyTest(int value) + { + await Task.Delay(1); // Simulate async work + + if (value * 0 != 0) + { + throw new InvalidOperationException("Multiplication by zero should always be zero"); + } + } + + [Test, FsCheckProperty] + public bool MultiplicationIsAssociative(int a, int b, int c) + { + // Using long to avoid overflow + var left = (long)a * ((long)b * c); + var right = ((long)a * b) * c; + return left == right; + } + + [Test, FsCheckProperty] + public bool SumOfFourNumbersIsCommutative(int a, int b, int c, int d) + { + var sum1 = a + b + c + d; + var sum2 = d + c + b + a; + return sum1 == sum2; + } + + [Test, FsCheckProperty] + public Property StringReversalProperty() + { + return Prop.ForAll(str => + { + var reversed = new string(str.Reverse().ToArray()); + var doubleReversed = new string(reversed.Reverse().ToArray()); + return str == doubleReversed; + }); + } + +} diff --git a/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.csproj b/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.csproj new file mode 100644 index 0000000000..f23b29646c --- /dev/null +++ b/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.csproj @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.sln b/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.sln new file mode 100644 index 0000000000..4ddb9e442f --- /dev/null +++ b/TUnit.Example.FsCheck.TestProject/TUnit.Example.FsCheck.TestProject.sln @@ -0,0 +1,24 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.5.2.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TUnit.Example.FsCheck.TestProject", "TUnit.Example.FsCheck.TestProject.csproj", "{41C48729-CBC0-9C84-9E2E-AD18967D3F54}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {41C48729-CBC0-9C84-9E2E-AD18967D3F54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {41C48729-CBC0-9C84-9E2E-AD18967D3F54}.Debug|Any CPU.Build.0 = Debug|Any CPU + {41C48729-CBC0-9C84-9E2E-AD18967D3F54}.Release|Any CPU.ActiveCfg = Release|Any CPU + {41C48729-CBC0-9C84-9E2E-AD18967D3F54}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {29D619C4-A78D-44B6-9D5C-304546DF9B20} + EndGlobalSection +EndGlobal diff --git a/TUnit.FsCheck/FsCheckPropertyAttribute.cs b/TUnit.FsCheck/FsCheckPropertyAttribute.cs new file mode 100644 index 0000000000..7433a2618a --- /dev/null +++ b/TUnit.FsCheck/FsCheckPropertyAttribute.cs @@ -0,0 +1,100 @@ +using TUnit.Core; +using TUnit.Core.Interfaces; + +namespace TUnit.FsCheck; + +/// +/// Marks a test method as an FsCheck property-based test. +/// The test method parameters will be generated by FsCheck and the test +/// will be run multiple times with different generated values. +/// +/// +/// This attribute must be used together with the . +/// Example: +/// +/// [Test, FsCheckProperty] +/// public bool MyProperty(int value) => value * 0 == 0; +/// +/// +[AttributeUsage(AttributeTargets.Method)] +public class FsCheckPropertyAttribute : Attribute, ITestRegisteredEventReceiver, IDataSourceAttribute +{ + + /// + /// The maximum number of tests to run. Default is 100. + /// + public int MaxTest { get; set; } = 100; + + /// + /// The maximum number of rejected tests (tests that failed the precondition) before failing. + /// + public int MaxFail { get; set; } = 1000; + + /// + /// The starting size for test generation. Size increases linearly between StartSize and EndSize. + /// + public int StartSize { get; set; } = 1; + + /// + /// The ending size for test generation. Size increases linearly between StartSize and EndSize. + /// + public int EndSize { get; set; } = 100; + + /// + /// If set, replay the test using this seed. Format: "seed1,seed2" or just "seed1". + /// Useful for reproducing failures. + /// + public string? Replay { get; set; } + + /// + /// If true, output all generated arguments to the test output. + /// + public bool Verbose { get; set; } + + /// + /// If true, suppress output on passing tests. + /// + public bool QuietOnSuccess { get; set; } + + /// + /// The level of parallelism to use when running tests. + /// Default is 1 (no parallelism within property execution). + /// + public int Parallelism { get; set; } = 1; + + /// + /// Types containing Arbitrary instances to use for generating test data. + /// + public Type[]? Arbitrary { get; set; } + + /// + /// Gets the order in which this event receiver is executed. + /// + public int Order => 0; + + /// + /// Not used - FsCheck generates its own data during test execution. + /// This property exists to satisfy the IDataSourceAttribute interface. + /// + public bool SkipIfEmpty { get; set; } + + /// + /// Called when the test is registered. Sets up the FsCheck property executor. + /// + public ValueTask OnTestRegistered(TestRegisteredContext context) + { + context.SetTestExecutor(new FsCheckPropertyTestExecutor(this)); + return default; + } + + /// + /// Returns placeholder data - actual test data is generated by FsCheck during test execution. + /// +#pragma warning disable CS1998 // Async method lacks 'await' operators + async IAsyncEnumerable>> IDataSourceAttribute.GetDataRowsAsync(DataGeneratorMetadata dataGeneratorMetadata) +#pragma warning restore CS1998 + { + // Return null array as placeholder - the FsCheckPropertyTestExecutor will generate actual data + yield return () => Task.FromResult(null); + } +} diff --git a/TUnit.FsCheck/FsCheckPropertyTestExecutor.cs b/TUnit.FsCheck/FsCheckPropertyTestExecutor.cs new file mode 100644 index 0000000000..1ab58810ef --- /dev/null +++ b/TUnit.FsCheck/FsCheckPropertyTestExecutor.cs @@ -0,0 +1,281 @@ +using System.Diagnostics.CodeAnalysis; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Text; +using FsCheck; +using FsCheck.Fluent; +using TUnit.Core; +using TUnit.Core.Interfaces; + +namespace TUnit.FsCheck; + +/// +/// A test executor that runs FsCheck property-based tests. +/// +#pragma warning disable IL2046 // RequiresUnreferencedCode attribute mismatch +#pragma warning disable IL3051 // RequiresDynamicCode attribute mismatch +#pragma warning disable IL2072 // DynamicallyAccessedMembers warning +public class FsCheckPropertyTestExecutor : ITestExecutor +{ + private readonly FsCheckPropertyAttribute _propertyAttribute; + + public FsCheckPropertyTestExecutor(FsCheckPropertyAttribute propertyAttribute) + { + _propertyAttribute = propertyAttribute; + } + + public ValueTask ExecuteTest(TestContext context, Func action) + { + var testDetails = context.Metadata.TestDetails; + var classInstance = testDetails.ClassInstance; + var classType = testDetails.ClassType; + var methodName = testDetails.MethodName; + + // Get MethodInfo via reflection from the class type + var methodInfo = GetMethodInfo(classType, methodName, testDetails.MethodMetadata.Parameters); + + var config = CreateConfig(); + + RunPropertyCheck(methodInfo, classInstance, config); + + return default; + } + + [UnconditionalSuppressMessage("Trimming", "IL2070", Justification = "FsCheck requires reflection")] + private static MethodInfo GetMethodInfo( + [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicMethods)] + Type classType, + string methodName, + ParameterMetadata[] parameters) + { + // Try to find the method by name and parameter count + var methods = classType + .GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static) + .Where(m => m.Name == methodName && m.GetParameters().Length == parameters.Length) + .ToArray(); + + if (methods.Length == 0) + { + throw new InvalidOperationException($"Could not find method '{methodName}' on type '{classType.FullName}'"); + } + + if (methods.Length == 1) + { + return methods[0]; + } + + // Multiple overloads - try to match by parameter types + foreach (var method in methods) + { + var methodParams = method.GetParameters(); + var match = true; + for (var i = 0; i < methodParams.Length; i++) + { + if (methodParams[i].ParameterType != parameters[i].Type) + { + match = false; + break; + } + } + + if (match) + { + return method; + } + } + + // Just return the first one if no exact match + return methods[0]; + } + + private Config CreateConfig() + { + var config = Config.QuickThrowOnFailure + .WithMaxTest(_propertyAttribute.MaxTest) + .WithMaxRejected(_propertyAttribute.MaxFail) + .WithStartSize(_propertyAttribute.StartSize) + .WithEndSize(_propertyAttribute.EndSize); + + if (!string.IsNullOrEmpty(_propertyAttribute.Replay)) + { + var parts = _propertyAttribute.Replay!.Split(','); + if (parts.Length >= 1 && ulong.TryParse(parts[0].Trim(), out var seed1)) + { + var seed2 = parts.Length >= 2 && ulong.TryParse(parts[1].Trim(), out var s2) ? s2 : 0UL; + config = config.WithReplay(seed1, seed2); + } + } + + if (_propertyAttribute.Arbitrary != null && _propertyAttribute.Arbitrary.Length > 0) + { + config = config.WithArbitrary(_propertyAttribute.Arbitrary); + } + + return config; + } + + [UnconditionalSuppressMessage("Trimming", "IL2060", Justification = "FsCheck requires reflection")] + [UnconditionalSuppressMessage("AOT", "IL3050", Justification = "FsCheck requires dynamic code")] + private static void RunPropertyCheck(MethodInfo methodInfo, object classInstance, Config config) + { + try + { + Check.Method(config, methodInfo, classInstance); + } + catch (Exception ex) + { + throw new PropertyFailedException(FormatCounterexample(methodInfo, ex)); + } + } + + private static string FormatCounterexample(MethodInfo methodInfo, Exception ex) + { + var parameters = methodInfo.GetParameters(); + var args = parameters + .Select((p, i) => p.Name ?? $"arg{i}") + .ToArray(); + + var methodName = methodInfo.Name; + + var sb = new StringBuilder(); + sb.AppendLine($"Property '{methodName}' failed with counterexample:"); + + // Unwrap TargetInvocationException to get to the actual FsCheck exception + var innerEx = ex; + while (innerEx is TargetInvocationException { InnerException: not null } tie) + { + innerEx = tie.InnerException; + } + + // Try to extract shrunk values from FsCheck message + var shrunkValues = TryParseShrunkValues(innerEx?.Message); + + // Display args, using shrunk values if available + for (int i = 0; i < args.Length; i++) + { + var name = args[i]; + var value = shrunkValues?[i]; + if (value != null) + { + sb.AppendLine($" {name} = {value}"); + } + } + + // Append the FsCheck message for full details + if (innerEx != null && !string.IsNullOrEmpty(innerEx.Message)) + { + sb.AppendLine(); + sb.AppendLine("FsCheck output:"); + // Indent each line of the FsCheck message + foreach (var line in innerEx.Message.Split('\n')) + { + sb.Append(" "); + sb.AppendLine(line.TrimEnd('\r')); + } + } + + return sb.ToString(); + } + + private static string[]? TryParseShrunkValues(string? message) + { + if (string.IsNullOrEmpty(message)) + return null; + + // Look for "Shrunk:" followed by values on the next line + var shrunkIndex = message!.IndexOf("Shrunk:", StringComparison.Ordinal); + if (shrunkIndex < 0) + return null; + + var afterShrunk = message[(shrunkIndex + 7)..].TrimStart(); + + // Take only the first line + var newlineIndex = afterShrunk.IndexOfAny(['\r', '\n']); + var shrunkLine = newlineIndex >= 0 ? afterShrunk[..newlineIndex] : afterShrunk; + + if (shrunkLine.StartsWith('(')) + { + return ParseTupleValues(shrunkLine); + } + else + { + // Single value (no brackets) + return [shrunkLine.Trim()]; + } + } + + private static string[]? ParseTupleValues(string tupleString) + { + if (!tupleString.StartsWith('(')) + return null; + + var values = new List(); + var current = new StringBuilder(); + var depth = 0; + var inString = false; + var escaped = false; + + for (var i = 1; i < tupleString.Length; i++) + { + var c = tupleString[i]; + + if (escaped) + { + current.Append(c); + escaped = false; + continue; + } + + if (c == '\\' && inString) + { + current.Append(c); + escaped = true; + continue; + } + + if (c == '"') + { + inString = !inString; + current.Append(c); + continue; + } + + if (inString) + { + current.Append(c); + continue; + } + + switch (c) + { + case '(': + depth++; + current.Append(c); + break; + case ')': + if (depth == 0) + { + if (current.Length > 0) + values.Add(current.ToString().Trim()); + return values.ToArray(); + } + + depth--; + current.Append(c); + break; + case ',' when depth == 0: + values.Add(current.ToString().Trim()); + current.Clear(); + break; + default: + current.Append(c); + break; + } + } + + return values.ToArray(); + } +} +#pragma warning restore IL2046 +#pragma warning restore IL3051 +#pragma warning restore IL2072 diff --git a/TUnit.FsCheck/PropertyFailedException.cs b/TUnit.FsCheck/PropertyFailedException.cs new file mode 100644 index 0000000000..310c832db0 --- /dev/null +++ b/TUnit.FsCheck/PropertyFailedException.cs @@ -0,0 +1,15 @@ +namespace TUnit.FsCheck; + +/// +/// Exception thrown when an FsCheck property test fails. +/// +public class PropertyFailedException : Exception +{ + public PropertyFailedException(string message) : base(message) + { + } + + public PropertyFailedException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/TUnit.FsCheck/TUnit.FsCheck.csproj b/TUnit.FsCheck/TUnit.FsCheck.csproj new file mode 100644 index 0000000000..4e1db6d206 --- /dev/null +++ b/TUnit.FsCheck/TUnit.FsCheck.csproj @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + diff --git a/TUnit.sln b/TUnit.sln index 2bb8c46d11..3350695215 100644 --- a/TUnit.sln +++ b/TUnit.sln @@ -151,6 +151,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TUnit.AspNetCore.Analyzers. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TUnit.AspNetCore.Analyzers", "TUnit.AspNetCore.Analyzers\TUnit.AspNetCore.Analyzers.csproj", "{6134813B-F928-443F-A629-F6726A1112F9}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TUnit.FsCheck", "TUnit.FsCheck\TUnit.FsCheck.csproj", "{6846A70E-2232-4BEF-9CE5-03F28A221335}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TUnit.Example.FsCheck.TestProject", "TUnit.Example.FsCheck.TestProject\TUnit.Example.FsCheck.TestProject.csproj", "{3428D7AD-B362-4647-B1B0-72674CF3BC7C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -869,6 +873,30 @@ Global {6134813B-F928-443F-A629-F6726A1112F9}.Release|x64.Build.0 = Release|Any CPU {6134813B-F928-443F-A629-F6726A1112F9}.Release|x86.ActiveCfg = Release|Any CPU {6134813B-F928-443F-A629-F6726A1112F9}.Release|x86.Build.0 = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|x64.ActiveCfg = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|x64.Build.0 = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|x86.ActiveCfg = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Debug|x86.Build.0 = Debug|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|Any CPU.Build.0 = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|x64.ActiveCfg = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|x64.Build.0 = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|x86.ActiveCfg = Release|Any CPU + {6846A70E-2232-4BEF-9CE5-03F28A221335}.Release|x86.Build.0 = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|x64.ActiveCfg = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|x64.Build.0 = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|x86.ActiveCfg = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Debug|x86.Build.0 = Debug|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|Any CPU.Build.0 = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|x64.ActiveCfg = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|x64.Build.0 = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|x86.ActiveCfg = Release|Any CPU + {3428D7AD-B362-4647-B1B0-72674CF3BC7C}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -935,6 +963,8 @@ Global {D5C70ADD-B960-4E6C-836C-6041938D04BE} = {503DA9FA-045D-4910-8AF6-905E6048B1F1} {9B33972F-F5B9-4EC2-AE5C-4D48604DEB04} = {62AD1EAF-43C4-4AC0-B9FA-CD59739B3850} {6134813B-F928-443F-A629-F6726A1112F9} = {503DA9FA-045D-4910-8AF6-905E6048B1F1} + {3428D7AD-B362-4647-B1B0-72674CF3BC7C} = {0BA988BF-ADCE-4343-9098-B4EF65C43709} + {6846A70E-2232-4BEF-9CE5-03F28A221335} = {1B56B580-4D59-4E83-9F80-467D58DADAC1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {109D285A-36B3-4503-BCDF-8E26FB0E2C5B}