feat: Toon SpecGenerator for building unit tests to verify against spec

.gitattributes

@@ -0,0 +1,2 @@
+* text=auto
+*.cs text=lf

.specignore

@@ -0,0 +1,23 @@
+# reason: .net doesn't support hyphens in property/field names (full-name)
+skips folding when segment requires quotes (safe mode)
+
+# reason: .net doesn't support . syntax for properties/fields (data.meta.items)
+skips folding on sibling literal-key collision (safe mode)
+
+# reason: .net doesn't support special chars or colons/brackets/spaces in property/field names or spaces (order:id full name)
+encodes tabular arrays with keys needing quotes
+quotes key with colon
+quotes key with brackets
+quotes key with braces
+quotes key with comma
+quotes key with spaces
+quotes key with leading hyphen
+quotes key with leading and trailing spaces
+quotes numeric key
+quotes empty string key
+escapes newline in key
+escapes tab in key
+escapes quotes in key
+
+# reason: may need to rely on System.Numerics.BigInteger for large numbers
+encodes large number

CONTRIBUTING.md

@@ -63,6 +63,9 @@ We target .NET 8.0 and .NET 9.0 for broad compatibility.
   dotnet test
   ```
 
+Some tests are auto generated to comply with the [TOON specification](https://github.com/toon-format/spec/blob/main/SPEC.md). To ensure tests are 
+aligned with the spec, execute the `specgen.sh` or `specgen.ps1` script.
+
 ## SPEC Compliance
 
 All implementations must comply with the [TOON specification](https://github.com/toon-format/spec/blob/main/SPEC.md).

ToonFormat.sln

@@ -11,6 +11,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToonFormat.Tests", "tests\ToonFormat.Tests\ToonFormat.Tests.csproj", "{25E10B0C-CAC4-475E-90C9-B4F85BB56C55}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ToonFormat.SpecGenerator", "tests\ToonFormat.SpecGenerator\ToonFormat.SpecGenerator.csproj", "{DB25DA28-D02C-4D8F-8C4A-D581D544607C}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -45,12 +47,25 @@ Global
 		{25E10B0C-CAC4-475E-90C9-B4F85BB56C55}.Release|x64.Build.0 = Release|Any CPU
 		{25E10B0C-CAC4-475E-90C9-B4F85BB56C55}.Release|x86.ActiveCfg = Release|Any CPU
 		{25E10B0C-CAC4-475E-90C9-B4F85BB56C55}.Release|x86.Build.0 = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|x64.Build.0 = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Debug|x86.Build.0 = Debug|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|x64.ActiveCfg = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|x64.Build.0 = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|x86.ActiveCfg = Release|Any CPU
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{1951209F-73E0-4B33-AB40-8D80CACCB507} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{25E10B0C-CAC4-475E-90C9-B4F85BB56C55} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
+		{DB25DA28-D02C-4D8F-8C4A-D581D544607C} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
 	EndGlobalSection
 EndGlobal

specgen.ps1

@@ -0,0 +1,8 @@
+# generate spec
+$GH_REPO = "https://github.com/toon-format/spec.git"
+$OUT_DIR = "./tests/ToonFormat.Tests"
+
+# build and execute spec generator
+dotnet build tests/ToonFormat.SpecGenerator
+
+dotnet run --project tests/ToonFormat.SpecGenerator -- --url="$GH_REPO" --output="$OUT_DIR" --branch="v2.0.0" --loglevel="Information"

specgen.sh

@@ -0,0 +1,8 @@
+# generate spec
+GH_REPO="https://github.com/toon-format/spec.git"
+OUT_DIR="./tests/ToonFormat.Tests"
+
+# build and execute spec generator
+dotnet build tests/ToonFormat.SpecGenerator
+
+dotnet run --project tests/ToonFormat.SpecGenerator -- --url="$GH_REPO" --output="$OUT_DIR" --branch="v2.0.0" --loglevel="Information"

src/ToonFormat/Constants.cs

@@ -3,23 +3,23 @@
 namespace ToonFormat
 {
 
     public static class Constants
     {
         public const char LIST_ITEM_MARKER = '-';
 
-        public const string LIST_ITEM_PREFIX  = "- ";
+        public const string LIST_ITEM_PREFIX = "- ";
 
         // #region Structural characters
         public const char COMMA = ',';
         public const char COLON = ':';
         public const char SPACE = ' ';
         public const char PIPE = '|';
         public const char HASH = '#';
         // #endregion
 
         // #region Brackets and braces
         public const char OPEN_BRACKET = '[';
         public const char CLOSE_BRACKET = ']';
         public const char OPEN_BRACE = '{';
         public const char CLOSE_BRACE = '}';
         // #endregion

src/ToonFormat/Internal/Decode/Parser.cs

@@ -202,7 +202,7 @@ private static BracketSegmentResult ParseBracketSegment(string seg, char default
         /// </summary>
         public static List<string> ParseDelimitedValues(string input, char delimiter)
         {
-            var values = new List<string>(16); // 预分配一些容量
+            var values = new List<string>(16); // pre-allocate for performance
             var current = new System.Text.StringBuilder(input.Length);
             bool inQuotes = false;
 
@@ -212,7 +212,7 @@ public static List<string> ParseDelimitedValues(string input, char delimiter)
 
                 if (ch == Constants.BACKSLASH && inQuotes && i + 1 < input.Length)
                 {
-                    // 转义处理
+                    // Escape sequence in quoted string
                     current.Append(ch);
                     current.Append(input[i + 1]);
                     i++;
@@ -244,7 +244,6 @@ public static List<string> ParseDelimitedValues(string input, char delimiter)
             return values;
         }
 
-
         /// <summary>
         /// Maps an array of string tokens to JSON primitive values.
         /// </summary>

src/ToonFormat/Internal/Decode/Scanner.cs

@@ -127,7 +127,7 @@ public static ScanResult ToParsedLines(string source, int indentSize, bool stric
             while (!span.IsEmpty)
             {
                 lineNumber++;
-                // 找到这一行的结束位置
+                // find the end of this line
                 int newlineIdx = span.IndexOf('\n');
                 ReadOnlySpan<char> lineSpan;
                 if (newlineIdx >= 0)
@@ -140,12 +140,12 @@ public static ScanResult ToParsedLines(string source, int indentSize, bool stric
                     lineSpan = span;
                     span = ReadOnlySpan<char>.Empty;
                 }
-                // 去掉结尾的换行符和回车
+                // remove trailing carriage return if present
                 if (!lineSpan.IsEmpty && lineSpan[lineSpan.Length - 1] == '\r')
                 {
                     lineSpan = lineSpan.Slice(0, lineSpan.Length - 1);
                 }
-                // 计算缩进
+                // calculate indentation
                 int indent = 0;
                 while (indent < lineSpan.Length && lineSpan[indent] == Constants.SPACE)
                 {

src/ToonFormat/Internal/Decode/Validation.cs

@@ -78,7 +78,7 @@ public static void ValidateNoExtraTabularRows(
                 return;
 
             var nextLine = cursor.Peek();
-            if (nextLine != null 
+            if (nextLine != null
                 && nextLine.Depth == rowDepth
                 && !nextLine.Content.StartsWith(Constants.LIST_ITEM_PREFIX)
                 && IsDataRow(nextLine.Content, header.Delimiter))

src/ToonFormat/Internal/Encode/Encoders.cs

@@ -193,7 +193,7 @@ public static string EncodeInlineArrayLine(
             bool lengthMarker = false)
         {
             var header = Primitives.FormatHeader(values.Count, prefix, null, delimiter, lengthMarker);
-            
+
             if (values.Count == 0)
             {
                 return header;
@@ -235,7 +235,7 @@ public static void EncodeArrayOfObjectsAsTabular(
 
             var firstRow = rows[0];
             var firstKeys = firstRow.Select(kvp => kvp.Key).ToList();
-            
+
             if (firstKeys.Count == 0)
                 return null;
 
@@ -323,7 +323,7 @@ public static void EncodeMixedArrayAsListItems(
         public static void EncodeObjectAsListItem(JsonObject obj, LineWriter writer, int depth, ResolvedEncodeOptions options)
         {
             var keys = obj.Select(kvp => kvp.Key).ToList();
-            
+
             if (keys.Count == 0)
             {
                 writer.Push(depth, Constants.LIST_ITEM_MARKER.ToString());
@@ -342,7 +342,7 @@ public static void EncodeObjectAsListItem(JsonObject obj, LineWriter writer, int
             else if (Normalize.IsJsonArray(firstValue))
             {
                 var arr = (JsonArray)firstValue!;
-                
+
                 if (Normalize.IsArrayOfPrimitives(arr))
                 {
                     // Inline format for primitive arrays
@@ -354,7 +354,7 @@ public static void EncodeObjectAsListItem(JsonObject obj, LineWriter writer, int
                     // Check if array of objects can use tabular format
                     var objects = arr.Cast<JsonObject>().ToList();
                     var header = ExtractTabularHeader(objects);
-                    
+
                     if (header != null)
                     {
                         // Tabular format for uniform arrays of objects
@@ -387,7 +387,7 @@ public static void EncodeObjectAsListItem(JsonObject obj, LineWriter writer, int
             else if (Normalize.IsJsonObject(firstValue))
             {
                 var nestedObj = (JsonObject)firstValue!;
-                
+
                 if (nestedObj.Count == 0)
                 {
                     writer.PushListItem(depth, $"{encodedKey}{Constants.COLON}");

src/ToonFormat/Internal/Encode/Primitives.cs

@@ -56,7 +56,7 @@ public static string EncodePrimitive(JsonNode? value, char delimiter = Constants
         public static string EncodeStringLiteral(string value, char delimiter = Constants.COMMA)
         {
             var delimiterEnum = Constants.FromDelimiterChar(delimiter);
-            
+
             if (ValidationShared.IsSafeUnquoted(value, delimiterEnum))
             {
                 return value;

tests/ToonFormat.SpecGenerator/Extensions/StringExtensions.cs

@@ -0,0 +1,20 @@
+using System.Globalization;
+
+namespace ToonFormat.SpecGenerator.Extensions;
+
+public static class StringExtensions
+{
+    public static string ToPascalCase(this string input)
+    {
+        if (string.IsNullOrWhiteSpace(input))
+        {
+            return input;
+        }
+
+        string formattedInput = input.Replace("-", " ");
+
+        TextInfo textInfo = CultureInfo.CurrentCulture.TextInfo;
+        string titleCase = textInfo.ToTitleCase(formattedInput.ToLower());
+        return titleCase.Replace(" ", "");
+    }
+}