Skip to content

Commit 921f394

Browse files
neueccneuecc
committed
impl Source Generator
1 parent 49ebe01 commit 921f394

File tree

12 files changed

+337
-17
lines changed

12 files changed

+337
-17
lines changed

‎ToonEncoder.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<Project Path="sandbox/ConsoleApp1/ConsoleApp1.csproj" />
55
</Folder>
66
<Folder Name="/src/">
7+
<Project Path="src/ToonEncoder.Generator/ToonEncoder.Generator.csproj" Id="a2d2b5e5-c45d-4a64-a860-37af7ed11b60" />
78
<Project Path="src/ToonEncoder/ToonEncoder.csproj" />
89
</Folder>
910
<Folder Name="/tests/">

‎sandbox/BenchmarkSuite/EncodeBenchmark.cs‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,16 @@
11
using BenchmarkDotNet.Attributes;
22
using BenchmarkDotNet.Order;
3+
using Cysharp.AI;
4+
using Cysharp.AI.Internal;
5+
using System;
6+
using System.Buffers;
7+
using System.IO;
8+
using System.IO.Pipelines;
39
using System.Linq;
10+
using System.Text;
11+
using System.Text.Json;
12+
using System.Threading;
13+
using System.Threading.Tasks;
414

515
namespace BenchmarkSuite
616
{
@@ -32,6 +42,12 @@ public string ToonEncoder_EncodeAsTabularArray()
3242
return global::Cysharp.AI.ToonEncoder.EncodeAsTabularArray(data);
3343
}
3444

45+
[Benchmark]
46+
public string ToonEncoder_EncodeAsTabularArray_SourceGenerated()
47+
{
48+
return PersonToonTabularArrayConverter.EncodeAsTabularArray(data);
49+
}
50+
3551
// https://github.com/StefH/Toon.NET
3652
[Benchmark]
3753
public string Toon_Encode()
@@ -62,9 +78,93 @@ public string ToonDotNet_Encode()
6278

6379
// Official impl: https://github.com/toon-format/toon-dotnet
6480
[Benchmark]
65-
public string ToonFormat_Encode()
81+
public string ToonOfficial_Encode()
6682
{
6783
return global::Toon.Format.ToonEncoder.Encode(data);
6884
}
6985
}
86+
87+
// TODO: test generation, finally need to remove.
88+
public class PersonToonTabularArrayConverter : System.Text.Json.Serialization.JsonConverter<Person[]>
89+
{
90+
static readonly ReadOnlyMemory<byte>[] utf8FieldNames = ["Id"u8.ToArray(), "Name"u8.ToArray(), "Age"u8.ToArray()];
91+
92+
public static string EncodeAsTabularArray(Person[] value)
93+
{
94+
var bufferWriter = new Cysharp.AI.Internal.ValueArrayPoolBufferWriter<byte>();
95+
try
96+
{
97+
EncodeAsTabularArray(ref bufferWriter, value);
98+
return Encoding.UTF8.GetString(bufferWriter.WrittenSpan);
99+
}
100+
finally
101+
{
102+
bufferWriter.Dispose();
103+
}
104+
}
105+
106+
public static byte[] EncodeAsTabularArrayToUtf8Bytes(Person[] value)
107+
{
108+
var bufferWriter = new ValueArrayPoolBufferWriter<byte>();
109+
try
110+
{
111+
EncodeAsTabularArray(ref bufferWriter, value);
112+
return bufferWriter.WrittenSpan.ToArray();
113+
}
114+
finally
115+
{
116+
bufferWriter.Dispose();
117+
}
118+
}
119+
120+
public static async ValueTask EncodeAsTabularArrayAsync(Stream utf8Stream, Person[] value, CancellationToken cancellationToken = default)
121+
{
122+
var writer = PipeWriter.Create(utf8Stream);
123+
EncodeAsTabularArray(ref writer, value);
124+
await writer.FlushAsync(cancellationToken);
125+
}
126+
127+
public static void EncodeAsTabularArray<TBufferWriter>(ref TBufferWriter bufferWriter, Person[] value)
128+
where TBufferWriter : IBufferWriter<byte>
129+
{
130+
var toonWriter = ToonWriter.Create(ref bufferWriter);
131+
EncodeAsTabularArray(ref toonWriter, value);
132+
toonWriter.Flush();
133+
}
134+
135+
public static void EncodeAsTabularArray<TBufferWriter>(ref ToonWriter<TBufferWriter> toonWriter, Person[] value)
136+
where TBufferWriter : IBufferWriter<byte>
137+
{
138+
toonWriter.WriteStartTabularArray(value.Length, utf8FieldNames, escaped: true);
139+
140+
foreach (var item in value)
141+
{
142+
toonWriter.WriteNextRowOfTabularArray();
143+
toonWriter.WriteNumber(item.Id);
144+
toonWriter.WriteString(item.Name);
145+
toonWriter.WriteNumber(item.Age);
146+
}
147+
148+
toonWriter.WriteEndTabularArray();
149+
}
150+
151+
public override void Write(Utf8JsonWriter utf8JsonWriter, Person[] value, JsonSerializerOptions options)
152+
{
153+
var bufferWriter = new Cysharp.AI.Internal.ValueArrayPoolBufferWriter<byte>();
154+
try
155+
{
156+
EncodeAsTabularArray(ref bufferWriter, value);
157+
utf8JsonWriter.WriteStringValue(bufferWriter.WrittenSpan);
158+
}
159+
finally
160+
{
161+
bufferWriter.Dispose();
162+
}
163+
}
164+
165+
public override Person[]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
166+
{
167+
throw new NotSupportedException("Toon serialization only supports Write.");
168+
}
169+
}
70170
}

‎sandbox/BenchmarkSuite/Program.cs‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,32 @@
44
using BenchmarkDotNet.Reports;
55
using BenchmarkDotNet.Running;
66
using BenchmarkDotNet.Toolchains.CsProj;
7+
using BenchmarkSuite;
78
using Perfolizer.Horology;
9+
using System;
10+
using System.Linq;
811

12+
#if DEBUG
913

14+
var bench = new EncodeBenchmark();
15+
var a = bench.ToonEncoder_EncodeAsTabularArray_SourceGenerated();
16+
var b = bench.ToonEncoder_Encode();
17+
var c = bench.ToonEncoder_EncodeAsTabularArray();
18+
var d = bench.Toon_Encode();
19+
var e = bench.ToonSharp_Serialize();
20+
var f = bench.ToonNet_Encode();
21+
var g = bench.ToonDotNet_Encode();
22+
var reference = bench.ToonOfficial_Encode();
23+
24+
Console.WriteLine(reference.SequenceEqual(a));
25+
Console.WriteLine(reference.SequenceEqual(b));
26+
Console.WriteLine(reference.SequenceEqual(c));
27+
Console.WriteLine(reference.SequenceEqual(d));
28+
Console.WriteLine(reference.SequenceEqual(e));
29+
Console.WriteLine(reference.SequenceEqual(f));
30+
Console.WriteLine(reference.SequenceEqual(g));
31+
32+
#else
1033
var config = DefaultConfig.Instance
1134
.WithSummaryStyle(SummaryStyle.Default)
1235
// .WithTimeUnit(TimeUnit.Millisecond))
@@ -19,4 +42,5 @@
1942
.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) // .NET 10
2043
.DontEnforcePowerPlan());
2144

22-
var _ = BenchmarkRunner.Run(typeof(Program).Assembly, config);
45+
var _ = BenchmarkRunner.Run(typeof(Program).Assembly, config);
46+
#endif

‎sandbox/ConsoleApp1/ConsoleApp1.csproj‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

3-
<PropertyGroup>
4-
<OutputType>Exe</OutputType>
5-
<TargetFramework>net10.0</TargetFramework>
6-
<ImplicitUsings>enable</ImplicitUsings>
7-
<Nullable>enable</Nullable>
8-
</PropertyGroup>
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net10.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
</PropertyGroup>
99

10-
<ItemGroup>
11-
<ProjectReference Include="..\..\src\ToonEncoder\ToonEncoder.csproj" />
12-
</ItemGroup>
10+
<ItemGroup>
11+
<ProjectReference Include="..\..\src\ToonEncoder\ToonEncoder.csproj" />
12+
<ProjectReference Include="..\..\src\ToonEncoder.Generator\ToonEncoder.Generator.csproj">
13+
<OutputItemType>Analyzer</OutputItemType>
14+
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
15+
</ProjectReference>
16+
</ItemGroup>
1317

1418
</Project>

‎sandbox/ConsoleApp1/Program.cs‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
using Cysharp.AI;
2+
using Cysharp.AI.Internal;
3+
using System.Buffers;
4+
using System.IO.Pipelines;
25
using System.Text;
36
using System.Text.Encodings.Web;
47
using System.Text.Json;
@@ -38,5 +41,90 @@
3841

3942

4043

44+
[Cysharp.AI.GenerateToonTabularArrayConverter]
4145
public record Person(int Id, string Name, int Age);
4246

47+
48+
49+
public class PersonToonTabularArrayConverter : System.Text.Json.Serialization.JsonConverter<Person[]>
50+
{
51+
static readonly ReadOnlyMemory<byte>[] utf8FieldNames = ["Id"u8.ToArray(), "Name"u8.ToArray(), "Age"u8.ToArray()];
52+
53+
public static string EncodeAsTabularArray(Person[] value)
54+
{
55+
var bufferWriter = new Cysharp.AI.Internal.ValueArrayPoolBufferWriter<byte>();
56+
try
57+
{
58+
EncodeAsTabularArray(ref bufferWriter, value);
59+
return Encoding.UTF8.GetString(bufferWriter.WrittenSpan);
60+
}
61+
finally
62+
{
63+
bufferWriter.Dispose();
64+
}
65+
}
66+
67+
public static byte[] EncodeAsTabularArrayToUtf8Bytes(Person[] value)
68+
{
69+
var bufferWriter = new ValueArrayPoolBufferWriter<byte>();
70+
try
71+
{
72+
EncodeAsTabularArray(ref bufferWriter, value);
73+
return bufferWriter.WrittenSpan.ToArray();
74+
}
75+
finally
76+
{
77+
bufferWriter.Dispose();
78+
}
79+
}
80+
81+
public static async ValueTask EncodeAsTabularArrayAsync(Stream utf8Stream, Person[] value, CancellationToken cancellationToken = default)
82+
{
83+
var writer = PipeWriter.Create(utf8Stream);
84+
EncodeAsTabularArray(ref writer, value);
85+
await writer.FlushAsync(cancellationToken);
86+
}
87+
88+
public static void EncodeAsTabularArray<TBufferWriter>(ref TBufferWriter bufferWriter, Person[] value)
89+
where TBufferWriter : IBufferWriter<byte>
90+
{
91+
var toonWriter = ToonWriter.Create(ref bufferWriter);
92+
EncodeAsTabularArray(ref toonWriter, value);
93+
toonWriter.Flush();
94+
}
95+
96+
public static void EncodeAsTabularArray<TBufferWriter>(ref ToonWriter<TBufferWriter> toonWriter, Person[] value)
97+
where TBufferWriter : IBufferWriter<byte>
98+
{
99+
toonWriter.WriteStartTabularArray(value.Length, utf8FieldNames, escaped: true);
100+
101+
foreach (var item in value)
102+
{
103+
toonWriter.WriteNextRowOfTabularArray();
104+
toonWriter.WriteNumber(item.Id);
105+
toonWriter.WriteString(item.Name);
106+
toonWriter.WriteNumber(item.Age);
107+
}
108+
109+
toonWriter.WriteEndTabularArray();
110+
}
111+
112+
public override void Write(Utf8JsonWriter utf8JsonWriter, Person[] value, JsonSerializerOptions options)
113+
{
114+
var bufferWriter = new Cysharp.AI.Internal.ValueArrayPoolBufferWriter<byte>();
115+
try
116+
{
117+
EncodeAsTabularArray(ref bufferWriter, value);
118+
utf8JsonWriter.WriteStringValue(bufferWriter.WrittenSpan);
119+
}
120+
finally
121+
{
122+
bufferWriter.Dispose();
123+
}
124+
}
125+
126+
public override Person[]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
127+
{
128+
throw new NotSupportedException("Toon serialization only supports Write.");
129+
}
130+
}

‎src/ToonEncoder.Generator/Properties/launchSettings.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"profiles": {
3+
"プロファイル 1": {
4+
"commandName": "DebugRoslynComponent",
5+
"targetProject": "..\\..\\sandbox\\ConsoleApp1\\ConsoleApp1.csproj"
6+
}
7+
}
8+
}

‎src/ToonEncoder.Generator/StringExtensions.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Text;
4+
5+
namespace Cysharp.AI;
6+
7+
internal static class StringExtensions
8+
{
9+
#if NETSTANDARD2_0
10+
public static string ReplaceLineEndings(this string input)
11+
{
12+
#pragma warning disable RS1035
13+
return ReplaceLineEndings(input, Environment.NewLine);
14+
#pragma warning restore RS1035
15+
}
16+
17+
public static string ReplaceLineEndings(this string text, string replacementText)
18+
{
19+
text = text.Replace("\r\n", "\n");
20+
21+
if (replacementText != "\n")
22+
text = text.Replace("\n", replacementText);
23+
24+
return text;
25+
}
26+
#endif
27+
}

‎src/ToonEncoder.Generator/ToonEncoder.Generator.csproj‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>netstandard2.0</TargetFramework>
5+
<LangVersion>14</LangVersion>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<RootNamespace>Cysharp.AI</RootNamespace>
9+
10+
<!-- Source Generator -->
11+
<IsRoslynComponent>true</IsRoslynComponent>
12+
<AnalyzerLanguage>cs</AnalyzerLanguage>
13+
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
14+
15+
</PropertyGroup>
16+
17+
<ItemGroup>
18+
<!-- Roslyn for .NET 8 / C# 12 -->
19+
<!-- https://learn.microsoft.com/en-us/visualstudio/extensibility/roslyn-version-support -->
20+
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />
21+
<PackageReference Include="PolySharp" Version="1.15.0">
22+
<PrivateAssets>all</PrivateAssets>
23+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
24+
</PackageReference>
25+
</ItemGroup>
26+
27+
</Project>

0 commit comments

Comments
 (0)