Skip to content

Commit 6d74c32

Browse files
neueccneuecc
committed
tabular array more fix
1 parent cb17e32 commit 6d74c32

File tree

5 files changed

+98
-65
lines changed

5 files changed

+98
-65
lines changed

‎ToonEncoder.slnx‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,14 @@
11
<Solution>
2+
<Folder Name="/items/">
3+
<File Path=".editorconfig" />
4+
<File Path=".gitignore" />
5+
<File Path="Directory.Build.props" />
6+
<File Path="exclusion.dic" />
7+
<File Path="global.json" />
8+
<File Path="LICENSE" />
9+
<File Path="opensource.snk" />
10+
<File Path="README.md" />
11+
</Folder>
212
<Folder Name="/sandbox/">
313
<Project Path="sandbox/BenchmarkSuite/BenchmarkSuite.csproj" Id="a601f4ad-3dfd-43cc-b51b-845076f73eef" />
414
<Project Path="sandbox/ConsoleApp1/ConsoleApp1.csproj" />

‎sandbox/ConsoleApp1/Program.cs‎

Lines changed: 24 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,40 @@
11
using Cysharp.AI;
2-
using Cysharp.AI.Internal;
32
using System.Buffers;
4-
using System.IO.Pipelines;
3+
using System.Collections.Generic;
4+
using System.Data;
55
using System.Text;
6-
using System.Text.Encodings.Web;
7-
using System.Text.Json;
8-
using System.Text.Json.Serialization;
6+
using System.Xml.Linq;
97

8+
var arrayBufferWriter = new ArrayBufferWriter<byte>();
9+
var toonWriter = ToonWriter.Create(ref arrayBufferWriter);
1010

11-
Console.OutputEncoding = Encoding.GetEncoding("utf-8");
11+
// write as NonUniform Array
12+
toonWriter.WriteStartNonUniformArray(2);
1213

13-
var options = new JsonSerializerOptions
14-
{
15-
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
16-
WriteIndented = false,
17-
DefaultIgnoreCondition = JsonIgnoreCondition.Never,
18-
Converters =
19-
{
20-
// new ToonTabularArrayConverter<Person[]>(),
21-
new Cysharp.AI.Converters.PersonTabularArrayConverter(),
22-
},
23-
};
24-
25-
var persons = new Person[]
26-
{
27-
new Person(1, "Alice", 30),
28-
new Person(2, "Bob", 25),
29-
new Person(3, "Charlie", 35),
30-
};
14+
toonWriter.WriteNextRowOfNonUniformArray();
15+
toonWriter.WriteNumber(1);
3116

17+
toonWriter.WriteNextRowOfNonUniformArray();
18+
toonWriter.WriteString(MyEnum.Orange);
3219

33-
var foo = JsonSerializer.Serialize(persons, options);
20+
toonWriter.WriteEndNonUniformArray();
3421

22+
toonWriter.Flush();
3523

36-
Console.WriteLine(foo);
24+
var uu = new User(Id: 1, Name: "Alice", Role: "Admin", dt: DateTime.Now, dt2: DateTimeOffset.Now, ts: TimeSpan.FromHours(1), me: MyEnum.Fruit);
25+
if (uu.dt == null)
26+
{
27+
}
3728

38-
Console.WriteLine("---");
29+
var str = Encoding.UTF8.GetString(arrayBufferWriter.WrittenSpan);
30+
Console.WriteLine(str);
3931

40-
var hoge = JsonSerializer.Deserialize<string>(foo);
41-
Console.WriteLine(hoge);
4232

43-
[Cysharp.AI.GenerateToonTabularArrayConverter]
44-
public record Person(int Id, string Name, int Age);
33+
[GenerateToonTabularArrayConverter]
34+
public record User(int Id, string Name, string Role, DateTime dt, DateTimeOffset dt2, TimeSpan ts, MyEnum me);
4535

46-
public class Dummy
36+
37+
public enum MyEnum
4738
{
39+
Fruit, Orange, Apple
4840
}
49-
50-

‎src/ToonEncoder.Generator/ToonEncoderGenerator.cs‎

Lines changed: 23 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
3131
static void EmitAttributes(IncrementalGeneratorPostInitializationContext context)
3232
{
3333
context.AddSource("GenerateToonTabularArrayConverter.g.cs", """
34+
// <auto-generated/>
35+
#pragma warning disable
36+
#nullable enable
37+
3438
using System;
3539
3640
namespace Cysharp.AI
@@ -60,14 +64,9 @@ static void EmitTabularArrayConverter(SourceProductionContext sourceProductionCo
6064
ToonPrimitiveKind.Boolean => $"toonWriter.WriteBoolean(item.{name});",
6165
ToonPrimitiveKind.String => $"toonWriter.WriteString(item.{name});",
6266
ToonPrimitiveKind.Number => $"toonWriter.WriteNumber(item.{name});",
63-
ToonPrimitiveKind.DateTime => $"toonWriter.WriteString(item.{name});",
64-
ToonPrimitiveKind.Guid => $"toonWriter.WriteString(item.{name});",
65-
ToonPrimitiveKind.Enum => $"toonWriter.WriteString(item.{name});",
6667
ToonPrimitiveKind.NullableBoolean => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteBoolean(item.{name}); }}",
6768
ToonPrimitiveKind.NullableNumber => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteNumber(item.{name}); }}",
68-
ToonPrimitiveKind.NullableDateTime => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteString(item.{name}); }}",
69-
ToonPrimitiveKind.NullableGuid => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteString(item.{name}); }}",
70-
ToonPrimitiveKind.NullableEnum => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteString(item.{name}); }}",
69+
ToonPrimitiveKind.NullableString => $"if (item.{name} == null) {{ toonWriter.WriteNull(); }} else {{ toonWriter.WriteString(item.{name}); }}",
7170
_ => throw new NotSupportedException($"Unsupported property type for Toon serialization: {kind}"),
7271
};
7372
return " "/* indent */ + str;
@@ -223,20 +222,15 @@ static ToonPrimitiveKind GetKind(ITypeSymbol t)
223222
{
224223
var underlyingType = nts.TypeArguments[0];
225224

226-
if (underlyingType.TypeKind == TypeKind.Enum)
227-
{
228-
return ToonPrimitiveKind.NullableEnum;
229-
}
230-
231-
if (IsGuid(underlyingType))
225+
if (underlyingType.TypeKind == TypeKind.Enum || IsGuid(underlyingType) || IsTimeSpan(underlyingType) || IsDateTimeOffset(underlyingType))
232226
{
233-
return ToonPrimitiveKind.NullableGuid;
227+
return ToonPrimitiveKind.NullableString;
234228
}
235229

236230
return underlyingType.SpecialType switch
237231
{
238232
SpecialType.System_Boolean => ToonPrimitiveKind.NullableBoolean,
239-
SpecialType.System_DateTime => ToonPrimitiveKind.NullableDateTime,
233+
SpecialType.System_DateTime => ToonPrimitiveKind.NullableString,
240234
SpecialType.System_Byte or
241235
SpecialType.System_SByte or
242236
SpecialType.System_Int16 or
@@ -253,20 +247,15 @@ SpecialType.System_Double or
253247
}
254248
else
255249
{
256-
if (t.TypeKind == TypeKind.Enum)
257-
{
258-
return ToonPrimitiveKind.Enum;
259-
}
260-
261-
if (IsGuid(t))
250+
if (t.TypeKind == TypeKind.Enum || IsGuid(t) || IsTimeSpan(t) || IsDateTimeOffset(t))
262251
{
263-
return ToonPrimitiveKind.Guid;
252+
return ToonPrimitiveKind.String;
264253
}
265254

266255
return t.SpecialType switch
267256
{
268257
SpecialType.System_Boolean => ToonPrimitiveKind.Boolean,
269-
SpecialType.System_DateTime => ToonPrimitiveKind.DateTime,
258+
SpecialType.System_DateTime => ToonPrimitiveKind.String,
270259
SpecialType.System_String => ToonPrimitiveKind.String,
271260
SpecialType.System_Byte or
272261
SpecialType.System_SByte or
@@ -288,6 +277,16 @@ static bool IsGuid(ITypeSymbol type)
288277
{
289278
return type is INamedTypeSymbol { Name: "Guid", ContainingNamespace: { Name: "System", ContainingNamespace.IsGlobalNamespace: true } };
290279
}
280+
281+
static bool IsTimeSpan(ITypeSymbol type)
282+
{
283+
return type is INamedTypeSymbol { Name: "TimeSpan", ContainingNamespace: { Name: "System", ContainingNamespace.IsGlobalNamespace: true } };
284+
}
285+
286+
static bool IsDateTimeOffset(ITypeSymbol type)
287+
{
288+
return type is INamedTypeSymbol { Name: "DateTimeOffset", ContainingNamespace: { Name: "System", ContainingNamespace.IsGlobalNamespace: true } };
289+
}
291290
})
292291
.ToArray();
293292

@@ -332,17 +331,11 @@ public bool Verify(SourceProductionContext sourceProductionContext)
332331
public enum ToonPrimitiveKind
333332
{
334333
Boolean,
335-
String,
336334
Number,
337-
DateTime, // write as string
338-
Guid, // write as string
339-
Enum, // write as string
340-
335+
String,
341336
NullableNumber,
342337
NullableBoolean,
343-
NullableDateTime,
344-
NullableGuid,
345-
NullableEnum,
338+
NullableString,
346339

347340
Unsupported
348341
}

‎src/ToonEncoder/ToonEncoder.csproj‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,21 @@
55
<ImplicitUsings>enable</ImplicitUsings>
66
<Nullable>enable</Nullable>
77
<RootNamespace>Cysharp.AI</RootNamespace>
8+
9+
<!-- NuGet -->
10+
<IsPackable>true</IsPackable>
11+
<PackageTags>llm,serialization,json</PackageTags>
12+
<Description>High performance Toon encoder for .NET.</Description>
813
</PropertyGroup>
914

1015
<ItemGroup>
1116
<PackageReference Include="System.IO.Hashing" Version="10.0.1" />
1217
</ItemGroup>
1318

19+
<!-- Bundle SourceGenerator -->
20+
<ItemGroup>
21+
<ProjectReference Include="..\ToonEncoder.Generator\ToonEncoder.Generator.csproj" ReferenceOutputAssembly="false" />
22+
<None Include="..\ToonEncoder.Generator\bin\$(Configuration)\netstandard2.0\ToonEncoder.Generator.dll" PackagePath="analyzers\dotnet\roslyn4.3\cs" Pack="true" Visible="false" />
23+
</ItemGroup>
24+
1425
</Project>

‎src/ToonEncoder/ToonWriter.cs‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
using Cysharp.AI.Internal;
2+
using System;
23
using System.Buffers;
34
using System.Buffers.Text;
45
using System.Diagnostics.CodeAnalysis;
56
using System.Globalization;
67
using System.Runtime.CompilerServices;
8+
using System.Text;
79
using System.Text.Unicode;
810

911
namespace Cysharp.AI;
@@ -202,6 +204,33 @@ public void WriteString(TimeSpan value)
202204
written += bytesWritten;
203205
}
204206

207+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
208+
public void WriteString<TEnum>(TEnum value)
209+
where TEnum : struct, Enum
210+
{
211+
TryWriteKeyValueSeparator();
212+
WriteDelimiter();
213+
214+
// currently Enum doesn't support IUtf8SpanFormattable: https://github.com/dotnet/runtime/issues/81500
215+
Span<char> dest = stackalloc char[256];
216+
int charsWritten = 0;
217+
while (!Enum.TryFormat(value, dest, out charsWritten))
218+
{
219+
if (dest.Length < 512)
220+
{
221+
#pragma warning disable CA2014 // Do not use stackalloc in loops
222+
dest = stackalloc char[dest.Length * 2];
223+
#pragma warning restore CA2014
224+
}
225+
else
226+
{
227+
dest = new char[dest.Length * 2]; // too large
228+
}
229+
}
230+
231+
WriteUtf16Core(dest.Slice(0, charsWritten));
232+
}
233+
205234
// TOON's escape character is similar as Json's one so JsonElement has already escaped string, don't need to escape.
206235
public void WriteEscapedString(ReadOnlySpan<byte> escapedUtf8Value)
207236
{
@@ -430,7 +459,7 @@ void WriteUtf8WithEscapeCore(ReadOnlySpan<byte> value)
430459
}
431460
}
432461

433-
void WriteUtf16Core(ReadOnlySpan<char> value)
462+
void WriteUtf16Core(scoped ReadOnlySpan<char> value)
434463
{
435464
while (value.Length > 0)
436465
{

0 commit comments

Comments
 (0)