Skip to content

Commit 2772a4c

Browse files
dameng324dameng324
authored
Read/Write Long Message Support(int.MaxValue to long.MaxValue) (#172)
* Write Super Long Message Support * Improve handling of large message sizes and add tests Refactored size calculation methods to consistently use long-based calculations and removed redundant overflow checks. Added overflow checks and exceptions for serialization methods that could exceed array or prefix size limits. Expanded test coverage for large messages, including overflow scenarios and additional assertions for size calculation consistency. * Update test to use CalculateSize instead of CalculateLongSize Replaces calls to CalculateLongSize with CalculateSize in LongLengthMessageTests to reflect updated method naming or API changes. * Limit LongLengthMessageTest to .NET 10 and reduce memory usage Wrapped LongLengthMessageTest in a .NET 10 conditional to avoid out-of-memory issues in CI. Increased the number of items but reduced the size of each item's data to lower overall memory usage. Added explicit GC.Collect call and updated assertions accordingly. * Add tests for overflow in long length message handling Updated LongLengthMessageTests to include assertions for OverflowException when calculating sizes for large messages. Fixed a typo in the preprocessor directive for .NET 10. Added a KeyValuePair test case in NonGenericSerializerTests. Corrected method usage in IntergrationTests for CalculateLongSize.
1 parent ff7b916 commit 2772a4c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+795
-127
lines changed

‎src/LightProto.Generator/LightProtoGenerator.cs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ string className
202202
$"void IProtoWriter.WriteTo(ref WriterContext output, object message) => WriteTo(ref output, ({proxyOrClassName})message);"
203203
);
204204
writer.WriteLine($"int IProtoWriter.CalculateSize(object message) => CalculateSize(({proxyOrClassName})message);");
205+
writer.WriteLine($"long IProtoWriter.CalculateLongSize(object message) => CalculateLongSize(({proxyOrClassName})message);");
205206
writer.WriteLine($"public bool IsMessage => true;");
206207
writer.WriteLine($"public WireFormat.WireType WireType => WireFormat.WireType.LengthDelimited;");
207208
GenerateMemberProtoParsers(writer, protoMembers, compilation, targetType, "Writer");
@@ -221,14 +222,14 @@ private void GenerateCommonCalculateSizeMethod(
221222
List<(uint RawTag, ProtoContract Contract)> derivedTypes
222223
)
223224
{
224-
writer.WriteLine($"public int CalculateSize({proxyOrClassName} value)");
225+
writer.WriteLine($"public long CalculateLongSize({proxyOrClassName} value)");
225226
using (writer.IndentScope())
226227
{
227228
var valueToMessage = proxyOrClassName.Equals(className, StringComparison.Ordinal)
228229
? $"ref {className} message = ref value;"
229230
: $"{className} message = value;";
230231
writer.WriteLine(valueToMessage);
231-
writer.WriteLine("int size=0;");
232+
writer.WriteLine("long size=0;");
232233
foreach (var member in protoMembers)
233234
{
234235
var tagSize = member.RawTagSize;
@@ -238,11 +239,11 @@ private void GenerateCommonCalculateSizeMethod(
238239
using (writer.IndentScope())
239240
{
240241
if (Helper.IsCollectionType(compilation, member.Type) || Helper.IsDictionaryType(compilation, member.Type))
241-
writer.WriteLine($"size += {member.Name}_ProtoWriter.CalculateSize({memberAccess}); ");
242+
writer.WriteLine($"size += {member.Name}_ProtoWriter.CalculateLongSize({memberAccess}); ");
242243
else if (TryGetInternalTypeName(member.Type, member.DataFormat, member.StringIntern, out var name))
243244
writer.WriteLine($"size += {tagSize} + CodedOutputStream.Compute{name}Size({memberAccess});");
244245
else
245-
writer.WriteLine($"size += {tagSize} + {member.Name}_ProtoWriter.CalculateMessageSize({memberAccess});");
246+
writer.WriteLine($"size += {tagSize} + {member.Name}_ProtoWriter.CalculateLongMessageSize({memberAccess});");
246247
}
247248
}
248249

@@ -252,12 +253,24 @@ private void GenerateCommonCalculateSizeMethod(
252253
using (writer.IndentScope(false))
253254
{
254255
writer.WriteLine(
255-
$"size += {ProtoMember.GetRawTagSize(member.RawTag)} + {member.Contract.Type}.MemberStructWriter.CalculateMessageSize(message.{member.Contract.Type.Name}_MemberStruct.Value); "
256+
$"size += {ProtoMember.GetRawTagSize(member.RawTag)} + {member.Contract.Type}.MemberStructWriter.CalculateLongMessageSize(message.{member.Contract.Type.Name}_MemberStruct.Value); "
256257
);
257258
}
258259
}
259260
writer.WriteLine("return size;");
260261
}
262+
263+
writer.WriteLine($"public int CalculateSize({proxyOrClassName} value)");
264+
using (writer.IndentScope())
265+
{
266+
writer.WriteLine("var longSize = CalculateLongSize(value);");
267+
writer.WriteLine("if (longSize > int.MaxValue)");
268+
using (writer.IndentScope())
269+
{
270+
writer.WriteLine("throw new OverflowException(\"Calculated size exceeds Int32.MaxValue\");");
271+
}
272+
writer.WriteLine("return (int)longSize;");
273+
}
261274
}
262275

263276
private void GenerateCalculateSizeMethod(
@@ -697,12 +710,14 @@ static void GenDerivedLightProtoWriter(CodeWriter writer, string proxyOrClassNam
697710
$"void IProtoWriter.WriteTo(ref WriterContext output, object message) => WriteTo(ref output, ({proxyOrClassName})message);"
698711
);
699712
writer.WriteLine($"int IProtoWriter.CalculateSize(object message) => CalculateSize(({proxyOrClassName})message);");
713+
writer.WriteLine($"long IProtoWriter.CalculateLongSize(object message) => CalculateLongSize(({proxyOrClassName})message);");
700714
writer.WriteLine($"public bool IsMessage => true;");
701715
writer.WriteLine($"public WireFormat.WireType WireType => WireFormat.WireType.LengthDelimited;");
702716
writer.WriteLine(
703717
$"public void WriteTo(ref WriterContext output, {proxyOrClassName} message) => {writerName}.WriteTo(ref output, {message});"
704718
);
705719
writer.WriteLine($"public int CalculateSize({proxyOrClassName} message) => {writerName}.CalculateSize({message});");
720+
writer.WriteLine($"public long CalculateLongSize({proxyOrClassName} message) => {writerName}.CalculateLongSize({message});");
706721
}
707722
}
708723

‎src/LightProto/CodedOutputStream.ComputeSize.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,16 @@ public static int ComputeLengthSize(int length)
244244
return ComputeRawVarint32Size((uint)length);
245245
}
246246

247+
/// <summary>
248+
/// Computes the number of bytes that would be needed to encode a length,
249+
/// as written by <see cref="WriteLength"/>.
250+
/// </summary>
251+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
252+
public static int ComputeLongLengthSize(long length)
253+
{
254+
return ComputeRawVarint64Size((ulong)length);
255+
}
256+
247257
/// <summary>
248258
/// Computes the number of bytes that would be needed to encode a varint.
249259
/// </summary>

‎src/LightProto/IProtoParser.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ public interface IProtoWriter<in T>
2222
public WireFormat.WireType WireType { get; }
2323
public bool IsMessage { get; }
2424
public int CalculateSize(T value);
25+
public long CalculateLongSize(T value);
2526
public void WriteTo(ref WriterContext output, T value);
2627
}
2728

@@ -37,6 +38,7 @@ public interface IProtoWriter
3738
public WireFormat.WireType WireType { get; }
3839
public bool IsMessage { get; }
3940
public int CalculateSize(object value);
41+
public long CalculateLongSize(object value);
4042
public void WriteTo(ref WriterContext output, object value);
4143
}
4244
}

‎src/LightProto/Parser/BigInteger.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,13 @@ sealed class BigIntegerProtoWriter : IProtoWriter, IProtoWriter<BigInteger>
3131

3232
public void WriteTo(ref WriterContext output, object value) => WriteTo(ref output, (BigInteger)value);
3333

34+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((BigInteger)value);
35+
36+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
37+
public int CalculateSize(BigInteger value) => (int)CalculateLongSize(value);
38+
3439
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
35-
public int CalculateSize(BigInteger value)
40+
public long CalculateLongSize(BigInteger value)
3641
{
3742
#if NET7_0_OR_GREATER
3843
var byteCount = value.GetByteCount();
@@ -45,7 +50,7 @@ public int CalculateSize(BigInteger value)
4550
public void WriteTo(ref WriterContext output, BigInteger value)
4651
{
4752
var bytes = value.ToByteArray();
48-
output.WriteLength(bytes.Length);
53+
output.WriteLongLength(bytes.Length);
4954
WritingPrimitives.WriteRawBytes(ref output.buffer, ref output.state, bytes);
5055
}
5156
}

‎src/LightProto/Parser/Bool.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ sealed class BooleanProtoWriter : IProtoWriter, IProtoWriter<bool>
2727
public WireFormat.WireType WireType => WireFormat.WireType.Varint;
2828
public bool IsMessage => false;
2929

30-
public int CalculateSize(bool value)
30+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((bool)value);
31+
32+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
33+
public int CalculateSize(bool value) => (int)CalculateLongSize(value);
34+
35+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
36+
public long CalculateLongSize(bool value)
3137
{
3238
return CodedOutputStream.ComputeBoolSize(value);
3339
}

‎src/LightProto/Parser/Byte.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ sealed class ByteProtoWriter : IProtoWriter, IProtoWriter<byte>
2727
public WireFormat.WireType WireType => WireFormat.WireType.Varint;
2828
public bool IsMessage => false;
2929

30-
public int CalculateSize(byte value)
30+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((byte)value);
31+
32+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
33+
public int CalculateSize(byte value) => (int)CalculateLongSize(value);
34+
35+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
36+
public long CalculateLongSize(byte value)
3137
{
3238
return CodedOutputStream.ComputeUInt32Size(value);
3339
}

‎src/LightProto/Parser/ByteArray.cs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,20 @@ sealed class ByteArrayProtoWriter : IProtoWriter, IProtoWriter<byte[]>
2828

2929
public void WriteTo(ref WriterContext output, object value) => WriteTo(ref output, (byte[])value);
3030

31+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((byte[])value);
32+
33+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
34+
public int CalculateSize(byte[] value) => (int)CalculateLongSize(value);
35+
3136
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
32-
public int CalculateSize(byte[] value)
37+
public long CalculateLongSize(byte[] value)
3338
{
34-
return CodedOutputStream.ComputeLengthSize(value.Length) + value.Length;
39+
return CodedOutputStream.ComputeLongLengthSize(value.Length) + value.Length;
3540
}
3641

3742
public void WriteTo(ref WriterContext output, byte[] value)
3843
{
39-
output.WriteLength(value.Length);
44+
output.WriteLongLength(value.Length);
4045
WritingPrimitives.WriteRawBytes(ref output.buffer, ref output.state, value);
4146
}
4247
}

‎src/LightProto/Parser/Char.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ sealed class CharProtoWriter : IProtoWriter, IProtoWriter<Char>
2727
public WireFormat.WireType WireType => WireFormat.WireType.Varint;
2828
public bool IsMessage => false;
2929

30-
public int CalculateSize(Char value)
30+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((Char)value);
31+
32+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
33+
public int CalculateSize(Char value) => (int)CalculateLongSize(value);
34+
35+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
36+
public long CalculateLongSize(Char value)
3137
{
3238
return CodedOutputStream.ComputeUInt32Size(value);
3339
}

‎src/LightProto/Parser/CultureInfo.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,13 @@ sealed class CultureInfoProtoWriter : IProtoWriter, IProtoWriter<CultureInfo>
3131
public WireFormat.WireType WireType => WireFormat.WireType.LengthDelimited;
3232
public bool IsMessage => false;
3333

34-
public int CalculateSize(CultureInfo value)
34+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((CultureInfo)value);
35+
36+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
37+
public int CalculateSize(CultureInfo value) => (int)CalculateLongSize(value);
38+
39+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
40+
public long CalculateLongSize(CultureInfo value)
3541
{
3642
return CodedOutputStream.ComputeStringSize(value.Name);
3743
}

‎src/LightProto/Parser/DateOnly.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,13 @@ sealed class DateOnlyProtoWriter : IProtoWriter, IProtoWriter<DateOnly>
2828
public WireFormat.WireType WireType => WireFormat.WireType.Varint;
2929
public bool IsMessage => false;
3030

31+
long IProtoWriter.CalculateLongSize(object value) => CalculateLongSize((DateOnly)value);
32+
33+
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
34+
public int CalculateSize(DateOnly value) => (int)CalculateLongSize(value);
35+
3136
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]
32-
public int CalculateSize(DateOnly value)
37+
public long CalculateLongSize(DateOnly value)
3338
{
3439
return CodedOutputStream.ComputeInt32Size(value.DayNumber);
3540
}

0 commit comments

Comments
 (0)