Skip to content

Commit efda405

Browse files
yfakariyayfakariya
authored
Merge pull request #250 from msgpack/wip/improve-packer-unpacker
Improve packer unpacker performance
2 parents bf4909e + 8167927 commit efda405

File tree

177 files changed

+76321
-54179
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

177 files changed

+76321
-54179
lines changed

‎CHANGES.txt‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -641,7 +641,20 @@ Release 0.9.0 beta2 2017/2/12
641641
* Fix types which implement IPackable and IUnpackable but do not have any members cannot be serialized. Issue #202
642642
* Fix Windows Native build error. Issue #206.
643643

644-
Release 0.9.0 RC1 T.B.D.
644+
Release 0.9.0 RC1 T.B.D
645+
646+
NEW FEATURES
647+
* ByteArrayPacker/ByteArrayUnpacker. They are suitable for fixed pattern serialization/deserialization.
648+
* Fast mode of unpacker which omits nested collection management. This option can be disabled with setting UnpackerOptions.ValidationLevel to ValidationLevel.Collection.
649+
650+
IMPROVEMENTS
651+
* Byte array based serialization API now uses ByteArrayPacker/ByteArrayUnpacker, improves about 40% faster than Stream based.
652+
* Deserialization now uses ValidationLevel.None, improves about 30% faster than validating one.
653+
* Packer/Unpacker performance improvements about 10-20%. As a result, new byte array "fast" unpacker is about 3x faster than previous unpacker.
645654

646655
BUG FIXES
647656
* Fix constructor deserialization fails if the constructor parameters order is not lexical. Issue #233
657+
* Fix asynchronous multi dimensional array deserialization corruption.
658+
* Fix enum serialization throws NullReferenceException in Unity. Issue #215.
659+
* Fix MessagePackSerializer.Capability does not work correctly in Unity.
660+
* Fix polymorphic serializer error in Unity.

‎Sync.Test.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
<Preserve Path="System\Collections\Generic\*" />
2727
<Preserve Path="App.xaml" />
2828
<Preserve Path="App.xaml.cs" />
29+
<Preserve Path="Augments.cs" />
2930
<Exclude Path="gen\*" />
3031
<Exclude Path="Serialization\PreGenerated*" />
3132
<Exclude Path="Serialization\_SetUpFixture.cs" />
@@ -50,6 +51,7 @@
5051
<Exclude Path="Serialization\RoslynCodeGeneration.cs" />
5152
<Exclude Path="Serialization\TempFileDependentAssemblyManager.cs" />
5253
<Exclude Path="_SetUpFixture.cs" />
54+
<Exclude Path="AssertEx.cs" />
5355
<Exclude Path="TestSuite.cs" />
5456
<Exclude File="*.ps1" />
5557
<Exclude File="*.tt" />
@@ -147,6 +149,7 @@
147149
<Exclude Path="Serialization\AppendableReadOnlyCollections.cs" />
148150
<Exclude Path="Serialization\RoslynCodeGeneration.cs" />
149151
<Preserve Path="Serialization\CodeDomCodeGeneration.cs" />
152+
<Preserve Path="Augments.cs" />
150153
<Preserve Path="Dummies\System.Threading.Tasks\Task.cs" />
151154
<Preserve Path="Dummies\System.Threading.Tasks\TaskFactory.cs" />
152155
<Preserve Path="Mono\System\AggregateException.cs" />
@@ -162,6 +165,7 @@
162165
<Exclude File="*.ps1" />
163166
<Exclude File="*.tt" />
164167
<Exclude File="*.ttinclude" />
168+
<Exclude Path="AssertEx.cs" />
165169
</Project>
166170

167171
<Project Name="MsgPack.UnitTest.CodeDom.Net35" Base="MsgPack.UnitTest.CodeDom">

‎Sync.xml‎

Lines changed: 14 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,5 @@
11
<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
22
<ProjectSync>
3-
<Project Name="MsgPack.Net35" Base="MsgPack">
4-
<Preserve Path="Properties\AssemblyInfo.cs" />
5-
<Exclude Path="Serialization\AbstractSerializers\SerializerBuilder`2.Tuple.cs" />
6-
<Exclude Path="Serialization\CollectionSerializers\ReadOnlyCollectionMessagePackSerializer`2.cs" />
7-
<Exclude Path="Serialization\CollectionSerializers\ReadOnlyDictionaryMessagePackSerializer`3.cs" />
8-
<Exclude Path="Serialization\DefaultSerializers\AbstractReadOnlyCollectionMessagePackSerializer`2.cs" />
9-
<Exclude Path="Serialization\DefaultSerializers\AbstractReadOnlyDictionaryMessagePackSerializer`3.cs" />
10-
<Exclude Path="Serialization\DefaultSerializers\FSharpCollectionSerializer`2.cs" />
11-
<Exclude Path="Serialization\DefaultSerializers\FSharpMapSerializer`3.cs" />
12-
<Exclude Path="Serialization\DefaultSerializers\ImmutableCollectionSerializer`2.cs" />
13-
<Exclude Path="Serialization\DefaultSerializers\ImmutableDictionarySerializer`3.cs" />
14-
<Exclude Path="Serialization\DefaultSerializers\ImmutableStackSerializer`2.cs" />
15-
<Exclude Path="Serialization\DefaultSerializers\SimdTypeSerializers.cs" />
16-
<Exclude Path="Serialization\DefaultSerializers\System_Numerics_ComplexMessagePackSerializer.cs" />
17-
<Exclude Path="Serialization\DefaultSerializers\System_Security_Cryptography_HashAlgorithmNameMessagePackSerializer.cs" />
18-
<Exclude Path="Serialization\EmittingSerializers\CallbackReadOnlyCollectionMessagePackSerializer`2.cs" />
19-
<Exclude Path="Serialization\EmittingSerializers\CallbackReadOnlyDictionaryMessagePackSerializer`3.cs" />
20-
<Exclude Path="Serialization\ExpressionSerializers\*" />
21-
<Exclude Path="Serialization\Metadata\_Type.cs" />
22-
<Exclude Path="Serialization\ReflectionSerializers\ReflectionTupleMessagePackSerializer`1.cs" />
23-
<Exclude Path="BufferedStream.cs" />
24-
<Exclude File="*.tt" />
25-
<Exclude File="*.ttinclude" />
26-
<Preserve Path="Contract.cs" />
27-
<Preserve Path="Delegates.tt" />
28-
<Preserve Path="Delegates.cs" />
29-
<Preserve Path="NetFxCompatibilities.cs" />
30-
<Preserve Path="Volatile.cs" />
31-
<Preserve Path="Tuple`n.tt" />
32-
<Preserve Path="Tuple`n.cs" />
33-
<Exclude Path="TupleItems.cs" />
34-
</Project>
35-
36-
<!-- TOOD: should be project.json -->
37-
<!-- .Uwp is testing purpose only, this is just .NET Native test target for .NET Standard 1.3 configuration -->
383
<Project Name="MsgPack.Uwp" Base="MsgPack">
394
<Preserve Path="Properties\AssemblyInfo.cs" />
405
<Preserve Path="BufferedStream.cs" />
@@ -72,13 +37,14 @@
7237
<Exclude File="*.ttinclude" />
7338
</Project>
7439

75-
<Project Name="MsgPack.Unity.Full" Base="MsgPack.Xamarin.iOS">
40+
<Project Name="MsgPack.Unity.Full" Base="MsgPack.Uwp">
7641
<Preserve Path="Properties\AssemblyInfo.cs" />
7742
<Preserve Path="Serialization\AotHelper*.tt" />
7843
<Preserve Path="Serialization\AotHelper*.cs" />
7944
<Preserve Path="Serialization\NonGenericMessagePackSerializer.cs" />
8045
<Preserve Path="Serialization\TypedMessagePackSerializerWrapper`1.cs" />
8146
<Preserve Path="Serialization\EnumTypedMessagePackSerializerWrapper`1.cs" />
47+
<Preserve Path="Serialization\Tracer.cs" />
8248
<Preserve Path="CorLibOnlyHelper*.tt" />
8349
<Preserve Path="CorLibOnlyHelper*.cs" />
8450
<Preserve Path="MPContract.cs" />
@@ -87,6 +53,11 @@
8753
<Exclude Path="Serialization\CollectionSerializers\DictionarySerializerBase`3.cs" />
8854
<Exclude Path="Serialization\CollectionSerializers\ReadOnlyCollectionMessagePackSerializer`2.cs" />
8955
<Exclude Path="Serialization\CollectionSerializers\ReadOnlyDictionaryMessagePackSerializer`3.cs" />
56+
<Exclude Path="Serialization\DefaultSerializers\FileTimeMessagePackSerializerProvider.cs" />
57+
<Exclude Path="Serialization\DefaultSerializers\NativeFileTimeMessagePackSerializer.cs" />
58+
<Exclude Path="Serialization\DefaultSerializers\UnixEpocFileTimeMessagePackSerializer.cs" />
59+
<Exclude Path="Serialization\DefaultSerializers\SimdTypeSerializers.cs" />
60+
<Exclude Path="Serialization\ReflectionExtensions.ConstructorDelegate.cs" />
9061
<Exclude Path="Serialization\DefaultSerializers\FSharpCollectionSerializer`2.cs" />
9162
<Exclude Path="Serialization\DefaultSerializers\FSharpMapSerializer`3.cs" />
9263
<Exclude Path="Serialization\DefaultSerializers\ImmutableCollectionSerializer`2.cs" />
@@ -98,8 +69,13 @@
9869
<Exclude Path="Serialization\DefaultSerializers\System_Numerics_ComplexMessagePackSerializer.cs" />
9970
<Exclude Path="Serialization\DefaultSerializers\System_Security_Cryptography_HashAlgorithmNameMessagePackSerializer.cs" />
10071
<Exclude Path="Serialization\ReflectionSerializers\ReflectionTupleMessagePackSerializer`1.cs" />
72+
<Exclude Path="BufferedStream.cs" />
73+
<Exclude Path="NetStandardCompatibility.cs" />
74+
<Exclude Path="TaskAugument.cs" />
10175
<Exclude Path="UnsafeNativeMethods.cs" />
10276
<Exclude Path="Validation.cs" />
77+
<Exclude Path="project.json" />
78+
<Exclude File="*.snk" />
10379
</Project>
10480

10581
<Project Name="MsgPack.Unity" Base="MsgPack.Unity.Full">
@@ -145,6 +121,7 @@
145121
<Exclude Path="Serialization\DefaultSerializers\System_Security_Cryptography_HashAlgorithmNameMessagePackSerializer.cs" />
146122
<Exclude Path="BufferedStream.cs" />
147123
<Exclude Path="NetStandardCompatibility.cs" />
124+
<Exclude Path="TaskAugument.cs" />
148125
<Exclude Path="project.json" />
149126
</Project>
150127

@@ -160,5 +137,6 @@
160137
<Exclude Path="Serialization\Reflection\ReflectionHelpers.cs" />
161138
<Exclude Path="BufferedStream.cs" />
162139
<Exclude Path="NetStandardCompatibility.cs" />
140+
<Exclude Path="TaskAugument.cs" />
163141
</Project>
164142
</ProjectSync>

‎build/SetBuildEnv.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ if ( $env:APPVEYOR_REPO_TAG -ne "True" )
77
{
88
$now = [DateTime]::UtcNow
99
$daysSpan = $now - ( New-Object DateTime( $now.Year, 1, 1 ) )
10-
$env:PackageVersion = "${version}-{0:yy}{1:000}" -f @( $now, $daysSpan.Days )
10+
$env:PackageVersion = "${version}-{0:yy}{1:000}-{2:000}" -f @( $now, $daysSpan.Days, ( $now.TimeOfDay.TotalMinutes / 2 ) )
1111
}
1212
elseif ( ${version} -match "^[\d.]+$" )
1313
{

‎samples/Samples/Sample10_ByteArrayBased.cs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#region -- License Terms --
2+
//
3+
// MessagePack for CLI
4+
//
5+
// Copyright (C) 2017 FUJIWARA, Yusuke
6+
//
7+
// Licensed under the Apache License, Version 2.0 (the "License");
8+
// you may not use this file except in compliance with the License.
9+
// You may obtain a copy of the License at
10+
//
11+
// http://www.apache.org/licenses/LICENSE-2.0
12+
//
13+
// Unless required by applicable law or agreed to in writing, software
14+
// distributed under the License is distributed on an "AS IS" BASIS,
15+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
// See the License for the specific language governing permissions and
17+
// limitations under the License.
18+
//
19+
#endregion -- License Terms --
20+
21+
using System;
22+
using System.Collections.Generic;
23+
using MsgPack;
24+
using MsgPack.Serialization;
25+
using NUnit.Framework; // For running checking
26+
27+
namespace Samples
28+
{
29+
[TestFixture]
30+
public class Sample10_ByteArrayBased
31+
{
32+
[Test]
33+
public void SimpleBufferCase()
34+
{
35+
// Assumes that we know maximum serialized data size, so just use it!
36+
var buffer = new byte[ 1024 * 64 ];
37+
38+
var context = new SerializationContext();
39+
var serializer = context.GetSerializer<PhotoEntry>();
40+
41+
var obj =
42+
new PhotoEntry
43+
{
44+
Id = 123,
45+
Title = "My photo",
46+
Date = DateTime.Now,
47+
Image = new byte[] { 1, 2, 3, 4 },
48+
Comment = "This is test object to be serialize/deserialize using MsgPack."
49+
};
50+
// Note that the packer automatically increse buffer.
51+
using ( var bytePacker = Packer.Create( buffer ) )
52+
{
53+
serializer.PackTo( bytePacker, obj );
54+
// Note: You can get actual bytes with GetResultBytes(), but it causes array copy.
55+
// You can avoid copying using original buffer (when you prohibits buffer allocation on Packer.Create) or GetFinalBuffers() instead.
56+
Console.WriteLine( "Serialized: {0}", BitConverter.ToString( buffer, 0, ( int )bytePacker.BytesUsed ) );
57+
58+
using ( var byteUnpacker = Unpacker.Create( buffer ) )
59+
{
60+
var deserialized = serializer.UnpackFrom( byteUnpacker );
61+
}
62+
}
63+
}
64+
}
65+
66+
public class MyArrayBufferManager
67+
{
68+
public IList<ArraySegment<byte>> Buffers { get; }
69+
}
70+
}

‎samples/Samples/Samples.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
<Compile Include="Sample08_Polymorphism.cs" />
5656
<Compile Include="Sample07_ConstructorBasedDeserialization.cs" />
5757
<Compile Include="Sample09_Async.cs" />
58+
<Compile Include="Sample10_ByteArrayBased.cs" />
5859
</ItemGroup>
5960
<ItemGroup>
6061
<ProjectReference Include="..\..\src\MsgPack\MsgPack.csproj">

‎src/MsgPack.Silverlight.5/MsgPack.Silverlight.5.csproj‎

Lines changed: 70 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -95,15 +95,33 @@
9595
<Compile Include="..\MsgPack\BufferManager.cs">
9696
<Link>BufferManager.cs</Link>
9797
</Compile>
98+
<Compile Include="..\MsgPack\ByteArrayPacker.cs">
99+
<Link>ByteArrayPacker.cs</Link>
100+
</Compile>
101+
<Compile Include="..\MsgPack\ByteArrayUnpacker.cs">
102+
<Link>ByteArrayUnpacker.cs</Link>
103+
</Compile>
104+
<Compile Include="..\MsgPack\ByteBufferAllocator.cs">
105+
<Link>ByteBufferAllocator.cs</Link>
106+
</Compile>
98107
<Compile Include="..\MsgPack\CollectionDebuggerProxy`1.cs">
99108
<Link>CollectionDebuggerProxy`1.cs</Link>
100109
</Compile>
101110
<Compile Include="..\MsgPack\CollectionOperation.cs">
102111
<Link>CollectionOperation.cs</Link>
103112
</Compile>
113+
<Compile Include="..\MsgPack\CollectionType.cs">
114+
<Link>CollectionType.cs</Link>
115+
</Compile>
104116
<Compile Include="..\MsgPack\DictionaryDebuggerProxy`2.cs">
105117
<Link>DictionaryDebuggerProxy`2.cs</Link>
106118
</Compile>
119+
<Compile Include="..\MsgPack\EncodingExtensions.cs">
120+
<Link>EncodingExtensions.cs</Link>
121+
</Compile>
122+
<Compile Include="..\MsgPack\FixedArrayBufferAllocator.cs">
123+
<Link>FixedArrayBufferAllocator.cs</Link>
124+
</Compile>
107125
<Compile Include="..\MsgPack\Float32Bits.cs">
108126
<Link>Float32Bits.cs</Link>
109127
</Compile>
@@ -119,23 +137,20 @@
119137
<Compile Include="..\MsgPack\IAsyncUnpackable.cs">
120138
<Link>IAsyncUnpackable.cs</Link>
121139
</Compile>
140+
<Compile Include="..\MsgPack\Int32OffsetValue`1.cs">
141+
<Link>Int32OffsetValue`1.cs</Link>
142+
</Compile>
143+
<Compile Include="..\MsgPack\Int64OffsetValue`1.cs">
144+
<Link>Int64OffsetValue`1.cs</Link>
145+
</Compile>
122146
<Compile Include="..\MsgPack\InvalidMessagePackStreamException.cs">
123147
<Link>InvalidMessagePackStreamException.cs</Link>
124148
</Compile>
125149
<Compile Include="..\MsgPack\IPackable.cs">
126150
<Link>IPackable.cs</Link>
127151
</Compile>
128-
<Compile Include="..\MsgPack\ItemsUnpacker.cs">
129-
<Link>ItemsUnpacker.cs</Link>
130-
</Compile>
131-
<Compile Include="..\MsgPack\ItemsUnpacker.Read.cs">
132-
<Link>ItemsUnpacker.Read.cs</Link>
133-
</Compile>
134-
<Compile Include="..\MsgPack\ItemsUnpacker.Skipping.cs">
135-
<Link>ItemsUnpacker.Skipping.cs</Link>
136-
</Compile>
137-
<Compile Include="..\MsgPack\ItemsUnpacker.Unpacking.cs">
138-
<Link>ItemsUnpacker.Unpacking.cs</Link>
152+
<Compile Include="..\MsgPack\IRootUnpacker.cs">
153+
<Link>IRootUnpacker.cs</Link>
139154
</Compile>
140155
<Compile Include="..\MsgPack\IUnpackable.cs">
141156
<Link>IUnpackable.cs</Link>
@@ -149,6 +164,18 @@
149164
<Compile Include="..\MsgPack\MessageNotSupportedException.cs">
150165
<Link>MessageNotSupportedException.cs</Link>
151166
</Compile>
167+
<Compile Include="..\MsgPack\MessagePackByteArrayPacker.cs">
168+
<Link>MessagePackByteArrayPacker.cs</Link>
169+
</Compile>
170+
<Compile Include="..\MsgPack\MessagePackByteArrayPacker.Pack.cs">
171+
<Link>MessagePackByteArrayPacker.Pack.cs</Link>
172+
</Compile>
173+
<Compile Include="..\MsgPack\MessagePackByteArrayUnpacker.cs">
174+
<Link>MessagePackByteArrayUnpacker.cs</Link>
175+
</Compile>
176+
<Compile Include="..\MsgPack\MessagePackByteArrayUnpacker.Unpack.cs">
177+
<Link>MessagePackByteArrayUnpacker.Unpack.cs</Link>
178+
</Compile>
152179
<Compile Include="..\MsgPack\MessagePackCode.cs">
153180
<Link>MessagePackCode.cs</Link>
154181
</Compile>
@@ -185,6 +212,18 @@
185212
<Compile Include="..\MsgPack\MessagePackObjectEqualityComparer.cs">
186213
<Link>MessagePackObjectEqualityComparer.cs</Link>
187214
</Compile>
215+
<Compile Include="..\MsgPack\MessagePackStreamPacker.cs">
216+
<Link>MessagePackStreamPacker.cs</Link>
217+
</Compile>
218+
<Compile Include="..\MsgPack\MessagePackStreamPacker.Pack.cs">
219+
<Link>MessagePackStreamPacker.Pack.cs</Link>
220+
</Compile>
221+
<Compile Include="..\MsgPack\MessagePackStreamUnpacker.cs">
222+
<Link>MessagePackStreamUnpacker.cs</Link>
223+
</Compile>
224+
<Compile Include="..\MsgPack\MessagePackStreamUnpacker.Unpack.cs">
225+
<Link>MessagePackStreamUnpacker.Unpack.cs</Link>
226+
</Compile>
188227
<Compile Include="..\MsgPack\MessagePackString.cs">
189228
<Link>MessagePackString.cs</Link>
190229
</Compile>
@@ -194,6 +233,9 @@
194233
<Compile Include="..\MsgPack\Packer.cs">
195234
<Link>Packer.cs</Link>
196235
</Compile>
236+
<Compile Include="..\MsgPack\Packer.Factory.cs">
237+
<Link>Packer.Factory.cs</Link>
238+
</Compile>
197239
<Compile Include="..\MsgPack\Packer.Nullable.cs">
198240
<Link>Packer.Nullable.cs</Link>
199241
</Compile>
@@ -215,6 +257,12 @@
215257
<Compile Include="..\MsgPack\PreserveAttribute.cs">
216258
<Link>PreserveAttribute.cs</Link>
217259
</Compile>
260+
<Compile Include="..\MsgPack\ReadValueResult.cs">
261+
<Link>ReadValueResult.cs</Link>
262+
</Compile>
263+
<Compile Include="..\MsgPack\ReadValueResults.cs">
264+
<Link>ReadValueResults.cs</Link>
265+
</Compile>
218266
<Compile Include="..\MsgPack\ReflectionAbstractions.cs">
219267
<Link>ReflectionAbstractions.cs</Link>
220268
</Compile>
@@ -722,8 +770,8 @@
722770
<Compile Include="..\MsgPack\SetOperation.cs">
723771
<Link>SetOperation.cs</Link>
724772
</Compile>
725-
<Compile Include="..\MsgPack\StreamPacker.cs">
726-
<Link>StreamPacker.cs</Link>
773+
<Compile Include="..\MsgPack\SingleArrayBufferAllocator.cs">
774+
<Link>SingleArrayBufferAllocator.cs</Link>
727775
</Compile>
728776
<Compile Include="..\MsgPack\StringEscape.cs">
729777
<Link>StringEscape.cs</Link>
@@ -743,9 +791,18 @@
743791
<Compile Include="..\MsgPack\Unpacker.cs">
744792
<Link>Unpacker.cs</Link>
745793
</Compile>
794+
<Compile Include="..\MsgPack\Unpacker.Leaf.cs">
795+
<Link>Unpacker.Leaf.cs</Link>
796+
</Compile>
746797
<Compile Include="..\MsgPack\Unpacker.Unpacking.cs">
747798
<Link>Unpacker.Unpacking.cs</Link>
748799
</Compile>
800+
<Compile Include="..\MsgPack\UnpackerOptions.cs">
801+
<Link>UnpackerOptions.cs</Link>
802+
</Compile>
803+
<Compile Include="..\MsgPack\UnpackerValidationLevel.cs">
804+
<Link>UnpackerValidationLevel.cs</Link>
805+
</Compile>
749806
<Compile Include="..\MsgPack\UnpackException.cs">
750807
<Link>UnpackException.cs</Link>
751808
</Compile>

0 commit comments

Comments
 (0)