Skip to content

Commit d5cf96d

Browse files
TwoTenPvPTwoTenPvP
authored
Merge pull request #37 from GabrielTofvesson/master
New binary serializer
2 parents 3b0d7b5 + 06fdaa9 commit d5cf96d

File tree

4 files changed

+499
-0
lines changed

4 files changed

+499
-0
lines changed

‎MLAPI/MLAPI.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@
7878
<Compile Include="GlobalSuppressions.cs" />
7979
<Compile Include="MonoBehaviours\Prototyping\NetworkedAnimator.cs" />
8080
<Compile Include="MonoBehaviours\Prototyping\NetworkedNavMeshAgent.cs" />
81+
<Compile Include="NetworkingManagerComponents\Binary\BinaryCollector.cs" />
82+
<Compile Include="NetworkingManagerComponents\Binary\BinaryDistributor.cs" />
83+
<Compile Include="NetworkingManagerComponents\Binary\BinaryHelpers.cs" />
8184
<Compile Include="NetworkingManagerComponents\Binary\BinarySerializer.cs" />
8285
<Compile Include="NetworkingManagerComponents\Cryptography\CryptographyHelper.cs" />
8386
<Compile Include="NetworkingManagerComponents\Cryptography\DiffieHellman.cs" />

‎MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,361 @@
1+
using MLAPI.NetworkingManagerComponents.Binary;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Reflection;
7+
using System.Runtime.InteropServices;
8+
using System.Text;
9+
10+
namespace Tofvesson.Common
11+
{
12+
public sealed class BinaryCollector : IDisposable
13+
{
14+
// Collects reusable
15+
private static readonly List<WeakReference> expired = new List<WeakReference>();
16+
17+
private static readonly float[] holder_f = new float[1];
18+
private static readonly double[] holder_d = new double[1];
19+
private static readonly ulong[] holder_u = new ulong[1];
20+
private static readonly uint[] holder_i = new uint[1];
21+
private static readonly List<Type> supportedTypes = new List<Type>()
22+
{
23+
typeof(bool),
24+
typeof(byte),
25+
typeof(sbyte),
26+
typeof(char),
27+
typeof(short),
28+
typeof(ushort),
29+
typeof(int),
30+
typeof(uint),
31+
typeof(long),
32+
typeof(ulong),
33+
typeof(float),
34+
typeof(double),
35+
typeof(decimal)
36+
};
37+
38+
private static readonly FieldInfo
39+
dec_lo,
40+
dec_mid,
41+
dec_hi,
42+
dec_flags;
43+
44+
static BinaryCollector()
45+
{
46+
dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic);
47+
dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic);
48+
dec_hi = typeof(decimal).GetField("hi", BindingFlags.NonPublic);
49+
dec_flags = typeof(decimal).GetField("flags", BindingFlags.NonPublic);
50+
}
51+
52+
private object[] collect;
53+
private readonly int bufferSize;
54+
private int collectCount = 0;
55+
56+
/// <summary>
57+
/// Allocates a new binary collector.
58+
/// </summary>
59+
public BinaryCollector(int bufferSize)
60+
{
61+
this.bufferSize = bufferSize;
62+
for (int i = expired.Count - 1; i >= 0; --i)
63+
if (expired[i].IsAlive)
64+
{
65+
collect = (object[])expired[i].Target;
66+
if (collect.Length >= bufferSize)
67+
{
68+
expired.RemoveAt(i); // This entry he been un-expired for now
69+
break;
70+
}
71+
}
72+
else expired.RemoveAt(i); // Entry has been collected by GC
73+
if (collect == null || collect.Length < bufferSize)
74+
collect = new object[bufferSize];
75+
}
76+
77+
private void Push<T>(T b)
78+
{
79+
if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
80+
collect[collectCount++] = b is string ? Encoding.UTF8.GetBytes(b as string) : b as object;
81+
//else
82+
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
83+
}
84+
85+
86+
public void WriteBool(bool b) => Push(b);
87+
public void WriteFloat(float f) => Push(f);
88+
public void WriteDouble(double d) => Push(d);
89+
public void WriteByte(byte b) => Push(b);
90+
public void WriteUShort(ushort s) => Push(s);
91+
public void WriteUInt(uint i) => Push(i);
92+
public void WriteULong(ulong l) => Push(l);
93+
public void WriteSByte(sbyte b) => Push(b);
94+
public void WriteShort(short s) => Push(s);
95+
public void WriteInt(int i) => Push(i);
96+
public void WriteLong(long l) => Push(l);
97+
public void WriteFloatArray(float[] f) => Push(f);
98+
public void WriteDoubleArray(double[] d) => Push(d);
99+
public void WriteByteArray(byte[] b) => Push(b);
100+
public void WriteUShortArray(ushort[] s) => Push(s);
101+
public void WriteUIntArray(uint[] i) => Push(i);
102+
public void WriteULongArray(ulong[] l) => Push(l);
103+
public void WriteSByteArray(sbyte[] b) => Push(b);
104+
public void WriteShortArray(short[] s) => Push(s);
105+
public void WriteIntArray(int[] i) => Push(i);
106+
public void WriteLongArray(long[] l) => Push(l);
107+
public void WriteString(string s) => Push(s);
108+
109+
public byte[] ToArray()
110+
{
111+
long bitCount = 0;
112+
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
113+
114+
byte[] alloc = new byte[(bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)];
115+
long bitOffset = 0;
116+
foreach (var item in collect)
117+
Serialize(item, alloc, ref bitOffset);
118+
119+
return alloc;
120+
}
121+
122+
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
123+
{
124+
Type type = t.GetType();
125+
bool size = false;
126+
if (type.IsArray)
127+
{
128+
var array = t as Array;
129+
Serialize((ushort)array.Length, writeTo, ref bitOffset);
130+
foreach (var element in array)
131+
Serialize(element, writeTo, ref bitOffset);
132+
}
133+
else if (IsSupportedType(type))
134+
{
135+
long offset = GetBitAllocation(type);
136+
if (type == typeof(bool))
137+
{
138+
WriteBit(writeTo, t as bool? ?? false, bitOffset);
139+
bitOffset += offset;
140+
}
141+
else if (type == typeof(decimal))
142+
{
143+
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset);
144+
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32);
145+
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64);
146+
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96);
147+
bitOffset += offset;
148+
}
149+
else if ((size = type == typeof(float)) || type == typeof(double))
150+
{
151+
int bytes = size ? 4 : 8;
152+
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
153+
Array result_holder = size ? holder_i as Array : holder_u as Array;
154+
lock (result_holder)
155+
lock (type_holder)
156+
{
157+
// Clear artifacts
158+
if (size) result_holder.SetValue(0U, 0);
159+
else result_holder.SetValue(0UL, 0);
160+
type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array
161+
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
162+
163+
// Since floating point flag bits are seemingly the highest bytes of the floating point values
164+
// and even very small values have them, we swap the endianness in the hopes of reducing the size
165+
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset);
166+
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset);
167+
}
168+
//bitOffset += offset;
169+
}
170+
else
171+
{
172+
bool signed = IsSigned(t.GetType());
173+
ulong value;
174+
if (signed)
175+
{
176+
Type t1 = t.GetType();
177+
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1);
178+
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2);
179+
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4);
180+
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
181+
}
182+
else if (t is byte) value = t as byte? ?? 0;
183+
else if (t is ushort) value = t as ushort? ?? 0;
184+
else if (t is uint) value = t as uint? ?? 0;
185+
else /*if (t is ulong)*/ value = t as ulong? ?? 0;
186+
187+
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset);
188+
else if (value <= 2287)
189+
{
190+
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
191+
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
192+
}
193+
else if (value <= 67823)
194+
{
195+
WriteByte(writeTo, 249, bitOffset);
196+
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
197+
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
198+
}
199+
else
200+
{
201+
WriteByte(writeTo, value & 255, bitOffset + 8);
202+
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
203+
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
204+
if (value > 16777215)
205+
{
206+
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
207+
if (value > 4294967295)
208+
{
209+
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
210+
if (value > 1099511627775)
211+
{
212+
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
213+
if (value > 281474976710655)
214+
{
215+
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
216+
if (value > 72057594037927935)
217+
{
218+
WriteByte(writeTo, 255, bitOffset);
219+
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
220+
}
221+
else WriteByte(writeTo, 254, bitOffset);
222+
}
223+
else WriteByte(writeTo, 253, bitOffset);
224+
}
225+
else WriteByte(writeTo, 252, bitOffset);
226+
}
227+
else WriteByte(writeTo, 251, bitOffset);
228+
}
229+
else WriteByte(writeTo, 250, bitOffset);
230+
}
231+
bitOffset += BytesToRead(value) * 8;
232+
}
233+
}
234+
}
235+
236+
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
237+
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
238+
239+
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
240+
241+
private static Type GetUnsignedType(Type t) =>
242+
t == typeof(sbyte) ? typeof(byte) :
243+
t == typeof(short) ? typeof(ushort) :
244+
t == typeof(int) ? typeof(uint) :
245+
t == typeof(long) ? typeof(ulong) :
246+
null;
247+
248+
private static ulong ZigZagEncode(long d, int bytes) => (ulong)(((d >> (bytes * 8 - 1))&1) | (d << 1));
249+
250+
private static long GetBitCount<T>(T t)
251+
{
252+
Type type = t.GetType();
253+
long count = 0;
254+
if (type.IsArray)
255+
{
256+
Type elementType = type.GetElementType();
257+
258+
count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
259+
foreach (var element in t as Array)
260+
count += GetBitCount(element);
261+
}
262+
else if (IsSupportedType(type))
263+
{
264+
long ba = GetBitAllocation(type);
265+
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
266+
else if (t is bool || t is decimal) count += ba;
267+
else count += BytesToRead(t) * 8;
268+
}
269+
//else
270+
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
271+
return count;
272+
}
273+
274+
private static void WriteBit(byte[] b, bool bit, long index)
275+
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
276+
private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index);
277+
private static void WriteByte(byte[] b, byte value, long index)
278+
{
279+
int byteIndex = (int)(index / 8);
280+
int shift = (int)(index % 8);
281+
byte upper_mask = (byte)(0xFF << shift);
282+
byte lower_mask = (byte)~upper_mask;
283+
284+
b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift));
285+
if(shift != 0 && byteIndex + 1 < b.Length)
286+
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
287+
}
288+
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
289+
{
290+
for (int i = 0; i < bits; ++i)
291+
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
292+
}
293+
private static void WriteDynamic(byte[] b, int value, int byteCount, long index)
294+
{
295+
for (int i = 0; i < byteCount; ++i)
296+
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
297+
}
298+
299+
private static int BytesToRead(object i)
300+
{
301+
bool size;
302+
ulong integer;
303+
if ((size = i is float) || i is double)
304+
{
305+
int bytes = size ? 4 : 8;
306+
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
307+
Array result_holder = size ? holder_i as Array : holder_u as Array;
308+
lock (result_holder)
309+
lock (type_holder)
310+
{
311+
// Clear artifacts
312+
if (size) result_holder.SetValue(0U, 0);
313+
else result_holder.SetValue(0UL, 0);
314+
315+
type_holder.SetValue(i, 0); // Insert the value to convert into the preallocated holder array
316+
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
317+
if(size) integer = BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0));
318+
else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0));
319+
}
320+
}
321+
else integer = i as ulong? ?? 0;
322+
return
323+
integer <= 240 ? 1 :
324+
integer <= 2287 ? 2 :
325+
integer <= 67823 ? 3 :
326+
integer <= 16777215 ? 4 :
327+
integer <= 4294967295 ? 5 :
328+
integer <= 1099511627775 ? 6 :
329+
integer <= 281474976710655 ? 7 :
330+
integer <= 72057594037927935 ? 8 :
331+
9;
332+
}
333+
334+
// Supported datatypes for serialization
335+
private static bool IsSupportedType(Type t) => supportedTypes.Contains(t);
336+
337+
// Specifies how many bits will be written
338+
private static long GetBitAllocation(Type t) =>
339+
t == typeof(bool) ? 1 :
340+
t == typeof(byte) ? 8 :
341+
t == typeof(sbyte) ? 8 :
342+
t == typeof(short) ? 16 :
343+
t == typeof(char) ? 16 :
344+
t == typeof(ushort) ? 16 :
345+
t == typeof(int) ? 32 :
346+
t == typeof(uint) ? 32 :
347+
t == typeof(long) ? 64 :
348+
t == typeof(ulong) ? 64 :
349+
t == typeof(float) ? 32 :
350+
t == typeof(double) ? 64 :
351+
t == typeof(decimal) ? 128 :
352+
0; // Unknown type
353+
354+
// Creates a weak reference to the allocated collector so that reuse may be possible
355+
public void Dispose()
356+
{
357+
expired.Add(new WeakReference(collect));
358+
collect = null;
359+
}
360+
}
361+
}

0 commit comments

Comments
 (0)