Skip to content

Commit 13470ef

Browse files
GabrielTofvessonGabrielTofvesson
committed
Added new serialization/deserialization system
 - Code might look a bit janky because it was adapted from a version that used dynamic types
1 parent d674606 commit 13470ef

File tree

4 files changed

+465
-0
lines changed

4 files changed

+465
-0
lines changed

‎MLAPI/MLAPI.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@
7878
<Compile Include="GlobalSuppressions.cs" />
7979
<Compile Include="MonoBehaviours\Prototyping\NetworkedAnimator.cs" />
8080
<Compile Include="MonoBehaviours\Prototyping\NetworkedNavMeshAgent.cs" />
81+
<Compile Include="NetworkingManagerComponents\Binary\BinaryCollector.cs" />
82+
<Compile Include="NetworkingManagerComponents\Binary\BinaryDistributor.cs" />
83+
<Compile Include="NetworkingManagerComponents\Binary\BinaryHelpers.cs" />
8184
<Compile Include="NetworkingManagerComponents\Binary\BinarySerializer.cs" />
8285
<Compile Include="NetworkingManagerComponents\Cryptography\CryptographyHelper.cs" />
8386
<Compile Include="NetworkingManagerComponents\Cryptography\DiffieHellman.cs" />

‎MLAPI/NetworkingManagerComponents/Binary/BinaryCollector.cs

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
using MLAPI.NetworkingManagerComponents.Binary;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Reflection;
7+
using System.Runtime.InteropServices;
8+
using System.Text;
9+
10+
namespace Tofvesson.Common
11+
{
12+
public sealed class BinaryCollector : IDisposable
13+
{
14+
// Collects reusable
15+
private static readonly List<WeakReference> expired = new List<WeakReference>();
16+
17+
private static readonly float[] holder_f = new float[1];
18+
private static readonly double[] holder_d = new double[1];
19+
private static readonly ulong[] holder_u = new ulong[1];
20+
private static readonly uint[] holder_i = new uint[1];
21+
private static readonly List<Type> supportedTypes = new List<Type>()
22+
{
23+
typeof(bool),
24+
typeof(byte),
25+
typeof(sbyte),
26+
typeof(char),
27+
typeof(short),
28+
typeof(ushort),
29+
typeof(int),
30+
typeof(uint),
31+
typeof(long),
32+
typeof(ulong),
33+
typeof(float),
34+
typeof(double),
35+
typeof(decimal)
36+
};
37+
38+
private static readonly FieldInfo
39+
dec_lo,
40+
dec_mid,
41+
dec_hi,
42+
dec_flags;
43+
44+
static BinaryCollector()
45+
{
46+
dec_lo = typeof(decimal).GetField("lo", BindingFlags.NonPublic);
47+
dec_mid = typeof(decimal).GetField("mid", BindingFlags.NonPublic);
48+
dec_hi = typeof(decimal).GetField("hi", BindingFlags.NonPublic);
49+
dec_flags = typeof(decimal).GetField("flags", BindingFlags.NonPublic);
50+
}
51+
52+
private object[] collect;
53+
private readonly int bufferSize;
54+
private int collectCount = 0;
55+
56+
/// <summary>
57+
/// Allocates a new binary collector.
58+
/// </summary>
59+
public BinaryCollector(int bufferSize)
60+
{
61+
this.bufferSize = bufferSize;
62+
for (int i = expired.Count - 1; i >= 0; --i)
63+
if (expired[i].IsAlive)
64+
{
65+
collect = (object[])expired[i].Target;
66+
if (collect.Length >= bufferSize)
67+
{
68+
expired.RemoveAt(i); // This entry he been un-expired for now
69+
break;
70+
}
71+
}
72+
else expired.RemoveAt(i); // Entry has been collected by GC
73+
if (collect == null || collect.Length < bufferSize)
74+
collect = new object[bufferSize];
75+
}
76+
77+
public void Push<T>(T b)
78+
{
79+
if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
80+
collect[collectCount++] = b is string ? Encoding.UTF8.GetBytes(b as string) : b as object;
81+
//else
82+
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
83+
}
84+
85+
public byte[] ToArray()
86+
{
87+
long bitCount = 0;
88+
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
89+
90+
byte[] alloc = new byte[(bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)];
91+
long bitOffset = 0;
92+
foreach (var item in collect)
93+
Serialize(item, alloc, ref bitOffset);
94+
95+
return alloc;
96+
}
97+
98+
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
99+
{
100+
Type type = t.GetType();
101+
bool size = false;
102+
if (type.IsArray)
103+
{
104+
var array = t as Array;
105+
Serialize((ushort)array.Length, writeTo, ref bitOffset);
106+
foreach (var element in array)
107+
Serialize(element, writeTo, ref bitOffset);
108+
}
109+
else if (IsSupportedType(type))
110+
{
111+
long offset = GetBitAllocation(type);
112+
if (type == typeof(bool))
113+
{
114+
WriteBit(writeTo, t as bool? ?? false, bitOffset);
115+
bitOffset += offset;
116+
}
117+
else if (type == typeof(decimal))
118+
{
119+
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset);
120+
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32);
121+
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64);
122+
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96);
123+
bitOffset += offset;
124+
}
125+
else if ((size = type == typeof(float)) || type == typeof(double))
126+
{
127+
int bytes = size ? 4 : 8;
128+
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
129+
Array result_holder = size ? holder_i as Array : holder_u as Array;
130+
lock (result_holder)
131+
lock (type_holder)
132+
{
133+
// Clear artifacts
134+
if (size) result_holder.SetValue(0U, 0);
135+
else result_holder.SetValue(0UL, 0);
136+
type_holder.SetValue(t, 0); // Insert the value to convert into the preallocated holder array
137+
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
138+
dynamic d = result_holder.GetValue(0);
139+
140+
// Since floating point flag bits are seemingly the highest bytes of the floating point values
141+
// and even very small values have them, we swap the endianness in the hopes of reducing the size
142+
Serialize(BinaryHelpers.SwapEndian(d), writeTo, ref bitOffset);
143+
}
144+
//bitOffset += offset;
145+
}
146+
else
147+
{
148+
bool signed = IsSigned(t.GetType());
149+
dynamic value;
150+
if (signed)
151+
{
152+
Type t1 = t.GetType();
153+
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1);
154+
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2);
155+
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4);
156+
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
157+
}
158+
else value = t;
159+
160+
if (value <= 240) WriteByte(writeTo, value, bitOffset);
161+
else if (value <= 2287)
162+
{
163+
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
164+
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
165+
}
166+
else if (value <= 67823)
167+
{
168+
WriteByte(writeTo, 249, bitOffset);
169+
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
170+
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
171+
}
172+
else
173+
{
174+
WriteByte(writeTo, value & 255, bitOffset + 8);
175+
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
176+
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
177+
if (value > 16777215)
178+
{
179+
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
180+
if (value > 4294967295)
181+
{
182+
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
183+
if (value > 1099511627775)
184+
{
185+
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
186+
if (value > 281474976710655)
187+
{
188+
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
189+
if (value > 72057594037927935)
190+
{
191+
WriteByte(writeTo, 255, bitOffset);
192+
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
193+
}
194+
else WriteByte(writeTo, 254, bitOffset);
195+
}
196+
else WriteByte(writeTo, 253, bitOffset);
197+
}
198+
else WriteByte(writeTo, 252, bitOffset);
199+
}
200+
else WriteByte(writeTo, 251, bitOffset);
201+
}
202+
else WriteByte(writeTo, 250, bitOffset);
203+
}
204+
bitOffset += BytesToRead(value) * 8;
205+
}
206+
}
207+
}
208+
209+
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
210+
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
211+
212+
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
213+
214+
private static Type GetUnsignedType(Type t) =>
215+
t == typeof(sbyte) ? typeof(byte) :
216+
t == typeof(short) ? typeof(ushort) :
217+
t == typeof(int) ? typeof(uint) :
218+
t == typeof(long) ? typeof(ulong) :
219+
null;
220+
221+
private static ulong ZigZagEncode(long d, int bytes) => (ulong)(((d >> (bytes * 8 - 1))&1) | (d << 1));
222+
223+
private static long GetBitCount<T>(T t)
224+
{
225+
Type type = t.GetType();
226+
long count = 0;
227+
if (type.IsArray)
228+
{
229+
Type elementType = type.GetElementType();
230+
231+
count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
232+
foreach (var element in t as Array)
233+
count += GetBitCount(element);
234+
}
235+
else if (IsSupportedType(type))
236+
{
237+
long ba = GetBitAllocation(type);
238+
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
239+
else if (t is bool || t is decimal) count += ba;
240+
else count += BytesToRead(t) * 8;
241+
}
242+
//else
243+
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
244+
return count;
245+
}
246+
247+
private static void WriteBit(byte[] b, bool bit, long index)
248+
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
249+
//private static void WriteByte(byte[] b, dynamic value, long index) => WriteByte(b, (byte)value, index);
250+
private static void WriteByte(byte[] b, byte value, long index)
251+
{
252+
int byteIndex = (int)(index / 8);
253+
int shift = (int)(index % 8);
254+
byte upper_mask = (byte)(0xFF << shift);
255+
byte lower_mask = (byte)~upper_mask;
256+
257+
b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift));
258+
if(shift != 0 && byteIndex + 1 < b.Length)
259+
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
260+
}
261+
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
262+
{
263+
for (int i = 0; i < bits; ++i)
264+
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
265+
}
266+
private static void WriteDynamic(byte[] b, int value, int byteCount, long index)
267+
{
268+
for (int i = 0; i < byteCount; ++i)
269+
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
270+
}
271+
272+
private static int BytesToRead(object i)
273+
{
274+
bool size;
275+
ulong integer;
276+
if ((size = i is float) || i is double)
277+
{
278+
int bytes = size ? 4 : 8;
279+
Array type_holder = size ? holder_f as Array : holder_d as Array; // Fetch the preallocated array
280+
Array result_holder = size ? holder_i as Array : holder_u as Array;
281+
lock (result_holder)
282+
lock (type_holder)
283+
{
284+
// Clear artifacts
285+
if (size) result_holder.SetValue(0U, 0);
286+
else result_holder.SetValue(0UL, 0);
287+
288+
type_holder.SetValue(i, 0); // Insert the value to convert into the preallocated holder array
289+
Buffer.BlockCopy(type_holder, 0, result_holder, 0, bytes); // Perform an internal copy to the byte-based holder
290+
if(size) integer = BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0));
291+
else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0));
292+
}
293+
}
294+
else integer = i as ulong? ?? 0;
295+
return
296+
integer <= 240 ? 1 :
297+
integer <= 2287 ? 2 :
298+
integer <= 67823 ? 3 :
299+
integer <= 16777215 ? 4 :
300+
integer <= 4294967295 ? 5 :
301+
integer <= 1099511627775 ? 6 :
302+
integer <= 281474976710655 ? 7 :
303+
integer <= 72057594037927935 ? 8 :
304+
9;
305+
}
306+
307+
// Supported datatypes for serialization
308+
private static bool IsSupportedType(Type t) => supportedTypes.Contains(t);
309+
310+
// Specifies how many bits will be written
311+
private static long GetBitAllocation(Type t) =>
312+
t == typeof(bool) ? 1 :
313+
t == typeof(byte) ? 8 :
314+
t == typeof(sbyte) ? 8 :
315+
t == typeof(short) ? 16 :
316+
t == typeof(char) ? 16 :
317+
t == typeof(ushort) ? 16 :
318+
t == typeof(int) ? 32 :
319+
t == typeof(uint) ? 32 :
320+
t == typeof(long) ? 64 :
321+
t == typeof(ulong) ? 64 :
322+
t == typeof(float) ? 32 :
323+
t == typeof(double) ? 64 :
324+
t == typeof(decimal) ? 128 :
325+
0; // Unknown type
326+
327+
// Creates a weak reference to the allocated collector so that reuse may be possible
328+
public void Dispose()
329+
{
330+
expired.Add(new WeakReference(collect));
331+
collect = null;
332+
}
333+
}
334+
}

0 commit comments

Comments
 (0)