Skip to content

Commit 435db1f

Browse files
GabrielTofvessonGabrielTofvesson
committed
Optimized byte serialization
Fixed various bugs Added option to pad misalignments for improved performance (at the possible cost of compression size if called in more than one non-consecutive instance)
1 parent 902fb77 commit 435db1f

File tree

2 files changed

+91
-89
lines changed

2 files changed

+91
-89
lines changed

‎MLAPI/NetworkingManagerComponents/Binary/BitReader.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public byte ReadByte()
3535
bitCount += 8;
3636
return result;
3737
}
38+
public void SkipPadded() => bitCount += (8 - (bitCount % 8)) % 8;
3839
public ushort ReadUShort() => (ushort)ReadULong();
3940
public uint ReadUInt() => (uint)ReadULong();
4041
public sbyte ReadSByte() => (sbyte)ZigZagDecode(ReadByte(), 1);

‎MLAPI/NetworkingManagerComponents/Binary/BitWriter.cs

Lines changed: 90 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Diagnostics;
34
using System.Reflection;
5+
using System.Runtime.InteropServices;
46
using System.Text;
57
using UnityEngine;
68

@@ -52,7 +54,6 @@ static BitWriter()
5254

5355
private List<object> collect = null;
5456
private bool tempAlloc = false;
55-
private int collectCount = 0;
5657

5758
/// <summary>
5859
/// Allocates a new binary collector.
@@ -73,10 +74,11 @@ public BitWriter()
7374

7475
private void Push<T>(T b)
7576
{
76-
if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
77+
if (b == null) collect.Add(b);
78+
else if (b is string || b.GetType().IsArray || IsSupportedType(b.GetType()))
7779
collect.Add(b is string ? Encoding.UTF8.GetBytes(b as string) : b as object);
78-
//else
79-
// Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
80+
else
81+
Debug.LogWarning("MLAPI: The type \"" + b.GetType() + "\" is not supported by the Binary Serializer. It will be ignored");
8082
}
8183

8284

@@ -87,11 +89,12 @@ private void Push<T>(T b)
8789
public void WriteUShort(ushort s) => Push(s);
8890
public void WriteUInt(uint i) => Push(i);
8991
public void WriteULong(ulong l) => Push(l);
90-
public void WriteSByte(sbyte b) => Push(b);
91-
public void WriteShort(short s) => Push(s);
92-
public void WriteInt(int i) => Push(i);
93-
public void WriteLong(long l) => Push(l);
92+
public void WriteSByte(sbyte b) => Push(ZigZagEncode(b, 8));
93+
public void WriteShort(short s) => Push(ZigZagEncode(s, 8));
94+
public void WriteInt(int i) => Push(ZigZagEncode(i, 8));
95+
public void WriteLong(long l) => Push(ZigZagEncode(l, 8));
9496
public void WriteString(string s) => Push(s);
97+
public void WriteAlignBits() => Push<object>(null);
9598
public void WriteFloatArray(float[] f, bool known = false) => PushArray(f, known);
9699
public void WriteDoubleArray(double[] d, bool known = false) => PushArray(d, known);
97100
public void WriteByteArray(byte[] b, bool known = false) => PushArray(b, known);
@@ -103,10 +106,12 @@ private void Push<T>(T b)
103106
public void WriteIntArray(int[] i, bool known = false) => PushArray(i, known);
104107
public void WriteLongArray(long[] l, bool known = false) => PushArray(l, known);
105108

106-
private void PushArray<T>(T[] t, bool knownSize = false)
109+
public void PushArray<T>(T[] t, bool knownSize = false)
107110
{
108-
if (!knownSize) Push(t);
109-
else foreach (T t1 in t) Push(t1);
111+
if (!knownSize) Push((uint)t.Length);
112+
bool signed = IsSigned(t.GetType().GetElementType());
113+
int size = Marshal.SizeOf(t.GetType().GetElementType());
114+
foreach (T t1 in t) Push(signed ? (object)ZigZagEncode(t1 as long? ?? t1 as int? ?? t1 as short? ?? t1 as sbyte? ?? 0, size) : (object)t1);
110115
}
111116

112117
public long Finalize(ref byte[] buffer)
@@ -117,52 +122,59 @@ public long Finalize(ref byte[] buffer)
117122
return 0;
118123
}
119124
long bitCount = 0;
120-
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
125+
for (int i = 0; i < collect.Count; ++i) bitCount += collect[i] == null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
121126

122-
if(buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)))
127+
if (buffer.Length < ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1)))
123128
{
124129
Debug.LogWarning("MLAPI: The buffer size is not large enough");
125130
return 0;
126131
}
127132
long bitOffset = 0;
133+
bool isAligned = true;
128134
foreach (var item in collect)
129-
Serialize(item, buffer, ref bitOffset);
135+
if (item == null)
136+
{
137+
bitOffset += (8 - (bitOffset % 8)) % 8;
138+
isAligned = true;
139+
}
140+
else Serialize(item, buffer, ref bitOffset, ref isAligned);
130141

131142
return (bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1);
132143
}
133144

134145
public long GetFinalizeSize()
135146
{
136147
long bitCount = 0;
137-
for (int i = 0; i < collectCount; ++i) bitCount += GetBitCount(collect[i]);
148+
for (int i = 0; i < collect.Count; ++i) bitCount += collect[i]==null ? (8 - (bitCount % 8)) % 8 : GetBitCount(collect[i]);
138149
return ((bitCount / 8) + (bitCount % 8 == 0 ? 0 : 1));
139150
}
140151

141-
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
152+
private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset, ref bool isAligned)
142153
{
143154
Type type = t.GetType();
144155
bool size = false;
145156
if (type.IsArray)
146157
{
147158
var array = t as Array;
148-
Serialize((uint)array.Length, writeTo, ref bitOffset);
159+
Serialize((uint)array.Length, writeTo, ref bitOffset, ref isAligned);
149160
foreach (var element in array)
150-
Serialize(element, writeTo, ref bitOffset);
161+
Serialize(element, writeTo, ref bitOffset, ref isAligned);
151162
}
152163
else if (IsSupportedType(type))
153164
{
154-
long offset = GetBitAllocation(type);
165+
long offset = t is bool ? 1 : BytesToRead(t) * 8;
155166
if (type == typeof(bool))
156167
{
157168
WriteBit(writeTo, t as bool? ?? false, bitOffset);
158169
bitOffset += offset;
170+
isAligned = bitOffset % 8 == 0;
159171
}
160172
else if (type == typeof(decimal))
161173
{
162-
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset);
163-
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32);
164-
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64);
165-
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96);
174+
WriteDynamic(writeTo, (int)dec_lo.GetValue(t), 4, bitOffset, isAligned);
175+
WriteDynamic(writeTo, (int)dec_mid.GetValue(t), 4, bitOffset + 32, isAligned);
176+
WriteDynamic(writeTo, (int)dec_hi.GetValue(t), 4, bitOffset + 64, isAligned);
177+
WriteDynamic(writeTo, (int)dec_flags.GetValue(t), 4, bitOffset + 96, isAligned);
166178
bitOffset += offset;
167179
}
168180
else if ((size = type == typeof(float)) || type == typeof(double))
@@ -181,71 +193,75 @@ private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
181193

182194
// Since floating point flag bits are seemingly the highest bytes of the floating point values
183195
// and even very small values have them, we swap the endianness in the hopes of reducing the size
184-
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset);
185-
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset);
196+
if(size) Serialize(BinaryHelpers.SwapEndian((uint)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
197+
else Serialize(BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0)), writeTo, ref bitOffset, ref isAligned);
186198
}
187-
//bitOffset += offset;
188199
}
189200
else
190201
{
191-
bool signed = IsSigned(t.GetType());
202+
//bool signed = IsSigned(t.GetType());
192203
ulong value;
193-
if (signed)
204+
/*if (signed)
194205
{
195206
Type t1 = t.GetType();
196207
if (t1 == typeof(sbyte)) value = (byte)ZigZagEncode(t as sbyte? ?? 0, 1);
197208
else if (t1 == typeof(short)) value = (ushort)ZigZagEncode(t as short? ?? 0, 2);
198209
else if (t1 == typeof(int)) value = (uint)ZigZagEncode(t as int? ?? 0, 4);
199-
else /*if (t1 == typeof(long))*/ value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
210+
else /*if (t1 == typeof(long)) value = (ulong)ZigZagEncode(t as long? ?? 0, 8);
211+
}
212+
else*/
213+
if (t is byte)
214+
{
215+
WriteByte(writeTo, t as byte? ?? 0, bitOffset, isAligned);
216+
return;
200217
}
201-
else if (t is byte) value = t as byte? ?? 0;
202218
else if (t is ushort) value = t as ushort? ?? 0;
203219
else if (t is uint) value = t as uint? ?? 0;
204220
else /*if (t is ulong)*/ value = t as ulong? ?? 0;
205221

206-
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset);
222+
if (value <= 240) WriteByte(writeTo, (byte)value, bitOffset, isAligned);
207223
else if (value <= 2287)
208224
{
209-
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset);
210-
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8);
225+
WriteByte(writeTo, (value - 240) / 256 + 241, bitOffset, isAligned);
226+
WriteByte(writeTo, (value - 240) % 256, bitOffset + 8, isAligned);
211227
}
212228
else if (value <= 67823)
213229
{
214-
WriteByte(writeTo, 249, bitOffset);
215-
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8);
216-
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16);
230+
WriteByte(writeTo, 249, bitOffset, isAligned);
231+
WriteByte(writeTo, (value - 2288) / 256, bitOffset + 8, isAligned);
232+
WriteByte(writeTo, (value - 2288) % 256, bitOffset + 16, isAligned);
217233
}
218234
else
219235
{
220-
WriteByte(writeTo, value & 255, bitOffset + 8);
221-
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16);
222-
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24);
236+
WriteByte(writeTo, value & 255, bitOffset + 8, isAligned);
237+
WriteByte(writeTo, (value >> 8) & 255, bitOffset + 16, isAligned);
238+
WriteByte(writeTo, (value >> 16) & 255, bitOffset + 24, isAligned);
223239
if (value > 16777215)
224240
{
225-
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32);
241+
WriteByte(writeTo, (value >> 24) & 255, bitOffset + 32, isAligned);
226242
if (value > 4294967295)
227243
{
228-
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40);
244+
WriteByte(writeTo, (value >> 32) & 255, bitOffset + 40, isAligned);
229245
if (value > 1099511627775)
230246
{
231-
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48);
247+
WriteByte(writeTo, (value >> 40) & 55, bitOffset + 48, isAligned);
232248
if (value > 281474976710655)
233249
{
234-
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56);
250+
WriteByte(writeTo, (value >> 48) & 255, bitOffset + 56, isAligned);
235251
if (value > 72057594037927935)
236252
{
237-
WriteByte(writeTo, 255, bitOffset);
238-
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64);
253+
WriteByte(writeTo, 255, bitOffset, isAligned);
254+
WriteByte(writeTo, (value >> 56) & 255, bitOffset + 64, isAligned);
239255
}
240-
else WriteByte(writeTo, 254, bitOffset);
256+
else WriteByte(writeTo, 254, bitOffset, isAligned);
241257
}
242-
else WriteByte(writeTo, 253, bitOffset);
258+
else WriteByte(writeTo, 253, bitOffset, isAligned);
243259
}
244-
else WriteByte(writeTo, 252, bitOffset);
260+
else WriteByte(writeTo, 252, bitOffset, isAligned);
245261
}
246-
else WriteByte(writeTo, 251, bitOffset);
262+
else WriteByte(writeTo, 251, bitOffset, isAligned);
247263
}
248-
else WriteByte(writeTo, 250, bitOffset);
264+
else WriteByte(writeTo, 250, bitOffset, isAligned);
249265
}
250266
bitOffset += BytesToRead(value) * 8;
251267
}
@@ -255,7 +271,7 @@ private static void Serialize<T>(T t, byte[] writeTo, ref long bitOffset)
255271
private static byte Read7BitRange(byte higher, byte lower, int bottomBits) => (byte)((higher << bottomBits) & (lower & (0xFF << (8-bottomBits))));
256272
private static byte ReadNBits(byte from, int offset, int count) => (byte)(from & ((0xFF >> (8-count)) << offset));
257273

258-
private static bool IsSigned(Type t) => Convert.ToBoolean(t.GetField("MinValue").GetValue(null));
274+
private static bool IsSigned(Type t) => t == typeof(sbyte) || t == typeof(short) || t == typeof(int) || t == typeof(long);
259275

260276
private static Type GetUnsignedType(Type t) =>
261277
t == typeof(sbyte) ? typeof(byte) :
@@ -274,13 +290,16 @@ private static long GetBitCount<T>(T t)
274290
{
275291
Type elementType = type.GetElementType();
276292

277-
count += 16; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
278-
foreach (var element in t as Array)
279-
count += GetBitCount(element);
293+
count += BytesToRead((t as Array).Length) * 8; // Int16 array size. Arrays shouldn't be syncing more than 65k elements
294+
295+
if (elementType == typeof(bool)) count += (t as Array).Length;
296+
else
297+
foreach (var element in t as Array)
298+
count += GetBitCount(element);
280299
}
281300
else if (IsSupportedType(type))
282301
{
283-
long ba = GetBitAllocation(type);
302+
long ba = t is bool ? 1 : BytesToRead(t)*8;
284303
if (ba == 0) count += Encoding.UTF8.GetByteCount(t as string);
285304
else if (t is bool || t is decimal) count += ba;
286305
else count += BytesToRead(t) * 8;
@@ -292,33 +311,32 @@ private static long GetBitCount<T>(T t)
292311

293312
private static void WriteBit(byte[] b, bool bit, long index)
294313
=> b[index / 8] = (byte)((b[index / 8] & ~(1 << (int)(index % 8))) | (bit ? 1 << (int)(index % 8) : 0));
295-
private static void WriteByte(byte[] b, ulong value, long index) => WriteByte(b, (byte)value, index);
296-
private static void WriteByte(byte[] b, byte value, long index)
314+
private static void WriteByte(byte[] b, ulong value, long index, bool isAligned) => WriteByte(b, (byte)value, index, isAligned);
315+
private static void WriteByte(byte[] b, byte value, long index, bool isAligned)
297316
{
298-
int byteIndex = (int)(index / 8);
299-
int shift = (int)(index % 8);
300-
byte upper_mask = (byte)(0xFF << shift);
301-
byte lower_mask = (byte)~upper_mask;
317+
if (isAligned) b[index / 8] = value;
318+
else
319+
{
320+
int byteIndex = (int)(index / 8);
321+
int shift = (int)(index % 8);
322+
byte upper_mask = (byte)(0xFF << shift);
302323

303-
b[byteIndex] = (byte)((b[byteIndex] & lower_mask) | (value << shift));
304-
if(shift != 0 && byteIndex + 1 < b.Length)
324+
b[byteIndex] = (byte)((b[byteIndex] & (byte)~upper_mask) | (value << shift));
305325
b[byteIndex + 1] = (byte)((b[byteIndex + 1] & upper_mask) | (value >> (8 - shift)));
326+
}
306327
}
307-
private static void WriteBits(byte[] b, byte value, int bits, int offset, long index)
308-
{
309-
for (int i = 0; i < bits; ++i)
310-
WriteBit(b, (value & (1 << (i + offset))) != 0, index + i);
311-
}
312-
private static void WriteDynamic(byte[] b, int value, int byteCount, long index)
328+
private static void WriteDynamic(byte[] b, int value, int byteCount, long index, bool isAligned)
313329
{
314330
for (int i = 0; i < byteCount; ++i)
315-
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i));
331+
WriteByte(b, (byte)((value >> (8 * i)) & 0xFF), index + (8 * i), isAligned);
316332
}
317333

318334
private static int BytesToRead(object i)
319335
{
336+
if (i is byte) return 1;
320337
bool size;
321338
ulong integer;
339+
if (i is decimal) return BytesToRead((int)dec_flags.GetValue(i)) + BytesToRead((int)dec_lo.GetValue(i)) + BytesToRead((int)dec_mid.GetValue(i)) + BytesToRead((int)dec_hi.GetValue(i));
322340
if ((size = i is float) || i is double)
323341
{
324342
int bytes = size ? 4 : 8;
@@ -337,7 +355,7 @@ private static int BytesToRead(object i)
337355
else integer = BinaryHelpers.SwapEndian((ulong)result_holder.GetValue(0));
338356
}
339357
}
340-
else integer = i as ulong? ?? 0;
358+
else integer = i as ulong? ?? i as uint? ?? i as ushort? ?? i as byte? ?? 0;
341359
return
342360
integer <= 240 ? 1 :
343361
integer <= 2287 ? 2 :
@@ -352,24 +370,7 @@ private static int BytesToRead(object i)
352370

353371
// Supported datatypes for serialization
354372
private static bool IsSupportedType(Type t) => supportedTypes.Contains(t);
355-
356-
// Specifies how many bits will be written
357-
private static long GetBitAllocation(Type t) =>
358-
t == typeof(bool) ? 1 :
359-
t == typeof(byte) ? 8 :
360-
t == typeof(sbyte) ? 8 :
361-
t == typeof(short) ? 16 :
362-
t == typeof(char) ? 16 :
363-
t == typeof(ushort) ? 16 :
364-
t == typeof(int) ? 32 :
365-
t == typeof(uint) ? 32 :
366-
t == typeof(long) ? 64 :
367-
t == typeof(ulong) ? 64 :
368-
t == typeof(float) ? 32 :
369-
t == typeof(double) ? 64 :
370-
t == typeof(decimal) ? 128 :
371-
0; // Unknown type
372-
373+
373374
// Creates a weak reference to the allocated collector so that reuse may be possible
374375
public void Dispose()
375376
{

0 commit comments

Comments
 (0)