Skip to content

Commit 5f23abe

Browse files
force-netforce-net
committed
Mega feature! Sampled compression (name can be changed). Also, some small bugs are fixed
1 parent 00c9069 commit 5f23abe

25 files changed

+784
-52
lines changed

‎Blazer.Exe/Program.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,6 @@ private static int ProcessCompress(CommandLineParser<BlazerCommandLineOptions> o
8484
var fileName = options.GetNonParamOptions(0) ?? string.Empty;
8585
string archiveName = null;
8686

87-
string customFileName = null;
88-
8987
var listFile = options.GetNonParamOptions().FirstOrDefault(x => x[0] == '@');
9088
if (listFile != null)
9189
{

‎Blazer.Native.Build/Blazer.Native.x64.dll‎

0 Bytes
Binary file not shown.

‎Blazer.Native.Build/Blazer.Native.x86.dll‎

0 Bytes
Binary file not shown.

‎Blazer.Native/BlazerBlock.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ extern "C" __declspec(dllexport) __int32 blazer_block_compress_block(unsigned ch
8888
unsigned __int32 mulEl = 0;
8989

9090
if (bufferInLength > 3)
91-
mulEl = (unsigned __int32)(bufferIn[0] << 16 | bufferIn[1] << 8 | bufferIn[2]);
91+
mulEl = (unsigned __int32)(bufferIn[idxIn] << 16 | bufferIn[idxIn + 1] << 8 | bufferIn[idxIn + 2]);
9292

9393
while (idxIn < iterMax)
9494
{

‎Blazer.Net.Tests/Blazer.Net.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
<Compile Include="IntegrityTests.cs" />
5555
<Compile Include="OptionsTests.cs" />
5656
<Compile Include="Properties\AssemblyInfo.cs" />
57+
<Compile Include="SampledCompressionTests.cs" />
5758
<Compile Include="StreamEncoderTests.cs" />
5859
</ItemGroup>
5960
<ItemGroup>

‎Blazer.Net.Tests/SampledCompressionTests.cs‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
using System;
2+
using System.Linq;
3+
4+
using Force.Blazer.Algorithms;
5+
using Force.Blazer.Algorithms.Sampled;
6+
7+
using NUnit.Framework;
8+
9+
namespace Blazer.Net.Tests
10+
{
11+
[TestFixture(1)]
12+
[TestFixture(2)]
13+
[TestFixture(3)]
14+
public class SampledCompressionTests
15+
{
16+
private readonly Type _compressorType;
17+
18+
private BaseSampledCompressor GetCompressor()
19+
{
20+
return (BaseSampledCompressor)Activator.CreateInstance(_compressorType);
21+
}
22+
23+
public SampledCompressionTests(int type)
24+
{
25+
if (type == 1) _compressorType = typeof(StreamSampledCompressor);
26+
else if (type == 2) _compressorType = typeof(StreamHighSampledCompressor);
27+
else if (type == 3) _compressorType = typeof(BlockSampledCompressor);
28+
else throw new NotImplementedException();
29+
}
30+
31+
[Test]
32+
public void DataWithSample_Should_Be_Correctly_Encoded_Decoded()
33+
{
34+
var ssed = GetCompressor();
35+
var sample = new byte[100];
36+
sample[1] = 42;
37+
ssed.PrepareSample(sample, 0, sample.Length);
38+
39+
var data1 = new byte[10];
40+
data1[0] = 42;
41+
42+
var data2 = new byte[10];
43+
data2[0] = 12;
44+
45+
var tmpOut = new byte[ssed.CalculateMaxCompressedBufferLength(10)];
46+
var tmpOut2 = new byte[ssed.CalculateMaxCompressedBufferLength(10)];
47+
48+
var cntSampled = ssed.EncodeWithSample(data1, 0, data1.Length, tmpOut, 0);
49+
var cntUnsampled = StreamEncoder.CompressData(data1);
50+
// TODO: uncomment
51+
Assert.That(cntSampled, Is.LessThan(cntUnsampled.Length));
52+
var cntSampledUnpacked = ssed.DecodeWithSample(tmpOut, 0, cntSampled, tmpOut2, 0);
53+
Assert.That(cntSampledUnpacked, Is.EqualTo(10));
54+
CollectionAssert.AreEqual(data1, tmpOut2.Take(cntSampledUnpacked));
55+
56+
// checking that we can repeat without failure
57+
cntSampled = ssed.EncodeWithSample(data2, 0, data2.Length, tmpOut, 0);
58+
cntSampledUnpacked = ssed.DecodeWithSample(tmpOut, 0, cntSampled, tmpOut2, 0);
59+
Assert.That(cntSampledUnpacked, Is.EqualTo(10));
60+
CollectionAssert.AreEqual(data2, tmpOut2.Take(cntSampledUnpacked));
61+
}
62+
63+
[Test]
64+
public void DataWithSample_Should_Be_Correctly_Encoded_Decoded_With_Offsets()
65+
{
66+
var ssed = GetCompressor();
67+
var sample = new byte[100];
68+
sample[1] = 42;
69+
ssed.PrepareSample(sample, 1, sample.Length - 1);
70+
71+
var data1 = new byte[10];
72+
data1[1] = 42;
73+
74+
var data2 = new byte[9];
75+
data1[0] = 42;
76+
77+
var tmpOut = new byte[ssed.CalculateMaxCompressedBufferLength(10)];
78+
var tmpOut2 = new byte[ssed.CalculateMaxCompressedBufferLength(10)];
79+
80+
var cntSampled1 = ssed.EncodeWithSample(data1, 1, data1.Length - 1, tmpOut, 1);
81+
var cntSampled2 = ssed.EncodeWithSample(data2, 0, data2.Length, tmpOut2, 0);
82+
Assert.That(cntSampled1, Is.EqualTo(cntSampled2));
83+
var cntSampledUnpacked = ssed.DecodeWithSample(tmpOut, 1, cntSampled1, tmpOut2, 1);
84+
Assert.That(cntSampledUnpacked, Is.EqualTo(9));
85+
CollectionAssert.AreEqual(data1.Skip(1), tmpOut2.Skip(1).Take(cntSampledUnpacked));
86+
}
87+
88+
[Test]
89+
public void DataWithSample_Should_Be_Correctly_Encoded_Decoded_With_Simple_Interface()
90+
{
91+
var ssed = GetCompressor();
92+
var sample = new byte[100];
93+
sample[1] = 42;
94+
ssed.PrepareSample(sample, 1, sample.Length - 1);
95+
96+
var data1 = new byte[10];
97+
data1[1] = 42;
98+
99+
var sampled1 = ssed.EncodeWithSample(data1);
100+
var sampledUnpacked = ssed.DecodeWithSample(sampled1);
101+
102+
CollectionAssert.AreEqual(data1, sampledUnpacked);
103+
}
104+
105+
[Test]
106+
public void Zero_Length_Should_not_Cause_Error()
107+
{
108+
var ssed = GetCompressor();
109+
var sample = new byte[100];
110+
sample[1] = 42;
111+
ssed.PrepareSample(sample, 1, sample.Length - 1);
112+
113+
var data1 = new byte[0];
114+
115+
var sampled1 = ssed.EncodeWithSample(data1);
116+
Assert.That(sampled1.Length, Is.EqualTo(1));
117+
var sampledUnpacked = ssed.DecodeWithSample(sampled1);
118+
119+
CollectionAssert.AreEqual(data1, sampledUnpacked);
120+
}
121+
}
122+
}

‎Blazer.Net/Algorithms/BlockDecoder.cs‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Diagnostics.CodeAnalysis;
23

34
namespace Force.Blazer.Algorithms
45
{
@@ -10,6 +11,10 @@ public class BlockDecoder : IDecoder
1011
{
1112
// should be equal with BlockEncoder
1213
private const int HASH_TABLE_BITS = 16;
14+
15+
/// <summary>
16+
/// Length of hashtable - 1
17+
/// </summary>
1318
protected const int HASH_TABLE_LEN = (1 << HASH_TABLE_BITS) - 1;
1419
private const int MIN_SEQ_LEN = 4;
1520
// carefully selected random number
@@ -19,6 +24,23 @@ public class BlockDecoder : IDecoder
1924

2025
private int _maxUncompressedBlockSize;
2126

27+
/// <summary>
28+
/// Hash array to store dictionary between iterations
29+
/// </summary>
30+
[SuppressMessage("StyleCop.CSharp.NamingRules", "SA1304:NonPrivateReadonlyFieldsMustBeginWithUpperCaseLetter", Justification = "Reviewed. Suppression is OK here.")]
31+
protected readonly int[] _hashArr = new int[HASH_TABLE_LEN + 1];
32+
33+
/// <summary>
34+
/// Returns internal hash array
35+
/// </summary>
36+
public int[] HashArr
37+
{
38+
get
39+
{
40+
return _hashArr;
41+
}
42+
}
43+
2244
/// <summary>
2345
/// Decodes given buffer
2446
/// </summary>
@@ -27,7 +49,7 @@ public BufferInfo Decode(byte[] buffer, int offset, int length, bool isCompresse
2749
if (!isCompressed)
2850
return new BufferInfo(buffer, offset, length);
2951

30-
var outLen = DecompressBlock(buffer, offset, length, _innerBuffer, 0, _maxUncompressedBlockSize);
52+
var outLen = DecompressBlock(buffer, offset, length, _innerBuffer, 0, _maxUncompressedBlockSize, true);
3153
return new BufferInfo(_innerBuffer, 0, outLen);
3254
}
3355

@@ -51,10 +73,14 @@ public BlazerAlgorithm GetAlgorithmId()
5173
/// <summary>
5274
/// Decompresses block of data
5375
/// </summary>
54-
protected virtual int DecompressBlock(
55-
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int bufferOutLength)
76+
public virtual int DecompressBlock(
77+
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int bufferOutLength, bool doCleanup)
5678
{
57-
return DecompressBlockExternal(bufferIn, bufferInOffset, bufferInLength, bufferOut, bufferOutOffset, bufferOutLength);
79+
var cnt = DecompressBlockExternal(bufferIn, bufferInOffset, bufferInLength, bufferOut, bufferOutOffset, bufferOutLength, _hashArr);
80+
if (doCleanup)
81+
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
82+
83+
return cnt;
5884
}
5985

6086
/// <summary>
@@ -66,10 +92,11 @@ protected virtual int DecompressBlock(
6692
/// <param name="bufferOut">Out buffer, should be enough size</param>
6793
/// <param name="bufferOutOffset">Out buffer offset</param>
6894
/// <param name="bufferOutLength">Out buffer maximum right offset (offset + count)</param>
95+
/// <param name="hashArr">Hash array. Can be null.</param>
6996
/// <returns>Bytes count of decompressed data</returns>
70-
public static int DecompressBlockExternal(byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int bufferOutLength)
97+
public static int DecompressBlockExternal(byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int bufferOutLength, int[] hashArr)
7198
{
72-
var hashArr = new int[HASH_TABLE_LEN + 1];
99+
hashArr = hashArr ?? new int[HASH_TABLE_LEN + 1];
73100
var idxIn = bufferInOffset;
74101
var idxOut = bufferOutOffset;
75102
uint mulEl = 0;

‎Blazer.Net/Algorithms/BlockDecoderNative.cs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,25 +13,23 @@ public class BlockDecoderNative : BlockDecoder
1313
private static extern int blazer_block_decompress_block(
1414
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int bufferOutLength, int[] hashArr);
1515

16-
private int[] _hashArr;
17-
1816
/// <summary>
1917
/// Initializes encoder with information about maximum uncompressed block size
2018
/// </summary>
2119
public override void Init(int maxInBlockSize)
2220
{
2321
base.Init(maxInBlockSize);
24-
_hashArr = new int[HASH_TABLE_LEN + 1];
2522
}
2623

2724
/// <summary>
2825
/// Decompresses block of data
2926
/// </summary>
30-
protected override int DecompressBlock(
31-
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int idxOut, int bufferOutLength)
27+
public override int DecompressBlock(
28+
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int idxOut, int bufferOutLength, bool doCleanup)
3229
{
3330
var res = blazer_block_decompress_block(bufferIn, bufferInOffset, bufferInLength, bufferOut, idxOut, bufferOutLength, _hashArr);
34-
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
31+
if (doCleanup)
32+
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
3533

3634
if (res < 0)
3735
throw new InvalidOperationException("Invalid compressed data");

‎Blazer.Net/Algorithms/BlockEncoder.cs‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ namespace Force.Blazer.Algorithms
1010
public class BlockEncoder : IEncoder
1111
{
1212
private const int HASH_TABLE_BITS = 16;
13+
14+
/// <summary>
15+
/// Length of hashtable - 1
16+
/// </summary>
1317
protected const int HASH_TABLE_LEN = (1 << HASH_TABLE_BITS) - 1;
1418
private const int MIN_SEQ_LEN = 4;
1519
// carefully selected random number
@@ -21,12 +25,29 @@ public class BlockEncoder : IEncoder
2125
[SuppressMessage("StyleCop.CSharp.MaintainabilityRules", "SA1401:FieldsMustBePrivate", Justification = "Reviewed. Suppression is OK here.")]
2226
protected byte[] _bufferOut;
2327

28+
/// <summary>
29+
/// Hash array to store dictionary between iterations
30+
/// </summary>
31+
[SuppressMessage("StyleCop.CSharp.NamingRules", "SA1304:NonPrivateReadonlyFieldsMustBeginWithUpperCaseLetter", Justification = "Reviewed. Suppression is OK here.")]
32+
protected readonly int[] _hashArr = new int[HASH_TABLE_LEN + 1];
33+
34+
/// <summary>
35+
/// Returns internal hash array
36+
/// </summary>
37+
public int[] HashArr
38+
{
39+
get
40+
{
41+
return _hashArr;
42+
}
43+
}
44+
2445
/// <summary>
2546
/// Encodes given buffer
2647
/// </summary>
2748
public BufferInfo Encode(byte[] buffer, int offset, int length)
2849
{
29-
var cnt = CompressBlock(buffer, offset, length, _bufferOut, 0);
50+
var cnt = CompressBlock(buffer, offset, length, _bufferOut, 0, true);
3051
return new BufferInfo(_bufferOut, 0, cnt);
3152
}
3253

@@ -41,10 +62,19 @@ public virtual void Init(int maxInBlockSize)
4162
/// <summary>
4263
/// Compresses block of data
4364
/// </summary>
44-
protected virtual int CompressBlock(
45-
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset)
65+
/// <param name="bufferIn">In buffer</param>
66+
/// <param name="bufferInOffset">In buffer offset</param>
67+
/// <param name="bufferInLength">In buffer right offset (offset + count)</param>
68+
/// <param name="bufferOut">Out buffer, should be enough size</param>
69+
/// <param name="bufferOutOffset">Out buffer offset</param>
70+
/// <param name="doCleanup">Cleanup internal data after compression</param>
71+
public virtual int CompressBlock(
72+
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, bool doCleanup)
4673
{
47-
return CompressBlockExternal(bufferIn, bufferInOffset, bufferInLength, bufferOut, bufferOutOffset);
74+
var cnt = CompressBlockExternal(bufferIn, bufferInOffset, bufferInLength, bufferOut, bufferOutOffset, _hashArr);
75+
if (doCleanup)
76+
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
77+
return cnt;
4878
}
4979

5080
/// <summary>
@@ -55,10 +85,11 @@ protected virtual int CompressBlock(
5585
/// <param name="bufferInLength">In buffer right offset (offset + count)</param>
5686
/// <param name="bufferOut">Out buffer, should be enough size</param>
5787
/// <param name="bufferOutOffset">Out buffer offset</param>
88+
/// <param name="hashArr">Hash array. Can be null.</param>
5889
/// <returns>Bytes count of compressed data</returns>
59-
public static int CompressBlockExternal(byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset)
90+
public static int CompressBlockExternal(byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int[] hashArr)
6091
{
61-
var hashArr = new int[HASH_TABLE_LEN + 1];
92+
hashArr = hashArr ?? new int[HASH_TABLE_LEN + 1];
6293
var idxIn = bufferInOffset;
6394
var lastProcessedIdxIn = bufferInOffset;
6495
var idxOut = bufferOutOffset;
@@ -70,7 +101,7 @@ public static int CompressBlockExternal(byte[] bufferIn, int bufferInOffset, int
70101
uint mulEl = 0;
71102

72103
if (bufferInLength > 3)
73-
mulEl = (uint)(bufferIn[0] << 16 | bufferIn[1] << 8 | bufferIn[2]);
104+
mulEl = (uint)(bufferIn[bufferInOffset] << 16 | bufferIn[bufferInOffset + 1] << 8 | bufferIn[bufferInOffset + 2]);
74105

75106
while (idxIn < iterMax)
76107
{

‎Blazer.Net/Algorithms/BlockEncoderNative.cs‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,36 +13,28 @@ public class BlockEncoderNative : BlockEncoder
1313
private static extern int blazer_block_compress_block(
1414
byte[] bufferIn, int bufferInOffset, int bufferInLength, byte[] bufferOut, int bufferOutOffset, int[] hashArr);
1515

16-
private int[] _hashArr;
17-
18-
/// <summary>
19-
/// Initializes encoder with information about maximum uncompressed block size
20-
/// </summary>
21-
public override void Init(int maxInBlockSize)
22-
{
23-
base.Init(maxInBlockSize);
24-
_hashArr = new int[HASH_TABLE_LEN + 1];
25-
}
26-
2716
/// <summary>
2817
/// Compresses block of data
2918
/// </summary>
30-
protected override int CompressBlock(
19+
public override int CompressBlock(
3120
byte[] bufferIn,
3221
int bufferInOffset,
3322
int bufferInCount,
3423
byte[] bufferOut,
35-
int bufferOutOffset)
24+
int bufferOutOffset,
25+
bool doCleanup)
3626
{
3727
var cnt = blazer_block_compress_block(
3828
bufferIn,
3929
bufferInOffset,
4030
bufferInCount,
41-
_bufferOut,
31+
bufferOut,
4232
bufferOutOffset,
4333
_hashArr);
4434

45-
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
35+
if (doCleanup)
36+
Array.Clear(_hashArr, 0, HASH_TABLE_LEN + 1);
37+
4638
return cnt;
4739
}
4840
}

0 commit comments

Comments
 (0)