Skip to content
This repository was archived by the owner on Jun 23, 2022. It is now read-only.

Commit 952d2b1

Browse files
jzebedeejzebedee
committed
Stream-based patching
1 parent 434887b commit 952d2b1

File tree

4 files changed

+119
-106
lines changed

4 files changed

+119
-106
lines changed

‎Installer/Installer.cs‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -647,7 +647,9 @@ private bool PatchMaster(string ESM)
647647
//TODO: change to a user-friendly condition and message
648648
Trace.Assert(File.Exists(dataPath));
649649

650+
//make sure we didn't include old patches by mistake
650651
Debug.Assert(patches.All(p => p.Metadata.Type == FileValidation.ChecksumType.Murmur128));
652+
651653
using (var dataChk = new FileValidation(dataPath))
652654
{
653655
var matchPatch = patches.SingleOrDefault(p => p.Metadata == dataChk);
@@ -658,21 +660,21 @@ private bool PatchMaster(string ESM)
658660
}
659661
else
660662
{
661-
byte[]
662-
dataBytes = File.ReadAllBytes(dataPath),
663-
outputBytes;
664-
665663
FileValidation outputChk;
666664

667-
if (matchPatch.PatchBytes(dataBytes, newChk, out outputBytes, out outputChk))
668-
{
669-
File.WriteAllBytes(finalPath, outputBytes);
670-
LogDual("\tPatch successful");
671-
return true;
672-
}
673-
else
665+
using (FileStream
666+
dataStream = File.OpenRead(dataPath),
667+
outputStream = File.Open(finalPath, FileMode.Create, FileAccess.ReadWrite))
674668
{
675-
LogFile("\tPatch failed");
669+
if (matchPatch.PatchStream(dataStream, newChk, outputStream, out outputChk))
670+
{
671+
LogDual("\tPatch successful");
672+
return true;
673+
}
674+
else
675+
{
676+
LogFile("\tPatch failed - " + outputChk);
677+
}
676678
}
677679
}
678680
}

‎Installer/Patching/Diff.cs‎

Lines changed: 90 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@ public const long
4343
SIG_LZDIFF41 = 0x3134464649445a4c,
4444
SIG_NONONONO = 0x4f4e4f4e4f4e4f4e;
4545
public const int HEADER_SIZE = 32;
46-
public const int BUFFER_SIZE = 1024 * 1024 * 8; //8MiB
4746

4847
private const int LZMA_DICTSIZE_ULTRA = 1024 * 1024 * 64; //64MiB, 7z 'Ultra'
4948

@@ -87,32 +86,28 @@ public static Stream GetEncodingStream(Stream stream, long signature, bool outpu
8786
/// the patch to be opened concurrently.</param>
8887
/// <param name="output">A <see cref="Stream"/> to which the patched data is written.</param>
8988
public static unsafe void Apply(byte* pInput, long length, byte* pPatch, long patchLength, Stream output)
89+
{
90+
Stream controlStream, diffStream, extraStream;
91+
var newSize = CreatePatchStreams(pPatch, patchLength, out controlStream, out diffStream, out extraStream);
92+
93+
// prepare to read three parts of the patch in parallel
94+
ApplyInternal(newSize, new UnmanagedMemoryStream(pInput, length), controlStream, diffStream, extraStream, output);
95+
}
96+
97+
public static unsafe void Apply(Stream input, byte* pPatch, long patchLength, Stream output)
98+
{
99+
Stream controlStream, diffStream, extraStream;
100+
var newSize = CreatePatchStreams(pPatch, patchLength, out controlStream, out diffStream, out extraStream);
101+
102+
// prepare to read three parts of the patch in parallel
103+
ApplyInternal(newSize, input, controlStream, diffStream, extraStream, output);
104+
}
105+
106+
static unsafe long CreatePatchStreams(byte* pPatch, long patchLength, out Stream ctrl, out Stream diff, out Stream extra)
90107
{
91108
Func<long, long, Stream> openPatchStream = (u_offset, u_length) =>
92109
new UnmanagedMemoryStream(pPatch + u_offset, u_length > 0 ? u_length : patchLength - u_offset);
93110

94-
// check arguments
95-
if (pInput == null)
96-
throw new ArgumentNullException("input");
97-
if (openPatchStream == null)
98-
throw new ArgumentNullException("openPatchStream");
99-
if (output == null)
100-
throw new ArgumentNullException("output");
101-
102-
/*
103-
File format:
104-
0 8 "BSDIFF40"
105-
8 8 X
106-
16 8 Y
107-
24 8 sizeof(newfile)
108-
32 X bzip2(control block)
109-
32+X Y bzip2(diff block)
110-
32+X+Y ??? bzip2(extra block)
111-
with control block a set of triples (x,y,z) meaning "add x bytes
112-
from oldfile to x bytes from the diff block; copy y bytes from the
113-
extra block; seek forwards in oldfile by z bytes".
114-
*/
115-
116111
// read header
117112
long controlLength, diffLength, newSize, signature;
118113
using (Stream patchStream = openPatchStream(0, HEADER_SIZE))
@@ -141,82 +136,71 @@ with control block a set of triples (x,y,z) meaning "add x bytes
141136
throw new InvalidOperationException("Corrupt patch.");
142137
}
143138

144-
// preallocate buffers for reading and writing
145-
var newData = new byte[BUFFER_SIZE];
146-
147139
// prepare to read three parts of the patch in parallel
148-
using (Stream
140+
Stream
149141
compressedControlStream = openPatchStream(HEADER_SIZE, controlLength),
150142
compressedDiffStream = openPatchStream(HEADER_SIZE + controlLength, diffLength),
151-
compressedExtraStream = openPatchStream(HEADER_SIZE + controlLength + diffLength, -1),
143+
compressedExtraStream = openPatchStream(HEADER_SIZE + controlLength + diffLength, -1);
144+
145+
// decompress each part (to read it)
146+
ctrl = GetEncodingStream(compressedControlStream, signature, false);
147+
diff = GetEncodingStream(compressedDiffStream, signature, false);
148+
extra = GetEncodingStream(compressedExtraStream, signature, false);
149+
150+
return newSize;
151+
}
152+
153+
static void ApplyInternal(long newSize, Stream input, Stream ctrl, Stream diff, Stream extra, Stream output)
154+
{
155+
long addSize, copySize, seekAmount;
156+
157+
using (ctrl)
158+
using (diff)
159+
using (extra)
160+
using (BinaryReader
161+
diffReader = new BinaryReader(diff),
162+
extraReader = new BinaryReader(extra))
163+
while (output.Position < newSize)
164+
{
165+
//read control data
166+
//set of triples (x,y,z) meaning
167+
// add x bytes from oldfile to x bytes from the diff block;
168+
// copy y bytes from the extra block;
169+
// seek forwards in oldfile by z bytes;
170+
addSize = ReadInt64(ctrl);
171+
copySize = ReadInt64(ctrl);
172+
seekAmount = ReadInt64(ctrl);
173+
174+
// sanity-check
175+
if (output.Position + addSize > newSize)
176+
throw new InvalidOperationException("Corrupt patch.");
152177

153-
// decompress each part (to read it)
154-
controlStream = GetEncodingStream(compressedControlStream, signature, false),
155-
diffStream = GetEncodingStream(compressedDiffStream, signature, false),
156-
extraStream = GetEncodingStream(compressedExtraStream, signature, false))
157-
{
158-
long[] control = new long[3];
159-
byte[] buffer = new byte[8];
160-
161-
int oldPosition = 0;
162-
int newPosition = 0;
163-
fixed (byte* pNew = newData)
164-
fixed (byte* pBuf = buffer)
165-
while (newPosition < newSize)
166178
{
167-
// read control data
168-
for (int i = 0; i < 3; i++)
169-
{
170-
controlStream.Read(buffer, 0, 8);
171-
control[i] = ReadInt64(pBuf);
172-
}
173-
174-
// sanity-check
175-
if (newPosition + control[0] > newSize)
176-
throw new InvalidOperationException("Corrupt patch.");
177-
178-
int bytesToCopy = (int)control[0];
179-
while (bytesToCopy > 0)
180-
{
181-
int actualBytesToCopy = Math.Min(bytesToCopy, BUFFER_SIZE);
182-
183-
// read diff string
184-
diffStream.Read(newData, 0, actualBytesToCopy);
185-
186-
// add old data to diff string
187-
int availableInputBytes = Math.Min(actualBytesToCopy, (int)(length - oldPosition));
188-
for (int i = 0; i < availableInputBytes; i++)
189-
pNew[i] += pInput[oldPosition + i];
190-
191-
output.Write(newData, 0, actualBytesToCopy);
192-
193-
// adjust counters
194-
newPosition += actualBytesToCopy;
195-
oldPosition += actualBytesToCopy;
196-
bytesToCopy -= actualBytesToCopy;
197-
}
198-
199-
// sanity-check
200-
if (newPosition + control[1] > newSize)
201-
throw new InvalidOperationException("Corrupt patch.");
202-
203-
// read extra string
204-
bytesToCopy = (int)control[1];
205-
while (bytesToCopy > 0)
206-
{
207-
int actualBytesToCopy = Math.Min(bytesToCopy, BUFFER_SIZE);
208-
209-
extraStream.Read(newData, 0, actualBytesToCopy);
210-
output.Write(newData, 0, actualBytesToCopy);
211-
212-
newPosition += actualBytesToCopy;
213-
bytesToCopy -= actualBytesToCopy;
214-
}
215-
216-
// adjust position
217-
oldPosition = (int)(oldPosition + control[2]);
179+
// read diff string
180+
var newData = diffReader.ReadBytes((int)addSize);
181+
182+
// add old data to diff string
183+
var availableInputBytes = (int)Math.Min(addSize, input.Length - input.Position);
184+
for (int i = 0; i < availableInputBytes; i++)
185+
newData[i] += (byte)input.ReadByte();
186+
187+
output.Write(newData, 0, (int)addSize);
188+
//input.Seek(addSize, SeekOrigin.Current);
218189
}
219-
}
190+
191+
// sanity-check
192+
if (output.Position + copySize > newSize)
193+
throw new InvalidOperationException("Corrupt patch.");
194+
195+
// read extra string
196+
{
197+
var newData = extraReader.ReadBytes((int)copySize);
198+
output.Write(newData, 0, (int)copySize);
199+
}
200+
201+
// adjust position
202+
input.Seek(seekAmount, SeekOrigin.Current);
203+
}
220204
}
221205

222206
public static unsafe long ReadInt64(byte* pb)
@@ -232,5 +216,18 @@ public static unsafe long ReadInt64(byte* pb)
232216

233217
return (pb[7] & 0x80) != 0 ? -y : y;
234218
}
219+
220+
public static long ReadInt64(Stream ps)
221+
{
222+
var buf = new byte[sizeof(long)];
223+
if (ps.Read(buf, 0, sizeof(long)) != sizeof(long))
224+
throw new InvalidOperationException("Could not read long from stream");
225+
226+
unsafe
227+
{
228+
fixed (byte* pb = buf)
229+
return ReadInt64(pb);
230+
}
231+
}
235232
}
236233
}

‎Installer/Patching/PatchInfo.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,20 @@ public bool PatchBytes(byte[] inputBytes, FileValidation targetChk, out byte[] o
6868
}
6969
}
7070

71+
public bool PatchStream(Stream input, FileValidation targetChk, Stream output, out FileValidation outputChk)
72+
{
73+
unsafe
74+
{
75+
fixed (byte* pPatch = Data)
76+
Diff.Apply(input, pPatch, Data.LongLength, output);
77+
}
78+
79+
output.Seek(0, SeekOrigin.Begin);
80+
outputChk = new FileValidation(output, targetChk.Type);
81+
82+
return targetChk == outputChk;
83+
}
84+
7185
#if LEGACY || DEBUG
7286
public static PatchInfo FromOldDiff(byte[] diffData, FileValidation oldChk)
7387
{

‎Installer/Tale Of Two Wastelands Installer.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
<DebugType>full</DebugType>
3535
<Optimize>false</Optimize>
3636
<OutputPath>bin\Debug\</OutputPath>
37-
<DefineConstants>TRACE;DEBUG</DefineConstants>
37+
<DefineConstants>TRACE;DEBUG;PARALLEL</DefineConstants>
3838
<ErrorReport>prompt</ErrorReport>
3939
<WarningLevel>4</WarningLevel>
4040
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>

0 commit comments

Comments
 (0)