Skip to content

Commit fbdadce

Browse files
HeroponRikiBestestmnadareski
HeroponRikiBestest
and
mnadareski
authored
Add Matroschka processing. (#23)
* Made changes * Temporary hack to not rely on models without significantly changing current code. Revert all of this with offset-based reading later. Also added unnecessary casting in wrapperfactory so serialization will build locally. Revert this, since I assume it somehow builds fine for GA/sabre/etc. * small fixes * Store matroschka section as PE extension * Move extractor out of deserializer, remove weird hack * Potential GA fix * More potential GA fixes. * I have no idea why GA hits that error but not me * Giving up on GA for now * fix locking issues * Fix GA building; thank you sabre * Minor improvements all around * Catch some braced single-line if statements * Use var more * Seperate deserializer into helper methods * Make file path reading much more sane * Removed MatroschkaHeaderType enum * Removed MatroschkaGapType enum, further simplify matgaphelper. * Remove MatroschkaHasUnknown enum, further simplify Unknown value reading. * Cache initial offset. * Remove TryCreate patterns. * Rename matroschka variable to package * Newline after object * Rename to obj * Remove a few unecessary TODOs * Seperate hexstring byte read to another line. * Fix documentation. * More private static * Changed data.position setting to seeking. NTS: check if this broke anything later * rename entries to obj * MatroschkaEntry to var * Newline * Alphabetical * More alphabetical. * section to package * Move private variables. * Move to extension properties. * Revert section finding. * Remove uneeded _dataSource lock and access. * combine lines and make var * Combine two null checks. * Packaged files, some past commits I think I forgot to push. * Missed two * newline * space * newline * Combine two lines * Removed comment * Return false explicitly * Change hashing string implementation * Fix order. * Use offset reading instead of filedataarray * Change file reading around a little preemptively for BOS --------- Co-authored-by: Matt Nadareski <mnadareski@outlook.com>
1 parent d3e7abf commit fbdadce

File tree

5 files changed

+528
-1
lines changed

5 files changed

+528
-1
lines changed

‎SabreTools.Serialization/Deserializers/SecuROMMatroschkaPackage.cs‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
using System.IO;
2+
using System.Text;
3+
using SabreTools.IO.Extensions;
4+
using SabreTools.Matching;
5+
using SabreTools.Models.SecuROM;
6+
using static SabreTools.Models.SecuROM.Constants;
7+
8+
namespace SabreTools.Serialization.Deserializers
9+
{
10+
public class SecuROMMatroschkaPackage : BaseBinaryDeserializer<MatroshkaPackage>
11+
{
12+
/// <inheritdoc/>
13+
public override MatroshkaPackage? Deserialize(Stream? data)
14+
{
15+
// If the data is invalid
16+
if (data == null || !data.CanRead)
17+
return null;
18+
19+
try
20+
{
21+
// Cache the initial offset
22+
long initialOffset = data.Position;
23+
24+
// TODO: Unify matroschka spelling. They spell it matroschka in all official stuff, as far as has been observed. Will double check.
25+
// Try to parse the header
26+
var package = ParsePreEntryHeader(data);
27+
if (package == null)
28+
return null;
29+
30+
var entries = ParseEntries(data, package);
31+
if (entries == null)
32+
return null;
33+
34+
package.Entries = entries;
35+
return package;
36+
}
37+
catch
38+
{
39+
// Ignore the actual error
40+
return null;
41+
}
42+
}
43+
44+
private static MatroshkaPackage? ParsePreEntryHeader(Stream data)
45+
{
46+
var obj = new MatroshkaPackage();
47+
48+
byte[] magic = data.ReadBytes(4);
49+
obj.Signature = Encoding.ASCII.GetString(magic);
50+
if (obj.Signature != MatroshkaMagicString)
51+
return null;
52+
53+
obj.EntryCount = data.ReadUInt32LittleEndian();
54+
if (obj.EntryCount == 0)
55+
return null;
56+
57+
// Check if "matrosch" section is a longer header one or not based on whether the next uint is 0 or 1. Anything
58+
// else will just already be starting the filename string, which is never going to start with this.
59+
// Previously thought that the longer header was correlated with RC, but at least one executable
60+
// (NecroVisioN.exe from the GamersGate patch NecroVisioN_Patch1.2_GG.exe) isn't RC and still has it.
61+
long tempPosition = data.Position;
62+
uint tempValue = data.ReadUInt32LittleEndian();
63+
data.Seek(tempPosition, SeekOrigin.Begin);
64+
65+
if (tempValue < 2) // Only little-endian 0 or 1 have been observed for long sections.
66+
{
67+
obj.UnknownRCValue1 = data.ReadUInt32LittleEndian();
68+
obj.UnknownRCValue2 = data.ReadUInt32LittleEndian();
69+
obj.UnknownRCValue3 = data.ReadUInt32LittleEndian();
70+
71+
// Exact byte count has to be used because non-RC executables have all 0x00 here.
72+
var keyHexBytes = data.ReadBytes(32);
73+
obj.KeyHexString = Encoding.ASCII.GetString(keyHexBytes);
74+
if (!data.ReadBytes(4).EqualsExactly([0x00, 0x00, 0x00, 0x00]))
75+
return null;
76+
}
77+
return obj;
78+
}
79+
80+
private static MatroshkaEntry[]? ParseEntries(Stream data, MatroshkaPackage package)
81+
{
82+
83+
// If we have any entries
84+
var obj = new MatroshkaEntry[package.EntryCount];
85+
86+
int matGapType = 0;
87+
bool? matHasUnknown = null;
88+
89+
// Read entries
90+
for (int i = 0; i < obj.Length; i++)
91+
{
92+
var entry = new MatroshkaEntry();
93+
// Determine if file path size is 256 or 512 bytes
94+
if (matGapType == 0)
95+
matGapType = GapHelper(data);
96+
97+
// TODO: Spaces/non-ASCII have not yet been observed. Still, probably safer to store as byte array?
98+
// TODO: Read as string and trim once models is bumped. For now, this needs to be trimmed by anything reading it.
99+
entry.Path = data.ReadBytes((int)matGapType);
100+
101+
// Entry type isn't currently validated as it's always predictable anyways, nor necessary to know.
102+
entry.EntryType = (MatroshkaEntryType)data.ReadUInt32LittleEndian();
103+
entry.Size = data.ReadUInt32LittleEndian();
104+
entry.Offset = data.ReadUInt32LittleEndian();
105+
106+
// Check for unknown 4-byte 0x00 value. Not correlated with 256 vs 512-byte gaps.
107+
if (matHasUnknown == null)
108+
matHasUnknown = UnknownHelper(data, entry);
109+
110+
if (matHasUnknown == true) // If already known, read or don't read the unknown value.
111+
entry.Unknown = data.ReadUInt32LittleEndian(); // TODO: Validate it's zero?
112+
113+
entry.ModifiedTime = data.ReadUInt64LittleEndian();
114+
entry.CreatedTime = data.ReadUInt64LittleEndian();
115+
entry.AccessedTime = data.ReadUInt64LittleEndian();
116+
entry.MD5 = data.ReadBytes(16);
117+
118+
obj[i] = entry;
119+
}
120+
121+
return obj;
122+
}
123+
124+
private static int GapHelper(Stream data)
125+
{
126+
var tempPosition = data.Position;
127+
data.Seek(data.Position + 256, SeekOrigin.Begin);
128+
var tempValue = data.ReadUInt32LittleEndian();
129+
data.Seek(tempPosition, SeekOrigin.Begin);
130+
if (tempValue <= 0) // Gap is 512 bytes. Actually just == 0, but ST prefers ranges.
131+
return 512;
132+
133+
// Otherwise, gap is 256 bytes.
134+
return 256;
135+
}
136+
137+
private static bool UnknownHelper(Stream data, MatroshkaEntry entry)
138+
{
139+
var tempPosition = data.Position;
140+
var tempValue = data.ReadUInt32LittleEndian();
141+
data.Seek(tempPosition, SeekOrigin.Begin);
142+
if (tempValue > 0) // Entry does not have the Unknown value.
143+
return false;
144+
145+
// Entry does have the unknown value.
146+
return true;
147+
}
148+
}
149+
}

‎SabreTools.Serialization/Wrappers/PortableExecutable.Extraction.cs‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ public partial class PortableExecutable : IExtractable
2121
public bool Extract(string outputDirectory, bool includeDebug)
2222
{
2323
bool cexe = ExtractCExe(outputDirectory, includeDebug);
24+
bool matroschka = ExtractMatroschka(outputDirectory, includeDebug);
2425
bool overlay = ExtractFromOverlay(outputDirectory, includeDebug);
2526
bool resources = ExtractFromResources(outputDirectory, includeDebug);
2627
bool wise = ExtractWise(outputDirectory, includeDebug);
2728

28-
return cexe || overlay || resources | wise;
29+
return cexe || matroschka || overlay || resources || wise;
2930
}
3031

3132
/// <summary>
@@ -373,6 +374,22 @@ public bool ExtractFromResources(string outputDirectory, bool includeDebug)
373374
return false;
374375
}
375376
}
377+
378+
/// <summary>
379+
/// Extract data from a SecuROM Matroschka Package
380+
/// </summary>
381+
/// <param name="outputDirectory">Output directory to write to</param>
382+
/// <param name="includeDebug">True to include debug data, false otherwise</param>
383+
/// <returns>True if extraction succeeded, false otherwise</returns>
384+
public bool ExtractMatroschka(string outputDirectory, bool includeDebug)
385+
{
386+
// Check if executable contains Matroschka package or not
387+
if (MatroschkaPackage == null)
388+
return false;
389+
390+
// Attempt to extract package
391+
return MatroschkaPackage.Extract(outputDirectory, includeDebug);
392+
}
376393

377394
/// <summary>
378395
/// Extract data from a Wise installer

‎SabreTools.Serialization/Wrappers/PortableExecutable.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,84 @@ public List<string> HeaderPaddingStrings
184184

185185
/// <inheritdoc cref="Executable.ImportTable"/>
186186
public ImportTable? ImportTable => Model.ImportTable;
187+
188+
public SecuROMMatroschkaPackage? MatroschkaPackage
189+
{
190+
get
191+
{
192+
lock (_matroschkaPackageLock)
193+
{
194+
// Use the cached data if possible
195+
if (_matroschkaPackage != null)
196+
return _matroschkaPackage;
197+
198+
// Check to see if creation has already been attempted
199+
if (_matroschkaPackageFailed)
200+
return null;
201+
202+
// Get the available source length, if possible
203+
var dataLength = Length;
204+
if (dataLength == -1)
205+
{
206+
_matroschkaPackageFailed = true;
207+
return null;
208+
}
209+
210+
// If the section table is missing
211+
if (SectionTable == null)
212+
{
213+
_matroschkaPackageFailed = true;
214+
return null;
215+
}
216+
217+
SectionHeader? section = null;
218+
219+
// Find the matrosch or rcpacker section
220+
foreach (var searchedSection in SectionTable)
221+
{
222+
string sectionName = Encoding.ASCII.GetString(searchedSection.Name ?? []).TrimEnd('\0');
223+
if (sectionName != "matrosch" && sectionName != "rcpacker")
224+
continue;
225+
226+
section = searchedSection;
227+
break;
228+
}
229+
230+
// Otherwise, it could not be found
231+
if (section == null)
232+
{
233+
_matroschkaPackageFailed = true;
234+
return null;
235+
}
236+
237+
// Get the offset
238+
long offset = section.VirtualAddress.ConvertVirtualAddress(SectionTable);
239+
if (offset < 0 || offset >= Length)
240+
{
241+
_matroschkaPackageFailed = true;
242+
return null;
243+
}
244+
245+
// Read the section into a local array
246+
var sectionLength = (int)section.VirtualSize;
247+
var sectionData = ReadRangeFromSource(offset, sectionLength);
248+
249+
// Parse the section header
250+
var header = SecuROMMatroschkaPackage.Create(sectionData, 0);
251+
252+
// If header creation failed, or if Entries does not exist
253+
if (header?.Entries == null)
254+
{
255+
_matroschkaPackageFailed = true;
256+
return null;
257+
}
258+
259+
// Otherwise, cache and return the data
260+
_matroschkaPackage = header;
261+
return _matroschkaPackage;
262+
}
263+
}
264+
}
187265

188266
/// <inheritdoc cref="Executable.OptionalHeader"/>
189267
public OptionalHeader? OptionalHeader => Model.OptionalHeader;
@@ -784,7 +862,22 @@ public string? AssemblyVersion
784862
/// Lock object for <see cref="_headerPaddingStrings"/>
785863
/// </summary>
786864
private readonly object _headerPaddingStringsLock = new();
865+
866+
/// <summary>
867+
/// Matroschka Package wrapper, if it exists
868+
/// </summary>
869+
private SecuROMMatroschkaPackage? _matroschkaPackage = null;
787870

871+
/// <summary>
872+
/// Lock object for <see cref="_matroschkaPackage"/>
873+
/// </summary>
874+
private readonly object _matroschkaPackageLock = new();
875+
876+
/// <summary>
877+
/// Cached attempt at creation for <see cref="_matroschkaPackage"/>
878+
/// </summary>
879+
private bool _matroschkaPackageFailed = false;
880+
788881
/// <summary>
789882
/// Address of the overlay, if it exists
790883
/// </summary>
@@ -1526,6 +1619,30 @@ public long FindWiseOverlayHeader()
15261619
}
15271620
}
15281621

1622+
/// <summary>
1623+
/// Find the location of a SecuROM Matroschka section, if it exists
1624+
/// </summary>
1625+
/// <returns>Matroschka section on success, null otherwise</returns>
1626+
public Models.PortableExecutable.SectionHeader? FindMatroschkaSection()
1627+
{
1628+
// If the section table is invalid
1629+
if (SectionTable == null)
1630+
return null;
1631+
1632+
// Find the matrosch or rcpacker section
1633+
foreach (var section in SectionTable)
1634+
{
1635+
var sectionName = Encoding.ASCII.GetString(section.Name ?? []).TrimEnd('\0');
1636+
if (sectionName != "matrosch" && sectionName != "rcpacker")
1637+
continue;
1638+
1639+
return section;
1640+
}
1641+
1642+
// Otherwise, it could not be found
1643+
return null;
1644+
}
1645+
15291646
#endregion
15301647

15311648
#region Resource Parsing

0 commit comments

Comments
 (0)