Add Matroschka processing.

SabreTools.Serialization/Deserializers/SecuROMMatroschkaPackage.cs

@@ -0,0 +1,149 @@
+using System.IO;
+using System.Text;
+using SabreTools.IO.Extensions;
+using SabreTools.Matching;
+using SabreTools.Models.SecuROM;
+using static SabreTools.Models.SecuROM.Constants;
+
+namespace SabreTools.Serialization.Deserializers
+{
+    public class SecuROMMatroschkaPackage : BaseBinaryDeserializer<MatroshkaPackage>
+    {
+        /// <inheritdoc/>
+        public override MatroshkaPackage? Deserialize(Stream? data)
+        {
+            // If the data is invalid
+            if (data == null || !data.CanRead)
+                return null;
+
+            try
+            {
+                // Cache the initial offset
+                long initialOffset = data.Position;
+
+                // TODO: Unify matroschka spelling. They spell it matroschka in all official stuff, as far as has been observed. Will double check.
+                // Try to parse the header
+                var package = ParsePreEntryHeader(data);
+                if (package == null)
+                    return null;
+
+                var entries = ParseEntries(data, package);
+                if (entries == null)
+                    return null;
+
+                package.Entries = entries;
+                return package;
+            }
+            catch
+            {
+                // Ignore the actual error
+                return null;
+            }
+        }
+
+        private static MatroshkaPackage? ParsePreEntryHeader(Stream data)
+        { 
+            var obj = new MatroshkaPackage();
+
+            byte[] magic = data.ReadBytes(4);
+            obj.Signature = Encoding.ASCII.GetString(magic);
+            if (obj.Signature != MatroshkaMagicString)
+                return null;
+
+            obj.EntryCount = data.ReadUInt32LittleEndian();
+            if (obj.EntryCount == 0)
+                return null;
+
+            // Check if "matrosch" section is a longer header one or not based on whether the next uint is 0 or 1. Anything
+            // else will just already be starting the filename string, which is never going to start with this.
+            // Previously thought that the longer header was correlated with RC, but at least one executable
+            // (NecroVisioN.exe from the GamersGate patch NecroVisioN_Patch1.2_GG.exe) isn't RC and still has it.
+            long tempPosition = data.Position;
+            uint tempValue = data.ReadUInt32LittleEndian();
+            data.Seek(tempPosition, SeekOrigin.Begin);
+
+            if (tempValue < 2) // Only little-endian 0 or 1 have been observed for long sections.
+            {
+                obj.UnknownRCValue1 = data.ReadUInt32LittleEndian();
+                obj.UnknownRCValue2 = data.ReadUInt32LittleEndian();
+                obj.UnknownRCValue3 = data.ReadUInt32LittleEndian();
+
+                // Exact byte count has to be used because non-RC executables have all 0x00 here.
+                var keyHexBytes = data.ReadBytes(32);
+                obj.KeyHexString = Encoding.ASCII.GetString(keyHexBytes);
+                if (!data.ReadBytes(4).EqualsExactly([0x00, 0x00, 0x00, 0x00]))
+                    return null;
+            }
+            return obj;
+        }
+
+        private static MatroshkaEntry[]? ParseEntries(Stream data, MatroshkaPackage package)
+        {
+
+                // If we have any entries
+                var obj = new MatroshkaEntry[package.EntryCount];
+
+                int matGapType = 0;
+                bool? matHasUnknown = null;
+
+                // Read entries
+                for (int i = 0; i < obj.Length; i++) 
+                {
+                    var entry = new MatroshkaEntry();
+                    // Determine if file path size is 256 or 512 bytes
+                    if (matGapType == 0)
+                        matGapType = GapHelper(data);
+
+                    // TODO: Spaces/non-ASCII have not yet been observed. Still, probably safer to store as byte array?
+                    // TODO: Read as string and trim once models is bumped. For now, this needs to be trimmed by anything reading it.
+                    entry.Path = data.ReadBytes((int)matGapType); 
+
+                    // Entry type isn't currently validated as it's always predictable anyways, nor necessary to know.
+                    entry.EntryType = (MatroshkaEntryType)data.ReadUInt32LittleEndian();
+                    entry.Size = data.ReadUInt32LittleEndian();
+                    entry.Offset = data.ReadUInt32LittleEndian();
+
+                    // Check for unknown 4-byte 0x00 value. Not correlated with 256 vs 512-byte gaps.
+                    if (matHasUnknown == null)
+                        matHasUnknown = UnknownHelper(data, entry);
+
+                    if (matHasUnknown == true) // If already known, read or don't read the unknown value.
+                        entry.Unknown = data.ReadUInt32LittleEndian(); // TODO: Validate it's zero?
+
+                    entry.ModifiedTime = data.ReadUInt64LittleEndian();
+                    entry.CreatedTime = data.ReadUInt64LittleEndian();
+                    entry.AccessedTime = data.ReadUInt64LittleEndian();
+                    entry.MD5 = data.ReadBytes(16);
+
+                    obj[i] = entry;
+                }
+
+                return obj;
+        }
+
+        private static int GapHelper(Stream data)
+        {
+            var tempPosition = data.Position;
+            data.Seek(data.Position + 256, SeekOrigin.Begin);
+            var tempValue = data.ReadUInt32LittleEndian();
+            data.Seek(tempPosition, SeekOrigin.Begin);
+            if (tempValue <= 0) // Gap is 512 bytes. Actually just == 0, but ST prefers ranges.
+                return 512;
+
+            // Otherwise, gap is 256 bytes. 
+            return 256;
+        }
+
+        private static bool UnknownHelper(Stream data, MatroshkaEntry entry)
+        {
+            var tempPosition = data.Position;
+            var tempValue = data.ReadUInt32LittleEndian();
+            data.Seek(tempPosition, SeekOrigin.Begin);
+            if (tempValue > 0) // Entry does not have the Unknown value.
+                return false;
+
+            // Entry does have the unknown value.
+            return true; 
+        }
+    }
+}

SabreTools.Serialization/Wrappers/PortableExecutable.Extraction.cs

@@ -21,11 +21,12 @@ public partial class PortableExecutable : IExtractable
         public bool Extract(string outputDirectory, bool includeDebug)
         {
             bool cexe = ExtractCExe(outputDirectory, includeDebug);
+            bool matroschka = ExtractMatroschka(outputDirectory, includeDebug);
             bool overlay = ExtractFromOverlay(outputDirectory, includeDebug);
             bool resources = ExtractFromResources(outputDirectory, includeDebug);
             bool wise = ExtractWise(outputDirectory, includeDebug);
 
-            return cexe || overlay || resources | wise;
+            return cexe || matroschka || overlay || resources || wise;
         }
 
         /// <summary>
@@ -373,6 +374,22 @@ public bool ExtractFromResources(string outputDirectory, bool includeDebug)
                 return false;
             }
         }
+
+        /// <summary>
+        /// Extract data from a SecuROM Matroschka Package
+        /// </summary>
+        /// <param name="outputDirectory">Output directory to write to</param>
+        /// <param name="includeDebug">True to include debug data, false otherwise</param>
+        /// <returns>True if extraction succeeded, false otherwise</returns>
+        public bool ExtractMatroschka(string outputDirectory, bool includeDebug)
+        {
+            // Check if executable contains Matroschka package or not
+            if (MatroschkaPackage == null)
+                return false;
+
+            // Attempt to extract package
+            return MatroschkaPackage.Extract(outputDirectory, includeDebug);
+        }
 
         /// <summary>
         /// Extract data from a Wise installer

SabreTools.Serialization/Wrappers/PortableExecutable.cs

@@ -184,6 +184,84 @@ public List<string> HeaderPaddingStrings
 
         /// <inheritdoc cref="Executable.ImportTable"/>
         public ImportTable? ImportTable => Model.ImportTable;
+
+        public SecuROMMatroschkaPackage? MatroschkaPackage
+        {
+            get
+            {
+                lock (_matroschkaPackageLock)
+                {
+                    // Use the cached data if possible
+                    if (_matroschkaPackage != null)
+                        return _matroschkaPackage;
+
+                    // Check to see if creation has already been attempted
+                    if (_matroschkaPackageFailed)
+                        return null;
+
+                    // Get the available source length, if possible
+                    var dataLength = Length;
+                    if (dataLength == -1)
+                    {
+                        _matroschkaPackageFailed = true;
+                        return null;
+                    }
+
+                    // If the section table is missing
+                    if (SectionTable == null)
+                    {
+                        _matroschkaPackageFailed = true;
+                        return null;
+                    }
+
+                    SectionHeader? section = null;
+
+                    // Find the matrosch or rcpacker section
+                    foreach (var searchedSection in SectionTable)
+                    {
+                        string sectionName = Encoding.ASCII.GetString(searchedSection.Name ?? []).TrimEnd('\0');
+                        if (sectionName != "matrosch" && sectionName != "rcpacker")
+                            continue;
+
+                        section = searchedSection;
+                        break;
+                    }
+
+                    // Otherwise, it could not be found
+                    if (section == null)
+                    {
+                        _matroschkaPackageFailed = true;
+                        return null;
+                    }
+
+                    // Get the offset
+                    long offset = section.VirtualAddress.ConvertVirtualAddress(SectionTable);
+                    if (offset < 0 || offset >= Length)
+                    {
+                        _matroschkaPackageFailed = true;
+                        return null;
+                    } 
+
+                    // Read the section into a local array
+                    var sectionLength = (int)section.VirtualSize;
+                    var sectionData = ReadRangeFromSource(offset, sectionLength);
+
+                    // Parse the section header
+                    var header = SecuROMMatroschkaPackage.Create(sectionData, 0);
+
+                    // If header creation failed, or if Entries does not exist
+                    if (header?.Entries == null)
+                    {
+                        _matroschkaPackageFailed = true;
+                        return null;
+                    }
+
+                    // Otherwise, cache and return the data
+                    _matroschkaPackage = header;
+                    return _matroschkaPackage;
+                }
+            }
+        }
 
         /// <inheritdoc cref="Executable.OptionalHeader"/>
         public OptionalHeader? OptionalHeader => Model.OptionalHeader;
@@ -784,7 +862,22 @@ public string? AssemblyVersion
         /// Lock object for <see cref="_headerPaddingStrings"/> 
         /// </summary>
         private readonly object _headerPaddingStringsLock = new();
+
+        /// <summary>
+        /// Matroschka Package wrapper, if it exists
+        /// </summary>
+        private SecuROMMatroschkaPackage? _matroschkaPackage = null;
 
+        /// <summary>
+        /// Lock object for <see cref="_matroschkaPackage"/> 
+        /// </summary>
+        private readonly object _matroschkaPackageLock = new();
+
+        /// <summary>
+        /// Cached attempt at creation for <see cref="_matroschkaPackage"/> 
+        /// </summary>
+        private bool _matroschkaPackageFailed = false;
+
         /// <summary>
         /// Address of the overlay, if it exists
         /// </summary>
@@ -1526,6 +1619,30 @@ public long FindWiseOverlayHeader()
             }
         }
 
+        /// <summary>
+        /// Find the location of a SecuROM Matroschka section, if it exists
+        /// </summary>
+        /// <returns>Matroschka section on success, null otherwise</returns>
+        public Models.PortableExecutable.SectionHeader? FindMatroschkaSection()
+        {
+            // If the section table is invalid
+            if (SectionTable == null)
+                return null;
+
+            // Find the matrosch or rcpacker section
+            foreach (var section in SectionTable)
+            {
+                var sectionName = Encoding.ASCII.GetString(section.Name ?? []).TrimEnd('\0');
+                if (sectionName != "matrosch" && sectionName != "rcpacker")
+                    continue;
+
+                return section;
+            }
+
+            // Otherwise, it could not be found
+            return null;
+        }
+
         #endregion
 
         #region Resource Parsing