diff --git a/Directory.Build.props b/Directory.Build.props index b7509f8..38c16fc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,5 +2,8 @@ latest true + + Copyright © VictorBush 2021 + 0.7.0 diff --git a/Directory.Build.targets b/Directory.Build.targets index eb3641e..8c119d5 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -1,14 +1,2 @@ - - - - - - - - - - - - diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 0000000..cfc04af --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,20 @@ + + + + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/NeFSedit.sln b/NeFSedit.sln index 600509d..e3fe8fa 100644 --- a/NeFSedit.sln +++ b/NeFSedit.sln @@ -12,8 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Directory.Build.props = Directory.Build.props Directory.Build.targets = Directory.Build.targets README.md = README.md - SharedAssemblyInfo.cs = SharedAssemblyInfo.cs TODO.md = TODO.md + Directory.Packages.props = Directory.Packages.props EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VictorBush.Ego.NefsLib", "VictorBush.Ego.NefsLib\VictorBush.Ego.NefsLib.csproj", "{C3A546A4-7C78-4FA2-98DD-D056858A57D9}" diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs deleted file mode 100644 index 9aecee8..0000000 --- a/SharedAssemblyInfo.cs +++ /dev/null @@ -1,18 +0,0 @@ -// See LICENSE.txt for license information. - -using System.Reflection; - -[assembly: AssemblyCopyright("Copyright © VictorBush 2021")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.7.0.0")] -[assembly: AssemblyFileVersion("0.7.0.0")] diff --git a/VictorBush.Ego.NefsCommon b/VictorBush.Ego.NefsCommon index 78b6f57..469e1d6 160000 --- a/VictorBush.Ego.NefsCommon +++ b/VictorBush.Ego.NefsCommon @@ -1 +1 @@ -Subproject commit 78b6f57330eb3e9b58f0d6303fa238248f3534ea +Subproject commit 469e1d62b4612223aba9a5c65e9cb51357885b43 diff --git a/VictorBush.Ego.NefsEdit.Tests/Properties/AssemblyInfo.cs b/VictorBush.Ego.NefsEdit.Tests/Properties/AssemblyInfo.cs deleted file mode 100644 index 81381f4..0000000 --- a/VictorBush.Ego.NefsEdit.Tests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,23 +0,0 @@ -// See LICENSE.txt for license information. - -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("VictorBush.Ego.NefsEdit.Tests")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("VictorBush.Ego.NefsEdit.Tests")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("40723aaf-dc4b-4dfe-be01-90c562c7e298")] diff --git a/VictorBush.Ego.NefsEdit.Tests/VictorBush.Ego.NefsEdit.Tests.csproj b/VictorBush.Ego.NefsEdit.Tests/VictorBush.Ego.NefsEdit.Tests.csproj index 600fa8b..ac4c31f 100644 --- a/VictorBush.Ego.NefsEdit.Tests/VictorBush.Ego.NefsEdit.Tests.csproj +++ b/VictorBush.Ego.NefsEdit.Tests/VictorBush.Ego.NefsEdit.Tests.csproj @@ -1,23 +1,15 @@ - net6.0-windows + net8.0-windows Library - false - - - Properties\SharedAssemblyInfo.cs - - - - diff --git a/VictorBush.Ego.NefsEdit/Properties/AssemblyInfo.cs b/VictorBush.Ego.NefsEdit/Properties/AssemblyInfo.cs deleted file mode 100644 index b1ddcea..0000000 --- a/VictorBush.Ego.NefsEdit/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,23 +0,0 @@ -// See LICENSE.txt for license information. - -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("NeFS Edit")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("VictorBush.Ego.NefsEdit")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("ffda0019-dfb4-4928-91f0-a0d7d18518d1")] diff --git a/VictorBush.Ego.NefsEdit/Source/Program.cs b/VictorBush.Ego.NefsEdit/Source/Program.cs index a70118a..8612d88 100644 --- a/VictorBush.Ego.NefsEdit/Source/Program.cs +++ b/VictorBush.Ego.NefsEdit/Source/Program.cs @@ -30,7 +30,7 @@ internal static class Program /// /// Gets the directory where the application exe is located. /// - internal static string ExeDirectory => Path.GetDirectoryName(typeof(Program).Assembly.Location); + internal static string ExeDirectory => Application.StartupPath; /// /// Gets the directory used by the application for writing temporary files. @@ -71,9 +71,7 @@ internal static void Main() }).Build(); // Run application - Application.EnableVisualStyles(); - Application.SetHighDpiMode(HighDpiMode.SystemAware); - Application.SetCompatibleTextRenderingDefault(false); + ApplicationConfiguration.Initialize(); Application.Run(host.Services.GetRequiredService()); } } diff --git a/VictorBush.Ego.NefsEdit/Source/UI/BrowseAllForm.cs b/VictorBush.Ego.NefsEdit/Source/UI/BrowseAllForm.cs index 16cd2f9..e2c3f23 100644 --- a/VictorBush.Ego.NefsEdit/Source/UI/BrowseAllForm.cs +++ b/VictorBush.Ego.NefsEdit/Source/UI/BrowseAllForm.cs @@ -143,7 +143,12 @@ private void ItemsListView_SelectedIndexChanged(object sender, EventArgs e) foreach (ListViewItem item in this.itemsListView.SelectedItems) { - selectedNefsItems.Add((NefsItem)item.Tag); + if (item.Tag is not NefsItem nefsItem) + { + continue; + } + + selectedNefsItems.Add(nefsItem); } // Tell the editor what items are selected diff --git a/VictorBush.Ego.NefsEdit/Source/UI/BrowseTreeForm.cs b/VictorBush.Ego.NefsEdit/Source/UI/BrowseTreeForm.cs index d428246..b451b23 100644 --- a/VictorBush.Ego.NefsEdit/Source/UI/BrowseTreeForm.cs +++ b/VictorBush.Ego.NefsEdit/Source/UI/BrowseTreeForm.cs @@ -87,8 +87,7 @@ private void FilesListView_DoubleClick(object sender, EventArgs e) { if (this.filesListView.SelectedItems.Count > 0) { - var item = (NefsItem)this.filesListView.SelectedItems[0].Tag; - if (item.Type == NefsItemType.Directory) + if (this.filesListView.SelectedItems[0].Tag is NefsItem { Type: NefsItemType.Directory } item) { OpenDirectory(item); } @@ -117,7 +116,12 @@ private void FilesListView_SelectedIndexChanged(object sender, EventArgs e) foreach (ListViewItem item in this.filesListView.SelectedItems) { - selectedNefsItems.Add((NefsItem)item.Tag); + if (item.Tag is not NefsItem nefsItem) + { + continue; + } + + selectedNefsItems.Add(nefsItem); } // Tell the editor what items are selected diff --git a/VictorBush.Ego.NefsEdit/Source/Workspace/NefsEditWorkspace.cs b/VictorBush.Ego.NefsEdit/Source/Workspace/NefsEditWorkspace.cs index bd9d3d7..e603248 100644 --- a/VictorBush.Ego.NefsEdit/Source/Workspace/NefsEditWorkspace.cs +++ b/VictorBush.Ego.NefsEdit/Source/Workspace/NefsEditWorkspace.cs @@ -341,7 +341,7 @@ public bool ReplaceItemByDialog(NefsItem item) return false; } - var fileSize = FileSystem.FileInfo.FromFileName(fileName).Length; + var fileSize = FileSystem.FileInfo.New(fileName).Length; var itemSize = new NefsItemSize((uint)fileSize); var newDataSource = new NefsFileDataSource(fileName, 0, itemSize, false); var cmd = new ReplaceFileCommand(item, item.DataSource, item.State, newDataSource); diff --git a/VictorBush.Ego.NefsEdit/VictorBush.Ego.NefsEdit.csproj b/VictorBush.Ego.NefsEdit/VictorBush.Ego.NefsEdit.csproj index e9e02d6..e6be5cd 100644 --- a/VictorBush.Ego.NefsEdit/VictorBush.Ego.NefsEdit.csproj +++ b/VictorBush.Ego.NefsEdit/VictorBush.Ego.NefsEdit.csproj @@ -1,22 +1,19 @@ - + - net6.0-windows + net8.0-windows WinExe true NefsEdit - false true - true enable + + NeFS Edit bin\Debug\NefsEdit.xml - - Properties\SharedAssemblyInfo.cs - Component @@ -26,24 +23,15 @@ - - - - 3.1.3 - - - 3.0.1 - - - 3.1.1 - + + + + - - - + - \ No newline at end of file + diff --git a/VictorBush.Ego.NefsLib.Tests/Properties/AssemblyInfo.cs b/VictorBush.Ego.NefsLib.Tests/Properties/AssemblyInfo.cs deleted file mode 100644 index 91b2a48..0000000 --- a/VictorBush.Ego.NefsLib.Tests/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,23 +0,0 @@ -// See LICENSE.txt for license information. - -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("VictorBush.Ego.NefsLib.Tests")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("VictorBush.Ego.NefsLib.Tests")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("2f06464e-5954-4a40-a3ce-a8529e7371e2")] diff --git a/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/LzssDecompressTests.cs b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/LzssDecompressTests.cs new file mode 100644 index 0000000..20c280e --- /dev/null +++ b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/LzssDecompressTests.cs @@ -0,0 +1,42 @@ +// See LICENSE.txt for license information. + +using System.Text; +using VictorBush.Ego.NefsLib.IO; +using Xunit; + +namespace VictorBush.Ego.NefsLib.Tests.IO; + +public class LzssDecompressTests +{ + [Fact] + public async Task Decompress_Test() + { + byte[] input = [ + 0xFF, 0x3C, 0x3F, 0x78, 0x6D, 0x6C, 0x20, 0x76, 0x65, 0xFF, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x3D, + 0x22, 0x31, 0xFF, 0x2E, 0x30, 0x22, 0x20, 0x73, 0x74, 0x61, 0x6E, 0xFF, 0x64, 0x61, 0x6C, 0x6F, + 0x6E, 0x65, 0x3D, 0x27, 0xFF, 0x79, 0x65, 0x73, 0x27, 0x20, 0x3F, 0x3E, 0x0D, 0xFF, 0x0A, 0x3C, + 0x53, 0x6B, 0x69, 0x70, 0x46, 0x72, 0xBF, 0x6F, 0x6E, 0x74, 0x45, 0x6E, 0x64, 0x14, 0x00, 0x09, + 0xFF, 0x3C, 0x50, 0x61, 0x72, 0x61, 0x6D, 0x65, 0x74, 0xEF, 0x65, 0x72, 0x20, 0x6E, 0x2C, 0x00, + 0x3D, 0x22, 0x73, 0xEE, 0x19, 0x00, 0x74, 0x6F, 0x67, 0x2C, 0x00, 0x22, 0x20, 0x76, 0x77, 0x61, + 0x6C, 0x75, 0x36, 0x00, 0x74, 0x72, 0x75, 0x42, 0x00, 0x05, 0x2F, 0x14, 0x01, 0x2F, 0x18, 0x0A + ]; + var expectedBytes = Encoding.ASCII.GetBytes(""" + + + + + """.ReplaceLineEndings("\r\n")); + + using var inputStream = new MemoryStream(input); + var lzss = new LzssDecompress(); + + // Test + using var outputStream = new MemoryStream(); + await lzss.DecompressAsync(inputStream, outputStream, CancellationToken.None) + .ConfigureAwait(false); + + // Verify + var actualBytes = outputStream.ToArray(); + Assert.Equal(expectedBytes, actualBytes); + } +} diff --git a/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsReaderTests.cs b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsReaderTests.cs index 3be8e01..86d37e9 100644 --- a/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsReaderTests.cs +++ b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsReaderTests.cs @@ -16,10 +16,49 @@ public class NefsReaderTests private readonly NefsProgress p = new NefsProgress(CancellationToken.None); + [Fact] + public async Task ReadHeaderIntroAsync_Dirt2V150Ps3Proto() + { + var expectedResult = + new NefsReader.DecryptHeaderIntroResult(true, IsEncrypted: false, IsXorEncoded: false, IsLittleEndian: false); + byte[] bytes = + { + // 5 bytes offset + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + + // Header intro + 0x53, 0x46, 0x65, 0x4E, 0x00, 0x11, 0x81, 0x10, 0x00, 0x01, 0x05, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x23, 0xA0, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, + 0x00, 0x03, 0x57, 0x70, 0x00, 0x07, 0x3C, 0xF0, 0x00, 0x09, 0xB8, 0xF4, 0x00, 0x11, 0x81, 0x00, + 0x38, 0x31, 0x38, 0x33, 0x41, 0x45, 0x32, 0x44, 0x33, 0x34, 0x44, 0x33, 0x33, 0x42, 0x44, 0x43, + 0x33, 0x41, 0x43, 0x35, 0x37, 0x36, 0x38, 0x43, 0x31, 0x36, 0x36, 0x39, 0x43, 0x38, 0x34, 0x41, + 0x36, 0x31, 0x31, 0x31, 0x37, 0x35, 0x30, 0x32, 0x42, 0x41, 0x34, 0x42, 0x39, 0x44, 0x33, 0x38, + 0x30, 0x38, 0x38, 0x44, 0x41, 0x42, 0x41, 0x44, 0x38, 0x41, 0x42, 0x32, 0x30, 0x44, 0x35, 0x30, + + // Extra bytes + 0xFF, 0xFF, + }; + + var stream = new MemoryStream(bytes); + var reader = new NefsReader(this.fileSystem); + var offset = 5; + + // Test + using var actualStream = new MemoryStream(); + var actualResult = await reader.ReadHeaderIntroAsync(stream, offset, actualStream, false, this.p) + .ConfigureAwait(false); + + // Verify + var actualBytes = actualStream.ToArray(); + Assert.Equal(expectedResult, actualResult); + Assert.Equal([], actualBytes); + } + [Fact] public async Task ReadHeaderIntroAsync_Dirt2V151() { - var expectedResult = new NefsReader.DecryptHeaderIntroResult(true, IsEncrypted: true, IsXorEncoded: true); + var expectedResult = + new NefsReader.DecryptHeaderIntroResult(true, IsEncrypted: true, IsXorEncoded: true, IsLittleEndian: true); byte[] bytes = { // 5 bytes offset @@ -65,6 +104,56 @@ public async Task ReadHeaderIntroAsync_Dirt2V151() Assert.Equal(expectedBytes, actualBytes); } + [Fact] + public async Task ReadHeaderIntroAsync_Dirt3V151X360() + { + var expectedResult = + new NefsReader.DecryptHeaderIntroResult(true, IsEncrypted: false, IsXorEncoded: true, IsLittleEndian: false); + byte[] bytes = + { + // 5 bytes offset + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + + // Header intro + 0x76, 0xB5, 0xEE, 0xC0, 0x0A, 0x65, 0xC5, 0x89, 0x0A, 0x64, 0x5B, 0x39, 0x0A, 0x65, 0x52, 0xC3, + 0x0A, 0x65, 0x5E, 0x38, 0x0A, 0x66, 0x5B, 0x39, 0x25, 0xFB, 0x0C, 0x5E, 0x0A, 0x65, 0x52, 0x43, + 0x25, 0xFB, 0x0C, 0x5E, 0x0A, 0x65, 0x52, 0xC1, 0x0A, 0x67, 0x31, 0x1D, 0x76, 0xBD, 0x9E, 0x20, + 0x25, 0xF3, 0x8B, 0x8E, 0x0A, 0x6D, 0xB4, 0x09, 0x2F, 0x9C, 0x7B, 0x6F, 0x67, 0xB8, 0x13, 0x6F, + 0x64, 0xCC, 0x63, 0x66, 0x61, 0xCD, 0x11, 0x13, 0x66, 0xCB, 0x17, 0x13, 0x17, 0xCF, 0x64, 0x67, + 0x60, 0xC9, 0x63, 0x12, 0x67, 0xCC, 0x19, 0x64, 0x12, 0xC8, 0x63, 0x62, 0x16, 0xC3, 0x63, 0x15, + 0x14, 0xCE, 0x10, 0x61, 0x66, 0xB9, 0x63, 0x60, 0x13, 0xC2, 0x64, 0x60, 0x1C, 0xBB, 0x64, 0x61, + 0x17, 0xC2, 0x15, 0x62, 0x61, 0xBF, 0x14, 0x6E, 0x1D, 0xC2, 0x13, 0x13, 0x4C, 0x31, 0xBB, 0x22, + + // Extra bytes + 0xFF, 0xFF, + }; + var expectedBytes = new byte[] + { + 0x53, 0x46, 0x65, 0x4E, 0x00, 0x08, 0x71, 0x80, 0x00, 0x01, 0x05, 0x01, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x0C, 0x7B, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x01, 0x2C, 0x08, 0x00, 0x02, 0x63, 0xDC, 0x00, 0x02, 0xF4, 0x94, 0x00, 0x08, 0x70, 0xE0, + 0x00, 0x08, 0x87, 0xD0, 0x7C, 0xD0, 0x2A, 0x29, 0x25, 0xFA, 0x20, 0x56, 0x42, 0x42, 0x33, 0x39, + 0x41, 0x36, 0x43, 0x30, 0x44, 0x37, 0x31, 0x45, 0x43, 0x31, 0x37, 0x45, 0x32, 0x35, 0x44, 0x31, + 0x45, 0x33, 0x43, 0x44, 0x42, 0x36, 0x39, 0x32, 0x37, 0x32, 0x43, 0x34, 0x33, 0x39, 0x43, 0x43, + 0x31, 0x34, 0x30, 0x37, 0x43, 0x43, 0x43, 0x36, 0x36, 0x38, 0x44, 0x36, 0x39, 0x41, 0x44, 0x37, + 0x32, 0x38, 0x35, 0x34, 0x44, 0x45, 0x34, 0x38, 0x38, 0x38, 0x33, 0x45, 0x4C, 0x31, 0xBB, 0x22, + }; + + var stream = new MemoryStream(bytes); + var reader = new NefsReader(this.fileSystem); + var offset = 5; + + // Test + using var actualStream = new MemoryStream(); + var actualResult = await reader.ReadHeaderIntroAsync(stream, offset, actualStream, false, this.p) + .ConfigureAwait(false); + + // Verify + var actualBytes = actualStream.ToArray(); + Assert.Equal(expectedResult, actualResult); + Assert.Equal(expectedBytes, actualBytes); + } + [Fact] public async void ReadHeaderPart1Async_ExtraBytesAtEnd_ExtraBytesIgnored() { @@ -473,9 +562,10 @@ public async void ReadHeaderPart5Async_ValidData_DataRead() var reader = new NefsReader(this.fileSystem); var size = 16; var offset = 5; + using var endianReader = new EndianBinaryReader(stream, true); // Test - var part5 = await reader.ReadHeaderPart5Async(stream, offset, size, this.p); + var part5 = await reader.ReadHeaderPart5Async(endianReader, offset, size, this.p); // Verify Assert.Equal((ulong)0x1817161514131211, part5.DataSize); diff --git a/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsWriterTests.cs b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsWriterTests.cs index c580fa6..34f3341 100644 --- a/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsWriterTests.cs +++ b/VictorBush.Ego.NefsLib.Tests/Source/Tests/IO/NefsWriterTests.cs @@ -332,7 +332,8 @@ public async Task WriteHeaderPart5Async_ValidData_Written() using (var ms = new MemoryStream()) { - await writer.WriteHeaderPartAsync(ms, offset, part5, new NefsProgress()); + var bw = new EndianBinaryWriter(ms); + await writer.WriteTocEntryAsync(bw, offset, part5.Data, new NefsProgress()); buffer = ms.ToArray(); } diff --git a/VictorBush.Ego.NefsLib.Tests/VictorBush.Ego.NefsLib.Tests.csproj b/VictorBush.Ego.NefsLib.Tests/VictorBush.Ego.NefsLib.Tests.csproj index 2c562d3..feb21eb 100644 --- a/VictorBush.Ego.NefsLib.Tests/VictorBush.Ego.NefsLib.Tests.csproj +++ b/VictorBush.Ego.NefsLib.Tests/VictorBush.Ego.NefsLib.Tests.csproj @@ -1,23 +1,15 @@ - net6 + net8.0 Library - false - - - Properties\SharedAssemblyInfo.cs - - - - diff --git a/VictorBush.Ego.NefsLib/Properties/AssemblyInfo.cs b/VictorBush.Ego.NefsLib/Properties/AssemblyInfo.cs deleted file mode 100644 index 479d906..0000000 --- a/VictorBush.Ego.NefsLib/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,28 +0,0 @@ -// See LICENSE.txt for license information. - -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("VictorBush.Ego.NefsLib")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("VictorBush.Ego.NefsLib")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("c3a546a4-7c78-4fa2-98dd-d056858a57d9")] - -// Allow tests to access internals -[assembly: InternalsVisibleTo("VictorBush.Ego.NefsLib.Tests")] -[assembly: InternalsVisibleTo("VictorBush.Ego.NefsEdit.Tests")] diff --git a/VictorBush.Ego.NefsLib/Source/DataSource/NefsDataTransform.cs b/VictorBush.Ego.NefsLib/Source/DataSource/NefsDataTransform.cs index c71965d..89fe4d7 100644 --- a/VictorBush.Ego.NefsLib/Source/DataSource/NefsDataTransform.cs +++ b/VictorBush.Ego.NefsLib/Source/DataSource/NefsDataTransform.cs @@ -42,6 +42,8 @@ public NefsDataTransform(uint fileSize) /// public uint ChunkSize { get; } + public bool IsLzssCompressed { get; init; } + /// /// Whether data chunks are AES encrypted. /// diff --git a/VictorBush.Ego.NefsLib/Source/DataTypes/UInt8Type.cs b/VictorBush.Ego.NefsLib/Source/DataTypes/UInt8Type.cs index 4168abc..9ab1ef7 100644 --- a/VictorBush.Ego.NefsLib/Source/DataTypes/UInt8Type.cs +++ b/VictorBush.Ego.NefsLib/Source/DataTypes/UInt8Type.cs @@ -29,7 +29,7 @@ public UInt8Type(int offset) public byte Value { get; set; } /// - public override byte[] GetBytes() => BitConverter.GetBytes(Value); + public override byte[] GetBytes() => [Value]; /// public override async Task ReadAsync(Stream file, long baseOffset, NefsProgress p) diff --git a/VictorBush.Ego.NefsLib/Source/Header/AesKeyBuffer.cs b/VictorBush.Ego.NefsLib/Source/Header/AesKeyBuffer.cs new file mode 100644 index 0000000..f0fe90d --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/AesKeyBuffer.cs @@ -0,0 +1,11 @@ +// See LICENSE.txt for license information. + +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header; + +[InlineArray(64)] +public struct AesKeyBuffer +{ + private byte element; +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/INefsHeaderIntro.cs b/VictorBush.Ego.NefsLib/Source/Header/INefsHeaderIntro.cs index 4898629..cf32169 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/INefsHeaderIntro.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/INefsHeaderIntro.cs @@ -7,6 +7,11 @@ namespace VictorBush.Ego.NefsLib.Header; /// public interface INefsHeaderIntro { + /// + /// Whether the file is in little endian format; otherwise, big endian. + /// + bool IsLittleEndian { get; } + /// /// Whether the header is encrypted. /// @@ -40,7 +45,7 @@ public interface INefsHeaderIntro /// /// 256-bit AES key stored as a hex string. /// - byte[] AesKeyHexString { get; } + ReadOnlySpan AesKeyHexString { get; } /// /// Gets the AES-256 key for this header. diff --git a/VictorBush.Ego.NefsLib/Source/Header/INefsTocData.cs b/VictorBush.Ego.NefsLib/Source/Header/INefsTocData.cs new file mode 100644 index 0000000..66a3080 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/INefsTocData.cs @@ -0,0 +1,10 @@ +// See LICENSE.txt for license information. + +namespace VictorBush.Ego.NefsLib.Header; + +public interface INefsTocData where T : unmanaged, INefsTocData +{ + static abstract int ByteCount { get; } + + void ReverseEndianness(); +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderIntro.cs b/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderIntro.cs index c47f96d..42bb86c 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderIntro.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderIntro.cs @@ -29,10 +29,10 @@ public NefsHeaderIntro() } /// - public byte[] AesKeyHexString + public ReadOnlySpan AesKeyHexString { get => Data0x24_AesKeyHexString.Value; - init => Data0x24_AesKeyHexString.Value = value; + init => value.CopyTo(Data0x24_AesKeyHexString.Value); } /// @@ -51,6 +51,9 @@ public uint HeaderSize init => Data0x64_HeaderSize.Value = value; } + /// + public bool IsLittleEndian { get; init; } + /// public bool IsEncrypted { get; init; } diff --git a/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderPart5.cs b/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderPart5.cs index 16d42a0..8677694 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderPart5.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/NefsHeaderPart5.cs @@ -1,6 +1,6 @@ // See LICENSE.txt for license information. -using VictorBush.Ego.NefsLib.DataTypes; +using VictorBush.Ego.NefsLib.Header.Version151; namespace VictorBush.Ego.NefsLib.Header; @@ -9,6 +9,8 @@ namespace VictorBush.Ego.NefsLib.Header; /// public sealed class NefsHeaderPart5 { + private readonly Nefs150TocVolumeInfo data; + /// /// The size of header part 5. /// @@ -17,19 +19,28 @@ public sealed class NefsHeaderPart5 /// /// Initializes a new instance of the class. /// - internal NefsHeaderPart5() + internal NefsHeaderPart5(Nefs150TocVolumeInfo? data = null) { - FirstDataOffset = Nefs20Header.DataOffsetDefault; + this.data = data ?? new Nefs150TocVolumeInfo(); + if (data is null) + { + FirstDataOffset = Nefs20Header.DataOffsetDefault; + } } + /// + /// The underlying data. + /// + public Nefs150TocVolumeInfo Data => this.data; + /// /// Offset into header part 3 for the name of the file containing the item data. For headless archives, it would be /// something like game.dat, game.bin, etc. For standard archives, it would be the name of the archive. /// public uint DataFileNameStringOffset { - get => Data0x08_DataFileNameStringOffset.Value; - init => Data0x08_DataFileNameStringOffset.Value = value; + get => this.data.NameOffset; + init => this.data.NameOffset = value; } /// @@ -37,8 +48,8 @@ public uint DataFileNameStringOffset /// public ulong DataSize { - get => Data0x00_TotalItemDataSize.Value; - init => Data0x00_TotalItemDataSize.Value = value; + get => this.data.Size; + init => this.data.Size = value; } /// @@ -46,16 +57,7 @@ public ulong DataSize /// public uint FirstDataOffset { - get => Data0x0C_FirstDataOffset.Value; - init => Data0x0C_FirstDataOffset.Value = value; + get => this.data.DataOffset; + init => this.data.DataOffset = value; } - - [FileData] - private UInt64Type Data0x00_TotalItemDataSize { get; } = new UInt64Type(0x00); - - [FileData] - private UInt32Type Data0x08_DataFileNameStringOffset { get; } = new UInt32Type(0x08); - - [FileData] - private UInt32Type Data0x0C_FirstDataOffset { get; } = new UInt32Type(0x0C); } diff --git a/VictorBush.Ego.NefsLib/Source/Header/NefsTocEntryFlags.cs b/VictorBush.Ego.NefsLib/Source/Header/NefsTocEntryFlags.cs new file mode 100644 index 0000000..2b27bab --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/NefsTocEntryFlags.cs @@ -0,0 +1,15 @@ +// See LICENSE.txt for license information. + +namespace VictorBush.Ego.NefsLib.Header; + +[Flags] +public enum NefsTocEntryFlags : ushort +{ + None = 0, + Transformed = 1 << 0, + Directory = 1 << 1, + Duplicated = 1 << 2, + Cacheable = 1 << 3, + LastSibling = 1 << 4, + Patched = 1 << 5 +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version 1.6/Nefs16HeaderPart4TransformType.cs b/VictorBush.Ego.NefsLib/Source/Header/Version 1.6/Nefs16HeaderPart4TransformType.cs index a321822..15baa76 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/Version 1.6/Nefs16HeaderPart4TransformType.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/Version 1.6/Nefs16HeaderPart4TransformType.cs @@ -12,6 +12,11 @@ public enum Nefs16HeaderPart4TransformType /// None = 0x0, + /// + /// Chunk is LZSS compressed. + /// + Lzss = 0x1, + /// /// Chunk is AES encrypted. /// diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150Header.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150Header.cs new file mode 100644 index 0000000..56846e9 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150Header.cs @@ -0,0 +1,138 @@ +// See LICENSE.txt for license information. + +using Microsoft.Extensions.Logging; +using VictorBush.Ego.NefsLib.DataSource; +using VictorBush.Ego.NefsLib.Item; +using VictorBush.Ego.NefsLib.Progress; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// A NeFS archive header. +/// +public sealed class Nefs150Header : INefsHeader +{ + private static readonly ILogger Log = NefsLog.GetLogger(); + + /// + /// Initializes a new instance of the class. + /// + /// Header intro. + /// Header part 1. + /// Header part 2. + /// Header part 3. + /// Header part 4. + /// Header part 5. + public Nefs150Header( + Nefs150HeaderIntro intro, + Nefs150HeaderPart1 part1, + Nefs150HeaderPart2 part2, + NefsHeaderPart3 part3, + Nefs150HeaderPart4 part4, + NefsHeaderPart5 part5) + { + Intro = intro ?? throw new ArgumentNullException(nameof(intro)); + Part1 = part1 ?? throw new ArgumentNullException(nameof(part1)); + Part2 = part2 ?? throw new ArgumentNullException(nameof(part2)); + Part3 = part3 ?? throw new ArgumentNullException(nameof(part3)); + Part4 = part4 ?? throw new ArgumentNullException(nameof(part4)); + Part5 = part5 ?? throw new ArgumentNullException(nameof(part5)); + } + + public Nefs150HeaderIntro Intro { get; } + + /// + public bool IsEncrypted => Intro.IsEncrypted; + + public Nefs150HeaderPart1 Part1 { get; } + public Nefs150HeaderPart2 Part2 { get; } + public NefsHeaderPart3 Part3 { get; } + public Nefs150HeaderPart4 Part4 { get; } + public NefsHeaderPart5 Part5 { get; } + + /// + public NefsItem CreateItemInfo(uint part1Index, NefsItemList dataSourceList) + { + return CreateItemInfo(Part1.EntriesByIndex[(int)part1Index].Guid, dataSourceList); + } + + /// + public NefsItem CreateItemInfo(Guid guid, NefsItemList dataSourceList) + { + var p1 = Part1.EntriesByGuid[guid]; + var p2 = Part2.EntriesByIndex[(int)p1.IndexPart2]; + var id = p2.Id; + + // Gather attributes + var attributes = p1.CreateAttributes(); + + // Find parent + var parentId = GetItemDirectoryId(p1.IndexPart2); + + // Offset and size + var dataOffset = (long)p1.OffsetToData; + var extractedSize = p2.ExtractedSize; + + // Transform + var transform = new NefsDataTransform(Intro.BlockSize, attributes.V20IsZlib, Intro.IsEncrypted ? Intro.GetAesKey() : null); + + // Data source + INefsDataSource dataSource; + if (attributes.IsDirectory) + { + // Item is a directory + dataSource = new NefsEmptyDataSource(); + transform = null; + } + else + { + var numChunks = Intro.ComputeNumChunks(p2.ExtractedSize); + var chunkSize = Intro.BlockSize; + var chunks = Part4.CreateChunksList(p1.IndexPart4, numChunks, chunkSize, Intro.GetAesKey()); + var size = new NefsItemSize(extractedSize, chunks); + dataSource = new NefsItemListDataSource(dataSourceList, dataOffset, size); + } + + // File name and path + var fileName = GetItemFileName(p1.IndexPart2); + + // Create item + return new NefsItem(p1.Guid, id, fileName, parentId, dataSource, transform, attributes); + } + + /// + public NefsItemList CreateItemList(string dataFilePath, NefsProgress p) + { + var items = new NefsItemList(dataFilePath); + + for (var i = 0; i < Part1.EntriesByIndex.Count; ++i) + { + p.CancellationToken.ThrowIfCancellationRequested(); + + try + { + var item = CreateItemInfo((uint)i, items); + items.Add(item); + } + catch (Exception) + { + Log.LogError($"Failed to create item with part 1 index {i}, skipping."); + } + } + + return items; + } + + /// + public NefsItemId GetItemDirectoryId(uint indexPart2) + { + return Part2.EntriesByIndex[(int)indexPart2].DirectoryId; + } + + /// + public string GetItemFileName(uint indexPart2) + { + var offsetIntoPart3 = Part2.EntriesByIndex[(int)indexPart2].OffsetIntoPart3; + return Part3.FileNamesByOffset[offsetIntoPart3]; + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderIntro.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderIntro.cs new file mode 100644 index 0000000..6ab401f --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderIntro.cs @@ -0,0 +1,160 @@ +// See LICENSE.txt for license information. + +using System.Text; +using VictorBush.Ego.NefsLib.Utility; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// Header introduction. Contains size, encryption, and verification info. +/// +public record Nefs150HeaderIntro : INefsHeaderIntro, INefsHeaderIntroToc +{ + private readonly Nefs150TocHeader data; + + /// + /// Initializes a new instance of the class. + /// + public Nefs150HeaderIntro(Nefs150TocHeader data) + { + this.data = data; + } + + /// + public bool IsLittleEndian { get; init; } + + /// + public bool IsEncrypted { get; init; } + + /// + public bool IsXorEncoded { get; init; } + + /// + public uint MagicNumber + { + get => this.data.Magic; + init => this.data.Magic = value; + } + + /// + public uint HeaderSize + { + get => this.data.TocSize; + init => this.data.TocSize = value; + } + + /// + public uint NefsVersion + { + get => this.data.Version; + init => this.data.Version = value; + } + + /// + /// The number of volumes. + /// + public uint NumVolumes + { + get => this.data.NumVolumes; + init => this.data.NumVolumes = value; + } + + /// + public uint NumberOfItems + { + get => this.data.NumEntries; + init => this.data.NumEntries = value; + } + + /// + /// Block size (chunk size). The size of chunks data is split up before any transforms are applied. + /// + public uint BlockSize + { + get => this.data.BlockSize; + init => this.data.BlockSize = value; + } + + /// + /// The split size. + /// + public uint SplitSize + { + get => this.data.SplitSize; + init => this.data.SplitSize = value; + } + + /// + public uint OffsetToPart1 + { + get => this.data.EntryTableStart; + init => this.data.EntryTableStart = value; + } + + /// + public uint OffsetToPart2 + { + get => this.data.SharedEntryInfoTableStart; + init => this.data.SharedEntryInfoTableStart = value; + } + + /// + public uint OffsetToPart3 + { + get => this.data.NameTableStart; + init => this.data.NameTableStart = value; + } + + /// + public uint OffsetToPart4 + { + get => this.data.BlockTableStart; + init => this.data.BlockTableStart = value; + } + + /// + public uint OffsetToPart5 + { + get => this.data.VolumeInfoTableStart; + init => this.data.VolumeInfoTableStart = value; + } + + /// + public uint Part1Size => OffsetToPart2 - OffsetToPart1; + + /// + public uint Part2Size => OffsetToPart3 - OffsetToPart2; + + /// + public uint Part3Size => OffsetToPart4 - OffsetToPart3; + + /// + public uint Part4Size => OffsetToPart5 - OffsetToPart4; + + /// + public ReadOnlySpan AesKeyHexString + { + get => this.data.AesKeyBuffer; + init => value.CopyTo(this.data.AesKeyBuffer); + } + + /// + public uint OffsetToPart6 => throw new NotSupportedException("Part6 not supported in 1.5.1."); + + /// + public uint OffsetToPart7 => throw new NotSupportedException("Part7 not supported in 1.5.1."); + + /// + public uint OffsetToPart8 => throw new NotSupportedException("Part8 not supported in 1.5.1."); + + /// + public byte[] GetAesKey() + { + var asciiKey = Encoding.ASCII.GetString(AesKeyHexString); + return StringHelper.FromHexString(asciiKey); + } + + /// + public uint ComputeNumChunks(uint extractedSize) => + (extractedSize + (BlockSize - 1)) / BlockSize; +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1.cs new file mode 100644 index 0000000..2d50590 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1.cs @@ -0,0 +1,79 @@ +// See LICENSE.txt for license information. + +using VictorBush.Ego.NefsLib.Item; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// Header part 1. The "master catalog" of items in the archive. +/// +public sealed class Nefs150HeaderPart1 +{ + private readonly Dictionary entriesByGuid; + private readonly List entriesByIndex; + + /// + /// Initializes a new instance of the class. + /// + /// A list of entries to instantiate this part with. + internal Nefs150HeaderPart1(IList entries) + { + this.entriesByIndex = new List(entries); + this.entriesByGuid = new Dictionary(entries.ToDictionary(e => e.Guid)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The list of items in the archive. + /// Header part 4. + internal Nefs150HeaderPart1(NefsItemList items, INefsHeaderPart4 part4) + { + this.entriesByGuid = new Dictionary(); + var indexPart2 = 0U; + + // Enumerate this list depth first. This determines the part 2 order. The part 1 entries will be sorted by item id. + foreach (var item in items.EnumerateDepthFirstByName()) + { + var flags = NefsTocEntryFlags.None; + flags |= item.Attributes.V16IsTransformed ? NefsTocEntryFlags.Transformed : 0; + flags |= item.Attributes.IsDirectory ? NefsTocEntryFlags.Directory : 0; + flags |= item.Attributes.IsDuplicated ? NefsTocEntryFlags.Duplicated : 0; + flags |= item.Attributes.IsCacheable ? NefsTocEntryFlags.Cacheable : 0; + flags |= item.Attributes.V16Unknown0x10 ? NefsTocEntryFlags.LastSibling : 0; + flags |= item.Attributes.IsPatched ? NefsTocEntryFlags.Patched : 0; + + var entry = new Nefs150HeaderPart1Entry(item.Guid) + { + Guid = item.Guid, + Id = new NefsItemId(item.Id.Value), + IndexPart2 = indexPart2++, + IndexPart4 = part4.GetIndexForItem(item), + OffsetToData = (ulong)item.DataSource.Offset, + Volume = item.Attributes.Part6Volume, + Flags = flags + }; + + this.entriesByGuid.Add(item.Guid, entry); + } + + // Sort part 1 by item id + this.entriesByIndex = new List(this.entriesByGuid.Values.OrderBy(e => e.Id)); + } + + /// + /// Gets entries for each item in the archive, accessible by Guid. + /// + public IReadOnlyDictionary EntriesByGuid => this.entriesByGuid; + + /// + /// Gets the list of entries in the order they appear in the header. Usually items are sorted by id, but this is not + /// guaranteed (for example, DiRT Rally has a header with items out of order). + /// + public IList EntriesByIndex => this.entriesByIndex; + + /// + /// Total size (in bytes) of part 1. + /// + public int Size => this.entriesByIndex.Count * Nefs150HeaderPart1Entry.EntrySize; +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1Entry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1Entry.cs new file mode 100644 index 0000000..2a09aa4 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart1Entry.cs @@ -0,0 +1,109 @@ +// See LICENSE.txt for license information. + +using VictorBush.Ego.NefsLib.Item; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// An entry in header part 1 for an item in an archive. +/// +public sealed class Nefs150HeaderPart1Entry : INefsHeaderPartEntry +{ + public static readonly int EntrySize = Nefs150TocEntry.ByteCount; + private readonly Nefs150TocEntry data; + + /// + /// Initializes a new instance of the class. + /// + /// The Guid of the item this metadata belongs to. + /// The underlying data. + internal Nefs150HeaderPart1Entry(Guid guid, Nefs150TocEntry? data = null) + { + Guid = guid; + this.data = data ?? new Nefs150TocEntry(); + } + + /// + /// The unique identifier of the item this data is for. + /// + public Guid Guid { get; init; } + + /// + /// The underlying data. + /// + public Nefs150TocEntry Data => this.data; + + /// + /// The absolute offset to the file's data in the archive. For directories, this is 0. + /// + public ulong OffsetToData + { + get => this.data.Start; + init => this.data.Start = value; + } + + /// + /// Unknown. + /// + public ushort Volume + { + get => this.data.Volume; + init => this.data.Volume = value; + } + + /// + /// A bitfield that has various flags. + /// + public NefsTocEntryFlags Flags + { + get => this.data.Flags; + init => this.data.Flags = value; + } + + /// + /// The index used for parts 2 for this item. + /// + public uint IndexPart2 + { + get => this.data.SharedInfo; + init => this.data.SharedInfo = value; + } + + /// + /// The index into header part 4 for this item. For the actual offset. + /// + public uint IndexPart4 + { + get => this.data.FirstBlock; + init => this.data.FirstBlock = value; + } + + /// + /// The id of the item. It is possible to have duplicate item's with the same id. + /// + public NefsItemId Id + { + get => new(this.data.NextDuplicate); + init => this.data.NextDuplicate = value.Value; + } + + public int Size => EntrySize; + + /// + /// Creates a object. + /// + public NefsItemAttributes CreateAttributes() + { + return new NefsItemAttributes( + v16IsTransformed: Flags.HasFlag(NefsTocEntryFlags.Transformed), + isDirectory: Flags.HasFlag(NefsTocEntryFlags.Directory), + isDuplicated: Flags.HasFlag(NefsTocEntryFlags.Duplicated), + isCacheable: Flags.HasFlag(NefsTocEntryFlags.Cacheable), + v16Unknown0x10: Flags.HasFlag(NefsTocEntryFlags.LastSibling), + isPatched: Flags.HasFlag(NefsTocEntryFlags.Patched), + v16Unknown0x40: false, + v16Unknown0x80: false, + part6Volume: Volume, + part6Unknown0x3: 0); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2.cs similarity index 57% rename from VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2.cs rename to VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2.cs index 8fedf7c..65da6bf 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2.cs @@ -7,36 +7,31 @@ namespace VictorBush.Ego.NefsLib.Header.Version151; /// /// Header part 2. /// -public sealed class Nefs151HeaderPart2 +public sealed class Nefs150HeaderPart2 { - /// - /// The size of a part 2 entry. This is used to get the offset into part 2 from an index into part 2. - /// - public const int EntrySize = 0x1C; - - private readonly List entriesByIndex; + private readonly List entriesByIndex; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// A list of entries to instantiate this part with. - internal Nefs151HeaderPart2(IList entries) + internal Nefs150HeaderPart2(IList entries) { - this.entriesByIndex = new List(entries); + this.entriesByIndex = new List(entries); } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The list of items in the archive. /// Header part 3. - internal Nefs151HeaderPart2(NefsItemList items, NefsHeaderPart3 part3) + internal Nefs150HeaderPart2(NefsItemList items, NefsHeaderPart3 part3) { - this.entriesByIndex = new List(); + this.entriesByIndex = new List(); foreach (var item in items.EnumerateDepthFirstByName()) { - var entry = new Nefs151HeaderPart2Entry + var entry = new Nefs150HeaderPart2Entry { DirectoryId = item.DirectoryId, ExtractedSize = item.DataSource.Size.ExtractedSize, @@ -54,10 +49,10 @@ internal Nefs151HeaderPart2(NefsItemList items, NefsHeaderPart3 part3) /// /// Gets the list of entries in the order they appear in the header. /// - public IList EntriesByIndex => this.entriesByIndex; + public IList EntriesByIndex => this.entriesByIndex; /// /// Total size (in bytes) of part 2. /// - public int Size => this.entriesByIndex.Count * EntrySize; + public int Size => this.entriesByIndex.Count * Nefs150HeaderPart2Entry.EntrySize; } diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2Entry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2Entry.cs similarity index 52% rename from VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2Entry.cs rename to VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2Entry.cs index 38824ed..889fde0 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart2Entry.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart2Entry.cs @@ -1,6 +1,5 @@ // See LICENSE.txt for license information. -using VictorBush.Ego.NefsLib.DataTypes; using VictorBush.Ego.NefsLib.Item; namespace VictorBush.Ego.NefsLib.Header.Version151; @@ -8,22 +7,32 @@ namespace VictorBush.Ego.NefsLib.Header.Version151; /// /// An entry in header part 2 for an item in an archive. /// -public sealed class Nefs151HeaderPart2Entry : INefsHeaderPartEntry +public sealed class Nefs150HeaderPart2Entry : INefsHeaderPartEntry { + public static readonly int EntrySize = Nefs150TocSharedEntryInfo.ByteCount; + private readonly Nefs150TocSharedEntryInfo data; + /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - internal Nefs151HeaderPart2Entry() + /// The underlying data. + internal Nefs150HeaderPart2Entry(Nefs150TocSharedEntryInfo? data = null) { + this.data = data ?? new Nefs150TocSharedEntryInfo(); } + /// + /// The underlying data. + /// + public Nefs150TocSharedEntryInfo Data => this.data; + /// /// The id of the directory this item belongs to. /// public NefsItemId DirectoryId { - get => new NefsItemId(Data_DirectoryId.Value); - init => Data_DirectoryId.Value = value.Value; + get => new(this.data.Parent); + init => this.data.Parent = value.Value; } /// @@ -32,8 +41,8 @@ public NefsItemId DirectoryId /// public NefsItemId SiblingId { - get => new NefsItemId(Data_SiblingId.Value); - init => Data_SiblingId.Value = value.Value; + get => new(this.data.NextSibling); + init => this.data.NextSibling = value.Value; } /// @@ -43,8 +52,8 @@ public NefsItemId SiblingId /// public NefsItemId FirstChildId { - get => new NefsItemId(Data_FirstChildId.Value); - init => Data_FirstChildId.Value = value.Value; + get => new(this.data.FirstChild); + init => this.data.FirstChild = value.Value; } /// @@ -52,8 +61,8 @@ public NefsItemId FirstChildId /// public uint OffsetIntoPart3 { - get => Data_OffsetIntoPart3.Value; - init => Data_OffsetIntoPart3.Value = value; + get => this.data.NameOffset; + init => this.data.NameOffset = value; } /// @@ -61,8 +70,8 @@ public uint OffsetIntoPart3 /// public uint ExtractedSize { - get => Data_ExtractedSize.Value; - init => Data_ExtractedSize.Value = value; + get => this.data.Size; + init => this.data.Size = value; } /// @@ -71,8 +80,8 @@ public uint ExtractedSize /// public NefsItemId Id { - get => new NefsItemId(Data_Id.Value); - init => Data_Id.Value = value.Value; + get => new(this.data.FirstDuplicate); + init => this.data.FirstDuplicate = value.Value; } /// @@ -80,30 +89,9 @@ public NefsItemId Id /// public NefsItemId Id2 { - get => new NefsItemId(Data_Id2.Value); - init => Data_Id2.Value = value.Value; + get => new(this.data.PatchedEntry); + init => this.data.PatchedEntry = value.Value; } - public int Size => Nefs151HeaderPart2.EntrySize; - - [FileData] - private UInt32Type Data_DirectoryId { get; } = new(0x00); - - [FileData] - private UInt32Type Data_SiblingId { get; } = new(0x04); - - [FileData] - private UInt32Type Data_FirstChildId { get; } = new(0x08); - - [FileData] - private UInt32Type Data_OffsetIntoPart3 { get; } = new(0x0C); - - [FileData] - private UInt32Type Data_ExtractedSize { get; } = new(0x10); - - [FileData] - private UInt32Type Data_Id { get; } = new(0x14); - - [FileData] - private UInt32Type Data_Id2 { get; } = new(0x18); + public int Size => EntrySize; } diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4.cs new file mode 100644 index 0000000..677459b --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4.cs @@ -0,0 +1,181 @@ +// See LICENSE.txt for license information. + +using Microsoft.Extensions.Logging; +using VictorBush.Ego.NefsLib.DataSource; +using VictorBush.Ego.NefsLib.Item; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// Header part 4. +/// +public sealed class Nefs150HeaderPart4 : INefsHeaderPart4 +{ + public const int LastValueSize = 0x4; + private static readonly ILogger Log = NefsLog.GetLogger(); + private readonly List entriesByIndex; + private readonly Dictionary indexLookup; + + /// + /// Initializes a new instance of the class. + /// + /// A collection of entries to initialize this object with. + /// + /// A dictionary that matches an item Guid to a part 4 index. This is used to find the correct index part 4 value + /// for an item. + /// + /// Last four bytes of part 4. + internal Nefs150HeaderPart4(IEnumerable entries, Dictionary indexLookup, uint unkownEndValue) + { + this.entriesByIndex = new List(entries); + this.indexLookup = new Dictionary(indexLookup); + UnkownEndValue = unkownEndValue; + } + + /// + /// Initializes a new instance of the class from a list of items. + /// + /// The items to initialize from. + /// Last four bytes of part 4. + internal Nefs150HeaderPart4(NefsItemList items, uint unkownEndValue) + { + this.entriesByIndex = new List(); + this.indexLookup = new Dictionary(); + UnkownEndValue = unkownEndValue; + + var nextStartIdx = 0U; + + foreach (var item in items.EnumerateById()) + { + if (item.Type == NefsItemType.Directory || item.DataSource.Size.Chunks.Count == 0) + { + // Item does not have a part 4 entry + continue; + } + + // Log this start index to item's Guid to allow lookup later + this.indexLookup.Add(item.Guid, nextStartIdx); + + // Create entry for each data chunk + foreach (var chunk in item.DataSource.Size.Chunks) + { + // Create entry + var entry = new Nefs150HeaderPart4Entry + { + CumulativeBlockSize = chunk.CumulativeSize, + TransformType = GetTransformType(chunk.Transform), + }; + this.entriesByIndex.Add(entry); + + nextStartIdx++; + } + } + } + + /// + /// List of data chunk info in order as they appear in the header. + /// + public IReadOnlyList EntriesByIndex => this.entriesByIndex; + + /// + IReadOnlyList INefsHeaderPart4.EntriesByIndex => this.entriesByIndex; + + /// + /// Gets the current size of header part 4. + /// + public int Size => (this.entriesByIndex.Count * Nefs150HeaderPart4Entry.EntrySize) + LastValueSize; + + /// + /// There is a 4-byte value at the end of header part 4. Purpose unknown. + /// + public uint UnkownEndValue { get; } + + /// + /// Creates a list of chunk metadata for an item. + /// + /// The part 4 index where the chunk list starts at. + /// The number of chunks. + /// The raw chunk size used in the transform. + /// The AES 256 key to use if chunk is encrypted. + /// A list of chunk data. + public List CreateChunksList(uint index, uint numChunks, uint chunkSize, byte[]? aes256key) + { + var chunks = new List(); + + for (var i = index; i < index + numChunks; ++i) + { + var entry = this.entriesByIndex[(int)i]; + var cumulativeSize = entry.CumulativeBlockSize; + var size = cumulativeSize; + + if (i > index) + { + size -= this.entriesByIndex[(int)i - 1].CumulativeBlockSize; + } + + // Determine transform + var transform = GetTransform(entry.TransformType, chunkSize, aes256key); + if (transform is null) + { + Log.LogError($"Found v1.5 data chunk with unknown transform ({entry.TransformType}); aborting."); + return new List(); + } + + // Create data chunk info + var chunk = new NefsDataChunk(size, cumulativeSize, transform); + chunks.Add(chunk); + } + + return chunks; + } + + /// + public uint GetIndexForItem(NefsItem item) + { + // Get index to part 4 + if (item.Type == NefsItemType.Directory) + { + // Item is a directory; the index 0 + return 0; + } + else + { + // Get index into part 4 + return this.indexLookup[item.Guid]; + } + } + + private NefsDataTransform? GetTransform(Nefs16HeaderPart4TransformType type, uint chunkSize, byte[]? aes256key) => + type switch + { + Nefs16HeaderPart4TransformType.Zlib => new NefsDataTransform(chunkSize, true), + Nefs16HeaderPart4TransformType.Aes => new NefsDataTransform(chunkSize, false, aes256key), + Nefs16HeaderPart4TransformType.Lzss => new NefsDataTransform(chunkSize, false) { IsLzssCompressed = true }, + Nefs16HeaderPart4TransformType.None => new NefsDataTransform(chunkSize, false), + _ => null, + }; + + private Nefs16HeaderPart4TransformType GetTransformType(NefsDataTransform transform) + { + // Can have both aes and zlib simulatneously? + if (transform.IsAesEncrypted && transform.IsZlibCompressed) + { + Log.LogWarning("Found multiple data transforms for header part 4 entry."); + } + + if (transform.IsAesEncrypted) + { + return Nefs16HeaderPart4TransformType.Aes; + } + else if (transform.IsZlibCompressed) + { + return Nefs16HeaderPart4TransformType.Zlib; + } + else if (transform.IsLzssCompressed) + { + return Nefs16HeaderPart4TransformType.Lzss; + } + + return Nefs16HeaderPart4TransformType.None; + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4Entry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4Entry.cs new file mode 100644 index 0000000..72b9ed7 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150HeaderPart4Entry.cs @@ -0,0 +1,45 @@ +// See LICENSE.txt for license information. + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// An entry in header part 4 for an item in an archive. +/// +public sealed class Nefs150HeaderPart4Entry : INefsHeaderPartEntry +{ + public static readonly int EntrySize = Nefs150TocBlock.ByteCount; + private readonly Nefs150TocBlock data; + + internal Nefs150HeaderPart4Entry(Nefs150TocBlock? data = null) + { + this.data = data ?? new Nefs150TocBlock(); + } + + /// + /// The underlying data. + /// + public Nefs150TocBlock Data => this.data; + + /// + /// Cumulative block size of this chunk. + /// + public uint CumulativeBlockSize + { + get => this.data.End; + init => this.data.End = value; + } + + /// + /// The size of a part 4 entry. This is used to get the offset into part 4 from an index into part 4. + /// + public int Size => EntrySize; + + /// + /// Transformation applied to this chunk. + /// + public Nefs16HeaderPart4TransformType TransformType + { + get => (Nefs16HeaderPart4TransformType)this.data.Transformation; + init => this.data.Transformation = (uint)value; + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocBlock.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocBlock.cs new file mode 100644 index 0000000..e3478da --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocBlock.cs @@ -0,0 +1,20 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs150TocBlock : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public uint End; + public uint Transformation; + + public void ReverseEndianness() + { + this.End = BinaryPrimitives.ReverseEndianness(this.End); + this.Transformation = BinaryPrimitives.ReverseEndianness(this.Transformation); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocEntry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocEntry.cs new file mode 100644 index 0000000..d04db00 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocEntry.cs @@ -0,0 +1,28 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs150TocEntry : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public ulong Start; + public ushort Volume; + public NefsTocEntryFlags Flags; + public uint SharedInfo; + public uint FirstBlock; + public uint NextDuplicate; + + public void ReverseEndianness() + { + this.Start = BinaryPrimitives.ReverseEndianness(this.Start); + this.Volume = BinaryPrimitives.ReverseEndianness(this.Volume); + this.Flags = (NefsTocEntryFlags)BinaryPrimitives.ReverseEndianness((ushort)this.Flags); + this.SharedInfo = BinaryPrimitives.ReverseEndianness(this.SharedInfo); + this.FirstBlock = BinaryPrimitives.ReverseEndianness(this.FirstBlock); + this.NextDuplicate = BinaryPrimitives.ReverseEndianness(this.NextDuplicate); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocHeader.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocHeader.cs new file mode 100644 index 0000000..0b95a14 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocHeader.cs @@ -0,0 +1,34 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs150TocHeader : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public uint Magic; + public uint TocSize; + public uint Version; + public uint NumVolumes; + public uint NumEntries; + public uint BlockSize; + public uint SplitSize; + public uint EntryTableStart; + public uint SharedEntryInfoTableStart; + public uint NameTableStart; + public uint BlockTableStart; + public uint VolumeInfoTableStart; + public AesKeyBuffer AesKeyBuffer; + + public unsafe void ReverseEndianness() + { + var buffer = new Span(Unsafe.AsPointer(ref this), 12); + for (var i = 0; i < buffer.Length; ++i) + { + buffer[i] = BinaryPrimitives.ReverseEndianness(buffer[i]); + } + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocSharedEntryInfo.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocSharedEntryInfo.cs new file mode 100644 index 0000000..e07410e --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocSharedEntryInfo.cs @@ -0,0 +1,30 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs150TocSharedEntryInfo : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public uint Parent; + public uint NextSibling; + public uint FirstChild; + public uint NameOffset; + public uint Size; + public uint FirstDuplicate; + public uint PatchedEntry; + + public void ReverseEndianness() + { + this.Parent = BinaryPrimitives.ReverseEndianness(this.Parent); + this.NextSibling = BinaryPrimitives.ReverseEndianness(this.NextSibling); + this.FirstChild = BinaryPrimitives.ReverseEndianness(this.FirstChild); + this.NameOffset = BinaryPrimitives.ReverseEndianness(this.NameOffset); + this.Size = BinaryPrimitives.ReverseEndianness(this.Size); + this.FirstDuplicate = BinaryPrimitives.ReverseEndianness(this.FirstDuplicate); + this.PatchedEntry = BinaryPrimitives.ReverseEndianness(this.PatchedEntry); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocVolumeInfo.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocVolumeInfo.cs new file mode 100644 index 0000000..d8fb9b3 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs150TocVolumeInfo.cs @@ -0,0 +1,22 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs150TocVolumeInfo : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public ulong Size; + public uint NameOffset; + public uint DataOffset; + + public void ReverseEndianness() + { + this.Size = BinaryPrimitives.ReverseEndianness(this.Size); + this.NameOffset = BinaryPrimitives.ReverseEndianness(this.NameOffset); + this.DataOffset = BinaryPrimitives.ReverseEndianness(this.DataOffset); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151Header.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151Header.cs index 4e53970..4e0bf9d 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151Header.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151Header.cs @@ -25,10 +25,10 @@ public sealed class Nefs151Header : INefsHeader /// Header part 5. public Nefs151Header( Nefs151HeaderIntro intro, - Nefs151HeaderPart1 part1, - Nefs151HeaderPart2 part2, + Nefs150HeaderPart1 part1, + Nefs150HeaderPart2 part2, NefsHeaderPart3 part3, - Nefs16HeaderPart4 part4, + Nefs151HeaderPart4 part4, NefsHeaderPart5 part5) { Intro = intro ?? throw new ArgumentNullException(nameof(intro)); @@ -44,10 +44,10 @@ public Nefs151Header( /// public bool IsEncrypted => Intro.IsEncrypted; - public Nefs151HeaderPart1 Part1 { get; } - public Nefs151HeaderPart2 Part2 { get; } + public Nefs150HeaderPart1 Part1 { get; } + public Nefs150HeaderPart2 Part2 { get; } public NefsHeaderPart3 Part3 { get; } - public Nefs16HeaderPart4 Part4 { get; } + public Nefs151HeaderPart4 Part4 { get; } public NefsHeaderPart5 Part5 { get; } /// @@ -61,7 +61,7 @@ public NefsItem CreateItemInfo(Guid guid, NefsItemList dataSourceList) { var p1 = Part1.EntriesByGuid[guid]; var p2 = Part2.EntriesByIndex[(int)p1.IndexPart2]; - var id = p1.Id; + var id = p2.Id; // Gather attributes var attributes = p1.CreateAttributes(); @@ -73,7 +73,7 @@ public NefsItem CreateItemInfo(Guid guid, NefsItemList dataSourceList) var dataOffset = (long)p1.OffsetToData; var extractedSize = p2.ExtractedSize; - // Transform TODO - what about AES here for 1.6? There is an aes attribute for chunks. + // Transform var transform = new NefsDataTransform(Intro.BlockSize, attributes.V20IsZlib, Intro.IsEncrypted ? Intro.GetAesKey() : null); // Data source diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderIntro.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderIntro.cs index 390e488..dc986ce 100644 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderIntro.cs +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderIntro.cs @@ -1,7 +1,6 @@ // See LICENSE.txt for license information. using System.Text; -using VictorBush.Ego.NefsLib.DataTypes; using VictorBush.Ego.NefsLib.Utility; namespace VictorBush.Ego.NefsLib.Header.Version151; @@ -11,14 +10,19 @@ namespace VictorBush.Ego.NefsLib.Header.Version151; /// public record Nefs151HeaderIntro : INefsHeaderIntro, INefsHeaderIntroToc { + private readonly Nefs151TocHeader data; + /// /// Initializes a new instance of the class. /// - public Nefs151HeaderIntro() + public Nefs151HeaderIntro(Nefs151TocHeader data) { - Data_MagicNumber.Value = NefsHeaderIntro.NefsMagicNumber; + this.data = data; } + /// + public bool IsLittleEndian { get; init; } + /// public bool IsEncrypted { get; init; } @@ -28,38 +32,38 @@ public Nefs151HeaderIntro() /// public uint MagicNumber { - get => Data_MagicNumber.Value; - init => Data_MagicNumber.Value = value; + get => this.data.Magic; + init => this.data.Magic = value; } /// public uint HeaderSize { - get => Data_HeaderSize.Value; - init => Data_HeaderSize.Value = value; + get => this.data.TocSize; + init => this.data.TocSize = value; } /// public uint NefsVersion { - get => Data_NefsVersion.Value; - init => Data_NefsVersion.Value = value; + get => this.data.Version; + init => this.data.Version = value; } /// - /// Unknown value. + /// The number of volumes. /// - public uint Unknown0x0C + public uint NumVolumes { - get => Data0x0C_Unknown.Value; - init => Data0x0C_Unknown.Value = value; + get => this.data.NumVolumes; + init => this.data.NumVolumes = value; } /// public uint NumberOfItems { - get => Data_NumberOfItems.Value; - init => Data_NumberOfItems.Value = value; + get => this.data.NumEntries; + init => this.data.NumEntries = value; } /// @@ -67,52 +71,52 @@ public uint NumberOfItems /// public uint BlockSize { - get => Data_BlockSize.Value; - init => Data_BlockSize.Value = value; + get => this.data.BlockSize; + init => this.data.BlockSize = value; } /// - /// Unknown value. + /// The split size. /// - public uint Unknown0x18 + public uint SplitSize { - get => Data0x18_Unknown.Value; - init => Data0x18_Unknown.Value = value; + get => this.data.SplitSize; + init => this.data.SplitSize = value; } /// public uint OffsetToPart1 { - get => Data_OffsetToPart1.Value; - init => Data_OffsetToPart1.Value = value; + get => this.data.EntryTableStart; + init => this.data.EntryTableStart = value; } /// public uint OffsetToPart2 { - get => Data_OffsetToPart2.Value; - init => Data_OffsetToPart2.Value = value; + get => this.data.SharedEntryInfoTableStart; + init => this.data.SharedEntryInfoTableStart = value; } /// public uint OffsetToPart3 { - get => Data_OffsetToPart3.Value; - init => Data_OffsetToPart3.Value = value; + get => this.data.NameTableStart; + init => this.data.NameTableStart = value; } /// public uint OffsetToPart4 { - get => Data_OffsetToPart4.Value; - init => Data_OffsetToPart4.Value = value; + get => this.data.BlockTableStart; + init => this.data.BlockTableStart = value; } /// public uint OffsetToPart5 { - get => Data_OffsetToPart5.Value; - init => Data_OffsetToPart5.Value = value; + get => this.data.VolumeInfoTableStart; + init => this.data.VolumeInfoTableStart = value; } /// @@ -130,44 +134,44 @@ public uint OffsetToPart5 /// /// Unknown value. /// - public uint Unknown0x30 + public uint Unknown1 { - get => Data0x30_Unknown.Value; - init => Data0x30_Unknown.Value = value; + get => this.data.Unknown1; + init => this.data.Unknown1 = value; } /// /// Unknown value. /// - public uint Unknown0x34 + public uint Unknown2 { - get => Data0x34_Unknown.Value; - init => Data0x34_Unknown.Value = value; + get => this.data.Unknown2; + init => this.data.Unknown2 = value; } /// /// Unknown value. /// - public uint Unknown0x38 + public uint Unknown3 { - get => Data0x38_Unknown.Value; - init => Data0x38_Unknown.Value = value; + get => this.data.Unknown3; + init => this.data.Unknown3 = value; } /// - public byte[] AesKeyHexString + public ReadOnlySpan AesKeyHexString { - get => Data_AesKeyHexString.Value; - init => Data_AesKeyHexString.Value = value; + get => this.data.AesKeyBuffer; + init => value.CopyTo(this.data.AesKeyBuffer); } /// /// Unknown value. /// - public uint Unknown0x7C + public uint Unknown4 { - get => Data0x7C_Unknown.Value; - init => Data0x7C_Unknown.Value = value; + get => this.data.Unknown4; + init => this.data.Unknown4 = value; } /// @@ -179,57 +183,6 @@ public uint Unknown0x7C /// public uint OffsetToPart8 => throw new NotSupportedException("Part8 not supported in 1.5.1."); - [FileData] - private UInt32Type Data_MagicNumber { get; } = new(0x0000); - - [FileData] - private UInt32Type Data_HeaderSize { get; } = new(0x0004); - - [FileData] - private UInt32Type Data_NefsVersion { get; } = new(0x0008); - - [FileData] - private UInt32Type Data0x0C_Unknown { get; } = new(0x000C); - - [FileData] - private UInt32Type Data_NumberOfItems { get; } = new(0x0010); - - [FileData] - private UInt32Type Data_BlockSize { get; } = new(0x0014); - - [FileData] - private UInt32Type Data0x18_Unknown { get; } = new(0x0018); - - [FileData] - private UInt32Type Data_OffsetToPart1 { get; } = new(0x001C); - - [FileData] - private UInt32Type Data_OffsetToPart2 { get; } = new(0x0020); - - [FileData] - private UInt32Type Data_OffsetToPart3 { get; } = new(0x0024); - - [FileData] - private UInt32Type Data_OffsetToPart4 { get; } = new(0x0028); - - [FileData] - private UInt32Type Data_OffsetToPart5 { get; } = new(0x002C); - - [FileData] - private UInt32Type Data0x30_Unknown { get; } = new(0x0030); - - [FileData] - private UInt32Type Data0x34_Unknown { get; } = new(0x0034); - - [FileData] - private UInt32Type Data0x38_Unknown { get; } = new(0x0038); - - [FileData] - private ByteArrayType Data_AesKeyHexString { get; } = new(0x003C, 0x40); - - [FileData] - private UInt32Type Data0x7C_Unknown { get; } = new(0x007C); - /// public byte[] GetAesKey() { diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1.cs deleted file mode 100644 index 9fb24e8..0000000 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1.cs +++ /dev/null @@ -1,87 +0,0 @@ -// See LICENSE.txt for license information. - -using VictorBush.Ego.NefsLib.Item; - -namespace VictorBush.Ego.NefsLib.Header.Version151; - -/// -/// Header part 1. The "master catalog" of items in the archive. -/// -public sealed class Nefs151HeaderPart1 -{ - /// - /// The size of a part 1 entry. - /// - public const int EntrySize = 0x18; - - private readonly Dictionary entriesByGuid; - private readonly List entriesByIndex; - - /// - /// Initializes a new instance of the class. - /// - /// A list of entries to instantiate this part with. - internal Nefs151HeaderPart1(IList entries) - { - this.entriesByIndex = new List(entries); - this.entriesByGuid = new Dictionary(entries.ToDictionary(e => e.Guid)); - } - - /// - /// Initializes a new instance of the class. - /// - /// The list of items in the archive. - /// Header part 4. - internal Nefs151HeaderPart1(NefsItemList items, INefsHeaderPart4 part4) - { - this.entriesByGuid = new Dictionary(); - var indexPart2 = 0U; - - // Enumerate this list depth first. This determines the part 2 order. The part 1 entries will be sorted by item id. - foreach (var item in items.EnumerateDepthFirstByName()) - { - var flags = Nefs16HeaderPart6Flags.None; - flags |= item.Attributes.V16IsTransformed ? Nefs16HeaderPart6Flags.IsTransformed : 0; - flags |= item.Attributes.IsDirectory ? Nefs16HeaderPart6Flags.IsDirectory : 0; - flags |= item.Attributes.IsDuplicated ? Nefs16HeaderPart6Flags.IsDuplicated : 0; - flags |= item.Attributes.IsCacheable ? Nefs16HeaderPart6Flags.IsCacheable : 0; - flags |= item.Attributes.V16Unknown0x10 ? Nefs16HeaderPart6Flags.Unknown0x10 : 0; - flags |= item.Attributes.IsPatched ? Nefs16HeaderPart6Flags.IsPatched : 0; - flags |= item.Attributes.V16Unknown0x40 ? Nefs16HeaderPart6Flags.Unknown0x40 : 0; - flags |= item.Attributes.V16Unknown0x80 ? Nefs16HeaderPart6Flags.Unknown0x80 : 0; - - var entry = new Nefs151HeaderPart1Entry(item.Guid) - { - Guid = item.Guid, - Id = new NefsItemId(item.Id.Value), - IndexPart2 = indexPart2++, - IndexPart4 = part4.GetIndexForItem(item), - OffsetToData = (ulong)item.DataSource.Offset, - Volume = item.Attributes.Part6Volume, - Flags = flags, - Unknown0x0B = item.Attributes.Part6Unknown0x3 - }; - - this.entriesByGuid.Add(item.Guid, entry); - } - - // Sort part 1 by item id - this.entriesByIndex = new List(this.entriesByGuid.Values.OrderBy(e => e.Id)); - } - - /// - /// Gets entries for each item in the archive, accessible by Guid. - /// - public IReadOnlyDictionary EntriesByGuid => this.entriesByGuid; - - /// - /// Gets the list of entries in the order they appear in the header. Usually items are sorted by id, but this is not - /// guaranteed (for example, DiRT Rally has a header with items out of order). - /// - public IList EntriesByIndex => this.entriesByIndex; - - /// - /// Total size (in bytes) of part 1. - /// - public int Size => this.entriesByIndex.Count * EntrySize; -} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1Entry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1Entry.cs deleted file mode 100644 index 025119f..0000000 --- a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart1Entry.cs +++ /dev/null @@ -1,130 +0,0 @@ -// See LICENSE.txt for license information. - -using VictorBush.Ego.NefsLib.DataTypes; -using VictorBush.Ego.NefsLib.Item; - -namespace VictorBush.Ego.NefsLib.Header.Version151; - -/// -/// An entry in header part 1 for an item in an archive. -/// -public sealed class Nefs151HeaderPart1Entry : INefsHeaderPartEntry -{ - /// - /// Initializes a new instance of the class. - /// - /// The Guid of the item this metadata belongs to. - internal Nefs151HeaderPart1Entry(Guid guid) - { - Guid = guid; - } - - /// - /// The unique identifier of the item this data is for. - /// - public Guid Guid { get; init; } - - /// - /// The absolute offset to the file's data in the archive. For directories, this is 0. - /// - public ulong OffsetToData - { - get => Data_OffsetToData.Value; - init => Data_OffsetToData.Value = value; - } - - /// - /// Unknown. - /// - public ushort Volume - { - get => Data_Volume.Value; - init => Data_Volume.Value = value; - } - - /// - /// A bitfield that has various flags. - /// - public Nefs16HeaderPart6Flags Flags - { - get => (Nefs16HeaderPart6Flags)Data_Flags.Value; - init => Data_Flags.Value = (byte)value; - } - - /// - /// Unknown. - /// - public byte Unknown0x0B - { - get => Data0x0B_Unknown.Value; - init => Data0x0B_Unknown.Value = value; - } - - /// - /// The index used for parts 2 for this item. - /// - public uint IndexPart2 - { - get => Data_IndexPart2.Value; - init => Data_IndexPart2.Value = value; - } - - /// - /// The index into header part 4 for this item. For the actual offset. - /// - public uint IndexPart4 - { - get => Data_IndexPart4.Value; - init => Data_IndexPart4.Value = value; - } - - /// - /// The id of the item. It is possible to have duplicate item's with the same id. - /// - public NefsItemId Id - { - get => new NefsItemId(Data_Id.Value); - init => Data_Id.Value = value.Value; - } - - public int Size => Nefs151HeaderPart1.EntrySize; - - [FileData] - private UInt64Type Data_OffsetToData { get; } = new(0x00); - - [FileData] - private UInt16Type Data_Volume { get; } = new(0x08); - - [FileData] - private UInt8Type Data_Flags { get; } = new(0x0A); - - [FileData] - private UInt8Type Data0x0B_Unknown { get; } = new(0x0B); - - [FileData] - private UInt32Type Data_IndexPart2 { get; } = new(0x0C); - - [FileData] - private UInt32Type Data_IndexPart4 { get; } = new(0x10); - - [FileData] - private UInt32Type Data_Id { get; } = new(0x14); - - /// - /// Creates a object. - /// - public NefsItemAttributes CreateAttributes() - { - return new NefsItemAttributes( - v16IsTransformed: Flags.HasFlag(Nefs16HeaderPart6Flags.IsTransformed), - isDirectory: Flags.HasFlag(Nefs16HeaderPart6Flags.IsDirectory), - isDuplicated: Flags.HasFlag(Nefs16HeaderPart6Flags.IsDuplicated), - isCacheable: Flags.HasFlag(Nefs16HeaderPart6Flags.IsCacheable), - v16Unknown0x10: Flags.HasFlag(Nefs16HeaderPart6Flags.Unknown0x10), - isPatched: Flags.HasFlag(Nefs16HeaderPart6Flags.IsPatched), - v16Unknown0x40: Flags.HasFlag(Nefs16HeaderPart6Flags.Unknown0x40), - v16Unknown0x80: Flags.HasFlag(Nefs16HeaderPart6Flags.Unknown0x80), - part6Volume: Volume, - part6Unknown0x3: Unknown0x0B); - } -} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4.cs new file mode 100644 index 0000000..a1749b5 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4.cs @@ -0,0 +1,177 @@ +// See LICENSE.txt for license information. + +using Microsoft.Extensions.Logging; +using VictorBush.Ego.NefsLib.DataSource; +using VictorBush.Ego.NefsLib.Item; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// Header part 4. +/// +public sealed class Nefs151HeaderPart4 : INefsHeaderPart4 +{ + public const int LastValueSize = 0x4; + private static readonly ILogger Log = NefsLog.GetLogger(); + private readonly List entriesByIndex; + private readonly Dictionary indexLookup; + + /// + /// Initializes a new instance of the class. + /// + /// A collection of entries to initialize this object with. + /// + /// A dictionary that matches an item Guid to a part 4 index. This is used to find the correct index part 4 value + /// for an item. + /// + /// Last four bytes of part 4. + internal Nefs151HeaderPart4(IEnumerable entries, Dictionary indexLookup, uint unkownEndValue) + { + this.entriesByIndex = new List(entries); + this.indexLookup = new Dictionary(indexLookup); + UnkownEndValue = unkownEndValue; + } + + /// + /// Initializes a new instance of the class from a list of items. + /// + /// The items to initialize from. + /// Last four bytes of part 4. + internal Nefs151HeaderPart4(NefsItemList items, uint unkownEndValue) + { + this.entriesByIndex = new List(); + this.indexLookup = new Dictionary(); + UnkownEndValue = unkownEndValue; + + var nextStartIdx = 0U; + + foreach (var item in items.EnumerateById()) + { + if (item.Type == NefsItemType.Directory || item.DataSource.Size.Chunks.Count == 0) + { + // Item does not have a part 4 entry + continue; + } + + // Log this start index to item's Guid to allow lookup later + this.indexLookup.Add(item.Guid, nextStartIdx); + + // Create entry for each data chunk + foreach (var chunk in item.DataSource.Size.Chunks) + { + // Create entry + var entry = new Nefs151HeaderPart4Entry + { + Checksum = 0x848, // TODO - How to compute this value is unknown. Writing bogus data for now. + CumulativeBlockSize = chunk.CumulativeSize, + TransformType = GetTransformType(chunk.Transform), + }; + this.entriesByIndex.Add(entry); + + nextStartIdx++; + } + } + } + + /// + /// List of data chunk info in order as they appear in the header. + /// + public IReadOnlyList EntriesByIndex => this.entriesByIndex; + + /// + IReadOnlyList INefsHeaderPart4.EntriesByIndex => this.entriesByIndex; + + /// + /// Gets the current size of header part 4. + /// + public int Size => (this.entriesByIndex.Count * Nefs151HeaderPart4Entry.EntrySize) + LastValueSize; + + /// + /// There is a 4-byte value at the end of header part 4. Purpose unknown. + /// + public uint UnkownEndValue { get; } + + /// + /// Creates a list of chunk metadata for an item. + /// + /// The part 4 index where the chunk list starts at. + /// The number of chunks. + /// The raw chunk size used in the transform. + /// The AES 256 key to use if chunk is encrypted. + /// A list of chunk data. + public List CreateChunksList(uint index, uint numChunks, uint chunkSize, byte[]? aes256key) + { + var chunks = new List(); + + for (var i = index; i < index + numChunks; ++i) + { + var entry = this.entriesByIndex[(int)i]; + var cumulativeSize = entry.CumulativeBlockSize; + var size = cumulativeSize; + + if (i > index) + { + size -= this.entriesByIndex[(int)i - 1].CumulativeBlockSize; + } + + // Determine transform + var transform = GetTransform(entry.TransformType, chunkSize, aes256key); + if (transform is null) + { + Log.LogError($"Found v1.5 data chunk with unknown transform ({entry.TransformType}); aborting."); + return new List(); + } + + // Create data chunk info + var chunk = new NefsDataChunk(size, cumulativeSize, transform); + chunks.Add(chunk); + } + + return chunks; + } + + /// + public uint GetIndexForItem(NefsItem item) + { + // Get index to part 4 + if (item.Type == NefsItemType.Directory) + { + // Item is a directory; the index 0 + return 0; + } + else + { + // Get index into part 4 + return this.indexLookup[item.Guid]; + } + } + + private NefsDataTransform? GetTransform(Nefs16HeaderPart4TransformType type, uint chunkSize, byte[]? aes256key) => type switch + { + Nefs16HeaderPart4TransformType.Zlib => new NefsDataTransform(chunkSize, true), + Nefs16HeaderPart4TransformType.Aes => new NefsDataTransform(chunkSize, false, aes256key), + Nefs16HeaderPart4TransformType.Lzss => new NefsDataTransform(chunkSize, false) { IsLzssCompressed = true }, + Nefs16HeaderPart4TransformType.None => new NefsDataTransform(chunkSize, false), + _ => null, + }; + + private Nefs16HeaderPart4TransformType GetTransformType(NefsDataTransform transform) + { + // Can have both aes and zlib simulatneously? + if (transform.IsAesEncrypted && transform.IsZlibCompressed) + { + Log.LogWarning("Found multiple data transforms for header part 4 entry."); + } + + if (transform.IsAesEncrypted) + { + return Nefs16HeaderPart4TransformType.Aes; + } + else if (transform.IsZlibCompressed) + { + return Nefs16HeaderPart4TransformType.Zlib; + } + + return Nefs16HeaderPart4TransformType.None; + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4Entry.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4Entry.cs new file mode 100644 index 0000000..5d56a19 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151HeaderPart4Entry.cs @@ -0,0 +1,54 @@ +// See LICENSE.txt for license information. + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +/// +/// An entry in header part 4 for an item in an archive. +/// +public sealed class Nefs151HeaderPart4Entry : INefsHeaderPartEntry +{ + public static readonly int EntrySize = Nefs151TocBlock.ByteCount; + private readonly Nefs151TocBlock data; + + internal Nefs151HeaderPart4Entry(Nefs151TocBlock? data = null) + { + this.data = data ?? new Nefs151TocBlock(); + } + + /// + /// The underlying data. + /// + public Nefs151TocBlock Data => this.data; + + /// + /// Cumulative block size of this chunk. + /// + public uint CumulativeBlockSize + { + get => this.data.End; + init => this.data.End = value; + } + + /// + /// The size of a part 4 entry. This is used to get the offset into part 4 from an index into part 4. + /// + public int Size => EntrySize; + + /// + /// Transformation applied to this chunk. + /// + public Nefs16HeaderPart4TransformType TransformType + { + get => (Nefs16HeaderPart4TransformType)this.data.Transformation; + init => this.data.Transformation = (ushort)value; + } + + /// + /// Checksum of the chunk. + /// + public ushort Checksum + { + get => this.data.Checksum; + init => this.data.Checksum = value; + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocBlock.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocBlock.cs new file mode 100644 index 0000000..7d605a2 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocBlock.cs @@ -0,0 +1,22 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs151TocBlock : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public uint End; + public ushort Transformation; + public ushort Checksum; + + public void ReverseEndianness() + { + this.End = BinaryPrimitives.ReverseEndianness(this.End); + this.Transformation = BinaryPrimitives.ReverseEndianness(this.Transformation); + this.Checksum = BinaryPrimitives.ReverseEndianness(this.Checksum); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocHeader.cs b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocHeader.cs new file mode 100644 index 0000000..54dad5a --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/Header/Version151/Nefs151TocHeader.cs @@ -0,0 +1,41 @@ +// See LICENSE.txt for license information. + +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace VictorBush.Ego.NefsLib.Header.Version151; + +public struct Nefs151TocHeader : INefsTocData +{ + public static int ByteCount { get; } = Unsafe.SizeOf(); + + public uint Magic; + public uint TocSize; + public uint Version; + public uint NumVolumes; + public uint NumEntries; + public uint BlockSize; + public uint SplitSize; + public uint EntryTableStart; + public uint SharedEntryInfoTableStart; + public uint NameTableStart; + public uint BlockTableStart; + public uint VolumeInfoTableStart; + public uint Unknown1; + public uint Unknown2; + public uint Unknown3; + public AesKeyBuffer AesKeyBuffer; + public uint Unknown4; + + public unsafe void ReverseEndianness() + { + var buffer = new Span(Unsafe.AsPointer(ref this), 15); + for (var i = 0; i < buffer.Length; ++i) + { + buffer[i] = BinaryPrimitives.ReverseEndianness(buffer[i]); + } + + // Leave Unknown4 alone for now. Not sure if actually part of header + } +} diff --git a/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryReader.cs b/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryReader.cs new file mode 100644 index 0000000..55c1b36 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryReader.cs @@ -0,0 +1,57 @@ +// See LICENSE.txt for license information. + +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using VictorBush.Ego.NefsLib.Header; + +namespace VictorBush.Ego.NefsLib.IO; + +public class EndianBinaryReader(Stream stream, bool littleEndian, ResizableBuffer buffer) : IDisposable +{ + private ResizableBuffer buffer = buffer; + + public Stream BaseStream { get; } = stream; + + public bool IsLittleEndian { get; } = littleEndian; + + public EndianBinaryReader(Stream stream) : this(stream, BitConverter.IsLittleEndian) + { + } + + public EndianBinaryReader(Stream stream, bool littleEndian) : this(stream, littleEndian, + new ResizableBuffer(ArrayPool.Shared, 16)) + { + } + + public async ValueTask ReadUInt32Async(CancellationToken cancellationToken = default) + { + await BaseStream.ReadExactlyAsync(this.buffer.Memory[..4], cancellationToken).ConfigureAwait(false); + return IsLittleEndian + ? BinaryPrimitives.ReadUInt32LittleEndian(this.buffer.Memory.Span) + : BinaryPrimitives.ReadUInt32BigEndian(this.buffer.Memory.Span); + } + + public async ValueTask ReadTocDataAsync(CancellationToken cancellationToken = default) + where T : unmanaged, INefsTocData + { + var size = T.ByteCount; + this.buffer.EnsureLength(size); + var buff = this.buffer.Memory[..size]; + + await BaseStream.ReadExactlyAsync(buff, cancellationToken).ConfigureAwait(false); + var data = Unsafe.As(ref MemoryMarshal.GetReference(this.buffer.Memory.Span)); + if (IsLittleEndian != BitConverter.IsLittleEndian) + { + data.ReverseEndianness(); + } + + return data; + } + + public void Dispose() + { + this.buffer.Dispose(); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryWriter.cs b/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryWriter.cs new file mode 100644 index 0000000..77294ae --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/IO/EndianBinaryWriter.cs @@ -0,0 +1,61 @@ +// See LICENSE.txt for license information. + +using System.Buffers; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using VictorBush.Ego.NefsLib.Header; + +namespace VictorBush.Ego.NefsLib.IO; + +public class EndianBinaryWriter(Stream stream, bool littleEndian, ResizableBuffer buffer) : IDisposable +{ + private ResizableBuffer buffer = buffer; + + public Stream BaseStream { get; } = stream; + + public bool IsLittleEndian { get; } = littleEndian; + + public EndianBinaryWriter(Stream stream) : this(stream, BitConverter.IsLittleEndian) + { + } + + public EndianBinaryWriter(Stream stream, bool littleEndian) : this(stream, littleEndian, + new ResizableBuffer(ArrayPool.Shared, 16)) + { + } + + public ValueTask WriteAsync(uint value, CancellationToken cancellationToken = default) + { + if (IsLittleEndian) + { + BinaryPrimitives.WriteUInt32LittleEndian(this.buffer.Memory.Span, value); + } + else + { + BinaryPrimitives.WriteUInt32BigEndian(this.buffer.Memory.Span, value); + } + + return BaseStream.WriteAsync(this.buffer.Memory[..4], cancellationToken); + } + + public ValueTask WriteTocDataAsync(T data, CancellationToken cancellationToken = default) + where T : unmanaged, INefsTocData + { + if (IsLittleEndian != BitConverter.IsLittleEndian) + { + data.ReverseEndianness(); + } + + var size = T.ByteCount; + this.buffer.EnsureLength(size); + var buff = this.buffer.Memory[..size]; + Unsafe.As(ref MemoryMarshal.GetReference(buff.Span)) = data; + return BaseStream.WriteAsync(buff, cancellationToken); + } + + public void Dispose() + { + this.buffer.Dispose(); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/IO/LzssDecompress.cs b/VictorBush.Ego.NefsLib/Source/IO/LzssDecompress.cs new file mode 100644 index 0000000..74beb49 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/IO/LzssDecompress.cs @@ -0,0 +1,130 @@ +// See LICENSE.txt for license information. + +using System.Buffers; +using System.Diagnostics; + +namespace VictorBush.Ego.NefsLib.IO; + +public class LzssDecompress +{ + private const byte BufferDefaultValue = 32; + private const int BufferHeadStart = 4078; + private const int RingBufferSize = 4113; + + private readonly byte[] ringBuffer; + private int ringBufferHead; + private bool history; + private byte historyLsb; + private int historyLength; + private int historyOffset; + private int controlFlags; + + public LzssDecompress() + { + this.ringBuffer = new byte[RingBufferSize]; + Reset(); + } + + public void Reset() + { + Array.Fill(this.ringBuffer, BufferDefaultValue); + this.controlFlags = 0; + this.historyLsb = 0; + this.ringBufferHead = BufferHeadStart; + this.history = false; + this.historyLength = 0; + this.historyOffset = 0; + } + + public (int Consumed, int Written) Decompress(Span input, Span output) + { + var i = 0; + var io = 0; + while (true) + { + while (this.historyLength > 0) + { + if (io >= output.Length) + { + break; + } + + var v14 = this.ringBuffer[this.historyOffset]; + --this.historyLength; + this.historyOffset = (this.historyOffset + 1) & 0xFFF; + output[io++] = v14; + this.ringBuffer[this.ringBufferHead] = v14; + this.ringBufferHead = (this.ringBufferHead + 1) & 0xFFF; + } + + if (i >= input.Length || io >= output.Length) + { + break; + } + + var b = input[i++]; + if (this.controlFlags > 255) + { + if ((this.controlFlags & 1) != 0) + { + output[io++] = b; + this.controlFlags >>= 1; + this.ringBuffer[this.ringBufferHead] = b; + this.ringBufferHead = (this.ringBufferHead + 1) & 0xFFF; + } + else if (this.history) + { + this.controlFlags >>= 1; + this.history = false; + this.historyOffset = this.historyLsb | (16 * (b & 0xF0)); + this.historyLength = (b & 0xF) + 3; + } + else + { + this.historyLsb = b; + this.history = true; + } + } + else + { + this.controlFlags = b | 0xFF00; + } + } + + return (i, io); + } + + public async Task DecompressAsync(Stream source, Stream destination, CancellationToken cancellationToken) + { + var buffer = ArrayPool.Shared.Rent(8192); + var outBuffer = ArrayPool.Shared.Rent(8192); + try + { + int bytesRead; + while ((bytesRead = + await source.ReadAsync(new Memory(buffer), cancellationToken).ConfigureAwait(false)) != 0) + { + var consumed = 0; + while (consumed < bytesRead) + { + // loop in case output buffer wasn't large enough + var pos = Decompress(buffer.AsSpan(consumed..bytesRead), outBuffer.AsSpan()); + await destination.WriteAsync(new ReadOnlyMemory(outBuffer, 0, pos.Written), cancellationToken) + .ConfigureAwait(false); + consumed += pos.Consumed; + } + } + + if (this.historyLength > 0) + { + Debug.Fail("LZSS decompression failed."); + throw new IOException("Failed to decompress data"); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(outBuffer); + } + } +} diff --git a/VictorBush.Ego.NefsLib/Source/IO/NefsReader.cs b/VictorBush.Ego.NefsLib/Source/IO/NefsReader.cs index 052b74a..06d490d 100644 --- a/VictorBush.Ego.NefsLib/Source/IO/NefsReader.cs +++ b/VictorBush.Ego.NefsLib/Source/IO/NefsReader.cs @@ -1,8 +1,11 @@ // See LICENSE.txt for license information. +using System.Buffers.Binary; +using System.Diagnostics; using Microsoft.Extensions.Logging; using System.IO.Abstractions; using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security.Cryptography; using System.Text; @@ -245,7 +248,7 @@ internal async Task DecryptHeaderIntroAsync( } internal readonly record struct DecryptHeaderIntroResult(bool Succeeded, bool IsEncrypted = false, - bool IsXorEncoded = false); + bool IsXorEncoded = false, bool IsLittleEndian = false); internal async Task ReadHeaderIntroAsync(Stream stream, long offset, Stream outDecryptStream, @@ -254,11 +257,11 @@ internal async Task ReadHeaderIntroAsync(Stream stream var isXorEncoded = false; byte[]? decodedData = null; - var validMagicNum = await ValidateMagicNumberAsync(stream, offset, p); + var (validMagicNum, isLittleEndian) = await ValidateMagicNumberAsync(stream, offset, p); if (!validMagicNum) { // Check for v1.5.1 xor encoding - validMagicNum = await ValidateXorMagicNumberAsync(stream, offset, p); + (validMagicNum, isLittleEndian) = await ValidateXorMagicNumberAsync(stream, offset, p); if (validMagicNum) { isXorEncoded = true; @@ -284,11 +287,11 @@ internal async Task ReadHeaderIntroAsync(Stream stream } Log.LogError("Failed to decrypt header."); - return new DecryptHeaderIntroResult(false, encrypted); + return new DecryptHeaderIntroResult(false, encrypted, IsLittleEndian:isLittleEndian); } else { - return new DecryptHeaderIntroResult(false, encrypted); + return new DecryptHeaderIntroResult(false, encrypted, IsLittleEndian:isLittleEndian); } } @@ -302,7 +305,7 @@ internal async Task ReadHeaderIntroAsync(Stream stream await stream.CopyToAsync(outDecryptStream, p.CancellationToken); } - return new DecryptHeaderIntroResult(true, encrypted, isXorEncoded); + return new DecryptHeaderIntroResult(true, encrypted, isXorEncoded, isLittleEndian); } internal async Task ReadHeaderIntroAsync(Stream stream, long offset, Stream outDecryptStream, @@ -326,15 +329,18 @@ internal async Task ReadHeaderIntroAsync(Stream stream, long o using (p.BeginTask(0.8f, "Reading header intro")) { - var oldVersion = await ReadVersionV151Async(introStream, introOffset, p); + var oldVersion = await ReadVersionV151Async(introStream, introOffset, readResult.IsLittleEndian, p); if (oldVersion is (uint)NefsVersion.Version140) { throw new NotImplementedException("Support for version 1.4.0 is not implemented."); } - if (oldVersion is (uint)NefsVersion.Version151) + if (oldVersion is (uint)NefsVersion.Version150) + { + intro = await ReadHeaderIntroV150Async(introStream, introOffset, readResult, p); + } + else if (oldVersion is (uint)NefsVersion.Version151) { - // this must be version < 1.6.0 (so far known to be 1.5.1) intro = await ReadHeaderIntroV151Async(introStream, introOffset, readResult, p); } else @@ -384,7 +390,7 @@ internal async Task ReadHeaderIntroAsync(Stream stream, long o var hiPart = new UInt16Type(NefsHeaderIntro.Size - 2); await hiPart.ReadAsync(introStream, introOffset, p); var intro151 = (Nefs151HeaderIntro)intro; - intro = intro151 with { Unknown0x7C = intro151.Unknown0x7C | ((uint)hiPart.Value << 16) }; + intro = intro151 with { Unknown4 = intro151.Unknown4 | ((uint)hiPart.Value << 16) }; } // Debug: for copying to hex editor @@ -443,6 +449,12 @@ internal async Task ReadHeaderAsync(Stream stream, long offset, Nef Log.LogInformation("Detected NeFS version 1.5.1."); header = await ReadHeaderV151Async(headerStream, offset, (Nefs151HeaderIntro)intro, p); } + else if (intro.NefsVersion is (uint)NefsVersion.Version150) + { + // 1.5.0 + Log.LogInformation("Detected NeFS version 1.5.0."); + header = await ReadHeaderV150Async(headerStream, offset, (Nefs150HeaderIntro)intro, p); + } else { Log.LogError($"Detected unknown NeFS version {intro.NefsVersion}. Treating as 2.0."); @@ -461,6 +473,36 @@ internal async Task ReadHeaderAsync(Stream stream, long offset, Nef return header; } + /// + /// Reads the 1.5.0 header from an input stream. + /// + /// + /// Reads the header intro from an input stream. This is for non-encrypted headers only. + /// + /// The stream to read from. + /// The offset to the header intro from the beginning of the stream. + /// Decryption state. + /// Progress info. + /// The loaded header intro. + internal static async Task ReadHeaderIntroV150Async( + Stream stream, + long offset, + DecryptHeaderIntroResult decryptResult, + NefsProgress p) + { + stream.Seek(offset, SeekOrigin.Begin); + using var reader = new EndianBinaryReader(stream, decryptResult.IsLittleEndian); + var header = await reader.ReadTocDataAsync(p.CancellationToken).ConfigureAwait(false); + + var intro = new Nefs150HeaderIntro(header) + { + IsEncrypted = decryptResult.IsEncrypted, + IsXorEncoded = decryptResult.IsXorEncoded, + IsLittleEndian = decryptResult.IsLittleEndian + }; + return intro; + } + /// /// Reads the 1.5.1 header from an input stream. /// @@ -479,10 +521,15 @@ internal static async Task ReadHeaderIntroV151Async( NefsProgress p) { stream.Seek(offset, SeekOrigin.Begin); + using var reader = new EndianBinaryReader(stream, decryptResult.IsLittleEndian); + var header = await reader.ReadTocDataAsync(p.CancellationToken).ConfigureAwait(false); - var intro = new Nefs151HeaderIntro - { IsEncrypted = decryptResult.IsEncrypted, IsXorEncoded = decryptResult.IsXorEncoded }; - await FileData.ReadDataAsync(stream, offset, intro, p); + var intro = new Nefs151HeaderIntro(header) + { + IsEncrypted = decryptResult.IsEncrypted, + IsXorEncoded = decryptResult.IsXorEncoded, + IsLittleEndian = decryptResult.IsLittleEndian + }; return intro; } @@ -540,41 +587,53 @@ internal async Task ReadHeaderIntroTocVersion20Async(Strea } /// - /// Reads 1.5.1 header part 1 from an input stream. + /// Reads ToC entries from an input stream. /// - /// The stream to read from. + /// The reader to use. /// The offset to the header part from the beginning of the stream. /// The size of the header part. /// Progress info. - /// The loaded header part. - internal async Task Read151HeaderPart1Async(Stream stream, long offset, int size, + /// The ToC entries. + internal async ValueTask ReadTocEntriesAsync(EndianBinaryReader reader, long offset, int size, NefsProgress p) + where T : unmanaged, INefsTocData { - var entries = new List(); + var stream = reader.BaseStream; // Validate inputs - if (!ValidateHeaderPartStream(stream, offset, size, "1")) + if (!ValidateHeaderPartStream(stream, offset, size, nameof(T))) { - return new Nefs151HeaderPart1(entries); + return []; } - // Get entries in part 1 - var numEntries = size / Nefs151HeaderPart1.EntrySize; - var entryOffset = offset; - + // Get entries + stream.Seek(offset, SeekOrigin.Begin); + var numEntries = size / T.ByteCount; + var entries = new T[numEntries]; for (var i = 0; i < numEntries; ++i) { using (p.BeginTask(1.0f / numEntries)) { - var guid = Guid.NewGuid(); - var entry = new Nefs151HeaderPart1Entry(guid); - await FileData.ReadDataAsync(stream, entryOffset, entry, p); - entryOffset += Nefs151HeaderPart1.EntrySize; - entries.Add(entry); + entries[i] = await reader.ReadTocDataAsync(p.CancellationToken).ConfigureAwait(false); } } - return new Nefs151HeaderPart1(entries); + return entries; + } + + /// + /// Reads 1.5.0 header part 1 from an input stream. + /// + /// The reader to use. + /// The offset to the header part from the beginning of the stream. + /// The size of the header part. + /// Progress info. + /// The loaded header part. + internal async ValueTask Read150HeaderPart1Async(EndianBinaryReader reader, long offset, int size, + NefsProgress p) + { + var entries = await ReadTocEntriesAsync(reader, offset, size, p).ConfigureAwait(false); + return new Nefs150HeaderPart1(entries.Select(x => new Nefs150HeaderPart1Entry(Guid.NewGuid(), x)).ToList()); } /// @@ -615,47 +674,19 @@ internal async Task ReadHeaderPart1Async(Stream stream, long of } /// - /// Reads 1.5.1 header part 2 from an input stream. + /// Reads 1.5.0 header part 2 from an input stream. /// - /// The stream to read from. + /// The reader to use. /// The offset to the header part from the beginning of the stream. /// The size of the header part. /// Progress info. /// The loaded header part. - internal async Task Read151HeaderPart2Async(Stream stream, long offset, int size, + internal async Task Read150HeaderPart2Async(EndianBinaryReader reader, long offset, int size, NefsProgress p) { - var entries = new List(); - - // Validate inputs - if (!ValidateHeaderPartStream(stream, offset, size, "2")) - { - return new Nefs151HeaderPart2(entries); - } - - // Get entries in part 2 - var numEntries = size / Nefs151HeaderPart2.EntrySize; - var entryOffset = offset; - - for (var i = 0; i < numEntries; ++i) - { - using (p.BeginTask(1.0f / numEntries)) - { - var entry = new Nefs151HeaderPart2Entry(); - await FileData.ReadDataAsync(stream, entryOffset, entry, p); - entryOffset += Nefs151HeaderPart2.EntrySize; - - entries.Add(entry); - - // TODO: figure out id vs id2, for now throw - if (entry.Id != entry.Id2) - { - throw new NotImplementedException("Proper understanding of part 2 ids not implemented."); - } - } - } - - return new Nefs151HeaderPart2(entries); + var entries = await ReadTocEntriesAsync(reader, offset, size, p).ConfigureAwait(false); + Debug.Assert(entries.All(x => x.FirstDuplicate == x.PatchedEntry)); + return new Nefs150HeaderPart2(entries.Select(x => new Nefs150HeaderPart2Entry(x)).ToList()); } /// @@ -760,43 +791,49 @@ internal async Task ReadHeaderPart3Async(Stream stream, long of } /// - /// Reads 1.5.1 header part 4 from an input stream. + /// Reads 1.5.0 header part 4 from an input stream. /// - /// The stream to read from. + /// The reader to use. /// The offset to the header part from the beginning of the stream. /// The size of the header part. /// Header part 1. /// Progress info. /// The loaded header part. - internal async Task Read151HeaderPart4Async(Stream stream, long offset, int size, - Nefs151HeaderPart1 part1, NefsProgress p) + internal async Task Read150HeaderPart4Async(EndianBinaryReader reader, long offset, int size, + Nefs150HeaderPart1 part1, NefsProgress p) { - var entries = new List(); - var indexLookup = new Dictionary(); + var entries = await ReadTocEntriesAsync(reader, offset, size, p).ConfigureAwait(false); - // Validate inputs - if (!ValidateHeaderPartStream(stream, offset, size, "4")) + // Create a table to allow looking up a part 4 index by item Guid + var indexLookup = new Dictionary(); + foreach (var p1 in part1.EntriesByIndex) { - return new Nefs16HeaderPart4(entries, indexLookup, 0); + indexLookup.Add(p1.Guid, p1.IndexPart4); } - // Get entries in part 4 - var numEntries = size / Nefs16HeaderPart4.EntrySize; - var entryOffset = offset; + // TODO: I believe this is padding to reach multiple of EntrySize boundary + // Get the unknown last value at the end of part 4 + var endValue = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); - for (var i = 0; i < numEntries; ++i) - { - using (p.BeginTask(1.0f / numEntries)) - { - var entry = new Nefs16HeaderPart4Entry(); - await FileData.ReadDataAsync(stream, entryOffset, entry, p); - entryOffset += Nefs16HeaderPart4.EntrySize; + return new Nefs150HeaderPart4(entries.Select(x => new Nefs150HeaderPart4Entry(x)), indexLookup, endValue); + } - entries.Add(entry); - } - } + /// + /// Reads 1.5.1 header part 4 from an input stream. + /// + /// The reader to use. + /// The offset to the header part from the beginning of the stream. + /// The size of the header part. + /// Header part 1. + /// Progress info. + /// The loaded header part. + internal async Task Read151HeaderPart4Async(EndianBinaryReader reader, long offset, int size, + Nefs150HeaderPart1 part1, NefsProgress p) + { + var entries = await ReadTocEntriesAsync(reader, offset, size, p).ConfigureAwait(false); // Create a table to allow looking up a part 4 index by item Guid + var indexLookup = new Dictionary(); foreach (var p1 in part1.EntriesByIndex) { indexLookup.Add(p1.Guid, p1.IndexPart4); @@ -804,10 +841,9 @@ internal async Task Read151HeaderPart4Async(Stream stream, lo // TODO: I believe this is padding to reach multiple of EntrySize boundary // Get the unknown last value at the end of part 4 - var endValue = new UInt32Type(0); - await endValue.ReadAsync(stream, stream.Position, p); + var endValue = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); - return new Nefs16HeaderPart4(entries, indexLookup, endValue.Value); + return new Nefs151HeaderPart4(entries.Select(x => new Nefs151HeaderPart4Entry(x)), indexLookup, endValue); } /// @@ -908,24 +944,17 @@ internal async Task ReadHeaderPart4Version20Async(Stream stre /// /// Reads header part 5 from an input stream. /// - /// The stream to read from. + /// The reader to use. /// The offset to the header part from the beginning of the stream. /// The size of the header part. /// Progress info. /// The loaded header part. - internal async Task ReadHeaderPart5Async(Stream stream, long offset, int size, NefsProgress p) + internal async Task ReadHeaderPart5Async(EndianBinaryReader reader, long offset, int size, + NefsProgress p) { - var part5 = new NefsHeaderPart5(); - - // Validate inputs - if (!ValidateHeaderPartStream(stream, offset, size, "5")) - { - return part5; - } - - // Read part 5 data - await FileData.ReadDataAsync(stream, offset, part5, p); - return part5; + // TODO: fix reading multiple volumes + var entries = await ReadTocEntriesAsync(reader, offset, size, p).ConfigureAwait(false); + return entries.Select(x => new NefsHeaderPart5(x)).First(); } /// @@ -1095,29 +1124,75 @@ internal async Task ReadHeaderPart8Async(Stream stream, long of return part8; } + internal async Task ReadHeaderV150Async( + Stream stream, + long primaryOffset, + Nefs150HeaderIntro intro, + NefsProgress p) + { + Nefs150HeaderPart1 part1; + Nefs150HeaderPart2 part2; + NefsHeaderPart3 part3; + Nefs150HeaderPart4 part4; + NefsHeaderPart5 part5; + + // Calc weight of each task (5 parts) + var weight = 1.0f / 5.0f; + + using var reader = new EndianBinaryReader(stream, intro.IsLittleEndian); + using (p.BeginTask(weight, "Reading header part 1")) + { + part1 = await Read150HeaderPart1Async(reader, primaryOffset + intro.OffsetToPart1, (int)intro.Part1Size, p); + } + + using (p.BeginTask(weight, "Reading header part 2")) + { + part2 = await Read150HeaderPart2Async(reader, primaryOffset + intro.OffsetToPart2, (int)intro.Part2Size, p); + } + + using (p.BeginTask(weight, "Reading header part 3")) + { + part3 = await ReadHeaderPart3Async(stream, primaryOffset + intro.OffsetToPart3, (int)intro.Part3Size, p); + } + + using (p.BeginTask(weight, "Reading header part 4")) + { + part4 = await Read150HeaderPart4Async(reader, primaryOffset + intro.OffsetToPart4, + (int)intro.Part4Size, part1, p); + } + + using (p.BeginTask(weight, "Reading header part 5")) + { + part5 = await ReadHeaderPart5Async(reader, primaryOffset + intro.OffsetToPart5, NefsHeaderPart5.Size, p); + } + + return new Nefs150Header(intro, part1, part2, part3, part4, part5); + } + internal async Task ReadHeaderV151Async( Stream stream, long primaryOffset, Nefs151HeaderIntro intro, NefsProgress p) { - Nefs151HeaderPart1 part1; - Nefs151HeaderPart2 part2; + Nefs150HeaderPart1 part1; + Nefs150HeaderPart2 part2; NefsHeaderPart3 part3; - Nefs16HeaderPart4 part4; + Nefs151HeaderPart4 part4; NefsHeaderPart5 part5; // Calc weight of each task (5 parts) var weight = 1.0f / 5.0f; + using var reader = new EndianBinaryReader(stream, intro.IsLittleEndian); using (p.BeginTask(weight, "Reading header part 1")) { - part1 = await Read151HeaderPart1Async(stream, primaryOffset + intro.OffsetToPart1, (int)intro.Part1Size, p); + part1 = await Read150HeaderPart1Async(reader, primaryOffset + intro.OffsetToPart1, (int)intro.Part1Size, p); } using (p.BeginTask(weight, "Reading header part 2")) { - part2 = await Read151HeaderPart2Async(stream, primaryOffset + intro.OffsetToPart2, (int)intro.Part2Size, p); + part2 = await Read150HeaderPart2Async(reader, primaryOffset + intro.OffsetToPart2, (int)intro.Part2Size, p); } using (p.BeginTask(weight, "Reading header part 3")) @@ -1127,13 +1202,13 @@ internal async Task ReadHeaderV151Async( using (p.BeginTask(weight, "Reading header part 4")) { - part4 = await Read151HeaderPart4Async(stream, primaryOffset + intro.OffsetToPart4, + part4 = await Read151HeaderPart4Async(reader, primaryOffset + intro.OffsetToPart4, (int)intro.Part4Size, part1, p); } using (p.BeginTask(weight, "Reading header part 5")) { - part5 = await ReadHeaderPart5Async(stream, primaryOffset + intro.OffsetToPart5, NefsHeaderPart5.Size, p); + part5 = await ReadHeaderPart5Async(reader, primaryOffset + intro.OffsetToPart5, NefsHeaderPart5.Size, p); } return new Nefs151Header(intro, part1, part2, part3, part4, part5); @@ -1159,6 +1234,7 @@ internal async Task ReadHeaderV16Async( // Calc weight of each task (8 parts + table of contents) var weight = 1.0f / 9.0f; + using var reader = new EndianBinaryReader(stream, BitConverter.IsLittleEndian); using (p.BeginTask(weight, "Reading header intro table of contents")) { toc = await ReadHeaderIntroTocVersion16Async(stream, primaryOffset + Nefs16HeaderIntroToc.Offset, p); @@ -1186,7 +1262,7 @@ internal async Task ReadHeaderV16Async( using (p.BeginTask(weight, "Reading header part 5")) { - part5 = await ReadHeaderPart5Async(stream, primaryOffset + toc.OffsetToPart5, NefsHeaderPart5.Size, p); + part5 = await ReadHeaderPart5Async(reader, primaryOffset + toc.OffsetToPart5, NefsHeaderPart5.Size, p); } using (p.BeginTask(weight, "Reading header part 6")) @@ -1231,6 +1307,7 @@ internal async Task ReadHeaderV20Async( // Calc weight of each task (8 parts + table of contents) var weight = 1.0f / 9.0f; + using var reader = new EndianBinaryReader(stream, BitConverter.IsLittleEndian); using (p.BeginTask(weight, "Reading header intro table of contents")) { toc = await ReadHeaderIntroTocVersion20Async(stream, primaryOffset + Nefs20HeaderIntroToc.Offset, p); @@ -1258,7 +1335,7 @@ internal async Task ReadHeaderV20Async( using (p.BeginTask(weight, "Reading header part 5")) { - part5 = await ReadHeaderPart5Async(stream, primaryOffset + toc.OffsetToPart5, NefsHeaderPart5.Size, p); + part5 = await ReadHeaderPart5Async(reader, primaryOffset + toc.OffsetToPart5, NefsHeaderPart5.Size, p); } using (p.BeginTask(weight, "Reading header part 6")) @@ -1299,7 +1376,7 @@ internal async Task ReadSplitHeaderAsync( INefsHeader header; NefsHeaderIntro intro; - var validMagicNum = await ValidateMagicNumberAsync(stream, primaryOffset, p); + var (validMagicNum, _) = await ValidateMagicNumberAsync(stream, primaryOffset, p); if (!validMagicNum) { throw new InvalidOperationException("Header magic number mismatch, aborting."); @@ -1335,33 +1412,50 @@ internal async Task ReadSplitHeaderAsync( return header; } - internal async Task ReadVersionV151Async(Stream stream, long offset, NefsProgress p) + internal async ValueTask ReadVersionV151Async(Stream stream, long offset, bool isLittleEndian, NefsProgress p) { - stream.Seek(offset, SeekOrigin.Begin); - var verNum = new UInt32Type(8); - await verNum.ReadAsync(stream, offset, p); - return verNum.Value; + stream.Seek(offset + 8, SeekOrigin.Begin); + using var reader = new EndianBinaryReader(stream, isLittleEndian); + var verNum = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); + return verNum; + } + + private static (bool Valid, bool LittleEndian) IsMagicNumber(uint value, bool valueLittleEndian) + { + if (value == NefsHeaderIntro.NefsMagicNumber) + { + return (true, valueLittleEndian); + } + + if (BinaryPrimitives.ReverseEndianness(value) == NefsHeaderIntro.NefsMagicNumber) + { + return (true, !valueLittleEndian); + } + + return (false, true); } - internal async Task ValidateMagicNumberAsync(Stream stream, long offset, NefsProgress p) + internal async Task<(bool Valid, bool LittleEndian)> ValidateMagicNumberAsync(Stream stream, long offset, + NefsProgress p) { // Read magic number (first four bytes) stream.Seek(offset, SeekOrigin.Begin); - var magicNum = new UInt32Type(0); - await magicNum.ReadAsync(stream, offset, p); - return magicNum.Value == NefsHeaderIntro.NefsMagicNumber; + using var reader = new EndianBinaryReader(stream); + var magicNum = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); + return IsMagicNumber(magicNum, reader.IsLittleEndian); } - internal async Task ValidateXorMagicNumberAsync(Stream stream, long offset, NefsProgress p) + internal async Task<(bool Valid, bool LittleEndian)> ValidateXorMagicNumberAsync(Stream stream, long offset, + NefsProgress p) { // Read magic number (first four bytes) stream.Seek(offset, SeekOrigin.Begin); - var magicNum = new UInt32Type(0); - await magicNum.ReadAsync(stream, offset, p); - var modNum = new UInt32Type(48); - await modNum.ReadAsync(stream, offset, p); - var magic = magicNum.Value ^ modNum.Value; - return magic == NefsHeaderIntro.NefsMagicNumber; + using var reader = new EndianBinaryReader(stream); + var magicNum = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); + stream.Seek(offset + 48, SeekOrigin.Begin); + var modNum = await reader.ReadUInt32Async(p.CancellationToken).ConfigureAwait(false); + var magic = magicNum ^ modNum; + return IsMagicNumber(magic, reader.IsLittleEndian); } private async Task ValidateEncryptedHeaderAsync(Stream stream, long offset, NefsHeaderIntro intro) diff --git a/VictorBush.Ego.NefsLib/Source/IO/NefsTransformer.cs b/VictorBush.Ego.NefsLib/Source/IO/NefsTransformer.cs index da7030d..2925921 100644 --- a/VictorBush.Ego.NefsLib/Source/IO/NefsTransformer.cs +++ b/VictorBush.Ego.NefsLib/Source/IO/NefsTransformer.cs @@ -108,6 +108,20 @@ public async Task DetransformChunkAsync( detransformedStream.SetLength(tempStream.Length); } + if (chunk.Transform.IsLzssCompressed) + { + var lzss = new LzssDecompress(); + using var tempStream = new MemoryStream(); + + await lzss.DecompressAsync(detransformedStream, tempStream, p.CancellationToken).ConfigureAwait(false); + tempStream.Seek(0, SeekOrigin.Begin); + + detransformedStream.Seek(0, SeekOrigin.Begin); + await tempStream.CopyToAsync(detransformedStream, p.CancellationToken); + detransformedStream.Seek(0, SeekOrigin.Begin); + detransformedStream.SetLength(tempStream.Length); + } + // Copy detransformed chunk to output stream var chunkSize = Math.Min(detransformedStream.Length, maxOutputSize); await detransformedStream.CopyPartialAsync(output, chunkSize, p.CancellationToken); diff --git a/VictorBush.Ego.NefsLib/Source/IO/NefsWriter.cs b/VictorBush.Ego.NefsLib/Source/IO/NefsWriter.cs index 7ee9484..60bca4c 100644 --- a/VictorBush.Ego.NefsLib/Source/IO/NefsWriter.cs +++ b/VictorBush.Ego.NefsLib/Source/IO/NefsWriter.cs @@ -139,6 +139,14 @@ internal async Task WriteHeaderPart3Async(Stream stream, long offset, NefsHeader } } + internal async Task WriteTocEntryAsync(EndianBinaryWriter writer, long offset, T entry, NefsProgress p) + where T : unmanaged, INefsTocData + { + using var t = p.BeginTask(1.0f); + writer.BaseStream.Seek(offset, SeekOrigin.Begin); + await writer.WriteTocDataAsync(entry, p.CancellationToken).ConfigureAwait(false); + } + internal async Task WriteHeaderPartAsync(Stream stream, long offset, object part, NefsProgress p) { using (var t = p.BeginTask(1.0f)) @@ -481,6 +489,7 @@ private async Task WriteHeaderVersion20Async(Stream stream, long headerOffset, N // Get table of contents var toc = header.TableOfContents; + using var writer = new EndianBinaryWriter(stream, true); using (var t = p.BeginTask(weight, "Writing header intro")) { var offset = headerOffset + Nefs20Header.IntroOffset; @@ -520,7 +529,7 @@ private async Task WriteHeaderVersion20Async(Stream stream, long headerOffset, N using (var t = p.BeginTask(weight, "Writing header part 5")) { var offset = headerOffset + toc.OffsetToPart5; - await WriteHeaderPartAsync(stream, offset, header.Part5, p); + await WriteTocEntryAsync(writer, offset, header.Part5.Data, p); } using (var t = p.BeginTask(weight, "Writing header part 6")) @@ -918,6 +927,7 @@ private async Task WriteNefsInjectHeaderVersion16Async(Stream var primaryOffset = NefsInjectHeader.Size; var secondaryOffset = primaryOffset + primarySize; + using var writer = new EndianBinaryWriter(stream, true); using (p.BeginTask(weight, "Writing NesfInject header")) { nefsInject = new NefsInjectHeader(primaryOffset, primarySize, secondaryOffset, secondarySize); @@ -968,7 +978,7 @@ private async Task WriteNefsInjectHeaderVersion16Async(Stream using (var t = p.BeginTask(weight, "Writing header part 5")) { var offset = primaryOffset + toc.OffsetToPart5; - await WriteHeaderPartAsync(stream, offset, header.Part5, p); + await WriteTocEntryAsync(writer, offset, header.Part5.Data, p); } using (var t = p.BeginTask(weight, "Writing header part 8")) @@ -1004,6 +1014,7 @@ private async Task WriteNefsInjectHeaderVersion20Async(Stream var primaryOffset = NefsInjectHeader.Size; var secondaryOffset = primaryOffset + primarySize; + using var writer = new EndianBinaryWriter(stream, true); using (p.BeginTask(weight, "Writing NesfInject header")) { nefsInject = new NefsInjectHeader(primaryOffset, primarySize, secondaryOffset, secondarySize); @@ -1049,7 +1060,7 @@ private async Task WriteNefsInjectHeaderVersion20Async(Stream using (var t = p.BeginTask(weight, "Writing header part 5")) { var offset = primaryOffset + toc.OffsetToPart5; - await WriteHeaderPartAsync(stream, offset, header.Part5, p); + await WriteTocEntryAsync(writer, offset, header.Part5.Data, p); } using (var t = p.BeginTask(weight, "Writing header part 8")) diff --git a/VictorBush.Ego.NefsLib/Source/IO/ResizableBuffer.cs b/VictorBush.Ego.NefsLib/Source/IO/ResizableBuffer.cs new file mode 100644 index 0000000..d6e4770 --- /dev/null +++ b/VictorBush.Ego.NefsLib/Source/IO/ResizableBuffer.cs @@ -0,0 +1,41 @@ +// See LICENSE.txt for license information. + +using System.Buffers; + +namespace VictorBush.Ego.NefsLib.IO; + +public class ResizableBuffer : IDisposable +{ + private readonly ArrayPool arrayPool; + private byte[] buffer; + + public Memory Memory { get; private set; } + + public ResizableBuffer() : this(ArrayPool.Shared, 16) + { + } + + public ResizableBuffer(ArrayPool arrayPool, int startingMinimumLength) + { + this.arrayPool = arrayPool; + this.buffer = arrayPool.Rent(startingMinimumLength); + Memory = this.buffer.AsMemory(); + } + + public void EnsureLength(int length) + { + if (length <= this.buffer.Length) + { + return; + } + + this.arrayPool.Return(this.buffer); + this.buffer = this.arrayPool.Rent(length); + Memory = this.buffer.AsMemory(); + } + + public void Dispose() + { + this.arrayPool.Return(this.buffer); + } +} diff --git a/VictorBush.Ego.NefsLib/Source/Item/NefsItemId.cs b/VictorBush.Ego.NefsLib/Source/Item/NefsItemId.cs index 4f37513..b7e7101 100644 --- a/VictorBush.Ego.NefsLib/Source/Item/NefsItemId.cs +++ b/VictorBush.Ego.NefsLib/Source/Item/NefsItemId.cs @@ -45,7 +45,7 @@ public int CompareTo(NefsItemId other) } /// - public override bool Equals(object obj) => obj is NefsItemId id && id == this; + public override bool Equals(object? obj) => obj is NefsItemId id && id == this; /// public override int GetHashCode() => Value.GetHashCode(); diff --git a/VictorBush.Ego.NefsLib/Source/NefsVersion.cs b/VictorBush.Ego.NefsLib/Source/NefsVersion.cs index ecb7da5..0848dc5 100644 --- a/VictorBush.Ego.NefsLib/Source/NefsVersion.cs +++ b/VictorBush.Ego.NefsLib/Source/NefsVersion.cs @@ -12,6 +12,11 @@ public enum NefsVersion /// Version140 = 0x10400, + /// + /// Version 1.5.0. + /// + Version150 = 0x10500, + /// /// Version 1.5.1. /// diff --git a/VictorBush.Ego.NefsLib/Source/Utility/Sha256Hash.cs b/VictorBush.Ego.NefsLib/Source/Utility/Sha256Hash.cs index 1c76020..f702e5d 100644 --- a/VictorBush.Ego.NefsLib/Source/Utility/Sha256Hash.cs +++ b/VictorBush.Ego.NefsLib/Source/Utility/Sha256Hash.cs @@ -36,7 +36,7 @@ public Sha256Hash(byte[] value) public static bool operator ==(Sha256Hash a, Sha256Hash b) => a.Equals(b); /// - public override bool Equals(object obj) => obj is Sha256Hash hash && Value.SequenceEqual(hash.Value); + public override bool Equals(object? obj) => obj is Sha256Hash hash && Value.SequenceEqual(hash.Value); /// public override int GetHashCode() => Value.GetHashCode(); diff --git a/VictorBush.Ego.NefsLib/Source/Utility/StreamExtensions.cs b/VictorBush.Ego.NefsLib/Source/Utility/StreamExtensions.cs index f11332c..5b98d75 100644 --- a/VictorBush.Ego.NefsLib/Source/Utility/StreamExtensions.cs +++ b/VictorBush.Ego.NefsLib/Source/Utility/StreamExtensions.cs @@ -55,43 +55,4 @@ public static async Task CopyPartialAsync( ArrayPool.Shared.Return(buffer); } } - - /// - /// Copies data from a stream to another stream. - /// - /// The input stream to copy from. - /// The destination stream to write to. - /// A cancellation token. - /// An async task. - public static async Task CopyToAsync( - this Stream stream, - Stream destination, - CancellationToken cancelToken) - { - await stream.CopyToAsync(destination, CopyBufferSize, cancelToken); - } - - /// - /// Asynchronously reads bytes from the current stream, advances the position within the stream until the is filled, - /// and monitors cancellation requests. - /// - /// The input stream to copy from. - /// The buffer to write the data into. - /// The token to monitor for cancellation requests. - /// A task that represents the asynchronous read operation. - public static async ValueTask ReadExactlyAsync(this Stream stream, Memory buffer, - CancellationToken cancellationToken = default) - { - var totalRead = 0; - while (totalRead < buffer.Length) - { - var read = await stream.ReadAsync(buffer[totalRead..], cancellationToken).ConfigureAwait(false); - if (read == 0) - { - throw new EndOfStreamException(); - } - - totalRead += read; - } - } } diff --git a/VictorBush.Ego.NefsLib/Source/Utility/StringHelper.cs b/VictorBush.Ego.NefsLib/Source/Utility/StringHelper.cs index a006ffa..f81eeae 100644 --- a/VictorBush.Ego.NefsLib/Source/Utility/StringHelper.cs +++ b/VictorBush.Ego.NefsLib/Source/Utility/StringHelper.cs @@ -14,7 +14,7 @@ public static class StringHelper /// /// The bytes to print. /// The string. - public static string ByteArrayToString(byte[] bytes) + public static string ByteArrayToString(ReadOnlySpan bytes) { if (bytes == null) { diff --git a/VictorBush.Ego.NefsLib/VictorBush.Ego.NefsLib.csproj b/VictorBush.Ego.NefsLib/VictorBush.Ego.NefsLib.csproj index ead489f..a4184d3 100644 --- a/VictorBush.Ego.NefsLib/VictorBush.Ego.NefsLib.csproj +++ b/VictorBush.Ego.NefsLib/VictorBush.Ego.NefsLib.csproj @@ -1,24 +1,25 @@  - net6.0 + net8.0 Library - false enable + true bin\Debug\VictorBush.Ego.NefsLib.xml - - Properties\SharedAssemblyInfo.cs - + + <_Parameter1>$(MSBuildProjectName).Tests + + + <_Parameter1>VictorBush.Ego.NefsEdit.Tests + - - - +