Skip to content

Revive --extract-shaders for dxbc bytecode extraction #186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PachiMako wants to merge 1 commit into
base: develop
Choose a base branch
from
+58 −91
Open

Revive --extract-shaders for dxbc bytecode extraction #186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 27 additions & 11 deletions DataTool/FindLogic/Combo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -671,19 +671,35 @@ public static ComboInfo Find(ComboInfo info, ulong guid, Dictionary<ulong, ulong
materialInfo.m_materialIDs.Add(context.MaterialID);
materialInfo.m_shaderSourceGUID = GetReplacement(material.Header.ShaderSource, replacements);
materialInfo.m_shaderGroupGUID = GetReplacement(material.Header.ShaderGroup, replacements);
try {
if (Program.Flags.ExtractShaders) {
teShaderGroup shaderGroup = new teShaderGroup(OpenFile(materialInfo.m_shaderGroupGUID));
foreach (teResourceGUID shaderGuid in shaderGroup.Instances) {
ulong shaderInstanceGuid = GetReplacement(shaderGuid, replacements);
teShaderInstance shaderInstance = new teShaderInstance(OpenFile(shaderInstanceGuid));
ulong shaderCodeGuid = GetReplacement(shaderInstance.Header.ShaderCode, replacements);
teShaderCode shaderCode = new teShaderCode(OpenFile(shaderCodeGuid));
materialInfo.m_shaders.Add((shaderInstanceGuid, shaderCodeGuid, shaderCode.ByteCode));

if (Program.Flags.ExtractShaders) {
// Local function to prevent nesting on error handling, could be hoisted into a static function
void ExtractShaders() {
using Stream shaderGroupStream = OpenFile(materialInfo.m_shaderGroupGUID);
if (shaderGroupStream == null) {
Logger.Error("Combo", $"ExtractShaders: can't open shader group {materialInfo.m_shaderGroupGUID:X16}");
return;
}

teShaderGroup shaderGroup = new teShaderGroup(shaderGroupStream);
if (shaderGroup.Shaders == null || shaderGroup.Instances == null) {
Logger.Error("Combo", $"ExtractShaders: group {materialInfo.m_shaderGroupGUID:X16} has no shader/instance arrays");
return;
}

for (int i = 0; i < shaderGroup.Shaders.Length; i++) {
ulong shaderCodeGuid = GetReplacement(shaderGroup.Shaders[i], replacements);
using Stream shaderCodeStream = OpenFile(shaderCodeGuid);
if (shaderCodeStream == null) {
Logger.Error("Combo", $"ExtractShaders: can't open shader code {shaderCodeGuid:X16}");
continue;
}

teShaderCode shaderCode = new teShaderCode(shaderCodeStream);
materialInfo.m_shaders.Add((GetReplacement(shaderGroup.Instances[i], replacements), shaderCodeGuid, shaderCode.ByteCode));
}
}
} catch {
// lol xd
ExtractShaders();
}

if (context.ModelLook == 0 && context.Model != 0) {
Expand Down
8 changes: 5 additions & 3 deletions DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using TankLib.STU.Types;
using static DataTool.Program;
using static DataTool.Helper.STUHelper;
using TankLib.Helpers;

namespace DataTool.ToolLogic.Extract.Debug;

Expand Down Expand Up @@ -286,9 +287,10 @@ public static void SaveShaderGroup(teShaderGroup shaderGroup, string path) {
int i = 0;
foreach (ulong shaderGroupInstance in shaderGroup.Instances) {
string name = null;
if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
name = shaderGroup.Hashes[i].ToString("X8");
}
Logger.Error("SaveShaderGroup", $"teShaderGroup no longer associates a hash with teShaderInstance, see documentation on GetShaderByHash.");
//if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
// name = shaderGroup.Hashes[i].ToString("X8");
//}

SaveShaderInstance(path, shaderGroupInstance, name);
i++;
Expand Down
27 changes: 6 additions & 21 deletions TankLib/teShaderCode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,16 @@ namespace TankLib {
/// <summary>Tank ShaderCode, file type 087</summary>
public class teShaderCode {
/// <summary>ShaderCode Header </summary>
[StructLayout(LayoutKind.Sequential, Pack = 4)]
[StructLayout(LayoutKind.Explicit)]
public struct ShaderCodeHeader {
/// <summary>Shader type flags</summary>
[FieldOffset(8)] public Enums.teSHADER_TYPE ShaderType; // 36 -> 8
/// <summary>Offset to data</summary>
public long DataOffset; // 0

/// <summary>Offset to unknown data</summary>
public long OffsetB; // 8

public uint UnknownFF; // 16

/// <summary>Unknown data count</summary>
public uint StreamOutDescCount; // 20

[FieldOffset(112)] public long DataOffset; // 0 -> 112
/// <summary>Size of compressed DXBC data</summary>
public int CompressedSize; // 24

[FieldOffset(132)] public int CompressedSize; // 24 -> 132
/// <summary>Size of decompressed DXBC data</summary>
public int UncompressedSize; // 28

public uint Unknown; // 32

/// <summary>Shader type</summary>
public Enums.teSHADER_TYPE ShaderType; // 36

// etc
[FieldOffset(136)] public int UncompressedSize; // 28 -> 136
}

/// <summary>Header Data</summary>
Expand Down
76 changes: 20 additions & 56 deletions TankLib/teShaderGroup.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,18 @@
using System.IO;
using System.Runtime.InteropServices;
using TankLib.Helpers;

namespace TankLib {
/// <summary>Tank ShaderGroup, file type 085</summary>
public class teShaderGroup {
[StructLayout(LayoutKind.Sequential, Pack = 4)]
[StructLayout(LayoutKind.Explicit)]
public struct GroupHeader {
/// <summary>Number of entries for both ShadersOffset and InstancesOffset</summary>
[FieldOffset(100)] public int NumShaders; // 64 -> 76 -> 100
/// <summary>ShaderCode array offset</summary>
[FieldOffset(104)] public long ShadersOffset; // na -> 104
/// <summary>ShaderInstance array offset</summary>
public long InstanceOffset; // 0

public long HashOffset; // 8
public long FlagsOffset; // 16

public long NewIdk; // n/a -> 24

public long OffsetD; // 24 -> 32
public long OffsetE; // 32 -> 40

/// <summary>teShaderSource that this group was generated from</summary>
/// <remarks>088 GUID</remarks>
public teResourceGUID SourceGUID; // 40 -> 48

/// <summary>A virtual reference. Usage unknown</summary>
/// <remarks>00F GUID</remarks>
public teResourceGUID CacheGUID; // 48 -> 56

public ulong Flags; // 56 -> 64

public int NumIdk; // n/a -> 72
public int NumShaders; // 64 -> 76

// ...
[FieldOffset(120)] public long InstancesOffset; // 0 -> 100
}

[StructLayout(LayoutKind.Sequential, Pack = 4)]
Expand All @@ -54,17 +36,15 @@ public struct ShaderUnk {
/// <summary>Header Data</summary>
public GroupHeader Header;

/// <summary>ShaderInstances</summary>
/// <summary>teShaderCode GUIDs (087)</summary>
public teResourceGUID[] Shaders;

/// <summary>teShaderInstance info GUIDs (040)</summary>
public teResourceGUID[] Instances;

/// <summary>Flags that are used to select the correct ShaderInstance pair based on parameters</summary>
public ulong[] InstanceFlags;

/// <summary>
/// ShaderInstance "hashes". Used to identify instances in 088 groups.
/// </summary>
public uint[] Hashes;

public ShaderQuality[] ShaderQualities;
public ShaderUnk[] ShaderUnks;

Expand All @@ -89,43 +69,27 @@ public teShaderGroup(BinaryReader reader) {
private void Read(BinaryReader reader) {
Header = reader.Read<GroupHeader>();

if (Header.InstanceOffset > 0) {
reader.BaseStream.Position = Header.InstanceOffset;
Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
}

if (Header.HashOffset > 0) {
reader.BaseStream.Position = Header.HashOffset;
Hashes = reader.ReadArray<uint>(Header.NumShaders);
if (Header.ShadersOffset > 0) {
reader.BaseStream.Position = Header.ShadersOffset;
Shaders = reader.ReadArray<teResourceGUID>(Header.NumShaders);
}

if (Header.FlagsOffset > 0) {
reader.BaseStream.Position = Header.FlagsOffset;
InstanceFlags = reader.ReadArray<ulong>(Header.NumShaders);
}

{
reader.BaseStream.Position = 72;
ShaderQualities = reader.ReadArray<ShaderQuality>(5);

reader.BaseStream.Position = 104;
ShaderUnks = reader.ReadArray<ShaderUnk>(5);
if (Header.InstancesOffset > 0) {
reader.BaseStream.Position = Header.InstancesOffset;
Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
}
}

/// <summary>
/// DEPRECATED: teShaderGroup used to store a hash to match with a specific teShaderInstance. This mapping has since been broken as
/// teShaderGroup now holds guids to both teShaderInstance and teShaderCode mapping them with a simple positional index instead.
/// Get a ShaderInstance GUID from a "hash"
/// </summary>
/// <note>Mostly used on 088 groups</note>
/// <param name="hash">"Hash" associated with a specific ShaderInstance</param>
/// <returns></returns>
public teResourceGUID GetShaderByHash(uint hash) {
if (Hashes == null) return (teResourceGUID) 0;
for (int i = 0; i < Header.NumShaders; i++) {
if (Hashes[i] == hash) {
return Instances[i];
}
}
Logger.Error("teShaderGroup", $"GetShaderByHash doesn't work any longer for {hash:X16}, see documentation on GetShaderByHash.");
return (teResourceGUID) 0;
}
}
Expand Down