From 8597241a0850cbbfed8b0f102e3e5464c817c581 Mon Sep 17 00:00:00 2001
From: PachiMako <299646743+PachiMako@users.noreply.github.com>
Date: Fri, 10 Jul 2026 18:45:12 +0200
Subject: [PATCH] Revive --extract-shaders for dxbc bytecode extraction

teShaderCode, teShaderInstance and teShaderGroup all changed structure since at least the patch of Aug 10 '23 (Manifest 3320652641754470004).

Changes:
 - Update offsets of used fields in teShaderGroup and teShaderCode headers
 - Improves error handling and logging for --extract-shaders
 - Adds errors in codepaths that are not longer functional such as resolving file names in SaveShaderGroup and resolving previously extracted shaders using their now stale hash

Verified working on overwatch build prometheus-2_23_0_0-150480 with valid dxbc for all entries of teShaderCode with the following distributions across shader types:
PIXEL (2): 112210
VERTEX (1): 21777
COMPUTE (0): 147
GEOMETRY (4): 24
VERTEX, UNKNOWN_C (129): 13
GEOMETRY, UNKNOWN_B (20): 1
---
 DataTool/FindLogic/Combo.cs                   | 38 +++++++---
 .../Extract/Debug/ExtractDebugShaders.cs      |  8 +-
 TankLib/teShaderCode.cs                       | 27 ++-----
 TankLib/teShaderGroup.cs                      | 76 +++++--------------
 4 files changed, 58 insertions(+), 91 deletions(-)

diff --git a/DataTool/FindLogic/Combo.cs b/DataTool/FindLogic/Combo.cs
index 8364d460e..2b424965f 100644
--- a/DataTool/FindLogic/Combo.cs
+++ b/DataTool/FindLogic/Combo.cs
@@ -671,19 +671,35 @@ public static ComboInfo Find(ComboInfo info, ulong guid, Dictionary<ulong, ulong
                 materialInfo.m_materialIDs.Add(context.MaterialID);
                 materialInfo.m_shaderSourceGUID = GetReplacement(material.Header.ShaderSource, replacements);
                 materialInfo.m_shaderGroupGUID = GetReplacement(material.Header.ShaderGroup, replacements);
-                try {
-                    if (Program.Flags.ExtractShaders) {
-                        teShaderGroup shaderGroup = new teShaderGroup(OpenFile(materialInfo.m_shaderGroupGUID));
-                        foreach (teResourceGUID shaderGuid in shaderGroup.Instances) {
-                            ulong shaderInstanceGuid = GetReplacement(shaderGuid, replacements);
-                            teShaderInstance shaderInstance = new teShaderInstance(OpenFile(shaderInstanceGuid));
-                            ulong shaderCodeGuid = GetReplacement(shaderInstance.Header.ShaderCode, replacements);
-                            teShaderCode shaderCode = new teShaderCode(OpenFile(shaderCodeGuid));
-                            materialInfo.m_shaders.Add((shaderInstanceGuid, shaderCodeGuid, shaderCode.ByteCode));
+
+                if (Program.Flags.ExtractShaders) {
+                    // Local function to prevent nesting on error handling, could be hoisted into a static function
+                    void ExtractShaders() {
+                        using Stream shaderGroupStream = OpenFile(materialInfo.m_shaderGroupGUID);
+                        if (shaderGroupStream == null) {
+                            Logger.Error("Combo", $"ExtractShaders: can't open shader group {materialInfo.m_shaderGroupGUID:X16}");
+                            return;
+                        }
+
+                        teShaderGroup shaderGroup = new teShaderGroup(shaderGroupStream);
+                        if (shaderGroup.Shaders == null || shaderGroup.Instances == null) {
+                            Logger.Error("Combo", $"ExtractShaders: group {materialInfo.m_shaderGroupGUID:X16} has no shader/instance arrays");
+                            return;
+                        }
+
+                        for (int i = 0; i < shaderGroup.Shaders.Length; i++) {
+                            ulong shaderCodeGuid = GetReplacement(shaderGroup.Shaders[i], replacements);
+                            using Stream shaderCodeStream = OpenFile(shaderCodeGuid);
+                            if (shaderCodeStream == null) { 
+                                Logger.Error("Combo", $"ExtractShaders: can't open shader code {shaderCodeGuid:X16}"); 
+                                continue; 
+                            }
+
+                            teShaderCode shaderCode = new teShaderCode(shaderCodeStream);
+                            materialInfo.m_shaders.Add((GetReplacement(shaderGroup.Instances[i], replacements), shaderCodeGuid, shaderCode.ByteCode));
                         }
                     }
-                } catch {
-                    // lol xd
+                    ExtractShaders();
                 }
 
                 if (context.ModelLook == 0 && context.Model != 0) {
diff --git a/DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs b/DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs
index 78865b6c9..beb53d73a 100644
--- a/DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs
+++ b/DataTool/ToolLogic/Extract/Debug/ExtractDebugShaders.cs
@@ -9,6 +9,7 @@
 using TankLib.STU.Types;
 using static DataTool.Program;
 using static DataTool.Helper.STUHelper;
+using TankLib.Helpers;
 
 namespace DataTool.ToolLogic.Extract.Debug;
 
@@ -286,9 +287,10 @@ public static void SaveShaderGroup(teShaderGroup shaderGroup, string path) {
         int i = 0;
         foreach (ulong shaderGroupInstance in shaderGroup.Instances) {
             string name = null;
-            if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
-                name = shaderGroup.Hashes[i].ToString("X8");
-            }
+            Logger.Error("SaveShaderGroup", $"teShaderGroup no longer associates a hash with teShaderInstance, see documentation on GetShaderByHash."); 
+            //if (shaderGroup.Hashes != null && shaderGroup.Hashes[i] != 0) {
+            //    name = shaderGroup.Hashes[i].ToString("X8");
+            //}
 
             SaveShaderInstance(path, shaderGroupInstance, name);
             i++;
diff --git a/TankLib/teShaderCode.cs b/TankLib/teShaderCode.cs
index 387926194..98423762b 100644
--- a/TankLib/teShaderCode.cs
+++ b/TankLib/teShaderCode.cs
@@ -7,31 +7,16 @@ namespace TankLib {
     /// <summary>Tank ShaderCode, file type 087</summary>
     public class teShaderCode {
         /// <summary>ShaderCode Header </summary>
-        [StructLayout(LayoutKind.Sequential, Pack = 4)]
+        [StructLayout(LayoutKind.Explicit)]
         public struct ShaderCodeHeader {
+            /// <summary>Shader type flags</summary>
+            [FieldOffset(8)] public Enums.teSHADER_TYPE ShaderType; // 36 -> 8
             /// <summary>Offset to data</summary>
-            public long DataOffset; // 0
-            
-            /// <summary>Offset to unknown data</summary>
-            public long OffsetB; // 8
-            
-            public uint UnknownFF; // 16
-            
-            /// <summary>Unknown data count</summary>
-            public uint StreamOutDescCount; // 20
-            
+            [FieldOffset(112)] public long DataOffset; // 0 -> 112
             /// <summary>Size of compressed DXBC data</summary>
-            public int CompressedSize;  // 24
-
+            [FieldOffset(132)] public int CompressedSize; // 24 -> 132
             /// <summary>Size of decompressed DXBC data</summary>
-            public int UncompressedSize;  // 28
-
-            public uint Unknown; // 32
-            
-            /// <summary>Shader type</summary>
-            public Enums.teSHADER_TYPE ShaderType;  // 36
-            
-            // etc
+            [FieldOffset(136)] public int UncompressedSize; // 28 -> 136
         }
 
         /// <summary>Header Data</summary>
diff --git a/TankLib/teShaderGroup.cs b/TankLib/teShaderGroup.cs
index ec480633f..ec367b0d6 100644
--- a/TankLib/teShaderGroup.cs
+++ b/TankLib/teShaderGroup.cs
@@ -1,36 +1,18 @@
 using System.IO;
 using System.Runtime.InteropServices;
+using TankLib.Helpers;
 
 namespace TankLib {
     /// <summary>Tank ShaderGroup, file type 085</summary>
     public class teShaderGroup {
-        [StructLayout(LayoutKind.Sequential, Pack = 4)]
+        [StructLayout(LayoutKind.Explicit)]
         public struct GroupHeader {
+            /// <summary>Number of entries for both ShadersOffset and InstancesOffset</summary>
+            [FieldOffset(100)] public int NumShaders; // 64 -> 76 -> 100
+            /// <summary>ShaderCode array offset</summary>
+            [FieldOffset(104)] public long ShadersOffset; // na -> 104
             /// <summary>ShaderInstance array offset</summary>
-            public long InstanceOffset; // 0
-            
-            public long HashOffset; // 8
-            public long FlagsOffset; // 16
-            
-            public long NewIdk; // n/a -> 24
-            
-            public long OffsetD; // 24 -> 32
-            public long OffsetE; // 32 -> 40
-            
-            /// <summary>teShaderSource that this group was generated from</summary>
-            /// <remarks>088 GUID</remarks>
-            public teResourceGUID SourceGUID; // 40 -> 48
-            
-            /// <summary>A virtual reference. Usage unknown</summary>
-            /// <remarks>00F GUID</remarks>
-            public teResourceGUID CacheGUID; // 48 -> 56
-
-            public ulong Flags; // 56 -> 64
-            
-            public int NumIdk; // n/a -> 72
-            public int NumShaders;  // 64 -> 76
-            
-            // ...
+            [FieldOffset(120)] public long InstancesOffset; // 0 -> 100
         }
 
         [StructLayout(LayoutKind.Sequential, Pack = 4)]
@@ -54,17 +36,15 @@ public struct ShaderUnk {
         /// <summary>Header Data</summary>
         public GroupHeader Header;
         
-        /// <summary>ShaderInstances</summary>
+        /// <summary>teShaderCode GUIDs (087)</summary>
+        public teResourceGUID[] Shaders;
+
+        /// <summary>teShaderInstance info GUIDs (040)</summary>
         public teResourceGUID[] Instances;
 
         /// <summary>Flags that are used to select the correct ShaderInstance pair based on parameters</summary>
         public ulong[] InstanceFlags;
 
-        /// <summary>
-        /// ShaderInstance "hashes". Used to identify instances in 088 groups.
-        /// </summary>
-        public uint[] Hashes;
-
         public ShaderQuality[] ShaderQualities;
         public ShaderUnk[] ShaderUnks;
         
@@ -89,43 +69,27 @@ public teShaderGroup(BinaryReader reader) {
         private void Read(BinaryReader reader) {
             Header = reader.Read<GroupHeader>();
             
-            if (Header.InstanceOffset > 0) {
-                reader.BaseStream.Position = Header.InstanceOffset;
-                Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
-            }
-
-            if (Header.HashOffset > 0) {
-                reader.BaseStream.Position = Header.HashOffset;
-                Hashes = reader.ReadArray<uint>(Header.NumShaders);
+            if (Header.ShadersOffset > 0) {
+                reader.BaseStream.Position = Header.ShadersOffset;
+                Shaders = reader.ReadArray<teResourceGUID>(Header.NumShaders);
             }
 
-            if (Header.FlagsOffset > 0) {
-                reader.BaseStream.Position = Header.FlagsOffset;
-                InstanceFlags = reader.ReadArray<ulong>(Header.NumShaders);
-            }
-
-            {
-                reader.BaseStream.Position = 72;
-                ShaderQualities = reader.ReadArray<ShaderQuality>(5);
-
-                reader.BaseStream.Position = 104;
-                ShaderUnks = reader.ReadArray<ShaderUnk>(5);
+            if (Header.InstancesOffset > 0) {
+                reader.BaseStream.Position = Header.InstancesOffset;
+                Instances = reader.ReadArray<teResourceGUID>(Header.NumShaders);
             }
         }
 
         /// <summary>
+        /// DEPRECATED: teShaderGroup used to store a hash to match with a specific teShaderInstance. This mapping has since been broken as
+        /// teShaderGroup now holds guids to both teShaderInstance and teShaderCode mapping them with a simple positional index instead.
         /// Get a ShaderInstance GUID from a "hash"
         /// </summary>
         /// <note>Mostly used on 088 groups</note>
         /// <param name="hash">"Hash" associated with a specific ShaderInstance</param>
         /// <returns></returns>
         public teResourceGUID GetShaderByHash(uint hash) {
-            if (Hashes == null) return (teResourceGUID) 0;
-            for (int i = 0; i < Header.NumShaders; i++) {
-                if (Hashes[i] == hash) {
-                    return Instances[i];
-                }
-            }
+            Logger.Error("teShaderGroup", $"GetShaderByHash doesn't work any longer for {hash:X16}, see documentation on GetShaderByHash."); 
             return (teResourceGUID) 0;
         }
     }