Merge pull request #76 from senko-forks/pr-atch

PMP stuff + avfx stuff + neck morph stuff

Author: Lunaretic

xivModdingFramework/Models/DataContainers/LevelOfDetail.cs

@@ -97,6 +97,7 @@ public EMeshType GetMeshType(int offset)
 
         /// <summary>
         /// Unknown Usage
+        /// This appears to be multiple individual byte values, with the first 2 being related to neck morph data
         /// </summary>
         public int Unknown7 { get; set; }
 

xivModdingFramework/Models/DataContainers/MdlModelData.cs

@@ -172,7 +172,7 @@ public class MdlModelData
         /// </summary>
         public byte BgCrestChangeMaterialIndex { get; set; }
 
-        public byte Unknown12 { get; set; }
+        public byte NeckMorphTableSize { get; set; }
 
         /// <summary>
         /// Unknown Usage
@@ -241,7 +241,7 @@ public static MdlModelData Read(BinaryReader br)
                 BgChangeMaterialIndex = br.ReadByte(),
                 BgCrestChangeMaterialIndex = br.ReadByte(),
 
-                Unknown12 = br.ReadByte(),
+                NeckMorphTableSize = br.ReadByte(),
                 BoneSetSize = br.ReadInt16(),
 
                 Unknown13 = br.ReadInt16(),
@@ -278,7 +278,7 @@ public void Write(BinaryWriter br)
             br.Write((byte)Flags3);
             br.Write(BgChangeMaterialIndex);
             br.Write(BgCrestChangeMaterialIndex);
-            br.Write(Unknown12);
+            br.Write(NeckMorphTableSize);
             br.Write(BoneSetSize);
             br.Write(Unknown13);
             br.Write(Unknown14);

xivModdingFramework/Models/DataContainers/NeckMorphEntry.cs

@@ -0,0 +1,56 @@
+// xivModdingFramework
+// Copyright © 2018 Rafael Gonzalez - All Rights Reserved
+// 
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+// 
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+using SharpDX;
+using System.Collections.Generic;
+
+namespace xivModdingFramework.Models.DataContainers
+{
+    /// <summary>
+    /// Class representing a neck morph table entry in a model, which modifies a vertex on the neck seam of a character's body when active.
+    /// Typically 10 of these entries exist on a head mesh.
+    /// It is unclear how each entry relates to individual vertex on the body mesh.
+    /// </summary>
+    public class NeckMorphEntry
+    {
+        /// <summary>
+        /// Relative adjustment of the Position of the vertex.
+        /// Its unclear how this is applied, but it seems affected by the referenced bones.
+        /// </summary>
+        public Vector3 PositionAdjust { get; set; }
+
+        /// <summary>
+        /// This value is unclear but preserving it is important.
+        /// If it is incorrect, the vertex adjustments disappear when viewed from certain camera angles.
+        /// For all known working examples, it just has the value 0x00006699.
+        /// </summary>
+        public uint Unknown { get; set; }
+
+        /// <summary>
+        /// Relative adjustment of the Normal of the vertex.
+        /// Its unclear how this is applied.
+        /// </summary>
+        public Vector3 NormalAdjust { get; set; }
+
+        /// <summary>
+        /// A list of bones, stored as indexes in to MdlPathData.BoneList.
+        /// For all known working examples, the referenced bones are ["j_kubi", "j_sebo_c"].
+        /// If the incorrect bones are referenced, the neck behavior becomes erratic.
+        /// NOTE: In the model file these are actually stored as indexes in to BoneSet 0.
+        /// </summary>
+        public List<short> Bones { get; set; }
+    }
+}

xivModdingFramework/Models/DataContainers/XivMdl.cs

@@ -141,5 +141,10 @@ public class XivMdl
         /// This happens when the sum of all LoD mesh counts is less than the model data mesh count
         /// </remarks>
         public List<MeshData> ExtraMeshData { get; set; }
+
+        /// <summary>
+        /// This data is present on heads and seems to affect the shape of the neck on the body mesh
+        /// </summary>
+        public List<NeckMorphEntry> NeckMorphTable { get; set; }
     }
 }

xivModdingFramework/Models/FileTypes/Mdl.cs

@@ -894,7 +894,7 @@ public static XivMdl GetXivMdl(byte[] mdlData, string mdlPath = "")
 
                 #endregion
 
-                #region Part Bone Sets & Padding
+                #region Part Bone Sets
                 // Bone index for Parts
                 var partBoneSet = new BoneSet
                 {
@@ -908,7 +908,43 @@ public static XivMdl GetXivMdl(byte[] mdlData, string mdlPath = "")
                 }
 
                 xivMdl.PartBoneSets = partBoneSet;
+                #endregion
+
+                #region Neck Morph Data
+                // Neck morph data (appears on face models new in Patch 7.1)
+                xivMdl.NeckMorphTable = new List<NeckMorphEntry>();
+                for (var i = 0; i < xivMdl.ModelData.NeckMorphTableSize; ++i)
+                {
+                    var neckMorphDataEntry = new NeckMorphEntry
+                    {
+                        PositionAdjust = new Vector3(br.ReadSingle(), br.ReadSingle(), br.ReadSingle()),
+                        Unknown = br.ReadUInt32(),
+                        NormalAdjust = new Vector3(br.ReadSingle(), br.ReadSingle(), br.ReadSingle()),
+                        Bones = new List<short>()
+                    };
+                    byte[] neckBoneTable = br.ReadBytes(4);
+                    // Weird code alert:
+                    // - Most vanilla heads legitimately have a zero value in the second slot of the table.
+                    // - Female Hrothgar heads legitimately have a zero value in the first slot of the table.
+                    // - Values in the third and fourth slot seem to be unused, and also seem to use 0 as a padding or null value.
+                    // - However, at least one vanilla model seems to have a non-zero value in the third slot of the table.
+                    // Therefore, only in the third and fourth slot is a zero value treated as an early list terminator.
+                    // This means the table should always contain at least two bones.
+                    for (int j = 0; j < neckBoneTable.Length; ++j)
+                    {
+                        int boneset0_index = neckBoneTable[j];
+                        if (j >= 2 && boneset0_index == 0) break; // Treat this case as an early list terminator.
+                        // Resolve the bone to an index in the bone path table here, to make the in-memory representation a little more normal
+                        if (xivMdl.MeshBoneSets.Count > 0 && xivMdl.MeshBoneSets[0].BoneIndices.Count() > boneset0_index)
+                        {
+                            neckMorphDataEntry.Bones.Add(xivMdl.MeshBoneSets[0].BoneIndices[boneset0_index]);
+                        }
+                    }
+                    xivMdl.NeckMorphTable.Add(neckMorphDataEntry);
+                }
+                #endregion
 
+                #region Padding
                 // Padding
                 xivMdl.PaddingSize = br.ReadByte();
                 xivMdl.PaddedBytes = br.ReadBytes(xivMdl.PaddingSize);
@@ -2987,8 +3023,10 @@ public static byte[] MakeUncompressedMdlFile(TTModel ttModel, XivMdl ogMdl, Acti
                 basicModelBlock.Add(bgChangeIdx);
                 basicModelBlock.Add(crestChangeIdx);
 
-                // More Unknowns
-                basicModelBlock.Add(ogModelData.Unknown12);
+                // Using neck morph data from original modal
+                // The field currently named LevelOfDetail.Unknown7 also contains this number, and also gets copied from the original model
+                var neckMorphTableSizePointer = basicModelBlock.Count; // we want to reset this to 0 later if the neck data cannot be preserved
+                basicModelBlock.Add(ogModelData.NeckMorphTableSize);
 
                 // We fix this pointer later after bone table is done.
                 var boneSetSizePointer = basicModelBlock.Count;
@@ -3537,6 +3575,64 @@ public static byte[] MakeUncompressedMdlFile(TTModel ttModel, XivMdl ogMdl, Acti
 
                 #endregion
 
+                // Neck Morph Data
+                #region Neck Morph Data
+                var neckMorphDataBlock = new List<byte>();
+                // Preserve the original model's neck morph data if present -- but update the bone references inside of it
+                // Bone references are made via BoneSet[0]
+                for (int i = 0; i < ogMdl.NeckMorphTable.Count; ++i)
+                {
+                    // Extract the original data (except the bone list)
+                    var positionAdjust = ogMdl.NeckMorphTable[i].PositionAdjust;
+                    var unknown = ogMdl.NeckMorphTable[i].Unknown;
+                    var normalAdjust = ogMdl.NeckMorphTable[i].NormalAdjust;
+                    var bones = new List<byte>();
+
+                    // Look up the originally referenced bone by name, and map it to the same bone in the imported model
+                    for (int j = 0; j < ogMdl.NeckMorphTable[i].Bones.Count; ++j)
+                    {
+                        string ogBoneName = ogMdl.PathData.BoneList[ogMdl.NeckMorphTable[i].Bones[j]];
+                        int boneset0Index = -1;
+
+                        if (ttModel.MeshGroups.Count > 0)
+                        {
+                            boneset0Index = ttModel.MeshGroups[0].Bones.FindIndex(x => x == ogBoneName);
+                        }
+
+                        // If a bone can't be located in the new model, just discard all of the neck morph data
+                        if (boneset0Index == -1 || boneset0Index > 255)
+                        {
+                            loggingFunction(true, "Could not match bones in neck morph data, so the data was discarded!");
+                            // Reset the table size to 0 in the model header and drop the data
+                            // (Two bytes are also cleared later on when writing the LODs)
+                            basicModelBlock[neckMorphTableSizePointer] = 0;
+                            neckMorphDataBlock = new List<byte>();
+                            break;
+                        }
+
+                        bones.Add((byte)boneset0Index);
+                    }
+
+                    // Serialize
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(positionAdjust.X));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(positionAdjust.Y));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(positionAdjust.Z));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(unknown));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(normalAdjust.X));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(normalAdjust.Y));
+                    neckMorphDataBlock.AddRange(BitConverter.GetBytes(normalAdjust.Z));
+
+                    // Bone list is always 4 bytes -- pad with zeroes
+                    for (int j = 0; j < 4; ++j)
+                    {
+                        if (j < bones.Count)
+                            neckMorphDataBlock.Add(bones[j]);
+                        else
+                            neckMorphDataBlock.Add(0);
+                    }
+                }
+                #endregion
+
                 // Padding 
                 #region Padding Data Block
 
@@ -3692,7 +3788,7 @@ public static byte[] MakeUncompressedMdlFile(TTModel ttModel, XivMdl ogMdl, Acti
                 // This is the offset to the beginning of the vertex data
                 var combinedDataBlockSize = _MdlHeaderSize + vertexInfoBlock.Count + pathInfoBlock.Count + basicModelBlock.Count + unknownDataBlock0.Length + (60 * ogMdl.LoDList.Count) + extraMeshesBlock.Count + meshDataBlock.Count +
                     attributePathDataBlock.Count + (unknownDataBlock1?.Length ?? 0) + meshPartDataBlock.Count + unknownDataBlock2.Length + matPathOffsetDataBlock.Count + bonePathOffsetDataBlock.Count +
-                    boneSetsBlock.Count + FullShapeDataBlock.Count + partBoneSetsBlock.Count + paddingDataBlock.Count + boundingBoxDataBlock.Count + boneBoundingBoxDataBlock.Count;
+                    boneSetsBlock.Count + FullShapeDataBlock.Count + partBoneSetsBlock.Count + neckMorphDataBlock.Count + paddingDataBlock.Count + boundingBoxDataBlock.Count + boneBoundingBoxDataBlock.Count;
 
                 var lodDataBlock = new List<byte>();
                 List<int> indexStartInjectPointers = new List<int>();
@@ -3735,6 +3831,14 @@ public static byte[] MakeUncompressedMdlFile(TTModel ttModel, XivMdl ogMdl, Acti
                 lodDataBlock.AddRange(BitConverter.GetBytes(ogMdl.LoDList[0].Unknown6));
                 lodDataBlock.AddRange(BitConverter.GetBytes(ogMdl.LoDList[0].Unknown7));
 
+                // If the neck morph data was discarded -- clear the morph counts here too
+                // (The first 2 bytes of what is currently named "Unknown7" seem to refer to the size of this table)
+                if (ogMdl.NeckMorphTable.Count > 0 && neckMorphDataBlock.Count == 0)
+                {
+                    lodDataBlock[lodDataBlock.Count - 4] = 0;
+                    lodDataBlock[lodDataBlock.Count - 3] = 0;
+                }
+
                 // Vertex & Index Sizes
                 lodDataBlock.AddRange(BitConverter.GetBytes(vertexDataSize));
                 lodDataBlock.AddRange(BitConverter.GetBytes(indexDataSize));
@@ -3771,6 +3875,7 @@ public static byte[] MakeUncompressedMdlFile(TTModel ttModel, XivMdl ogMdl, Acti
                 modelDataBlock.AddRange(boneSetsBlock);
                 modelDataBlock.AddRange(FullShapeDataBlock);
                 modelDataBlock.AddRange(partBoneSetsBlock);
+                modelDataBlock.AddRange(neckMorphDataBlock);
                 modelDataBlock.AddRange(paddingDataBlock);
                 modelDataBlock.AddRange(boundingBoxDataBlock);
                 modelDataBlock.AddRange(boneBoundingBoxDataBlock);

xivModdingFramework/Mods/FileTypes/PmpManipulation.cs

@@ -25,6 +25,7 @@ namespace xivModdingFramework.Mods.FileTypes
     [JsonSubtypes.KnownSubType(typeof(PMPEqdpManipulationWrapperJson), "Eqdp")]
     [JsonSubtypes.KnownSubType(typeof(PMPGmpManipulationWrapperJson), "Gmp")]
     [JsonSubtypes.KnownSubType(typeof(PMPRspManipulationWrapperJson), "Rsp")]
+    [JsonSubtypes.KnownSubType(typeof(PMPAtchManipulationWrapperJson), "Atch")]
     [JsonSubtypes.KnownSubType(typeof(PMPGlobalEqpManipulationWrapperJson), "GlobalEqp")]
     public class PMPManipulationWrapperJson
     {
@@ -168,6 +169,23 @@ public override string GetNiceName()
             return base.GetNiceName() + " - " + Manipulation.SubRace + " " + Manipulation.Attribute;
         }
     }
+    public class PMPAtchManipulationWrapperJson : PMPManipulationWrapperJson
+    {
+        [JsonProperty(Order = 2)]
+        public PMPAtchManipulationJson Manipulation = new PMPAtchManipulationJson();
+        public override object GetManipulation()
+        {
+            return Manipulation;
+        }
+        public override void SetManipulation(object o)
+        {
+            Manipulation = o as PMPAtchManipulationJson;
+        }
+        public override string GetNiceName()
+        {
+            return base.GetNiceName() + " - " + Manipulation.Race + " " + Manipulation.Gender + " " + Manipulation.Type + " [" + Manipulation.Index + "]";
+        }
+    }
     public class PMPGlobalEqpManipulationWrapperJson : PMPManipulationWrapperJson
     {
         [JsonProperty(Order = 2)]
@@ -793,6 +811,29 @@ public static List<PMPRspManipulationJson> FromRgspEntry(RacialGenderScalingPara
         }
 
     }
+    public class PMPAtchManipulationJson
+    {
+        public struct PMPAtchEntry
+        {
+            public string Bone;
+            public float Scale;
+
+            public float OffsetX;
+            public float OffsetY;
+            public float OffsetZ;
+
+            public float RotationX;
+            public float RotationY;
+            public float RotationZ;
+        }
+
+        public PMPAtchEntry Entry;
+        public PMPGender Gender;
+        public PMPModelRace Race;
+        public string Type;
+        public int Index;
+    }
+
     public class PMPGlobalEqpManipulationJson
     {
         public GlobalEqpType Type;

xivModdingFramework/Mods/WizardData.cs

@@ -87,6 +87,7 @@ protected override bool CheckHasData()
             "Est",
             "Gmp",
             "Rsp",
+            "Atch",
             "GlobalEqp"
         };
 
@@ -496,6 +497,12 @@ public async Task<ModOption> ToModOption()
                     };
                     mo.Mods.Add(path, mData);
                 }
+
+                if (manips.OtherManipulations.Count > 0)
+                {
+                    var manip0 = manips.OtherManipulations[0];
+                    throw new InvalidDataException("TTMP Does not support " + manip0.Type + " manipulations.");
+                }
             }
 
 
@@ -546,6 +553,7 @@ public class WizardImcGroupData
         public ushort Variant;
         public XivImc BaseEntry = new XivImc();
         public bool AllVariants;
+        public bool OnlyAttributes;
     }
 
     /// <summary>
@@ -774,6 +782,7 @@ public static async Task<WizardGroupEntry> FromPMPGroup(PMPGroupJson pGroup, str
                     Root = imcGroup.Identifier.GetRoot(),
                     BaseEntry = imcGroup.DefaultEntry.ToXivImc(),
                     AllVariants = imcGroup.AllVariants,
+                    OnlyAttributes = imcGroup.OnlyAttributes,
                 };
             }
 
@@ -889,6 +898,7 @@ public async Task<PMPGroupJson> ToPmpGroup(string tempFolder, Dictionary<string,
                 imcG.Identifier = PmpIdentifierJson.FromRoot(ImcData.Root.Info, ImcData.Variant);
                 imcG.DefaultEntry = PMPImcManipulationJson.PMPImcEntry.FromXivImc(ImcData.BaseEntry);
                 imcG.AllVariants = ImcData.AllVariants;
+                imcG.OnlyAttributes = ImcData.OnlyAttributes;
             }
             else
             {

xivModdingFramework/VFX/FileTypes/Avfx.cs

@@ -52,10 +52,9 @@ await Task.Run(() =>
                         {
                             data = br.ReadInt32();
                         }
-                        catch (EndOfStreamException e)
+                        catch (EndOfStreamException)
                         {
-                            throw new System.Exception("VFX textures were detected but no texture paths could be found.\n" +
-                                "VFX textures will not be accessible.");
+                            return;
                         }                        
                     }