- Item Convert On Import initial work

- Perf fixes for item loading.

Author: Lunaretic

xivModdingFramework/Cache/XivDependencyGraph.cs

@@ -17,7 +17,10 @@
 using xivModdingFramework.Models.DataContainers;
 using xivModdingFramework.Models.FileTypes;
 using xivModdingFramework.Mods;
+using xivModdingFramework.Mods.DataContainers;
+using xivModdingFramework.Mods.FileTypes;
 using xivModdingFramework.Resources;
+using xivModdingFramework.SqPack.DataContainers;
 using xivModdingFramework.SqPack.FileTypes;
 using xivModdingFramework.Variants.FileTypes;
 using static xivModdingFramework.Cache.XivCache;
@@ -492,14 +495,25 @@ public override int GetHashCode()
         /// Gets all the model files in this dependency chain.
         /// </summary>
         /// <returns></returns>
-        public async Task<List<string>> GetModelFiles()
+        public async Task<List<string>> GetModelFiles(IndexFile index = null, ModList modlist = null)
         {
             // Some chains have no meta entries, and jump straight to models.
             // Try to resolve Meta files first.
             if (Info.PrimaryType == XivItemType.equipment || Info.PrimaryType == XivItemType.accessory)
             {
                 var _eqp = new Eqp(XivCache.GameInfo.GameDirectory);
-                var races = await _eqp.GetAvailableRacialModels(Info.PrimaryId, Info.Slot, false, true);
+
+                List<XivRace> races = null;
+                if (index != null)
+                {
+                    var metadata = await ItemMetadata.GetFromCachedIndex(this, index);
+                    races = metadata.EqdpEntries.Where(x => x.Value.bit1).Select(x => x.Key).ToList();
+                }
+                else
+                {
+                    races = await _eqp.GetAvailableRacialModels(Info.PrimaryId, Info.Slot, false, true);
+                }
+
                 var models = new List<string>();
                 foreach(var race in races)
                 {
@@ -523,8 +537,13 @@ public async Task<List<string>> GetModelFiles()
                 if (Info.PrimaryType == XivItemType.human && Info.SecondaryType != XivItemType.hair && Info.SecondaryId / 100 >= 1)
                 {
                     // For human types, if their model is missing, the version 00xx is used instead.
-                    var index = new Index(XivCache.GameInfo.GameDirectory);
-                    if(!(await index.FileExists(modelPath)))
+                    if(index == null)
+                    {
+                        var _index = new Index(XivCache.GameInfo.GameDirectory);
+                        index = await _index.GetIndexFile(IOUtil.GetDataFileFromPath(modelPath), false, true);
+                    }
+
+                    if(!(index.FileExists(modelPath)))
                     {
                         var replacementNumber = (Info.SecondaryId % 100);
                         var alteredRoot = new XivDependencyRoot(Info.PrimaryType, Info.PrimaryId, Info.SecondaryType, replacementNumber, Info.Slot);
@@ -546,8 +565,9 @@ public async Task<List<string>> GetModelFiles()
         /// Subsets of this data may be accessed with XivDependencyGraph::GetChildFiles(internalFilePath).
         /// </summary>
         /// <returns></returns>
-        public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
+        public async Task<List<string>> GetMaterialFiles(int materialVariant = -1, IndexFile index = null, ModList modlist = null)
         {
+            var useCache = index == null;
 
             var materials = new HashSet<string>();
             if (Info.PrimaryType == XivItemType.human && Info.SecondaryType == XivItemType.body)
@@ -560,8 +580,13 @@ public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
                 var path= $"chara/human/c{primary}/obj/body/b{body}/material/v0001/mt_c{primary}b{body}_a.mtrl";
 
                 // Just validate it exists and call it a day.
-                var index = new Index(XivCache.GameInfo.GameDirectory);
-                var exists = await index.FileExists(path);
+                if (index == null)
+                {
+                    var _index = new Index(XivCache.GameInfo.GameDirectory);
+                    index = await _index.GetIndexFile(IOUtil.GetDataFileFromPath(path), false, true);
+                }
+
+                var exists = index.FileExists(path);
                 if(exists)
                 {
                     materials.Add(path);
@@ -582,7 +607,7 @@ public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
             }
             else
             {
-                var models = await GetModelFiles();
+                var models = await GetModelFiles(index, modlist);
                 if (models != null && models.Count > 0)
                 {
                     var dataFile = IOUtil.GetDataFileFromPath(models[0]);
@@ -598,7 +623,15 @@ public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
 
                     foreach (var model in models)
                     {
-                        var mdlMats = await XivCache.GetChildFiles(model);
+                        List<string> mdlMats = null;
+                        if (useCache)
+                        {
+                            mdlMats = await XivCache.GetChildFiles(model);
+                        } else
+                        {
+                            mdlMats = await _mdl.GetReferencedMaterialPaths(model, -1, false, false, index, modlist);
+                        }
+
                         if (materialVariant <= 0)
                         {
                             foreach (var mat in mdlMats)
@@ -638,11 +671,15 @@ public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
             }
 
             // Here we get to get a little fancy.
-            var _modding = new Modding(XivCache.GameInfo.GameDirectory);
-            var modList = _modding.GetModList();
+            if (modlist == null)
+            {
+                var _modding = new Modding(XivCache.GameInfo.GameDirectory);
+                modlist = _modding.GetModList();
+            }
+
             var rootFolder = Info.GetRootFolder();
             var variantRep = "v" + materialVariant.ToString().PadLeft(4, '0');
-            foreach (var mod in modList.Mods)
+            foreach (var mod in modlist.Mods)
             {
                 if (!mod.enabled) continue;
 
@@ -673,15 +710,25 @@ public async Task<List<string>> GetMaterialFiles(int materialVariant = -1)
         /// Subsets of this data may be accessed with XivDependencyGraph::GetChildFiles(internalFilePath).
         /// </summary>
         /// <returns></returns>
-        public async Task<List<string>> GetTextureFiles(int materialVariant = -1)
+        public async Task<List<string>> GetTextureFiles(int materialVariant = -1, IndexFile index = null, ModList modlist = null)
         {
-            var materials = await GetMaterialFiles(materialVariant);
+            var materials = await GetMaterialFiles(materialVariant, index, modlist);
             var textures = new HashSet<string>();
             if (materials != null && materials.Count > 0)
             {
                 foreach (var mat in materials)
                 {
-                    var mtrlTexs = await XivCache.GetChildFiles(mat);
+                    List<string> mtrlTexs = null;
+                    if (index == null)
+                    {
+                        mtrlTexs = await XivCache.GetChildFiles(mat);
+                    } else
+                    {
+                        var dataFile = IOUtil.GetDataFileFromPath(mat);
+                        var _mtrl = new Mtrl(XivCache.GameInfo.GameDirectory, dataFile, XivCache.GameInfo.GameLanguage);
+                        mtrlTexs = await _mtrl.GetTexturePathsFromMtrlPath(mat, false, false, index, modlist);
+                    }
+
                     foreach (var tex in mtrlTexs)
                     {
                         textures.Add(tex);

xivModdingFramework/Materials/FileTypes/Mtrl.cs

@@ -295,10 +295,10 @@ public async Task<XivMtrl> GetMtrlData(string mtrlFile, int materialSet = -1, in
         /// </summary>
         /// <param name="mtrlPath"></param>
         /// <returns></returns>
-        public async Task<List<string>> GetTexturePathsFromMtrlPath(string mtrlPath, bool includeDummies = false, bool forceOriginal = false)
+        public async Task<List<string>> GetTexturePathsFromMtrlPath(string mtrlPath, bool includeDummies = false, bool forceOriginal = false, IndexFile index = null, ModList modlist = null)
         {
             var dat = new Dat(_gameDirectory);
-            var mtrlData = await dat.GetType2Data(mtrlPath, forceOriginal);
+            var mtrlData = await dat.GetType2Data(mtrlPath, forceOriginal, index, modlist);
             var uniqueTextures = new HashSet<string>();
             var texRegex = new Regex(".*\\.tex$");
 

xivModdingFramework/Models/FileTypes/Mdl.cs

@@ -56,6 +56,8 @@
 using System.Text.RegularExpressions;
 using xivModdingFramework.Materials.DataContainers;
 using xivModdingFramework.Cache;
+using xivModdingFramework.SqPack.DataContainers;
+using xivModdingFramework.Mods.DataContainers;
 
 namespace xivModdingFramework.Models.FileTypes
 {
@@ -1691,18 +1693,24 @@ public async Task<XivMdl> GetRawMdlData(string mdlPath, bool getOriginal = false
         /// <param name="mdlPath"></param>
         /// <param name="getOriginal"></param>
         /// <returns></returns>
-        public async Task<List<string>> GetReferencedMaterialPaths(string mdlPath, int materialVariant = -1, bool getOriginal = false, bool includeSkin = true)
+        public async Task<List<string>> GetReferencedMaterialPaths(string mdlPath, int materialVariant = -1, bool getOriginal = false, bool includeSkin = true, IndexFile index = null, ModList modlist = null)
         {
             // Language is irrelevant here.
             var dataFile = IOUtil.GetDataFileFromPath(mdlPath);
             var _mtrl = new Mtrl(_gameDirectory, dataFile, XivLanguage.None);
             var _imc = new Imc(_gameDirectory);
-            var _index = new Index(_gameDirectory);
+            if (index == null)
+            {
+                var _index = new Index(_gameDirectory);
+                var _modding = new Modding(_gameDirectory);
+                index = await _index.GetIndexFile(dataFile, false, true);
+                modlist = await _modding.GetModListAsync();
+            }
 
             var materials = new List<string>();
 
             // Read the raw Material names from the file.
-            var materialNames = await GetReferencedMaterialNames(mdlPath, getOriginal);
+            var materialNames = await GetReferencedMaterialNames(mdlPath, getOriginal, index, modlist);
             if(materialNames.Count == 0)
             {
                 return materials;
@@ -1729,7 +1737,7 @@ public async Task<List<string>> GetReferencedMaterialPaths(string mdlPath, int m
                     // We need to get the IMC info for this MDL so that we can pull every possible Material Variant.
                     try
                     {
-                        var info = await _imc.GetFullImcInfo(imcPath);
+                        var info = await _imc.GetFullImcInfo(imcPath, index, modlist);
                         var slotRegex = new Regex("_([a-z]{3}).mdl$");
                         var slot = "";
                         var m = slotRegex.Match(mdlPath);
@@ -1802,19 +1810,30 @@ public async Task<List<string>> GetReferencedMaterialPaths(string mdlPath, int m
         /// <param name="mdlPath"></param>
         /// <param name="getOriginal"></param>
         /// <returns></returns>
-        public async Task<List<string>> GetReferencedMaterialNames(string mdlPath, bool getOriginal = false)
+        public async Task<List<string>> GetReferencedMaterialNames(string mdlPath, bool getOriginal = false, IndexFile index = null, ModList modlist = null)
         {
             var materials = new List<string>();
-            var index = new Index(_gameDirectory);
             var dat = new Dat(_gameDirectory);
             var modding = new Modding(_gameDirectory);
-            var mod = await modding.TryGetModEntry(mdlPath);
-            var offset = await index.GetDataOffset(mdlPath);
 
-            var modded = mod != null && mod.enabled;
+
+            if (index == null)
+            {
+                var _index = new Index(_gameDirectory);
+                index = await _index.GetIndexFile(IOUtil.GetDataFileFromPath(mdlPath), false, true);
+                
+            }
+
+            var offset = index.Get8xDataOffset(mdlPath);
             if (getOriginal)
             {
-                if (modded)
+                if(modlist == null)
+                {
+                    modlist = await modding.GetModListAsync();
+                }
+
+                var mod = modlist.Mods.FirstOrDefault(x => x.fullPath == mdlPath);
+                if(mod != null)
                 {
                     offset = mod.data.originalOffset;
                 }

xivModdingFramework/Mods/FileTypes/ItemMetadata.cs

@@ -172,6 +172,36 @@ public static async Task<ItemMetadata> GetMetadata(XivDependencyRoot root, bool
             }
         }
 
+        /// <summary>
+        /// Retrieves the item metadata from a cached index setup (or raw)
+        /// </summary>
+        /// <param name="root"></param>
+        /// <param name="index"></param>
+        /// <returns></returns>
+        public static async Task<ItemMetadata> GetFromCachedIndex(XivDependencyRoot root, IndexFile index)
+        {
+
+            var _dat = new Dat(XivCache.GameInfo.GameDirectory);
+            var df = IOUtil.GetDataFileFromPath(root.Info.GetRootFile());
+
+            long offset = 0;
+            if (index != null)
+            {
+                offset = index.Get8xDataOffset(root.Info.GetRootFile());
+            }
+
+            ItemMetadata mData = null;
+            if (offset == 0)
+            {
+                mData = await ItemMetadata.GetMetadata(root);
+            }
+            else
+            {
+                var data = await _dat.GetType2Data(offset, df);
+                mData = await ItemMetadata.Deserialize(data);
+            }
+            return mData;
+        }
 
         /// <summary>
         /// Creates a new ItemMetaData entry from the constituent files around the FFXIV file system.