Merge pull request #28 from lmcintyre/texfix

Fix on both ends (create, import) of TT for uncompressed Tex sizes

Author: Lunaretic

xivModdingFramework/Cache/XivCache.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Data.SQLite;
@@ -15,6 +15,7 @@
 using xivModdingFramework.Items.Interfaces;
 using xivModdingFramework.Models.FileTypes;
 using xivModdingFramework.Mods;
+using xivModdingFramework.Mods.DataContainers;
 using xivModdingFramework.Resources;
 using xivModdingFramework.SqPack.FileTypes;
 
@@ -31,7 +32,7 @@ public static class XivCache
         private static GameInfo _gameInfo;
         private static DirectoryInfo _dbPath;
         private static DirectoryInfo _rootCachePath;
-        public static readonly Version CacheVersion = new Version("1.0.2.1");
+        public static readonly Version CacheVersion = new Version("1.0.2.2");
         private const string dbFileName = "mod_cache.db";
         private const string rootCacheFileName = "item_sets.db";
         private const string creationScript = "CreateCacheDB.sql";
@@ -187,7 +188,11 @@ public static void SetGameInfo(GameInfo gameInfo = null, bool enableCacheWorker
                 var reason = CacheNeedsRebuild();
                 if (reason != CacheRebuildReason.CacheOK && !_REBUILDING)
                 {
-                    RebuildCache(reason);
+					var ver = 
+						reason == CacheRebuildReason.CacheVersionUpdate 
+							? new Version(GetMetaValue("cache_version")) : CacheVersion;
+
+					RebuildCache(ver, reason);
                 }
             }
 
@@ -277,7 +282,7 @@ private static CacheRebuildReason CacheNeedsRebuild()
         /// help ensure it's never accidentally called
         /// without an await.
         /// </summary>
-        public static void RebuildCache(CacheRebuildReason reason = CacheRebuildReason.ManualRequest)
+        public static void RebuildCache(Version previousVersion, CacheRebuildReason reason = CacheRebuildReason.ManualRequest)
         {
             CacheWorkerEnabled = false;
             _REBUILDING = true;
@@ -309,7 +314,13 @@ public static void RebuildCache(CacheRebuildReason reason = CacheRebuildReason.M
                     tasks.Add(RebuildFurnitureCache());
                     tasks.Add(BuildModdedItemDependencies());
 
-                    await Task.WhenAll(tasks);
+                    // This was originally running only if the reason was cache update,
+                    // but if the cache gets messed up in one way or another, and has to
+                    // rebuild on a new TT version for any reason other than CacheUpdate
+                    // or whatever, it will prevent the migration from occurring properly
+	                tasks.Add(MigrateCache(previousVersion));
+
+	                await Task.WhenAll(tasks);
 
                     var post = (DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1))).TotalSeconds;
 
@@ -451,7 +462,47 @@ private static void CreateCache()
             }
         }
 
+        private static async Task MigrateCache(Version lastCacheVersion) {
+
+            // Tex file fix migration
+            // This technically has nothing to do with the cache version,
+            // but I think this is one of the only places that this can go
+            if (lastCacheVersion < new Version(1, 0, 2, 2)) {
+	            var m = new Modding(_gameInfo.GameDirectory);
+	            var modList = m.GetModList();
+	            foreach (var mod in modList.Mods) {
+		            if (mod.data.dataType != 4) continue;
+
+		            var datNum = (int)((mod.data.modOffset / 8) & 0x0F) / 2;
+		            var dat = XivDataFiles.GetXivDataFile(mod.datFile);
 
+		            var datPath = $"{_gameInfo.GameDirectory}/{dat.GetDataFileName()}{Dat.DatExtension}{datNum}";
+
+		            int uncompressedSize = -1;
+                    long seekTo = Dat.OffsetCorrection(datNum, mod.data.modOffset) + 8;
+
+                    // Seek to and read the uncompressed texture size catching any exceptions
+                    // because we handle further processing with the initial value of -1
+                    try
+                    {
+	                    using var reader = new BinaryReader(File.OpenRead(datPath));
+	                    reader.BaseStream.Position = seekTo;
+
+	                    uncompressedSize = reader.ReadInt32();
+                    } catch (Exception) {}
+                    
+                    // If we read an uncompressed size, seek to the same position and write the fixed uncompressed texture size
+                    if (uncompressedSize != -1)
+                    {
+	                    using var writer = new BinaryWriter(File.OpenWrite(datPath));
+	                    writer.BaseStream.Position = seekTo;
+
+	                    var tmp = BitConverter.GetBytes(uncompressedSize + 80);
+                        writer.Write(tmp);
+                    }
+	            }
+            }
+        }
 
         /// <summary>
         /// Populate the ui table.

xivModdingFramework/Helpers/ProblemChecker.cs

@@ -258,7 +258,7 @@ public Task PerformStartOver(DirectoryInfo backupsDirectory, IProgress<string> p
 
                     await Task.Run(async () =>
                     {
-                        XivCache.RebuildCache();
+                        XivCache.RebuildCache(XivCache.CacheVersion);
                     });
                 }
             });

xivModdingFramework/Mods/FileTypes/TTMP.cs

@@ -1,4 +1,4 @@
-// xivModdingFramework
+// xivModdingFramework
 // Copyright © 2018 Rafael Gonzalez - All Rights Reserved
 // 
 // This program is free software: you can redistribute it and/or modify
@@ -48,9 +48,9 @@ public class TTMP
             ".cmp", ".imc", ".eqdp", ".eqp", ".gmp", ".est"
         };
 
-        private readonly string _currentWizardTTMPVersion = "1.2w";
-        private readonly string _currentSimpleTTMPVersion = "1.2s";
-        private const string _minimumAssembly = "1.2.0.0";
+        private readonly string _currentWizardTTMPVersion = "1.3w";
+        private readonly string _currentSimpleTTMPVersion = "1.3s";
+        private const string _minimumAssembly = "1.3.0.0";
 
         private string _tempMPD, _tempMPL, _source;
         private readonly DirectoryInfo _modPackDirectory;
@@ -423,7 +423,7 @@ public Task<List<OriginalModPackJson>> GetOriginalModPackJsonData(DirectoryInfo
         /// </summary>
         /// <param name="modPackDirectory">The mod pack directory</param>
         /// <returns>The version of the mod pack as a string</returns>
-        public string GetVersion(DirectoryInfo modPackDirectory)
+        public static string GetVersion(DirectoryInfo modPackDirectory)
         {
             ModPackJson modPackJson = null;
 
@@ -529,6 +529,7 @@ await Task.Run(async () =>
 
                     var _modding = new Modding(XivCache.GameInfo.GameDirectory);
                     var modList = _modding.GetModList();
+                    var needsTexFix = DoesTexNeedFixing(modPackDirectory);
 
                     // 0 - Extract the MPD file.
                     using (var zf = ZipFile.Read(modPackDirectory.FullName))
@@ -572,6 +573,10 @@ await Task.Run(async () =>
                             {
                                 binaryReader.BaseStream.Seek(modJson.ModOffset, SeekOrigin.Begin);
                                 var data = binaryReader.ReadBytes(modJson.ModSize);
+
+                                if (modJson.FullPath.EndsWith(".tex") && needsTexFix)
+	                                FixupTextoolsTex(data);
+
                                 var df = IOUtil.GetDataFileFromPath(modJson.FullPath);
 
                                 var size = data.Length;
@@ -953,6 +958,46 @@ await Task.Run(async () =>
             return (count, errorCount, importErrors, seconds);
         }
 
+        /// <summary>
+        /// Parse the version out of this modpack to determine whether or not we need
+        /// to add 80 to the uncompressed size of the Tex files contained within.
+        /// </summary>
+        /// <param name="mpd">The path to the modpack.</param>
+        /// <returns>True if we must modify tex header uncompressed sizes, false otherwise.</returns>
+        private static bool DoesTexNeedFixing(DirectoryInfo mpd) {
+
+	        var ver = GetVersion(mpd);
+	        if (string.IsNullOrEmpty(ver))
+		        return true;
+
+	        var newVer = ver;
+
+	        var lastChar = ver.Substring(ver.Length - 1)[0];
+	        if (char.IsLetter(lastChar))
+		        newVer = ver.Substring(0, ver.Length - 1);
+
+	        double.TryParse(newVer, out var verDouble);
+
+	        return verDouble < 1.3;
+        }
+
+        /// <summary>
+        /// Fix xivModdingFramework TEX quirks.
+        /// </summary>
+        /// <param name="tex">The TEX data to be fixed up.</param>
+        public static void FixupTextoolsTex(byte[] tex) {
+
+	        // Read the uncompressed size from the file
+	        var size = BitConverter.ToInt32(tex, 8);
+	        var newSize = size + 80;
+	        
+	        byte[] buffer = BitConverter.GetBytes(newSize);
+	        tex[8] = buffer[0];
+	        tex[9] = buffer[1];
+	        tex[10] = buffer[2];
+	        tex[11] = buffer[3];
+        }
+
         /// <summary>
         /// Gets the data type from an item path
         /// </summary>

xivModdingFramework/SqPack/FileTypes/Dat.cs

@@ -1,4 +1,4 @@
-// xivModdingFramework
+// xivModdingFramework
 // Copyright © 2018 Rafael Gonzalez - All Rights Reserved
 // 
 // This program is free software: you can redistribute it and/or modify
@@ -1266,7 +1266,7 @@ public byte[] MakeType4DatHeader(XivTexFormat format, List<short> mipPartOffsets
 
             headerData.AddRange(BitConverter.GetBytes(headerSize + headerPadding));
             headerData.AddRange(BitConverter.GetBytes(4));
-            headerData.AddRange(BitConverter.GetBytes(uncompressedLength));
+            headerData.AddRange(BitConverter.GetBytes(uncompressedLength + 80));
             headerData.AddRange(BitConverter.GetBytes(0));
             headerData.AddRange(BitConverter.GetBytes(0));
             headerData.AddRange(BitConverter.GetBytes(newMipCount));

xivModdingFramework/xivModdingFramework.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <AssemblyVersion>1.2.0.0</AssemblyVersion>
+    <AssemblyVersion>1.3.0.0</AssemblyVersion>
     <Version>1.0.8</Version>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <LangVersion>latest</LangVersion>