Skip to content

Commit 59e6108

Browse files
starstormhunstarstormhun
authored
[ME] Local / deduped textures (#370)
* Add config elements * Implement local saving * Change texture hashing to CRC64, up to 2048 bytes * Implement loading of local textures * Add local file auditing (Needs heavy debug, WIP) * Fix audit bugs, add audit button functionality * Export original textures if possible * Fix breaking GUI.skin * Fix autosave logic, add bigger warnings to options * Fix file saving and parsing * Switch over to KKAPI LocalTextures API * Update dependencies * Update dependencies of projects that broke out of nowhere * Implement deduped saving / loading * Improve hash collision safety * Improve local file audit via background threads * Revamp audit file processing * Fix local texture path customisation * Remove SceneLoadWatcher as it's unnecessary * Reprivatise _token and add Hash property * Move new texture save handling to new class * Clean some unnecessary usings * Address Copilot suggestions * Fix error message * Separate audit constants and saveload variables * Fix showing inaccurate load error * Never return null texdic when loading * Switch over to KKAPI's TextureSaveHandlerBase Still needs API nuget update whenever that happens * Update references, fix ME-dependent project nugets * Fix LocalTexturePath config, prune unused configs * Protect Save function against errors thrown in base * Add warning when falling back * Relative path support for Local texture folder override * Oops * Address copilot concerns again
1 parent 7ea654a commit 59e6108

File tree

26 files changed

+513
-192
lines changed

26 files changed

+513
-192
lines changed

‎src/HairShadowColorControl.KK/KK.HairShadowColorControl.csproj‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,5 @@
1414
<ItemGroup>
1515
<ProjectReference Include="..\MaterialEditor.KK\KK.MaterialEditor.csproj" />
1616
</ItemGroup>
17-
<ItemGroup>
18-
<PackageReference Include="ExtensibleSaveFormat.Koikatu" Version="19.3.3" />
19-
<PackageReference Include="IllusionLibs.BepInEx" Version="5.4.22" />
20-
<PackageReference Include="IllusionLibs.BepInEx.Harmony" Version="2.9.0" />
21-
<PackageReference Include="IllusionLibs.Koikatu.Assembly-CSharp" Version="2019.4.27.4" />
22-
<PackageReference Include="IllusionLibs.Koikatu.Assembly-CSharp-firstpass" Version="2019.4.27.4" />
23-
<PackageReference Include="IllusionLibs.Koikatu.UnityEngine" Version="5.6.2.4" />
24-
<PackageReference Include="IllusionLibs.Koikatu.UnityEngine.UI" Version="5.6.2.4" />
25-
<PackageReference Include="IllusionModdingAPI.KKAPI" Version="1.38.0" />
26-
<PackageReference Include="KoikatuCompatibilityAnalyzer" Version="1.1.0" />
27-
</ItemGroup>
2817
<Import Project="..\HairShadowColorControl.Core\HairShadowColorControl.Core.projitems" Label="Shared" />
2918
</Project>

‎src/HairShadowColorControl.KKS/KKS.HairShadowColorControl.csproj‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,5 @@
1414
<ItemGroup>
1515
<ProjectReference Include="..\MaterialEditor.KKS\KKS.MaterialEditor.csproj" />
1616
</ItemGroup>
17-
<ItemGroup>
18-
<PackageReference Include="ExtensibleSaveFormat.KoikatsuSunshine" Version="19.3.3" />
19-
<PackageReference Include="IllusionLibs.BepInEx" Version="5.4.22" />
20-
<PackageReference Include="IllusionLibs.BepInEx.Harmony" Version="2.9.0" />
21-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.Assembly-CSharp" Version="2021.9.17" />
22-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.Assembly-CSharp-firstpass" Version="2021.9.17" />
23-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UniRx" Version="2021.9.17" />
24-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UniTask" Version="2021.9.17" />
25-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UnityEngine.CoreModule" Version="2019.4.9" />
26-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UnityEngine.IMGUIModule" Version="2019.4.9" />
27-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UnityEngine.InputLegacyModule" Version="2019.4.9" />
28-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UnityEngine.UI" Version="2019.4.9" />
29-
<PackageReference Include="IllusionLibs.KoikatsuSunshine.UnityEngine.UIModule" Version="2019.4.9" />
30-
<PackageReference Include="IllusionModdingAPI.KKSAPI" Version="1.38.0" />
31-
</ItemGroup>
3217
<Import Project="..\HairShadowColorControl.Core\HairShadowColorControl.Core.projitems" Label="Shared" />
3318
</Project>

‎src/MaterialEditor.AI/AI.MaterialEditor.csproj‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,12 @@
1414
</PropertyGroup>
1515
<ItemGroup>
1616
<EmbeddedResource Include="Resources\default.xml" />
17-
</ItemGroup>
18-
<ItemGroup>
1917
<EmbeddedResource Include="Resources\EC_Shaders.unity3d" />
20-
</ItemGroup>
21-
<ItemGroup>
2218
<EmbeddedResource Include="Resources\AI_Shaders.unity3d" />
2319
</ItemGroup>
2420
<ItemGroup>
2521
<PackageReference Include="BepInEx.Analyzers" Version="1.0.4" />
26-
<PackageReference Include="ExtensibleSaveFormat.AIGirl" Version="19.3.3" />
22+
<PackageReference Include="ExtensibleSaveFormat.AIGirl" Version="21.1.2" />
2723
<PackageReference Include="IllusionLibs.AIGirl.Assembly-CSharp" Version="2020.5.29.5" />
2824
<PackageReference Include="IllusionLibs.AIGirl.Assembly-CSharp-firstpass" Version="2020.5.29.5" />
2925
<PackageReference Include="IllusionLibs.AIGirl.MessagePack" Version="2020.5.29.5" />
@@ -40,8 +36,8 @@
4036
<PackageReference Include="IllusionLibs.BepInEx.Harmony" Version="2.9.0" />
4137
<PackageReference Include="IllusionLibs.BepInEx.MonoMod" Version="22.1.29.1" />
4238
<PackageReference Include="IllusionLibs.XUnity.ResourceRedirector" Version="4.18.0" />
43-
<PackageReference Include="IllusionModdingAPI.AIAPI" Version="1.38.0" />
44-
<PackageReference Include="Microsoft.Unity.Analyzers" Version="1.21.0" />
39+
<PackageReference Include="IllusionModdingAPI.AIAPI" Version="1.45.0" />
40+
<PackageReference Include="Microsoft.Unity.Analyzers" Version="1.25.0" />
4541
<PackageReference Include="SharpZipLib" Version="[1.3.3,1.4)" />
4642
<PackageReference Include="Sideloader.AIGirl" Version="19.3.3" />
4743
</ItemGroup>

‎src/MaterialEditor.Base/Extensions.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ public static bool SafeProcObject<T>(this T self, Action<T> act) where T : Unity
149149
public static TResult Call<T1, T2, TResult>(this Func<T1, T2, TResult> func, T1 arg1, T2 arg2, TResult result = default) => func == null ? result : func(arg1, arg2);
150150

151151
public static TResult Call<T1, T2, T3, TResult>(this Func<T1, T2, T3, TResult> func, T1 arg1, T2 arg2, T3 arg3, TResult result = default) => func == null ? result : func(arg1, arg2, arg3);
152+
153+
public static IEnumerable<T> SubSet<T>(this T[] array, int start, int end)
154+
{
155+
for (int i = start; i < end; i++) yield return array[i];
156+
}
152157
}
153158

154159
internal static class MeshExtensions

‎src/MaterialEditor.Base/PluginBase.cs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using BepInEx;
2+
using BepInEx.Bootstrap;
23
using BepInEx.Configuration;
34
using BepInEx.Logging;
45
using System;
@@ -36,6 +37,14 @@ public partial class MaterialEditorPluginBase : BaseUnityPlugin
3637
/// </summary>
3738
public static string ExportPath = ExportPathDefault;
3839
/// <summary>
40+
/// Default path where local textures will be exported to / imported from
41+
/// </summary>
42+
public static string LocalTexturePathDefault = Path.Combine(Paths.GameRootPath, @"UserData\MaterialEditor\_LocalTextures");
43+
/// <summary>
44+
/// Path where local textures will be exported to / imported from
45+
/// </summary>
46+
public static string LocalTexturePath = LocalTexturePathDefault;
47+
/// <summary>
3948
/// Saved material edits
4049
/// </summary>
4150
public static CopyContainer CopyData = new CopyContainer();
@@ -133,6 +142,10 @@ public partial class MaterialEditorPluginBase : BaseUnityPlugin
133142
/// When enabled, normalmaps get converted from DXT5 compressed (red) normals back to normal OpenGL (blue/purple) normals
134143
/// </summary>
135144
public static ConfigEntry<bool> ConvertNormalmapsOnExport { get; set; }
145+
/// <summary>
146+
/// Local textures will be exported to / imported from this folder. If empty, defaults to {LocalTexturePathDefault}
147+
/// </summary>
148+
internal static ConfigEntry<string> ConfigLocalTexturePath { get; set; }
136149

137150
/// <summary>
138151
/// Init logic, do not call
@@ -160,7 +173,7 @@ public virtual void Awake()
160173
SortPropertiesByCategory = Config.Bind("Config", "Sort Properties by Category", true, "Whether to sort shader properties by their category.");
161174
ConvertNormalmapsOnExport = Config.Bind("Config", "Convert Normalmaps On Export", true, new ConfigDescription("When enabled, normalmaps get converted from DXT5 compressed (red) normals back to normal OpenGL (blue/purple) normals"));
162175

163-
//Everything in these games is 10x the size of KK/KKS
176+
// Everything in these games is 10x the size of KK/KKS
164177
#if AI || HS2 || PH
165178
ProjectorNearClipPlaneMax = Config.Bind("Projector", "Max Near Clip Plane", 100f, new ConfigDescription("Controls the max value of the slider for this projector property", new AcceptableValueRange<float>(0.01f, 1000f), new ConfigurationManagerAttributes { Order = 5 }));
166179
ProjectorFarClipPlaneMax = Config.Bind("Projector", "Max Far Clip Plane", 1000f, new ConfigDescription("Controls the max value of the slider for this projector property", new AcceptableValueRange<float>(0.01f, 1000f), new ConfigurationManagerAttributes { Order = 4 }));

‎src/MaterialEditor.Base/UI/UI.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ public abstract class MaterialEditorUI : BaseUnityPlugin
108108

109109
private protected IMaterialEditorColorPalette ColorPalette;
110110

111-
private GameObject CurrentGameObject;
112-
private object CurrentData;
111+
internal GameObject CurrentGameObject;
112+
internal object CurrentData;
113113
private static string CurrentFilter = "";
114114
private bool DoObjExport = false;
115115
private Renderer ObjRenderer;
@@ -967,7 +967,7 @@ protected IEnumerator PopulateListCoroutine(GameObject go, object data, string f
967967
PopulateList(go, data, filter);
968968
}
969969

970-
private static void ExportTexture(Material mat, string property)
970+
internal virtual void ExportTexture(Material mat, string property)
971971
{
972972
var tex = mat.GetTexture($"_{property}");
973973
if (tex == null) return;
@@ -980,6 +980,16 @@ private static void ExportTexture(Material mat, string property)
980980
Utilities.OpenFileInExplorer(filename);
981981
}
982982

983+
internal void ExportTextureOriginal(Material mat, string property, string ext, byte[] texData)
984+
{
985+
var matName = mat.NameFormatted();
986+
matName = string.Concat(matName.Split(Path.GetInvalidFileNameChars())).Trim();
987+
string filename = Path.Combine(ExportPath, $"_Export_{DateTime.Now:yyyy-MM-dd-HH-mm-ss}_{matName}_{property}.{ext}");
988+
System.IO.File.WriteAllBytes(filename, texData);
989+
MaterialEditorPluginBase.Logger.LogInfo($"Exported {filename}");
990+
Utilities.OpenFileInExplorer(filename);
991+
}
992+
983993
/// <summary>
984994
/// Gets the original value of a renderer property.
985995
/// </summary>

‎src/MaterialEditor.Core.Maker/Core.MaterialEditor.Maker.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@
33
using KKAPI;
44
using KKAPI.Maker;
55
using KKAPI.Maker.UI;
6+
using KKAPI.Utilities;
67
using MaterialEditorAPI;
78
using System.Collections;
89
using System.Collections.Generic;
910
using UnityEngine;
1011
using static Illusion.Utils;
1112
using static MaterialEditorAPI.MaterialAPI;
13+
using System.Linq;
14+
1215
#if AI || HS2
1316
using AIChara;
1417
using ChaClothesComponent = AIChara.CmpClothes;
@@ -300,6 +303,26 @@ public void UpdateUIHair(int index)
300303
PopulateList(hair, new ObjectData(index, MaterialEditorCharaController.ObjectType.Hair));
301304
}
302305

306+
internal override void ExportTexture(Material mat, string property)
307+
{
308+
byte[] texData = null;
309+
if (CurrentData is ObjectData objData)
310+
{
311+
var controller = (MaterialEditorCharaController)CurrentGameObject.GetComponentInParent(typeof(MaterialEditorCharaController));
312+
if (controller != null)
313+
{
314+
var textureProperty = controller.MaterialTexturePropertyList.FirstOrDefault(x => x.ObjectType == objData.ObjectType && x.CoordinateIndex == controller.GetCoordinateIndex(objData.ObjectType) && x.Slot == objData.Slot && x.Property == property && x.MaterialName == mat.NameFormatted());
315+
if (textureProperty?.TexID != null)
316+
texData = controller.TextureDictionary[textureProperty.TexID.Value].Data;
317+
}
318+
}
319+
string ext = ImageTypeIdentifier.Identify(texData, "XXX");
320+
if (texData != null && ext != "XXX")
321+
base.ExportTextureOriginal(mat, property, ext, texData);
322+
else
323+
base.ExportTexture(mat, property);
324+
}
325+
303326
public override string GetRendererPropertyValueOriginal(object data, Renderer renderer, RendererProperties property, GameObject go)
304327
{
305328
ObjectData objectData = (ObjectData)data;

‎src/MaterialEditor.Core.Studio/Core.MaterialEditor.SceneController.cs‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
using ExtensibleSaveFormat;
2-
using KKAPI.Maker;
2+
using KKAPI;
33
using KKAPI.Studio;
44
using KKAPI.Studio.SaveLoad;
55
using KKAPI.Utilities;
@@ -24,19 +24,21 @@ namespace KK_Plugins.MaterialEditor
2424
/// </summary>
2525
public class SceneController : SceneCustomFunctionController
2626
{
27+
public const string TexDicSaveKey = nameof(TextureDictionary);
28+
2729
private readonly List<RendererProperty> RendererPropertyList = new List<RendererProperty>();
2830
private readonly List<ProjectorProperty> ProjectorPropertyList = new List<ProjectorProperty>();
2931
private readonly List<MaterialNameProperty> MaterialNamePropertyList = new List<MaterialNameProperty>();
3032
private readonly List<MaterialFloatProperty> MaterialFloatPropertyList = new List<MaterialFloatProperty>();
3133
private readonly List<MaterialKeywordProperty> MaterialKeywordPropertyList = new List<MaterialKeywordProperty>();
3234
private readonly List<MaterialColorProperty> MaterialColorPropertyList = new List<MaterialColorProperty>();
33-
private readonly List<MaterialTextureProperty> MaterialTexturePropertyList = new List<MaterialTextureProperty>();
35+
internal readonly List<MaterialTextureProperty> MaterialTexturePropertyList = new List<MaterialTextureProperty>();
3436
private readonly List<MaterialShader> MaterialShaderList = new List<MaterialShader>();
3537
private readonly List<MaterialCopy> MaterialCopyList = new List<MaterialCopy>();
3638

3739
private readonly Dictionary<MaterialTextureProperty, MEAnimationController> AnimationControllerMap = new Dictionary<MaterialTextureProperty, MEAnimationController>();
3840

39-
private static Dictionary<int, TextureContainer> TextureDictionary = new Dictionary<int, TextureContainer>();
41+
internal static Dictionary<int, TextureContainer> TextureDictionary = new Dictionary<int, TextureContainer>();
4042

4143
private static string FileToSet;
4244
private static string PropertyToSet;
@@ -60,10 +62,13 @@ protected override void OnSceneSave()
6062

6163
PurgeUnusedTextures();
6264

63-
if (TextureDictionary.Count > 0)
64-
data.data.Add(nameof(TextureDictionary), MessagePackSerializer.Serialize(TextureDictionary.ToDictionary(pair => pair.Key, pair => pair.Value.Data)));
65+
if (TextureDictionary.Count > 0 || (
66+
SceneLocalTextures.SaveType == SceneTextureSaveType.Deduped
67+
&& MaterialEditorCharaController.charaControllers.Any(x => x.TextureDictionary.Count > 0)
68+
))
69+
TextureSaveHandler.Instance.Save(data, TexDicSaveKey, TextureDictionary, false);
6570
else
66-
data.data.Add(nameof(TextureDictionary), null);
71+
data.data.Add(TexDicSaveKey, null);
6772

6873
if (RendererPropertyList.Count > 0)
6974
data.data.Add(nameof(RendererPropertyList), MessagePackSerializer.Serialize(RendererPropertyList));
@@ -198,13 +203,16 @@ protected override void OnSceneLoad(SceneOperationKind operation, ReadOnlyDictio
198203
var importDictionary = new Dictionary<int, int>();
199204

200205
if (operation == SceneOperationKind.Load)
201-
if (data.data.TryGetValue(nameof(TextureDictionary), out var texDic) && texDic != null)
202-
TextureDictionary = MessagePackSerializer.Deserialize<Dictionary<int, byte[]>>((byte[])texDic).ToDictionary(pair => pair.Key, pair => new TextureContainer(pair.Value));
206+
{
207+
TextureDictionary = TextureSaveHandler.Instance.Load<Dictionary<int, TextureContainer>>(data, TexDicSaveKey, false);
208+
}
203209

204210
if (operation == SceneOperationKind.Import)
205-
if (data.data.TryGetValue(nameof(TextureDictionary), out var texDic) && texDic != null)
206-
foreach (var x in MessagePackSerializer.Deserialize<Dictionary<int, byte[]>>((byte[])texDic))
207-
importDictionary[x.Key] = SetAndGetTextureID(x.Value);
211+
{
212+
var importDictionaryTemp = TextureSaveHandler.Instance.Load<Dictionary<int, TextureContainer>>(data, TexDicSaveKey, false);
213+
foreach (var kvp in importDictionaryTemp)
214+
importDictionary[kvp.Key] = SetAndGetTextureID(kvp.Value.Data);
215+
}
208216

209217
if (data.data.TryGetValue(nameof(MaterialCopyList), out var materialCopyData) && materialCopyData != null)
210218
{

‎src/MaterialEditor.Core.Studio/Core.MaterialEditor.Studio.cs‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using KKAPI.Maker;
55
using KKAPI.Studio;
66
using KKAPI.Studio.SaveLoad;
7+
using KKAPI.Utilities;
78
using MaterialEditorAPI;
89
using Studio;
910
using System;
@@ -15,7 +16,6 @@
1516
using UnityEngine;
1617
using UnityEngine.SceneManagement;
1718
using UnityEngine.UI;
18-
using static Illusion.Utils;
1919
using static MaterialEditorAPI.MaterialAPI;
2020
#if AI || HS2
2121
using AIChara;
@@ -379,6 +379,36 @@ private int AccessoryStringToIndex(string s)
379379
return -1;
380380
}
381381

382+
internal override void ExportTexture(Material mat, string property)
383+
{
384+
byte[] texData = null;
385+
if (CurrentData is ObjectData objData)
386+
{
387+
var controller = (MaterialEditorCharaController)CurrentGameObject.GetComponentInParent(typeof(MaterialEditorCharaController));
388+
if (controller != null)
389+
{
390+
var textureProperty = controller.MaterialTexturePropertyList.FirstOrDefault(x => x.ObjectType == objData.ObjectType && x.CoordinateIndex == controller.GetCoordinateIndex(objData.ObjectType) && x.Slot == objData.Slot && x.Property == property && x.MaterialName == mat.NameFormatted());
391+
if (textureProperty?.TexID != null)
392+
texData = controller.TextureDictionary[textureProperty.TexID.Value].Data;
393+
}
394+
}
395+
else if (CurrentData is int id)
396+
{
397+
var controller = GetSceneController();
398+
if (controller != null)
399+
{
400+
var textureProperty = controller.MaterialTexturePropertyList.FirstOrDefault(x => x.ID == id && x.MaterialName == mat.NameFormatted() && x.Property == property);
401+
if (textureProperty?.TexID != null)
402+
texData = SceneController.TextureDictionary[textureProperty.TexID.Value].Data;
403+
}
404+
}
405+
string ext = ImageTypeIdentifier.Identify(texData, "XXX");
406+
if (texData != null && ext != "XXX")
407+
base.ExportTextureOriginal(mat, property, ext, texData);
408+
else
409+
base.ExportTexture(mat, property);
410+
}
411+
382412
/// <summary>
383413
/// Get the ID for the specified ObjectCtrlInfo
384414
/// </summary>

0 commit comments

Comments
 (0)