Skip to content

Commit f711c8e

Browse files
maddie480maddie480
committed
Add mod-wide texture lazy loading
1 parent f376e76 commit f711c8e

File tree

4 files changed

+283
-1
lines changed

4 files changed

+283
-1
lines changed

‎CollabModule.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public override void Load() {
4545
AreaCompleteInfoInLevel.Load();
4646
SilverBlock.Load();
4747
MiniHeartDoorUnlockCutsceneTrigger.Load();
48+
LazyLoadingHandler.Load();
4849

4950
Everest.Content.OnUpdate += onModAssetUpdate;
5051

@@ -66,6 +67,7 @@ public override void Unload() {
6667
AreaCompleteInfoInLevel.Unload();
6768
SilverBlock.Unload();
6869
MiniHeartDoorUnlockCutsceneTrigger.Unload();
70+
LazyLoadingHandler.Unload();
6971

7072
Everest.Content.OnUpdate -= onModAssetUpdate;
7173

‎DOCUMENTATION.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ If anything is wrong or unclear, yell at max480 (max480#4596 on [the Celeste Dis
2222
- [Chapter Panel Trigger](#chapter-panel-trigger)
2323
- [Journal Trigger](#journal-trigger)
2424
- [Map Metadata](#map-metadata)
25+
- [Lazy Loading](#lazy-loading)
2526

2627
## Setting up your mod as a collab
2728

@@ -288,3 +289,17 @@ CollabUtilsRandomizedFlags:
288289
flag2: 0.5
289290
```
290291
With that setup, when entering the map, flag1 will have a 50% chance to be set, and flag2 will have a 20% chance to be set. **Both cannot be set at the same time**: in that example, that means there is a 30% chance no flag will be set.
292+
293+
## Lazy Loading
294+
295+
Lazy Loading is useful for large collabs, to prevent the game from loading all of your mod's graphics, and instead only load what is required for the map you are playing.
296+
297+
You can enable Lazy Loading for your mod by creating an empty `CollabUtils2LazyLoading.txt` file at the root of your mod, next to `everest.yaml`.
298+
299+
By doing that and restarting the game, **gameplay sprites that are included in your mod will not be loaded on startup anymore**, speeding up the startup by a fair bit. Instead, textures will be loaded **when you run into them in-game**. This has a drawback though: loading the texture takes a bit of time, which can cause stutters during gameplay.
300+
301+
To help mitigate this, Collab Utils collects the list of textures it lazily loaded and saves them, so that next time you enter the map, **they are loaded when you enter the map instead**. No more stutter during gameplay! That list of textures is saved when you **leave** the map, in a file ending with `.texturecache.txt` in `Mods/Cache/CollabUtils2`.
302+
303+
But you probably don't want gameplay to be stuttery the first time players play each map! To prevent that, **you should ship the .texturecache.txt files with your maps**. You just need to move the files out of your Cache folder and to put them next to your map bins. You should play through the map in order to see every texture (including secrets) before doing that.
304+
305+
Note that if some textures are missing from the `.texturecache.txt` file you shipped with the map, Collab Utils will create the `.texturecache.txt` file in `Mods/Cache/CollabUtils2` again, and you will find a warning in log.txt saying: `Found X lazily loaded texture(s)! Saving them at [path].` This file only contains missing textures, so in order to add them to the `.texturecache.txt` file that ships with your mod, you should merge the files, rather than replacing the one shipping with your mod.

‎LazyLoadingHandler.cs‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
using Celeste.Mod.Core;
2+
using Microsoft.Xna.Framework;
3+
using Microsoft.Xna.Framework.Graphics;
4+
using Mono.Cecil.Cil;
5+
using Monocle;
6+
using MonoMod.Cil;
7+
using MonoMod.RuntimeDetour;
8+
using System;
9+
using System.Collections.Generic;
10+
using System.IO;
11+
using System.Linq;
12+
13+
namespace Celeste.Mod.CollabUtils2 {
14+
// This class allows turning on lazy loading on a map pack by dropping a CollabUtils2LazyLoading.txt file at its root.
15+
// After doing that, play the maps and you should see texturecache.txt files appear in your Mods/Cache/CollabUtils2 folder;
16+
// you should ship those in the Maps folder. This tells the game which graphics should be loaded with each map.
17+
public static class LazyLoadingHandler {
18+
// complete list of all textures in maps that have CollabUtils2LazyLoading = true
19+
private static HashSet<string> lazilyLoadedTextures = new HashSet<string>();
20+
21+
// map SID => list of texture paths that have to be loaded when entering that map
22+
private static Dictionary<string, HashSet<string>> pathsPerMap = new Dictionary<string, HashSet<string>>();
23+
24+
// map SID => list of textures that were preloaded matching those in pathsPerMap, so that we can actually load them when entering the map.
25+
private static Dictionary<string, HashSet<VirtualTexture>> texturesPerMap = new Dictionary<string, HashSet<VirtualTexture>>();
26+
27+
// textures that were lazily loaded, and therefore should have been loaded in advance!
28+
private static HashSet<string> newPaths = new HashSet<string>();
29+
30+
private static string latestMapSID = null;
31+
private static bool preloadingTextures = false;
32+
private static ILHook hookOnTextureSafe;
33+
34+
public static void Load() {
35+
IL.Celeste.Mod.Everest.Content.Crawl += registerLazyLoadingModsOnLoad;
36+
IL.Monocle.VirtualTexture.Preload += turnOnLazyLoadingSelectively;
37+
On.Celeste.LevelLoader.ctor += lazilyLoadTextures;
38+
On.Monocle.VirtualTexture.Reload += onTextureLazyLoad;
39+
Everest.Events.Level.OnExit += saveNewLazilyLoadedPaths;
40+
41+
hookOnTextureSafe = new ILHook(typeof(VirtualTexture).GetMethod("get_Texture_Safe"), lazyLoadTexturesOnAccess);
42+
43+
// check all mods that were registered before us.
44+
foreach (ModContent modContent in Everest.Content.Mods) {
45+
registerLazyLoadingMods(modContent);
46+
}
47+
}
48+
49+
public static void Unload() {
50+
IL.Celeste.Mod.Everest.Content.Crawl -= registerLazyLoadingModsOnLoad;
51+
IL.Monocle.VirtualTexture.Preload -= turnOnLazyLoadingSelectively;
52+
On.Celeste.LevelLoader.ctor -= lazilyLoadTextures;
53+
On.Monocle.VirtualTexture.Reload -= onTextureLazyLoad;
54+
Everest.Events.Level.OnExit -= saveNewLazilyLoadedPaths;
55+
56+
hookOnTextureSafe?.Dispose();
57+
hookOnTextureSafe = null;
58+
}
59+
60+
private static void registerLazyLoadingModsOnLoad(ILContext il) {
61+
ILCursor cursor = new ILCursor(il);
62+
63+
// this place is right after we listed the files in the mod, but right before we start loading them (if loading them after startup)
64+
// because that includes loading textures, and we want to have all lazily loaded textures listed before that.
65+
cursor.GotoNext(MoveType.After, instr => instr.MatchCallvirt<ModContent>("_Crawl"));
66+
cursor.Emit(OpCodes.Ldarg_0);
67+
cursor.EmitDelegate<Action<ModContent>>(registerLazyLoadingMods);
68+
}
69+
70+
private static void registerLazyLoadingMods(ModContent modContent) {
71+
Logger.Log("CollabUtils2/LazyLoadingHandler", "Checking mod " + modContent.Name);
72+
73+
if (modContent.Map.ContainsKey("CollabUtils2LazyLoading")) {
74+
// lazy loading activated!
75+
foreach (KeyValuePair<string, ModAsset> asset in modContent.Map) {
76+
if (asset.Value.Type == typeof(Texture2D) && asset.Key.StartsWith("Graphics/Atlases/Gameplay/")) {
77+
// we will lazily load this gameplay sprite.
78+
lazilyLoadedTextures.Add(asset.Key);
79+
80+
Logger.Log("CollabUtils2/LazyLoadingHandler", asset.Key + " was registered for lazy loading");
81+
}
82+
83+
if (asset.Value.Type == typeof(AssetTypeMap)) {
84+
// we want to read the texturecache files associated with this map.
85+
string mapName = asset.Key.Substring("Maps/".Length);
86+
87+
// look for a texturecache packaged along with the map
88+
if (modContent.Map.TryGetValue(asset.Key + ".texturecache", out ModAsset textureCachePackaged) && textureCachePackaged.Type == typeof(AssetTypeText)) {
89+
using (Stream assetStream = textureCachePackaged.Stream) {
90+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", "Loading texture list for " + asset.Key + " from " + textureCachePackaged.PathVirtual);
91+
fillInTexturesFromCache(mapName, assetStream);
92+
}
93+
}
94+
95+
// look for a texturecache in the Cache folder
96+
string textureCachePath = Everest.Loader.PathCache + "/CollabUtils2/" + mapName + ".texturecache.txt";
97+
if (File.Exists(textureCachePath)) {
98+
using (FileStream stream = File.OpenRead(textureCachePath)) {
99+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", "Loading texture list for " + asset.Key + " from " + textureCachePath);
100+
fillInTexturesFromCache(mapName, stream);
101+
}
102+
}
103+
}
104+
}
105+
}
106+
}
107+
108+
private static void fillInTexturesFromCache(string key, Stream input) {
109+
if (!pathsPerMap.TryGetValue(key, out HashSet<string> pathsForThisMap)) {
110+
pathsForThisMap = new HashSet<string>();
111+
pathsPerMap[key] = pathsForThisMap;
112+
}
113+
114+
using (StreamReader reader = new StreamReader(input)) {
115+
string line;
116+
while ((line = reader.ReadLine()) != null) {
117+
Logger.Log("CollabUtils2/LazyLoadingHandler", "Added " + line + " as a texture for " + key);
118+
pathsForThisMap.Add(line);
119+
}
120+
}
121+
}
122+
123+
// this turns on or off lazy loading based on which texture is being loaded.
124+
private static void turnOnLazyLoadingSelectively(ILContext il) {
125+
ILCursor cursor = new ILCursor(il);
126+
cursor.GotoNext(MoveType.After, instr => instr.MatchCallvirt<CoreModuleSettings>("get_LazyLoading"));
127+
128+
cursor.Emit(OpCodes.Ldarg_0);
129+
cursor.EmitDelegate<Func<bool, VirtualTexture, bool>>((orig, self) => {
130+
// don't do anything if lazy loading is actually turned on or for textures with (somehow) no name.
131+
if (orig || self.Name == null)
132+
return orig;
133+
134+
string name = self.Name.Replace("\\", "/");
135+
if (lazilyLoadedTextures.Contains(name)) {
136+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", name + " was skipped and will be lazily loaded later");
137+
138+
// look for maps that use this, so that we can fill out texturesPerMap as we go through all textures.
139+
foreach (KeyValuePair<string, HashSet<string>> mapGraphics in pathsPerMap) {
140+
if (mapGraphics.Value.Any(path => name == path)) {
141+
Logger.Log("CollabUtils2/LazyLoadingHandler", name + " is associated to map " + mapGraphics.Key);
142+
143+
// associate the (non-loaded) texture to the map so that it can be loaded more easily later.
144+
if (!texturesPerMap.TryGetValue(mapGraphics.Key, out HashSet<VirtualTexture> list)) {
145+
list = new HashSet<VirtualTexture>();
146+
texturesPerMap[mapGraphics.Key] = list;
147+
}
148+
list.Add(self);
149+
}
150+
}
151+
152+
// this triggers lazy loading: preload of the texture, but not actually load it in video RAM.
153+
return true;
154+
}
155+
156+
// this disables lazy loading, and the game will actually load the texture.
157+
return false;
158+
});
159+
}
160+
161+
private static void lazilyLoadTextures(On.Celeste.LevelLoader.orig_ctor orig, LevelLoader self, Session session, Vector2? startPosition) {
162+
if (latestMapSID != session.Area.GetSID()) {
163+
writeNewPaths(latestMapSID);
164+
165+
// get the textures specific to the map we come from (if any) and the map we go to.
166+
if (latestMapSID == null || !texturesPerMap.TryGetValue(latestMapSID, out HashSet<VirtualTexture> texturesInOldMap)) {
167+
texturesInOldMap = new HashSet<VirtualTexture>();
168+
}
169+
if (!texturesPerMap.TryGetValue(session.Area.GetSID(), out HashSet<VirtualTexture> texturesInNewMap)) {
170+
texturesInNewMap = new HashSet<VirtualTexture>();
171+
}
172+
173+
// we are loading textures, but this is NOT Everest lazily loading them!
174+
preloadingTextures = true;
175+
176+
// textures to unload = textures that are in the old map but not the new one.
177+
foreach (VirtualTexture tex in texturesInOldMap.Except(texturesInNewMap)) {
178+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", "Unloading texture: " + tex.Name);
179+
tex.Unload();
180+
}
181+
182+
// textures to load = textures that are in the new map but not the old one.
183+
foreach (VirtualTexture tex in texturesInNewMap.Except(texturesInOldMap)) {
184+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", "Loading texture: " + tex.Name);
185+
tex.Reload();
186+
}
187+
188+
preloadingTextures = false;
189+
}
190+
191+
latestMapSID = session.Area.GetSID();
192+
orig(self, session, startPosition);
193+
}
194+
195+
private static void lazyLoadTexturesOnAccess(ILContext il) {
196+
ILCursor cursor = new ILCursor(il);
197+
cursor.GotoNext(MoveType.After, instr => instr.MatchCallvirt<CoreModuleSettings>("get_LazyLoading"));
198+
199+
cursor.Emit(OpCodes.Ldarg_0);
200+
cursor.EmitDelegate<Func<bool, VirtualTexture, bool>>((orig, self) => {
201+
// don't do anything if lazy loading is actually turned on or for textures with (somehow) no name.
202+
if (orig || self.Name == null)
203+
return orig;
204+
205+
// texture is lazily loaded if it is in our list.
206+
// this will make Everest actually check if the texture is loaded, and call Reload() if it is not.
207+
string name = self.Name.Replace("\\", "/");
208+
return lazilyLoadedTextures.Contains(name);
209+
});
210+
}
211+
212+
private static void onTextureLazyLoad(On.Monocle.VirtualTexture.orig_Reload orig, VirtualTexture self) {
213+
// this is actually called on every texture load, so we need to check if this is a lazy load or not
214+
string name = self.Name.Replace("\\", "/");
215+
if (!preloadingTextures && lazilyLoadedTextures.Contains(name)) {
216+
string currentMap = (Engine.Scene as Level)?.Session?.Area.GetSID();
217+
218+
Logger.Log(LogLevel.Debug, "CollabUtils2/LazyLoadingHandler", name + " was lazily loaded by Everest! It will be associated to map " + currentMap + ".");
219+
newPaths.Add(name);
220+
221+
if (currentMap != null) {
222+
// add the texture to the lists associated to this map
223+
if (!pathsPerMap.TryGetValue(currentMap, out HashSet<string> pathsForThisMap)) {
224+
pathsForThisMap = new HashSet<string>();
225+
pathsPerMap[currentMap] = pathsForThisMap;
226+
}
227+
pathsForThisMap.Add(name);
228+
229+
if (!texturesPerMap.TryGetValue(currentMap, out HashSet<VirtualTexture> texturesForThisMap)) {
230+
texturesForThisMap = new HashSet<VirtualTexture>();
231+
texturesPerMap[currentMap] = texturesForThisMap;
232+
}
233+
texturesForThisMap.Add(self);
234+
}
235+
}
236+
237+
orig(self);
238+
}
239+
240+
private static void saveNewLazilyLoadedPaths(Level level, LevelExit exit, LevelExit.Mode mode, Session session, HiresSnow snow) {
241+
writeNewPaths(session.Area.GetSID());
242+
}
243+
244+
private static void writeNewPaths(string levelSID) {
245+
if (newPaths.Count > 0) {
246+
// write the new paths on disk, creating the file or appending to it.
247+
// we do that now because writing to disk while playing the map would only make the lazy load stuttering worse.
248+
string textureCachePath = Everest.Loader.PathCache + "/CollabUtils2/" + levelSID + ".texturecache.txt";
249+
Directory.CreateDirectory(textureCachePath.Substring(0, textureCachePath.LastIndexOf("/")));
250+
using (FileStream file = File.Open(textureCachePath, FileMode.OpenOrCreate)) {
251+
Logger.Log(LogLevel.Warn, "CollabUtils2/LazyLoadingHandler", "Found " + newPaths.Count + " lazily loaded texture(s)! Saving them at " + textureCachePath + ".");
252+
253+
file.Seek(0, SeekOrigin.End);
254+
using (var stream = new StreamWriter(file)) {
255+
foreach (string path in newPaths) {
256+
stream.WriteLine(path);
257+
}
258+
}
259+
}
260+
261+
newPaths.Clear();
262+
}
263+
}
264+
}
265+
}

‎everest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
- Name: CollabUtils2
2-
Version: 1.5.13
2+
Version: 1.6.0
33
DLL: bin/Debug/net452/CollabUtils2.dll
44
Dependencies:
55
- Name: Everest

0 commit comments

Comments
 (0)