add xxh64 fastpath

Author: r58Playz

frontend/src/fs.tsx

@@ -6,7 +6,7 @@ import {
 	toWeb as streamToWeb,
 	/* @ts-expect-error */
 } from "streamx-webstream";
-import { h64 as XXH64 } from "xxhashjs";
+import XXH64Worker from "./xxh64.worker?worker";
 
 import iconFolder from "@ktibow/iconset-material-symbols/folder";
 import iconDraft from "@ktibow/iconset-material-symbols/draft";
@@ -41,16 +41,25 @@ export const TAR_TYPES = [
 	} as FilePickerAcceptType,
 ];
 
-export async function calculateCelesteHash(): Promise<string> {
-	const celesteFile = await rootFolder.getFileHandle("CustomCeleste.dll");
-	const celeste = await celesteFile.getFile().then((r) => r.arrayBuffer());
+export async function calculateXXH64(path: string): Promise<string> {
+	let split = path.split("/")
+	let dir = await recursiveGetDirectory(rootFolder, split.slice(0, -1));
+	let file = await dir.getFileHandle(split.at(-1)!);
+	let buf = await file.getFile().then((r) => r.arrayBuffer());
 
-	const hash = XXH64();
-	hash.init(0);
-	hash.update(celeste);
-	const out = hash.digest();
+	let worker = new XXH64Worker({ name: `xxh64-${path}` });
+	let promise = new Promise<string>((res, rej) => {
+		worker.onmessage = ({ data }) => res(data.digest);
+		worker.onerror = (x) => rej(new Error(`failed to hash: ${x}`));
+	});
+	worker.postMessage({ buf, }, [buf]);
+	let res = await promise;
+	worker.terminate();
 
-	return out.toString(16).toUpperCase().padStart(16, "0");
+	return res;
+}
+export async function calculateCelesteHash(): Promise<string> {
+	return await calculateXXH64("CustomCeleste.dll")
 }
 
 export async function replaceHashes(hash: string) {

frontend/src/game/dotnet.ts

@@ -1,5 +1,5 @@
 import { DotnetHostBuilder, MonoConfig } from "./dotnetdefs";
-import { recursiveGetDirectory, rootFolder } from "../fs";
+import { calculateXXH64, recursiveGetDirectory, rootFolder } from "../fs";
 import { SteamJS } from "../achievements";
 import { JsSplash } from "./loading";
 import { epoxyFetch, EpxTcpWs, EpxWs, getWispUrl } from "../epoxy";
@@ -67,13 +67,13 @@ function hookfmod() {
 			for (let context of contexts) {
 				try {
 					await context.resume();
-				} catch {}
+				} catch { }
 			}
 		} else {
 			for (let context of contexts) {
 				try {
 					await context.suspend();
-				} catch {}
+				} catch { }
 			}
 		}
 	});
@@ -89,7 +89,7 @@ useChange([gameState.playing, gameState.initting], () => {
 			// @ts-expect-error
 			navigator.keyboard.unlock();
 		}
-	} catch (err) {}
+	} catch (err) { }
 });
 
 let nativefetch = window.fetch;
@@ -133,15 +133,15 @@ function encryptRSA(data: Uint8Array, n: bigint, e: bigint): Uint8Array {
 
 		return BigInt(
 			"0x" +
-				[
-					"00",
-					"02",
-					...padding.map((byte) => byte.toString(16).padStart(2, "0")),
-					"00",
-					...Array.from(messageBytes).map((byte: any) =>
-						byte.toString(16).padStart(2, "0")
-					),
-				].join("")
+			[
+				"00",
+				"02",
+				...padding.map((byte) => byte.toString(16).padStart(2, "0")),
+				"00",
+				...Array.from(messageBytes).map((byte: any) =>
+					byte.toString(16).padStart(2, "0")
+				),
+			].join("")
 		);
 	};
 	const paddedMessage = pkcs1v15Pad(data, n);
@@ -273,7 +273,7 @@ export async function preInit() {
 				let h = await file.getFile();
 				console.log("got file cached", last);
 				return new Response(h.stream());
-			} catch {}
+			} catch { }
 			dl.download = "cross origin lol";
 			dl.href = args[0];
 			dl.click();
@@ -286,7 +286,7 @@ export async function preInit() {
 					let h = await file.getFile();
 					console.log("got file", last);
 					return new Response(h.stream());
-				} catch {}
+				} catch { }
 				await new Promise((r) => setTimeout(r, 100));
 			}
 		}
@@ -314,6 +314,22 @@ export async function preInit() {
 			let encrypted = encryptRSA(data, modulus, exponent);
 			return new Uint8Array(encrypted);
 		},
+		XXHash64_Fast: async (path: string) => {
+			if (!path.startsWith("/libsdl/")) throw new Error("can't hash things not in opfs");
+			path = path.slice("/libsdl/".length);
+
+			let start = performance.now();
+			let hex = await calculateXXH64(path);
+			let bytes = new Uint8Array(8);
+			for (let i = 0; i < 8; i++) {
+				bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+			}
+			let end = performance.now();
+
+			console.debug("xxh64_fast:", path, (end - start).toFixed(1));
+
+			return { ret: bytes };
+		}
 	});
 
 	(self as any).wasm = {

frontend/src/xxh64.worker.ts

@@ -0,0 +1,10 @@
+import { h64 as XXH64 } from "xxhashjs";
+
+self.onmessage = ({ data: { buf } }) => {
+	let hash = XXH64();
+	hash.init(0);
+	hash.update(buf);
+	let out = hash.digest();
+
+	self.postMessage({ digest: out.toString(16).toUpperCase().padStart(16, "0") })
+};

loader/Celeste.cs

@@ -20,6 +20,9 @@ private static void Main()
     [DllImport("Emscripten")]
     public extern static void wasm_func_viil(Int32 x, Int32 y, Int64 l);
 
+	[JSImport("XXHash64_Fast", "interop.js")]
+	public static partial Task<JSObject> XXHash64_Fast(string file);
+
     internal static void CallPinvokeFixers()
     {
         wasm_func_viil(0, 0, 0);
@@ -113,6 +116,19 @@ internal static Task Init(bool tailcalls)
             {
                 var ParseArgs = Everest.GetMethod("ParseArgs", BindingFlags.Static | BindingFlags.NonPublic);
                 ParseArgs.Invoke(null, [new string[] { }]);
+
+				var XXH64_Fast = Everest.GetField("CelesteWasm_XXHash64Fast", BindingFlags.Static | BindingFlags.Public);
+				if (XXH64_Fast != null) {
+					Func<string, byte[]> XXH64 = (path) => {
+						try {
+							var ret = XXHash64_Fast(path).Result;
+							return ret.GetPropertyAsByteArray("ret");
+						} catch {
+							return null;
+						}
+					};
+					XXH64_Fast.SetValue(null, XXH64);
+				}
             }
 
             Console.WriteLine($"CelesteWasm on {RuntimeInformation.FrameworkDescription}");

patcher/EverestMods.cs

@@ -3,6 +3,8 @@
 using MonoMod;
 using MonoMod.Cil;
 using System;
+using System.Linq;
+using System.IO;
 using System.Collections.Generic;
 
 namespace Celeste.Mod
@@ -18,13 +20,34 @@ private static void InitMMFlags(MonoModder modder)
 
                 modder.DependencyDirs.Add("/bin/");
                 modder.Mods.Add(ModuleDefinition.ReadModule("/bin/Celeste.Wasm.mm.dll"));
-				modder.RemovePatchReferences = false;
+                modder.RemovePatchReferences = false;
             }
 
             [MonoModIgnore]
             [PatchRelinkModuleMapPath]
             public extern static Dictionary<string, ModuleDefinition> get_SharedRelinkModuleMap();
         }
+
+        [MonoModIgnore]
+        public extern static byte[] ComputeHash(Stream inputStream);
+
+        public static Func<string, byte[]> CelesteWasm_XXHash64Fast;
+
+        // forward to js
+        public static byte[] GetChecksum(string path)
+        {
+            if (CelesteWasm_XXHash64Fast != null)
+            {
+                var res = CelesteWasm_XXHash64Fast(path);
+                if (res != null)
+                    return res;
+            }
+
+            using (FileStream fs = File.OpenRead(path))
+            {
+                return ComputeHash(fs);
+            }
+        }
     }
 
     public sealed class patch_EverestModuleAssemblyContext