Improve load times, up to 4X in testing

Source/DivaModLoader/DatabaseLoader.cpp

@@ -15,27 +15,54 @@
 
 constexpr char MAGIC = 0x01;
 
-bool resolveModDatabaseFilePath(const prj::string& filePath, prj::string& destFilePath)
-{
-    const size_t magicIdx0 = filePath.find(MAGIC);
-    if (magicIdx0 == std::string::npos)
-        return false;
-
-    const size_t magicIdx1 = filePath.find(MAGIC, magicIdx0 + 1);
-    if (magicIdx1 == std::string::npos)
-        return false;
-
-    const prj::string left = filePath.substr(0, magicIdx0); // folder
-    const prj::string center = filePath.substr(magicIdx0 + 1, magicIdx1 - magicIdx0 - 1); // mod folder
-    const prj::string right = filePath.substr(magicIdx1 + 1); // file name
+std::unordered_map<prj::string, std::optional<prj::string>> filePathCache;
 
-    destFilePath = center;
-    destFilePath += "/";
-    destFilePath += left;
-    destFilePath += "mod";
-    destFilePath += right;
-
-    return true;
+void loadFilePathCacheSubDirs(std::string modRomDirectory) {
+    for (const auto& file : std::filesystem::recursive_directory_iterator(modRomDirectory))
+    {
+        if (!file.is_regular_file())
+            continue;
+
+        auto dir = file.path().lexically_relative(modRomDirectory).remove_filename().string();
+        auto filename = file.path().filename().string();
+        std::replace(dir.begin(), dir.end(), '\\', '/');
+
+        if (*(uint32_t*)filename.c_str() == *(uint32_t*)"mod_")
+        {
+            prj::string inPath;
+            inPath += dir;
+            inPath += MAGIC;
+            inPath += modRomDirectory;
+            inPath += MAGIC;
+            inPath += (filename.c_str() + 3);
+            std::transform(inPath.begin(), inPath.end(), inPath.begin(), tolower);
+
+            prj::string outPath;
+            outPath += modRomDirectory;
+            outPath += '/';
+            outPath += dir;
+            outPath += filename;
+
+            filePathCache.emplace(inPath, outPath);
+        }
+        else
+        {
+            prj::string path;
+            path += modRomDirectory;
+            path += '/';
+            path += dir;
+            path += filename;
+            std::transform(path.begin(), path.end(), path.begin(), tolower);
+
+            prj::string rootPath;
+            rootPath += dir;
+            rootPath += filename;
+            std::transform(rootPath.begin(), rootPath.end(), rootPath.begin(), tolower);
+
+            filePathCache.emplace(path, path);
+            filePathCache.emplace(rootPath, path);
+        }
+    }
 }
 
 SIG_SCAN
@@ -48,19 +75,143 @@ SIG_SCAN
 
 HOOK(size_t, __fastcall, ResolveFilePath, readInstrPtr(sigResolveFilePath(), 0, 0x5), prj::string& filePath, prj::string* destFilePath)
 {
-    if (resolveModDatabaseFilePath(filePath, destFilePath != nullptr ? *destFilePath : filePath))
+    prj::string filePathCopy = std::filesystem::path(filePath).lexically_normal().string().c_str();
+    std::replace(filePathCopy.begin(), filePathCopy.end(), '\\', '/');
+    std::transform(filePathCopy.begin(), filePathCopy.end(), filePathCopy.begin(), tolower);
+
+    auto cachedResult = filePathCache.find(filePathCopy);
+    if (cachedResult != filePathCache.end())
+    {
+        bool result = cachedResult->second.has_value();
+        if (result && destFilePath != nullptr)
+            *destFilePath = cachedResult->second.value();
+        return result;
+    }
+
+    // Because we cache all mod files, any path with MAGIC cannot exist outside of cache and we can simply skip
+    if (filePath.find(MAGIC) != std::string::npos)
+        return false;
+
+    prj::string out;
+    bool result = originalResolveFilePath(filePath, &out);
+
+    if (result && destFilePath != nullptr)
+        *destFilePath = out;
+
+    if (result)
+        filePathCache.emplace(filePathCopy, out);
+    else
+        filePathCache.emplace(filePathCopy, std::nullopt);
+
+    return result;
+}
+
+// both PvLoader and this now have these function ptrs, idk, put them in a header file?
+static FUNCTION_PTR(bool, __fastcall, asyncFileLoad, 0x1402A4710, void** fileHandler, const char* file, int);
+static FUNCTION_PTR(bool, __fastcall, asyncFileLoading, 0x151C03830, void** fileHandler);
+static FUNCTION_PTR(const void*, __fastcall, asyncFileGetData, 0x151C0EF70, void** fileHandler);
+static FUNCTION_PTR(size_t, __fastcall, asyncFileGetSize, 0x151C7ADA0, void** fileHandler);
+static FUNCTION_PTR(void, __fastcall, freeAsyncFileHandler, 0x1402A4E90, void** fileHandler);
+static FUNCTION_PTR(void, __fastcall, farcParse, 0x1402A0750, void *farc, const void *data, uint64_t size);
+static FUNCTION_PTR(void, __fastcall, farcGetFile, 0x1402A1020, void *farc, void *buffer, uint64_t size, int fileIndex);
+static FUNCTION_PTR(void, __fastcall, freeFarc, 0x1402A0E00, void *farc);
+static FUNCTION_PTR(void, __fastcall, itemTableHandlerParse, 0x1404E8690, void* itemTable, void** fileHandler);
+
+std::vector<void*> itmFileHandlers;
+prj::list<prj::string>* mdataPrefixes = nullptr;
+
+HOOK (void, __fastcall, ItemTableHandlerArrayRead, 0x158339FF0)
+{
+    for (auto it = mdataPrefixes->begin(); it != mdataPrefixes->end(); ++it)
+    {
+        char buf[MAX_PATH];
+        sprintf(buf, "rom/%schritm_prop.farc", it->c_str());
+        prj::string path(buf);
+        if (implOfResolveFilePath(path, &path))
+        {
+            void* fileHandler = nullptr;
+            asyncFileLoad(&fileHandler, path.c_str(), 1);
+            itmFileHandlers.push_back(fileHandler);
+        }
+    }
+}
+
+struct FarcFile {
+    char name[128];
+    int offset;
+    int compressedSize;
+    int uncompressedSize;
+    int flags;
+};
+
+HOOK(bool, __fastcall, ItemTableHandlerArrayLoad, 0x1404E7E60)
+{
+    for (auto it = itmFileHandlers.begin (); it != itmFileHandlers.end (); ++it)
+        if (asyncFileLoading (&*it))
+            return true;
+
+    std::vector<void*> farcs;
+    for (auto it = itmFileHandlers.begin(); it != itmFileHandlers.end(); ++it)
     {
-        // Probably should be using GetFileAttributesW, but the game doesn't work with unicode paths anyway.
-        const auto fileAttributes = GetFileAttributesA(destFilePath != nullptr ? destFilePath->c_str() : filePath.c_str());
-        return fileAttributes != INVALID_FILE_ATTRIBUTES && !(fileAttributes & FILE_ATTRIBUTE_DIRECTORY);
+        void *farc = operatorNew(0x60);
+        memset(farc, 0, 0x60);
+        farcParse(farc, asyncFileGetData(&*it), asyncFileGetSize(&*it));
+        farcs.push_back(farc);
+
+        freeAsyncFileHandler(&*it);
     }
+    itmFileHandlers.clear();
 
-    return originalResolveFilePath(filePath, destFilePath);
+    for (int i = 0; i < 10; i++)
+    {
+        auto ptr = 0x14175B620 + 0x108 * i;
+        for (auto it = farcs.begin (); it != farcs.end (); ++it)
+        {
+            void* buf  = nullptr;
+            uint64_t size = 0;
+            auto farcFiles = (prj::vector<FarcFile>*)((uint64_t)*it + 0x38);
+            for (int i = 0; i < farcFiles->size(); ++i)
+            {
+                auto file = farcFiles->at(i);
+                if (strcmp(file.name, *(char**)(ptr + 0x20)) == 0)
+                {
+                    size = file.uncompressedSize;
+                    buf = operatorNew(size);
+                    farcGetFile(*it, buf, size, i);
+                    break;
+                }
+            }
+            if (buf == nullptr)
+                continue;
+
+            void *fakeFileHandlerData[32] = {0};
+            fakeFileHandlerData[27] = (void*)size;
+            fakeFileHandlerData[28] = buf;
+            void* fakeFileHandler = (void*)&fakeFileHandlerData;
+
+            itemTableHandlerParse ((void*)ptr, &fakeFileHandler);
+
+            operatorDelete(buf);
+        }
+        *(bool*)(ptr + 0x18) = true;
+    }
+
+    for (auto it = farcs.begin (); it != farcs.end (); ++it)
+    {
+        freeFarc(*it);
+        operatorDelete(*it);
+    }
+    farcs.clear();
+
+    return false;
 }
 
 void DatabaseLoader::init()
 {
     INSTALL_HOOK(ResolveFilePath);
+
+    INSTALL_HOOK(ItemTableHandlerArrayRead);
+    INSTALL_HOOK(ItemTableHandlerArrayLoad);
 }
 
 SIG_SCAN
@@ -74,7 +225,7 @@ SIG_SCAN
 void DatabaseLoader::initMdataMgr(const std::vector<std::string>& modRomDirectoryPaths)
 {
     // Get the list address from the lea instruction that loads it.
-    auto& list = *(prj::list<prj::string>*)readInstrPtr(sigInitMdataMgr(), 0xFE, 0x7);
+    mdataPrefixes = (prj::list<prj::string>*)readInstrPtr(sigInitMdataMgr(), 0xFE, 0x7);
 
     // Traverse mod folders in reverse to have correct priority.
     for (auto it = modRomDirectoryPaths.rbegin(); it != modRomDirectoryPaths.rend(); ++it)
@@ -86,6 +237,9 @@ void DatabaseLoader::initMdataMgr(const std::vector<std::string>& modRomDirector
         path += MAGIC;
         path += "_";
 
-        list.push_back(path);
+        mdataPrefixes->push_back(path);
     }
+
+    for (auto it = modRomDirectoryPaths.begin(); it != modRomDirectoryPaths.end(); ++it)
+        loadFilePathCacheSubDirs(*it);
 }

Source/DivaModLoader/ModLoader.cpp

@@ -48,6 +48,11 @@ HOOK(void, __fastcall, InitRomDirectoryPaths, sigInitRomDirectoryPaths())
 
     // Initialize mount data manager prefixes for mod databases.
     DatabaseLoader::initMdataMgr(modRomDirectoryPaths);
+
+    // Skip modRomDirectory in ResolveFilePath
+    WRITE_MEMORY(0x1402A4450, uint8_t, 0x48, 0x81, 0xC3); // add r/m64, imm32
+    WRITE_MEMORY(0x1402A4453, int32_t, modRomDirectoryPaths.size() * sizeof(prj::string));
+    WRITE_NOP(0x1402A4457, 6); // Disable JE, assume romDirs.size() is always >0
 }
 
 std::vector<std::string> ModLoader::modDirectoryPaths;

Source/DivaModLoader/PvLoader.cpp

@@ -1,5 +1,67 @@
 #include "PvLoader.h"
 #include "SigScan.h"
+#include "Types.h"
+
+SIG_SCAN
+(
+    sigTaskPvDbLoop,
+    0x1404BB290,
+    "\x48\x89\x5C\x24\x10\x48\x89\x74\x24\x18\x48\x89\x7C\x24\x20\x55\x41\x54\x41\x55\x41\x56\x41\x57\x48\x8D\xAC\x24\x70\xFC",
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+);
+
+std::list<void*> fileHandlers;
+
+static FUNCTION_PTR(bool, __fastcall, asyncFileLoad, 0x1402A4710, void** fileHandler, const char* file, int);
+static FUNCTION_PTR(bool, __fastcall, asyncFileLoading, 0x151C03830, void** fileHandler);
+static FUNCTION_PTR(void, __fastcall, freeAsyncFileHandler, 0x1402A4E90, void** fileHandler);
+
+HOOK(bool, __fastcall, TaskPvDbLoop, sigTaskPvDbLoop(), uint64_t task)
+{
+    auto state = (int*)(task + 0x68);
+    auto paths = (prj::list<prj::string>*)(task + 0x70);
+
+    if (*state == 0)
+    {
+        if (paths->size() > 0)
+        {
+            for (auto it = paths->begin(); it != paths->end(); it++)
+            {
+                void* handler = nullptr;
+                asyncFileLoad(&handler, it->c_str(), 0);
+                fileHandlers.push_back(handler);
+            }
+
+            *state = 1;
+        }
+        else
+        {
+            return originalTaskPvDbLoop(task);
+        }
+    }
+    else
+    {
+        while (fileHandlers.size() > 0)
+        {
+            auto handler = fileHandlers.front();
+            if (asyncFileLoading(&handler))
+                break;
+
+            *state = 3;
+            *(void**)(task + 0x80) = handler;
+            originalTaskPvDbLoop(task);
+
+            freeAsyncFileHandler(&handler);
+            fileHandlers.pop_front();
+            paths->pop_front();
+
+            if (fileHandlers.size() == 0)
+                *state = 0;
+        }
+    }
+
+    return false;
+}
 
 SIG_SCAN
 (
@@ -137,6 +199,8 @@ SIG_SCAN
 
 void PvLoader::init()
 {
+    INSTALL_HOOK(TaskPvDbLoop);
+
     // Skip if checks that always return true but would access out of bounds data due to large IDs regardless
     WRITE_NOP(sigPvLoaderIfCheck1(), 0xE);
     WRITE_MEMORY(sigPvLoaderIfCheck2(), uint8_t, 0x90, 0x90, 0x90, 0xEB);