Merge branch 'navmeshtool_worldspaces' into 'master'

Somewhat reduce navmeshtool peak memory usage

See merge request OpenMW/openmw!5076

Author: Capostrophic

apps/navmeshtool/main.cpp

@@ -24,6 +24,7 @@
 #include <components/resource/imagemanager.hpp>
 #include <components/resource/niffilemanager.hpp>
 #include <components/resource/scenemanager.hpp>
+#include <components/sceneutil/workqueue.hpp>
 #include <components/settings/values.hpp>
 #include <components/toutf8/toutf8.hpp>
 #include <components/version/version.hpp>
@@ -39,9 +40,9 @@
 #include <filesystem>
 #include <iostream>
 #include <map>
+#include <regex>
 #include <string>
 #include <thread>
-#include <type_traits>
 #include <utility>
 #include <vector>
 
@@ -119,6 +120,10 @@ namespace NavMeshTool
             addOption("write-binary-log", bpo::value<bool>()->implicit_value(true)->default_value(false),
                 "write progress in binary messages to be consumed by the launcher");
 
+            addOption("worldspace-filter", bpo::value<std::string>()->default_value(".*"),
+                "Regular expression to filter in specified worldspaces in modified ECMAScript grammar (see "
+                "https://en.cppreference.com/w/cpp/regex/ecmascript.html)");
+
             Files::ConfigurationManager::addCommonOptions(result);
 
             return result;
@@ -180,6 +185,8 @@ namespace NavMeshTool
             const bool removeUnusedTiles = variables["remove-unused-tiles"].as<bool>();
             const bool writeBinaryLog = variables["write-binary-log"].as<bool>();
 
+            const std::regex worldspaceFilter(variables["worldspace-filter"].as<std::string>());
+
 #ifdef WIN32
             if (writeBinaryLog)
                 _setmode(_fileno(stderr), _O_BINARY);
@@ -229,11 +236,42 @@ namespace NavMeshTool
             navigatorSettings.mRecast.mSwimHeightScale
                 = EsmLoader::getGameSetting(esmData.mGameSettings, "fSwimHeightScale").getFloat();
 
-            WorldspaceData cellsData = gatherWorldspaceData(
-                navigatorSettings, readers, vfs, bulletShapeManager, esmData, processInteriorCells, writeBinaryLog);
+            const std::unordered_map<ESM::RefId, std::vector<std::size_t>> worldspaceCells
+                = collectWorldspaceCells(esmData, processInteriorCells, worldspaceFilter);
+
+            Status status = Status::Ok;
+            bool needVacuum = false;
+            std::size_t count = 0;
+
+            SceneUtil::WorkQueue workQueue(threadsNumber);
+
+            Log(Debug::Info) << "Using " << threadsNumber << " parallel workers...";
+
+            for (const auto& [worldspace, cells] : worldspaceCells)
+            {
+                const WorldspaceData worldspaceData = gatherWorldspaceData(
+                    navigatorSettings, readers, vfs, bulletShapeManager, esmData, writeBinaryLog, worldspace, cells);
+
+                const Result result = generateAllNavMeshTiles(
+                    agentBounds, navigatorSettings, removeUnusedTiles, writeBinaryLog, worldspaceData, db, workQueue);
 
-            const Status status = generateAllNavMeshTiles(agentBounds, navigatorSettings, threadsNumber,
-                removeUnusedTiles, writeBinaryLog, cellsData, std::move(db));
+                ++count;
+
+                Log(Debug::Info) << "Processed worldspace (" << count << "/" << worldspaceCells.size() << ") "
+                                 << worldspace;
+
+                status = result.mStatus;
+                needVacuum = needVacuum || result.mNeedVacuum;
+
+                if (status != Status::Ok)
+                    break;
+            }
+
+            if (status == Status::Ok && needVacuum)
+            {
+                Log(Debug::Info) << "Vacuuming the database...";
+                db.vacuum();
+            }
 
             switch (status)
             {

apps/navmeshtool/navmesh.cpp

@@ -26,7 +26,6 @@
 #include <cstddef>
 #include <random>
 #include <string_view>
-#include <utility>
 #include <vector>
 
 namespace NavMeshTool
@@ -78,8 +77,8 @@ namespace NavMeshTool
         public:
             std::atomic_size_t mExpected{ 0 };
 
-            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles, bool writeBinaryLog)
-                : mDb(std::move(db))
+            explicit NavMeshTileConsumer(NavMeshDb& db, bool removeUnusedTiles, bool writeBinaryLog)
+                : mDb(db)
                 , mRemoveUnusedTiles(removeUnusedTiles)
                 , mWriteBinaryLog(writeBinaryLog)
                 , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate))
@@ -211,12 +210,6 @@ namespace NavMeshTool
                 mTransaction.commit();
             }
 
-            void vacuum()
-            {
-                const std::lock_guard lock(mMutex);
-                mDb.vacuum();
-            }
-
             void removeTilesOutsideRange(ESM::RefId worldspace, const TilesPositionsRange& range)
             {
                 const std::lock_guard lock(mMutex);
@@ -233,7 +226,7 @@ namespace NavMeshTool
             std::size_t mDeleted = 0;
             Status mStatus = Status::Ok;
             mutable std::mutex mMutex;
-            NavMeshDb mDb;
+            NavMeshDb& mDb;
             const bool mRemoveUnusedTiles;
             const bool mWriteBinaryLog;
             Transaction mTransaction;
@@ -254,45 +247,39 @@ namespace NavMeshTool
         };
     }
 
-    Status generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, std::size_t threadsNumber,
-        bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& data, NavMeshDb&& db)
+    Result generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, bool removeUnusedTiles,
+        bool writeBinaryLog, const WorldspaceData& data, NavMeshDb& db, SceneUtil::WorkQueue& workQueue)
     {
-        Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
+        Log(Debug::Info) << "Generating navmesh tiles for " << data.mWorldspace << " worldspace...";
 
-        SceneUtil::WorkQueue workQueue(threadsNumber);
-        auto navMeshTileConsumer
-            = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles, writeBinaryLog);
-        std::size_t tiles = 0;
-        std::mt19937_64 random;
+        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(db, removeUnusedTiles, writeBinaryLog);
 
-        for (const std::unique_ptr<WorldspaceNavMeshInput>& input : data.mNavMeshInputs)
-        {
-            const auto range = DetourNavigator::makeTilesPositionsRange(Misc::Convert::toOsgXY(input->mAabb.m_min),
-                Misc::Convert::toOsgXY(input->mAabb.m_max), settings.mRecast);
-
-            if (removeUnusedTiles)
-                navMeshTileConsumer->removeTilesOutsideRange(input->mWorldspace, range);
+        const auto range = DetourNavigator::makeTilesPositionsRange(
+            Misc::Convert::toOsgXY(data.mAabb.m_min), Misc::Convert::toOsgXY(data.mAabb.m_max), settings.mRecast);
 
-            std::vector<TilePosition> worldspaceTiles;
+        if (removeUnusedTiles)
+            navMeshTileConsumer->removeTilesOutsideRange(data.mWorldspace, range);
 
-            DetourNavigator::getTilesPositions(
-                range, [&](const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); });
+        std::vector<TilePosition> worldspaceTiles = data.mTiles;
 
-            tiles += worldspaceTiles.size();
+        {
+            const std::size_t tiles = worldspaceTiles.size();
 
             if (writeBinaryLog)
                 serializeToStderr(ExpectedTiles{ static_cast<std::uint64_t>(tiles) });
 
             navMeshTileConsumer->mExpected = tiles;
+        }
 
+        {
+            std::mt19937_64 random;
             std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random);
-
-            for (const TilePosition& tilePosition : worldspaceTiles)
-                workQueue.addWorkItem(new GenerateNavMeshTile(input->mWorldspace, tilePosition,
-                    RecastMeshProvider(input->mTileCachedRecastMeshManager), agentBounds, settings,
-                    navMeshTileConsumer));
         }
 
+        for (const TilePosition& tilePosition : worldspaceTiles)
+            workQueue.addWorkItem(new GenerateNavMeshTile(data.mWorldspace, tilePosition,
+                RecastMeshProvider(*data.mTileCachedRecastMeshManager), agentBounds, settings, navMeshTileConsumer));
+
         const Status status = navMeshTileConsumer->wait();
         if (status == Status::Ok)
             navMeshTileConsumer->commit();
@@ -304,12 +291,9 @@ namespace NavMeshTool
         Log(Debug::Info) << "Generated navmesh for " << navMeshTileConsumer->getProvided() << " tiles, " << inserted
                          << " are inserted, " << updated << " updated and " << deleted << " deleted";
 
-        if (inserted + updated + deleted > 0)
-        {
-            Log(Debug::Info) << "Vacuuming the database...";
-            navMeshTileConsumer->vacuum();
-        }
-
-        return status;
+        return Result{
+            .mStatus = status,
+            .mNeedVacuum = inserted + updated + deleted > 0,
+        };
     }
 }

apps/navmeshtool/navmesh.hpp

@@ -1,15 +1,18 @@
 #ifndef OPENMW_NAVMESHTOOL_NAVMESH_H
 #define OPENMW_NAVMESHTOOL_NAVMESH_H
 
-#include <cstddef>
-
 namespace DetourNavigator
 {
     class NavMeshDb;
     struct Settings;
     struct AgentBounds;
 }
 
+namespace SceneUtil
+{
+    class WorkQueue;
+}
+
 namespace NavMeshTool
 {
     struct WorldspaceData;
@@ -21,9 +24,15 @@ namespace NavMeshTool
         NotEnoughSpace,
     };
 
-    Status generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds,
-        const DetourNavigator::Settings& settings, std::size_t threadsNumber, bool removeUnusedTiles,
-        bool writeBinaryLog, WorldspaceData& cellsData, DetourNavigator::NavMeshDb&& db);
+    struct Result
+    {
+        Status mStatus;
+        bool mNeedVacuum;
+    };
+
+    Result generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds,
+        const DetourNavigator::Settings& settings, bool removeUnusedTiles, bool writeBinaryLog,
+        const WorldspaceData& data, DetourNavigator::NavMeshDb& db, SceneUtil::WorkQueue& workQueue);
 }
 
 #endif