Merge pull request #5572 from Tomonobu3110/tomo_less_memory

Reduce memory usage for raster source handling.

Author: gardster

CHANGELOG.md

@@ -14,6 +14,8 @@
       - CHANGED: allow routing past `barrier=arch` [#5352](https://github.com/Project-OSRM/osrm-backend/pull/5352)
       - CHANGED: default car weight was reduced to 2000 kg. [#5371](https://github.com/Project-OSRM/osrm-backend/pull/5371)
       - CHANGED: default car height was reduced to 2 meters. [#5389](https://github.com/Project-OSRM/osrm-backend/pull/5389)
+    - Misc:
+      - CHANGED: Reduce memory usage for raster source handling. [#5572](https://github.com/Project-OSRM/osrm-backend/pull/5572)
 
 # 5.21.0
   - Changes from 5.20.0

include/extractor/raster_source.hpp

@@ -4,16 +4,21 @@
 #include "util/coordinate.hpp"
 #include "util/exception.hpp"
 
+#include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/trim.hpp>
 #include <boost/assert.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
+#include <boost/foreach.hpp>
 #include <boost/spirit/include/qi.hpp>
 #include <boost/spirit/include/qi_int.hpp>
+
 #include <storage/io.hpp>
 
 #include <iterator>
+#include <string>
 #include <unordered_map>
+using namespace std;
 
 namespace osrm
 {
@@ -43,37 +48,31 @@ class RasterGrid
         xdim = _xdim;
         ydim = _ydim;
         _data.reserve(ydim * xdim);
+        BOOST_ASSERT(ydim * xdim <= _data.capacity());
 
+        // Construct FileReader
         storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);
-
         std::string buffer;
-        buffer.resize(file_reader.GetSize());
-
-        BOOST_ASSERT(buffer.size() > 1);
-
-        file_reader.ReadInto(&buffer[0], buffer.size());
-
-        boost::algorithm::trim(buffer);
-
-        auto itr = buffer.begin();
-        auto end = buffer.end();
-
-        bool r = false;
-        try
-        {
-            r = boost::spirit::qi::parse(
-                itr, end, +boost::spirit::qi::int_ % +boost::spirit::qi::space, _data);
-        }
-        catch (std::exception const &ex)
-        {
-            throw util::exception("Failed to read from raster source " + filepath.string() + ": " +
-                                  ex.what() + SOURCE_REF);
-        }
+        buffer.resize(xdim * 11); // INT32_MAX = 2147483647 = 10 chars + 1 white space = 11
+        BOOST_ASSERT(xdim * 11 <= buffer.size());
 
-        if (!r || itr != end)
+        for (unsigned int y = 0; y < ydim; y++)
         {
-            throw util::exception("Failed to parse raster source: " + filepath.string() +
-                                  SOURCE_REF);
+            // read one line from file.
+            file_reader.ReadLine(&buffer[0], xdim * 11);
+            boost::algorithm::trim(buffer);
+
+            std::vector<std::string> result;
+            boost::split(
+                result, buffer, boost::is_any_of(" \r\n\0"), boost::algorithm::token_compress_on);
+            unsigned int x = 0;
+            for (const auto &s : result)
+            {
+                if (x < xdim)
+                    _data[(y * xdim) + x] = atoi(s.c_str());
+                ++x;
+            }
+            BOOST_ASSERT(x == xdim);
         }
     }
 
@@ -143,8 +142,36 @@ class RasterContainer
     RasterDatum GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat);
 
   private:
+};
+
+// << singletone >> RasterCache
+// The instance of RasterContainer is created for every threads osrm-extract uses.
+// To avoid multiple load of same file on each RasterContainer,
+// The LoadedSources and LoadedSourcePaths are separated to RasterCache class
+// and handled as the singletone pattern to avoid duplicate creation.
+class RasterCache
+{
+  public:
+    // class method to get the instance
+    static RasterCache &getInstance()
+    {
+        if (NULL == g_instance)
+        {
+            g_instance = new RasterCache();
+        }
+        return *g_instance;
+    }
+    // get reference of cache
+    std::vector<RasterSource> &getLoadedSources() { return LoadedSources; }
+    std::unordered_map<std::string, int> &getLoadedSourcePaths() { return LoadedSourcePaths; }
+  private:
+    // constructor
+    RasterCache() = default;
+    // member
     std::vector<RasterSource> LoadedSources;
     std::unordered_map<std::string, int> LoadedSourcePaths;
+    // the instance
+    static RasterCache *g_instance;
 };
 }
 }

include/storage/io.hpp

@@ -10,6 +10,7 @@
 #include "util/log.hpp"
 #include "util/version.hpp"
 
+#include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/iostreams/device/array.hpp>
 #include <boost/iostreams/seek.hpp>
@@ -60,29 +61,27 @@ class FileReader
 
     std::size_t GetSize()
     {
-        const boost::filesystem::ifstream::pos_type position = input_stream.tellg();
-        input_stream.seekg(0, std::ios::end);
-        const boost::filesystem::ifstream::pos_type file_size = input_stream.tellg();
-
-        if (file_size == boost::filesystem::ifstream::pos_type(-1))
+        const boost::filesystem::path path(filepath);
+        try
         {
-            throw util::RuntimeError("Unable to determine file size for " +
-                                         std::string(filepath.string()),
-                                     ErrorCode::FileIOError,
-                                     SOURCE_REF,
-                                     std::strerror(errno));
+            return std::size_t(boost::filesystem::file_size(path)) -
+                   ((fingerprint == FingerprintFlag::VerifyFingerprint) ? sizeof(util::FingerPrint)
+                                                                        : 0);
         }
-
-        // restore the current position
-        input_stream.seekg(position, std::ios::beg);
-
-        if (fingerprint == FingerprintFlag::VerifyFingerprint)
+        catch (const boost::filesystem::filesystem_error &ex)
         {
-            return std::size_t(file_size) - sizeof(util::FingerPrint);
+            std::cout << ex.what() << std::endl;
+            throw;
         }
-        else
+    }
+
+    /* Read one line */
+    template <typename T> void ReadLine(T *dest, const std::size_t count)
+    {
+        if (0 < count)
         {
-            return file_size;
+            memset(dest, 0, count * sizeof(T));
+            input_stream.getline(reinterpret_cast<char *>(dest), count * sizeof(T));
         }
     }
 

src/extractor/raster_source.cpp

@@ -92,15 +92,15 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
     const auto _ymin = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymin}));
     const auto _ymax = static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{ymax}));
 
-    const auto itr = LoadedSourcePaths.find(path_string);
-    if (itr != LoadedSourcePaths.end())
+    const auto itr = RasterCache::getInstance().getLoadedSourcePaths().find(path_string);
+    if (itr != RasterCache::getInstance().getLoadedSourcePaths().end())
     {
         util::Log() << "[source loader] Already loaded source '" << path_string << "' at source_id "
                     << itr->second;
         return itr->second;
     }
 
-    int source_id = static_cast<int>(LoadedSources.size());
+    int source_id = static_cast<int>(RasterCache::getInstance().getLoadedSources().size());
 
     util::Log() << "[source loader] Loading from " << path_string << "  ... ";
     TIMER_START(loading_source);
@@ -116,8 +116,8 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
 
     RasterSource source{std::move(rasterData), ncols, nrows, _xmin, _xmax, _ymin, _ymax};
     TIMER_STOP(loading_source);
-    LoadedSourcePaths.emplace(path_string, source_id);
-    LoadedSources.push_back(std::move(source));
+    RasterCache::getInstance().getLoadedSourcePaths().emplace(path_string, source_id);
+    RasterCache::getInstance().getLoadedSources().push_back(std::move(source));
 
     util::Log() << "[source loader] ok, after " << TIMER_SEC(loading_source) << "s";
 
@@ -127,10 +127,11 @@ int RasterContainer::LoadRasterSource(const std::string &path_string,
 // External function for looking up nearest data point from a specified source
 RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, double lon, double lat)
 {
-    if (LoadedSources.size() < source_id + 1)
+    if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(LoadedSources.size()) +
+                              ", but there are only " +
+                              std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 
@@ -139,7 +140,7 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou
     BOOST_ASSERT(lon < 180);
     BOOST_ASSERT(lon > -180);
 
-    const auto &found = LoadedSources[source_id];
+    const auto &found = RasterCache::getInstance().getLoadedSources()[source_id];
     return found.GetRasterData(static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{lon})),
                                static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{lat})));
 }
@@ -148,10 +149,11 @@ RasterDatum RasterContainer::GetRasterDataFromSource(unsigned int source_id, dou
 RasterDatum
 RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double lon, double lat)
 {
-    if (LoadedSources.size() < source_id + 1)
+    if (RasterCache::getInstance().getLoadedSources().size() < source_id + 1)
     {
         throw util::exception("Attempted to access source " + std::to_string(source_id) +
-                              ", but there are only " + std::to_string(LoadedSources.size()) +
+                              ", but there are only " +
+                              std::to_string(RasterCache::getInstance().getLoadedSources().size()) +
                               " loaded" + SOURCE_REF);
     }
 
@@ -160,10 +162,12 @@ RasterContainer::GetRasterInterpolateFromSource(unsigned int source_id, double l
     BOOST_ASSERT(lon < 180);
     BOOST_ASSERT(lon > -180);
 
-    const auto &found = LoadedSources[source_id];
+    const auto &found = RasterCache::getInstance().getLoadedSources()[source_id];
     return found.GetRasterInterpolate(
         static_cast<std::int32_t>(util::toFixed(util::FloatLongitude{lon})),
         static_cast<std::int32_t>(util::toFixed(util::FloatLatitude{lat})));
 }
+
+RasterCache *RasterCache::g_instance = NULL;
 }
 }