Allow building OSRM with 64-bit IDs so we can extract planet.osm.pbf using foot profile

Author: pxpeterxu

CMakeLists.txt

@@ -35,6 +35,7 @@ option(ENABLE_LTO "Use LTO if available" OFF)
 option(ENABLE_FUZZING "Fuzz testing using LLVM's libFuzzer" OFF)
 option(ENABLE_NODE_BINDINGS "Build NodeJs bindings" OFF)
 option(ENABLE_CLANG_TIDY "Enables clang-tidy checks" OFF)
+option(USE_64BIT_IDS "Whether to use 64-bit node and edge IDs" OFF)
 
 if (ENABLE_CLANG_TIDY)
   find_program(CLANG_TIDY_COMMAND NAMES clang-tidy)
@@ -89,6 +90,13 @@ else()
   add_definitions(-DOSRM_PROJECT_DIR="${CMAKE_CURRENT_SOURCE_DIR}")
 endif()
 
+if(USE_64BIT_IDS)
+  add_definitions(-DUSE_64BIT_IDS)
+  message(STATUS "Using 64-bit node and edge IDs")
+else()
+  message(STATUS "Using 32-bit node and edge IDs")
+endif()
+
 # these two functions build up custom variables:
 #   DEPENDENCIES_INCLUDE_DIRS and OSRM_DEFINES
 # These variables we want to pass to
@@ -151,7 +159,7 @@ add_library(UPDATER OBJECT ${UpdaterGlob})
 add_library(STORAGE OBJECT ${StorageGlob})
 add_library(ENGINE OBJECT ${EngineGlob})
 
-if (BUILD_ROUTED) 
+if (BUILD_ROUTED)
   add_library(SERVER OBJECT ${ServerGlob})
   add_executable(osrm-routed src/tools/routed.cpp $<TARGET_OBJECTS:SERVER> $<TARGET_OBJECTS:UTIL>)
 endif()
@@ -309,7 +317,7 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR}
 set(FMT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/fmt-9.1.0/include")
 add_compile_definitions(FMT_HEADER_ONLY)
 include_directories(SYSTEM ${FMT_INCLUDE_DIR})
-        
+
 
 # see https://stackoverflow.com/questions/70898030/boost-link-error-using-conan-find-package
 if (MSVC)
@@ -319,7 +327,7 @@ endif()
 if(ENABLE_CONAN)
   message(STATUS "Installing dependencies via Conan")
 
-  # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority 
+  # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority
   list(INSERT CMAKE_MODULE_PATH 0 ${CMAKE_BINARY_DIR})
   list(INSERT CMAKE_PREFIX_PATH 0 ${CMAKE_BINARY_DIR})
 
@@ -330,26 +338,26 @@ if(ENABLE_CONAN)
   set(CONAN_EXPAT_VERSION "2.2.10#916908d4a570ad839edd25322c3268cd")
   set(CONAN_LUA_VERSION "5.4.4#3ec62efc37cd0a5d80b9e5cb35277360")
   set(CONAN_TBB_VERSION "2021.3.0#507ec17cbd51a84167e143b20d170eea")
-  
+
   set(CONAN_SYSTEM_INCLUDES ON)
 
-  # TODO: 
-  # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol 
+  # TODO:
+  # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol
   # with exit code = -1073741515, which means that program cannot load required DLL.
   if (MSVC)
     set(TBB_SHARED False)
   else()
     set(TBB_SHARED True)
   endif()
 
-  set(CONAN_ARGS       
-    REQUIRES 
+  set(CONAN_ARGS
+    REQUIRES
     "boost/${CONAN_BOOST_VERSION}"
     "bzip2/${CONAN_BZIP2_VERSION}"
     "expat/${CONAN_EXPAT_VERSION}"
     "lua/${CONAN_LUA_VERSION}"
     "onetbb/${CONAN_TBB_VERSION}"
-    BASIC_SETUP 
+    BASIC_SETUP
     GENERATORS cmake_find_package json  # json generator generates a conanbuildinfo.json in the build folder so (non-CMake) projects can easily parse OSRM's dependencies
     KEEP_RPATHS
     NO_OUTPUT_DIRS
@@ -386,7 +394,7 @@ if(ENABLE_CONAN)
   set(Boost_ZLIB_LIBRARY "${Boost_zlib_LIB_TARGETS}")
   set(Boost_REGEX_LIBRARY "${Boost_regex_LIB_TARGETS}")
   set(Boost_UNIT_TEST_FRAMEWORK_LIBRARY "${Boost_unit_test_framework_LIB_TARGETS}")
-  
+
 
   find_package(BZip2 REQUIRED)
   find_package(EXPAT REQUIRED)
@@ -730,23 +738,23 @@ if (ENABLE_FUZZING)
   add_subdirectory(fuzz)
 endif ()
 
-# add headers sanity check target that includes all headers independently	
-set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers")	
-file(GLOB_RECURSE headers_to_check	
-  ${PROJECT_BINARY_DIR}/*.hpp	
-  ${PROJECT_SOURCE_DIR}/include/*.hpp)	
-foreach(header ${headers_to_check})	
+# add headers sanity check target that includes all headers independently
+set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers")
+file(GLOB_RECURSE headers_to_check
+  ${PROJECT_BINARY_DIR}/*.hpp
+  ${PROJECT_SOURCE_DIR}/include/*.hpp)
+foreach(header ${headers_to_check})
   if ("${header}" MATCHES ".*/include/nodejs/.*")
     # we do not check NodeJS bindings headers
     continue()
   endif()
-  get_filename_component(filename ${header} NAME_WE)	
-  set(filename "${check_headers_dir}/${filename}.cpp")	
-  if (NOT EXISTS ${filename})	
-    file(WRITE ${filename} "#include \"${header}\"\n")	
-  endif()	
-  list(APPEND sources ${filename})	
-endforeach()	
-add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources})	
-set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir})	
+  get_filename_component(filename ${header} NAME_WE)
+  set(filename "${check_headers_dir}/${filename}.cpp")
+  if (NOT EXISTS ${filename})
+    file(WRITE ${filename} "#include \"${header}\"\n")
+  endif()
+  list(APPEND sources ${filename})
+endforeach()
+add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources})
+set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir})
 

include/contractor/graph_contractor_adaptors.hpp

@@ -145,7 +145,7 @@ template <class Edge, typename GraphT> inline std::vector<Edge> toEdges(GraphT g
         util::Percent p(log, graph.GetNumberOfNodes());
         const NodeID number_of_nodes = graph.GetNumberOfNodes();
         std::size_t edge_index = 0;
-        for (const auto node : util::irange(0u, number_of_nodes))
+        for (const auto node : util::irange(MIN_NODEID, number_of_nodes))
         {
             p.PrintStatus(node);
             for (auto edge : graph.GetAdjacentEdgeRange(node))

include/contractor/query_edge.hpp

@@ -45,7 +45,11 @@ struct QueryEdge
         // this ID is either the middle node of the shortcut, or the ID of the edge based node (node
         // based edge) storing the appropriate data. If `shortcut` is set to true, we get the middle
         // node. Otherwise we see the edge based node to access node data.
+        #ifdef USE_64BIT_IDS
+        NodeID turn_id : 63;
+        #else
         NodeID turn_id : 31;
+        #endif
         bool shortcut : 1;
         EdgeWeight weight;
         EdgeDuration::value_type duration : 30;

include/engine/datafacade/algorithm_datafacade.hpp

@@ -30,9 +30,9 @@ template <> class AlgorithmDataFacade<CH>
     virtual ~AlgorithmDataFacade() = default;
 
     // search graph access
-    virtual unsigned GetNumberOfNodes() const = 0;
+    virtual NodeID GetNumberOfNodes() const = 0;
 
-    virtual unsigned GetNumberOfEdges() const = 0;
+    virtual EdgeID GetNumberOfEdges() const = 0;
 
     virtual unsigned GetOutDegree(const NodeID edge_based_node_id) const = 0;
 
@@ -67,11 +67,11 @@ template <> class AlgorithmDataFacade<MLD>
     virtual ~AlgorithmDataFacade() = default;
 
     // search graph access
-    virtual unsigned GetNumberOfNodes() const = 0;
+    virtual NodeID GetNumberOfNodes() const = 0;
 
-    virtual unsigned GetMaxBorderNodeID() const = 0;
+    virtual NodeID GetMaxBorderNodeID() const = 0;
 
-    virtual unsigned GetNumberOfEdges() const = 0;
+    virtual EdgeID GetNumberOfEdges() const = 0;
 
     virtual unsigned GetOutDegree(const NodeID edge_based_node_id) const = 0;
 

include/engine/datafacade/contiguous_internalmem_datafacade.hpp

@@ -65,9 +65,9 @@ class ContiguousInternalMemoryAlgorithmDataFacade<CH> : public datafacade::Algor
     }
 
     // search graph access
-    unsigned GetNumberOfNodes() const override final { return m_query_graph.GetNumberOfNodes(); }
+    NodeID GetNumberOfNodes() const override final { return m_query_graph.GetNumberOfNodes(); }
 
-    unsigned GetNumberOfEdges() const override final { return m_query_graph.GetNumberOfEdges(); }
+    EdgeID GetNumberOfEdges() const override final { return m_query_graph.GetNumberOfEdges(); }
 
     unsigned GetOutDegree(const NodeID edge_based_node_id) const override final
     {
@@ -609,11 +609,11 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
     const customizer::CellMetricView &GetCellMetric() const override { return mld_cell_metric; }
 
     // search graph access
-    unsigned GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
+    NodeID GetNumberOfNodes() const override final { return query_graph.GetNumberOfNodes(); }
 
-    unsigned GetMaxBorderNodeID() const override final { return query_graph.GetMaxBorderNodeID(); }
+    NodeID GetMaxBorderNodeID() const override final { return query_graph.GetMaxBorderNodeID(); }
 
-    unsigned GetNumberOfEdges() const override final { return query_graph.GetNumberOfEdges(); }
+    EdgeID GetNumberOfEdges() const override final { return query_graph.GetNumberOfEdges(); }
 
     unsigned GetOutDegree(const NodeID edge_based_node_id) const override final
     {

include/engine/guidance/assemble_leg.hpp

@@ -32,7 +32,7 @@ struct NamedSegment
 {
     EdgeDuration duration;
     std::uint32_t position;
-    std::uint32_t name_id;
+    NameID name_id;
 };
 
 template <std::size_t SegmentNumber>

include/engine/guidance/route_step.hpp

@@ -56,7 +56,7 @@ inline IntermediateIntersection getInvalidIntersection()
 struct RouteStep
 {
     NodeID from_id;
-    unsigned name_id;
+    NameID name_id;
     bool is_segregated;
     std::string name;
     std::string ref;

include/engine/hint.hpp

@@ -76,11 +76,16 @@ struct Hint
     static Hint FromBase64(const std::string &base64Hint);
 };
 
+#ifdef USE_64BIT_IDS
+static_assert(sizeof(SegmentHint) == 96 + 8, "Hint is bigger than expected");
+// 140 == ceil(104 / 3) * 4 since we use integer division
+constexpr std::size_t ENCODED_SEGMENT_HINT_SIZE = 140;
+#else
 static_assert(sizeof(SegmentHint) == 80 + 4, "Hint is bigger than expected");
 constexpr std::size_t ENCODED_SEGMENT_HINT_SIZE = 112;
+#endif
 static_assert(ENCODED_SEGMENT_HINT_SIZE / 4 * 3 >= sizeof(SegmentHint),
               "ENCODED_SEGMENT_HINT_SIZE does not match size of SegmentHint");
-
 } // namespace osrm::engine
 
 #endif

include/engine/phantom_node.hpp

@@ -222,7 +222,11 @@ struct PhantomNode
     unsigned short bearing : 12;
 };
 
+#ifdef USE_64BIT_IDS
+static_assert(sizeof(PhantomNode) == 96, "PhantomNode has more padding then expected");
+#else
 static_assert(sizeof(PhantomNode) == 80, "PhantomNode has more padding then expected");
+#if
 
 using PhantomNodeCandidates = std::vector<PhantomNode>;
 using PhantomCandidateAlternatives = std::pair<PhantomNodeCandidates, PhantomNodeCandidates>;

include/extractor/compressed_edge_container.hpp

@@ -46,7 +46,7 @@ class CompressedEdgeContainer
                              const EdgeDuration duration);
 
     void InitializeBothwayVector();
-    unsigned ZipEdges(const unsigned f_edge_pos, const unsigned r_edge_pos);
+    unsigned ZipEdges(const EdgeID f_edge_pos, const EdgeID r_edge_pos);
 
     bool HasEntryForID(const EdgeID edge_id) const;
     bool HasZippedEntryForForwardID(const EdgeID edge_id) const;