Always read .osrm.enw file in updater

oxidase · Patrick Niklaus · commit db18e8669fc5 · 2018-04-28T08:59:18.000+02:00
diff --git a/include/customizer/customizer_config.hpp b/include/customizer/customizer_config.hpp
@@ -21,7 +21,8 @@ struct CustomizationConfig final : storage::IOConfig
                     ".osrm.partition",
                     ".osrm.cells",
                     ".osrm.ebg_nodes",
-                    ".osrm.properties"},
+                    ".osrm.properties",
+                    ".osrm.enw"},
                    {},
                    {".osrm.cell_metrics", ".osrm.mldgr"}),
           requested_num_threads(0)
diff --git a/include/customizer/edge_based_graph.hpp b/include/customizer/edge_based_graph.hpp
@@ -74,6 +74,8 @@ class MultiLevelGraph : public partitioner::MultiLevelGraph<EdgeDataT, Ownership
         // TODO: add EdgeArrayEntry shaving
     }
 
+    EdgeWeight GetNodeWeight(NodeID node) const { return node_weights[node]; }
+
     friend void
     serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,
                                               const std::string &name,
diff --git a/include/engine/datafacade/algorithm_datafacade.hpp b/include/engine/datafacade/algorithm_datafacade.hpp
@@ -71,12 +71,16 @@ template <> class AlgorithmDataFacade<MLD>
 
     virtual unsigned GetOutDegree(const NodeID n) const = 0;
 
+    virtual EdgeRange GetAdjacentEdgeRange(const NodeID node) const = 0;
+
+    virtual EdgeWeight GetNodeWeight(const NodeID node) const = 0;
+
+    virtual EdgeWeight GetNodeDuration(const NodeID node) const = 0; // TODO: to be removed
+
     virtual NodeID GetTarget(const EdgeID e) const = 0;
 
     virtual const EdgeData &GetEdgeData(const EdgeID e) const = 0;
 
-    virtual EdgeRange GetAdjacentEdgeRange(const NodeID node) const = 0;
-
     virtual const partitioner::MultiLevelPartitionView &GetMultiLevelPartition() const = 0;
 
     virtual const partitioner::CellStorageView &GetCellStorage() const = 0;
diff --git a/include/engine/datafacade/contiguous_internalmem_datafacade.hpp b/include/engine/datafacade/contiguous_internalmem_datafacade.hpp
@@ -282,13 +282,13 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade
         return segment_data.GetReverseDatasources(id);
     }
 
-    TurnPenalty GetWeightPenaltyForEdgeID(const unsigned id) const override final
+    TurnPenalty GetWeightPenaltyForEdgeID(const EdgeID id) const override final
     {
         BOOST_ASSERT(m_turn_weight_penalties.size() > id);
         return m_turn_weight_penalties[id];
     }
 
-    TurnPenalty GetDurationPenaltyForEdgeID(const unsigned id) const override final
+    TurnPenalty GetDurationPenaltyForEdgeID(const EdgeID id) const override final
     {
         BOOST_ASSERT(m_turn_duration_penalties.size() > id);
         return m_turn_duration_penalties[id];
@@ -682,16 +682,26 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo
         return query_graph.GetOutDegree(n);
     }
 
-    NodeID GetTarget(const EdgeID e) const override final { return query_graph.GetTarget(e); }
+    EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
+    {
+        return query_graph.GetAdjacentEdgeRange(node);
+    }
 
-    const EdgeData &GetEdgeData(const EdgeID e) const override final
+    EdgeWeight GetNodeWeight(const NodeID node) const override final
     {
-        return query_graph.GetEdgeData(e);
+        return query_graph.GetNodeWeight(node);
     }
 
-    EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final
+    EdgeDuration GetNodeDuration(const NodeID) const override final
     {
-        return query_graph.GetAdjacentEdgeRange(node);
+        return 0; // TODO: query_graph.GetNodeduration(node);
+    }
+
+    NodeID GetTarget(const EdgeID e) const override final { return query_graph.GetTarget(e); }
+
+    const EdgeData &GetEdgeData(const EdgeID e) const override final
+    {
+        return query_graph.GetEdgeData(e);
     }
 
     EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const override final
diff --git a/include/engine/datafacade/datafacade_base.hpp b/include/engine/datafacade/datafacade_base.hpp
@@ -86,9 +86,9 @@ class BaseDataFacade
     virtual NodeForwardRange GetUncompressedForwardGeometry(const EdgeID id) const = 0;
     virtual NodeReverseRange GetUncompressedReverseGeometry(const EdgeID id) const = 0;
 
-    virtual TurnPenalty GetWeightPenaltyForEdgeID(const unsigned id) const = 0;
+    virtual TurnPenalty GetWeightPenaltyForEdgeID(const EdgeID id) const = 0;
 
-    virtual TurnPenalty GetDurationPenaltyForEdgeID(const unsigned id) const = 0;
+    virtual TurnPenalty GetDurationPenaltyForEdgeID(const EdgeID id) const = 0;
 
     // Gets the weight values for each segment in an uncompressed geometry.
     // Should always be 1 shorter than GetUncompressedGeometry
diff --git a/include/engine/routing_algorithms/routing_base_mld.hpp b/include/engine/routing_algorithms/routing_base_mld.hpp
@@ -206,15 +206,21 @@ void relaxOutgoingEdges(const DataFacade<Algorithm> &facade,
     for (const auto edge : facade.GetBorderEdgeRange(level, node))
     {
         const auto &edge_data = facade.GetEdgeData(edge);
+
         if (DIRECTION == FORWARD_DIRECTION ? edge_data.forward : edge_data.backward)
         {
             const NodeID to = facade.GetTarget(edge);
 
             if (!facade.ExcludeNode(to) &&
                 checkParentCellRestriction(partition.GetCell(level + 1, to), args...))
             {
-                BOOST_ASSERT_MSG(edge_data.weight > 0, "edge_weight invalid");
-                const EdgeWeight to_weight = weight + edge_data.weight;
+                const auto node_weight =
+                    facade.GetNodeWeight(DIRECTION == FORWARD_DIRECTION ? node : to);
+                const auto turn_penalty = facade.GetWeightPenaltyForEdgeID(edge_data.turn_id);
+
+                BOOST_ASSERT(edge_data.weight == node_weight + turn_penalty);
+
+                const EdgeWeight to_weight = weight + node_weight + turn_penalty;
 
                 if (!forward_heap.WasInserted(to))
                 {
diff --git a/include/partitioner/partitioner_config.hpp b/include/partitioner/partitioner_config.hpp
@@ -16,7 +16,7 @@ namespace partitioner
 struct PartitionerConfig final : storage::IOConfig
 {
     PartitionerConfig()
-        : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes"},
+        : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},
                    {".osrm.hsgr", ".osrm.cnbg"},
                    {".osrm.ebg",
                     ".osrm.cnbg",
diff --git a/include/updater/updater_config.hpp b/include/updater/updater_config.hpp
@@ -53,7 +53,8 @@ struct UpdaterConfig final : storage::IOConfig
                     ".osrm.geometry",
                     ".osrm.fileIndex",
                     ".osrm.properties",
-                    ".osrm.restrictions"},
+                    ".osrm.restrictions",
+                    ".osrm.enw"},
                    {},
                    {".osrm.datasource_names"}),
           valid_now(0)
diff --git a/scripts/gdb_printers.py b/scripts/gdb_printers.py
@@ -9,6 +9,8 @@
 
 def call(this, method, *args):
     """Call this.method(args)"""
+    if (str(this) == '<optimized out>'):
+        raise BaseException('"this" is optimized out')
     command = '(*({})({})).{}({})'.format(this.type.target().pointer(), this.address, method, ','.join((str(x) for x in args)))
     return gdb.parse_and_eval(command)
 
@@ -234,7 +236,6 @@ def Facade(facade, width, height, arg):
             mld_facade = facade.cast(gdb.lookup_type('osrm::engine::datafacade::ContiguousInternalMemoryAlgorithmDataFacade<osrm::engine::routing_algorithms::mld::Algorithm>'))
             mld_partition = mld_facade['mld_partition']
             mld_levels = call(mld_partition, 'GetNumberOfLevels')
-            print (mld_level, mld_levels)
             if mld_level < mld_levels:
                 sentinel_node = call(mld_partition['partition'], 'size') - 1 # GetSentinelNode
                 number_of_cells = call(mld_partition, 'GetCell', mld_level, sentinel_node) # GetNumberOfCells
@@ -272,6 +273,7 @@ def Facade(facade, width, height, arg):
         for node in nodes:
             geometry_id = call(facade, 'GetGeometryIndex', node)
             direction = 'forward' if geometry_id['forward'] else 'reverse'
+            print (geometry_id, direction)
             geometry = SVGPrinter.getByGeometryId(facade, geometry_id, 'Geometry')
             weights = SVGPrinter.getByGeometryId(facade, geometry_id, 'Weights')
 
diff --git a/src/customize/customizer.cpp b/src/customize/customizer.cpp
@@ -74,18 +74,20 @@ void printUnreachableStatistics(const Partition &partition,
 
 auto LoadAndUpdateEdgeExpandedGraph(const CustomizationConfig &config,
                                     const partitioner::MultiLevelPartition &mlp,
+                                    std::vector<EdgeWeight> &node_weights,
                                     std::uint32_t &connectivity_checksum)
 {
     updater::Updater updater(config.updater_config);
 
-    EdgeID num_nodes;
     std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
-    std::tie(num_nodes, edge_based_edge_list, connectivity_checksum) =
-        updater.LoadAndUpdateEdgeExpandedGraph();
+    EdgeID num_nodes = updater.LoadAndUpdateEdgeExpandedGraph(
+        edge_based_edge_list, node_weights, connectivity_checksum);
 
     auto directed = partitioner::splitBidirectionalEdges(edge_based_edge_list);
+
     auto tidied = partitioner::prepareEdgesForUsageInGraph<
         typename partitioner::MultiLevelEdgeBasedGraph::InputEdge>(std::move(directed));
+
     auto edge_based_graph =
         partitioner::MultiLevelEdgeBasedGraph(mlp, num_nodes, std::move(tidied));
 
@@ -120,8 +122,11 @@ int Customizer::Run(const CustomizationConfig &config)
     partitioner::MultiLevelPartition mlp;
     partitioner::files::readPartition(config.GetPath(".osrm.partition"), mlp);
 
+    std::vector<EdgeWeight> node_weights;
     std::uint32_t connectivity_checksum = 0;
-    auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp, connectivity_checksum);
+    auto graph = LoadAndUpdateEdgeExpandedGraph(config, mlp, node_weights, connectivity_checksum);
+    BOOST_ASSERT(graph.GetNumberOfNodes() == node_weights.size());
+    std::for_each(node_weights.begin(), node_weights.end(), [](auto &w) { w &= 0x7fffffff; });
     util::Log() << "Loaded edge based graph: " << graph.GetNumberOfEdges() << " edges, "
                 << graph.GetNumberOfNodes() << " nodes";
 
@@ -158,8 +163,7 @@ int Customizer::Run(const CustomizationConfig &config)
     util::Log() << "MLD customization writing took " << TIMER_SEC(writing_mld_data) << " seconds";
 
     TIMER_START(writing_graph);
-    std::vector<EdgeWeight> node_weights;
-    std::vector<EdgeDuration> node_durations;
+    std::vector<EdgeDuration> node_durations; // TODO: save an empty vector, to be removed later
     MultiLevelEdgeBasedGraph shaved_graph{
         std::move(graph), std::move(node_weights), std::move(node_durations)};
     customizer::files::writeGraph(
diff --git a/src/engine/routing_algorithms/many_to_many_mld.cpp b/src/engine/routing_algorithms/many_to_many_mld.cpp
@@ -169,6 +169,8 @@ void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
         }
     }
 
+    const auto node_weight = facade.GetNodeWeight(node);
+    const auto node_duration = facade.GetNodeDuration(node); // TODO: remove later
     for (const auto edge : facade.GetBorderEdgeRange(level, node))
     {
         const auto &data = facade.GetEdgeData(edge);
@@ -180,8 +182,9 @@ void relaxOutgoingEdges(const DataFacade<mld::Algorithm> &facade,
                 continue;
             }
 
-            const auto edge_weight = data.weight;
-            const auto edge_duration = data.duration;
+            const auto turn_id = data.turn_id;
+            const auto edge_weight = node_weight + facade.GetWeightPenaltyForEdgeID(turn_id);
+            const auto edge_duration = node_duration + facade.GetDurationPenaltyForEdgeID(turn_id);
 
             BOOST_ASSERT_MSG(edge_weight > 0, "edge_weight invalid");
             const auto to_weight = weight + edge_weight;
diff --git a/src/partitioner/partitioner.cpp b/src/partitioner/partitioner.cpp
@@ -144,6 +144,12 @@ int Partitioner::Run(const PartitionerConfig &config)
         renumber(node_data, permutation);
         extractor::files::writeNodeData(config.GetPath(".osrm.ebg_nodes"), node_data);
     }
+    {
+        std::vector<EdgeWeight> node_weights;
+        extractor::files::readEdgeBasedNodeWeights(config.GetPath(".osrm.enw"), node_weights);
+        util::inplacePermutation(node_weights.begin(), node_weights.end(), permutation);
+        extractor::files::writeEdgeBasedNodeWeights(config.GetPath(".osrm.enw"), node_weights);
+    }
     {
         const auto &filename = config.GetPath(".osrm.maneuver_overrides");
         std::vector<extractor::StorageManeuverOverride> maneuver_overrides;
diff --git a/src/updater/updater.cpp b/src/updater/updater.cpp
@@ -517,7 +517,6 @@ updateConditionalTurns(std::vector<TurnPenalty> &turn_weight_penalties,
     {
         if (IsRestrictionValid(time_zone_handler, penalty))
         {
-            std::cout << "Disabling: " << penalty.turn_offset << std::endl;
             turn_weight_penalties[penalty.turn_offset] = INVALID_TURN_PENALTY;
             updated_turns.push_back(penalty.turn_offset);
         }
@@ -526,17 +525,6 @@ updateConditionalTurns(std::vector<TurnPenalty> &turn_weight_penalties,
 }
 }
 
-Updater::NumNodesAndEdges Updater::LoadAndUpdateEdgeExpandedGraph() const
-{
-    std::vector<EdgeWeight> node_weights;
-    std::vector<extractor::EdgeBasedEdge> edge_based_edge_list;
-    std::uint32_t connectivity_checksum;
-    auto number_of_edge_based_nodes = Updater::LoadAndUpdateEdgeExpandedGraph(
-        edge_based_edge_list, node_weights, connectivity_checksum);
-    return std::make_tuple(
-        number_of_edge_based_nodes, std::move(edge_based_edge_list), connectivity_checksum);
-}
-
 EdgeID
 Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
                                         std::vector<EdgeWeight> &node_weights,
@@ -548,6 +536,8 @@ Updater::LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &e
     std::vector<util::Coordinate> coordinates;
     extractor::PackedOSMIDs osm_node_ids;
 
+    extractor::files::readEdgeBasedNodeWeights(config.GetPath(".osrm.enw"), node_weights);
+
     extractor::files::readEdgeBasedGraph(config.GetPath(".osrm.ebg"),
                                          number_of_edge_based_nodes,
                                          edge_based_edge_list,
diff --git a/unit_tests/engine/offline_facade.cpp b/unit_tests/engine/offline_facade.cpp
@@ -325,6 +325,11 @@ class ContiguousInternalMemoryDataFacade<routing_algorithms::offline::Algorithm>
         return {};
     }
 
+    EdgeWeight GetNodeWeight(const NodeID /*node*/) const
+    {
+        return 0;
+    }
+
     bool HasLaneData(const EdgeID /*id*/) const override { return false; }
     NameID GetNameIndex(const NodeID /*nodeID*/) const override { return EMPTY_NAMEID; }
     StringView GetNameForID(const NameID /*id*/) const override { return StringView{}; }

Original file line number	Diff line number	Diff line change
`@@ -74,6 +74,8 @@ class MultiLevelGraph : public partitioner::MultiLevelGraph<EdgeDataT, Ownership`
`74`	`74`	`// TODO: add EdgeArrayEntry shaving`
`75`	`75`	`}`
`76`	`76`
	`77`	`+ EdgeWeight GetNodeWeight(NodeID node) const { return node_weights[node]; }`
	`78`	`+`
`77`	`79`	`friend void`
`78`	`80`	`serialization::read<EdgeDataT, Ownership>(storage::tar::FileReader &reader,`
`79`	`81`	`const std::string &name,`
Original file line number	Diff line number	Diff line change
`@@ -282,13 +282,13 @@ class ContiguousInternalMemoryDataFacadeBase : public BaseDataFacade`
`282`	`282`	`return segment_data.GetReverseDatasources(id);`
`283`	`283`	`}`
`284`	`284`
`285`		`- TurnPenalty GetWeightPenaltyForEdgeID(const unsigned id) const override final`
	`285`	`+ TurnPenalty GetWeightPenaltyForEdgeID(const EdgeID id) const override final`
`286`	`286`	`{`
`287`	`287`	`BOOST_ASSERT(m_turn_weight_penalties.size() > id);`
`288`	`288`	`return m_turn_weight_penalties[id];`
`289`	`289`	`}`
`290`	`290`
`291`		`- TurnPenalty GetDurationPenaltyForEdgeID(const unsigned id) const override final`
	`291`	`+ TurnPenalty GetDurationPenaltyForEdgeID(const EdgeID id) const override final`
`292`	`292`	`{`
`293`	`293`	`BOOST_ASSERT(m_turn_duration_penalties.size() > id);`
`294`	`294`	`return m_turn_duration_penalties[id];`
`@@ -682,16 +682,26 @@ template <> class ContiguousInternalMemoryAlgorithmDataFacade<MLD> : public Algo`
`682`	`682`	`return query_graph.GetOutDegree(n);`
`683`	`683`	`}`
`684`	`684`
`685`		`- NodeID GetTarget(const EdgeID e) const override final { return query_graph.GetTarget(e); }`
	`685`	`+ EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final`
	`686`	`+ {`
	`687`	`+ return query_graph.GetAdjacentEdgeRange(node);`
	`688`	`+ }`
`686`	`689`
`687`		`- const EdgeData &GetEdgeData(const EdgeID e) const override final`
	`690`	`+ EdgeWeight GetNodeWeight(const NodeID node) const override final`
`688`	`691`	`{`
`689`		`- return query_graph.GetEdgeData(e);`
	`692`	`+ return query_graph.GetNodeWeight(node);`
`690`	`693`	`}`
`691`	`694`
`692`		`- EdgeRange GetAdjacentEdgeRange(const NodeID node) const override final`
	`695`	`+ EdgeDuration GetNodeDuration(const NodeID) const override final`
`693`	`696`	`{`
`694`		`- return query_graph.GetAdjacentEdgeRange(node);`
	`697`	`+ return 0; // TODO: query_graph.GetNodeduration(node);`
	`698`	`+ }`
	`699`	`+`
	`700`	`+ NodeID GetTarget(const EdgeID e) const override final { return query_graph.GetTarget(e); }`
	`701`	`+`
	`702`	`+ const EdgeData &GetEdgeData(const EdgeID e) const override final`
	`703`	`+ {`
	`704`	`+ return query_graph.GetEdgeData(e);`
`695`	`705`	`}`
`696`	`706`
`697`	`707`	`EdgeRange GetBorderEdgeRange(const LevelID level, const NodeID node) const override final`
Original file line number	Diff line number	Diff line change
`@@ -206,15 +206,21 @@ void relaxOutgoingEdges(const DataFacade<Algorithm> &facade,`
`206`	`206`	`for (const auto edge : facade.GetBorderEdgeRange(level, node))`
`207`	`207`	`{`
`208`	`208`	`const auto &edge_data = facade.GetEdgeData(edge);`
	`209`	`+`
`209`	`210`	`if (DIRECTION == FORWARD_DIRECTION ? edge_data.forward : edge_data.backward)`
`210`	`211`	`{`
`211`	`212`	`const NodeID to = facade.GetTarget(edge);`
`212`	`213`
`213`	`214`	`if (!facade.ExcludeNode(to) &&`
`214`	`215`	`checkParentCellRestriction(partition.GetCell(level + 1, to), args...))`
`215`	`216`	`{`
`216`		`- BOOST_ASSERT_MSG(edge_data.weight > 0, "edge_weight invalid");`
`217`		`- const EdgeWeight to_weight = weight + edge_data.weight;`
	`217`	`+ const auto node_weight =`
	`218`	`+ facade.GetNodeWeight(DIRECTION == FORWARD_DIRECTION ? node : to);`
	`219`	`+ const auto turn_penalty = facade.GetWeightPenaltyForEdgeID(edge_data.turn_id);`
	`220`	`+`
	`221`	`+ BOOST_ASSERT(edge_data.weight == node_weight + turn_penalty);`
	`222`	`+`
	`223`	`+ const EdgeWeight to_weight = weight + node_weight + turn_penalty;`
`218`	`224`
`219`	`225`	`if (!forward_heap.WasInserted(to))`
`220`	`226`	`{`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ namespace partitioner`
`16`	`16`	`struct PartitionerConfig final : storage::IOConfig`
`17`	`17`	`{`
`18`	`18`	`PartitionerConfig()`
`19`		`- : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes"},`
	`19`	`+ : IOConfig({".osrm", ".osrm.fileIndex", ".osrm.ebg_nodes", ".osrm.enw"},`
`20`	`20`	`{".osrm.hsgr", ".osrm.cnbg"},`
`21`	`21`	`{".osrm.ebg",`
`22`	`22`	`".osrm.cnbg",`