stash relations

Author: MarcelloPerathoner

docs/areas.md

@@ -112,15 +112,19 @@ Argument | Type     | Notes
 ---------|----------|-------------------------------------------
 way      | OSMWay   | The same way as passed into `process_way`.
 
-#### area_manager:get_relations(way)
-Call this function inside `process_way()`.  If this way is a member of a relation that
-was registered for meshing, that relation will be returned. Since the member ways of a
-multipolygon relation are as a rule untagged, you have to copy at least the defining tag
-(and maybe the name) from the relation to the way. OSRM discards untagged ways.
+#### area_manager:get_relations(node), area_manager:get_relations(way)
+Call this functions inside `process_node()` and `process_way()` respectively.  If this
+node or way is a member of a relation that was registered for meshing, those relations
+will be returned.
+
+Since the member ways of a multipolygon relation are as a rule untagged, and since OSRM
+discards untagged ways, you have to copy at least the defining tag (and maybe the name
+tags) from the relation to the way.
 
 Argument | Type     | Notes
----------|----------|-------------------------------------------
-way      | OSMWay   | The same way as passed into `process_way`.
+---------|----------|-----------------------------------------------
+node     | OSMNode  | The same node as passed into `process_node()`.
+way      | OSMWay   | The same way as passed into `process_way()`.
 
 Usage example:
 

include/extractor/area/area_manager.hpp

@@ -37,12 +37,10 @@ class AreaManager
 
     void way(const osmium::Way &way);
     void relation(const osmium::Relation &relation);
-    bool new_member(const osmium::Relation &,
-                    const osmium::RelationMember &member,
-                    std::size_t /*n*/) const noexcept;
     void prepare_for_lookup(extractor::NodeLocationsForWays &node_locations_for_ways);
     bool is_registered_closed_way(osmium::object_id_type way_id) const;
-    ExtractionRelationContainer::RelationIDList get_relations(const osmium::Way &way) const;
+    ExtractionRelationContainer::RelationIDList get_relations_node(const osmium::Node &) const;
+    ExtractionRelationContainer::RelationIDList get_relations_way(const osmium::Way &) const;
 
     void complete_relation(const osmium::Relation &relation);
     void after_way(const osmium::Way &way);
@@ -55,8 +53,10 @@ class AreaManager
      * registered for meshing. They are also used for collecting the intersecting ways.
      */
     tbb::concurrent_vector<osmium::object_id_type> registered_closed_ways;
-    /** way_id -> rel_id: if way is a ring of rel */
-    tbb::concurrent_map<osmium::object_id_type, osmium::object_id_type> m_ring_of;
+    /** Map of way_id -> rel_id: if way is a member of rel */
+    tbb::concurrent_map<osmium::object_id_type, osmium::object_id_type> m_way_relation;
+    /** Map of node_id -> rel_id: if node is a member of rel */
+    tbb::concurrent_map<osmium::object_id_type, osmium::object_id_type> m_node_relation;
 };
 
 } // namespace osrm::extractor::area

include/extractor/area/dijkstra.hpp

@@ -59,12 +59,14 @@ template <class vertex_t> class DijkstraImpl
      */
     size_t add_vertex(const vertex_t &v)
     {
-        if (seen_vertices.contains(v))
-            return seen_vertices.at(v);
         size_t pos = vertices.size();
-        seen_vertices.emplace(v, pos);
-        vertices.push_back(v);
-        return pos;
+        auto [_, inserted] = seen_vertices.emplace(v, pos);
+        if (inserted)
+        {
+            vertices.push_back(v);
+            return pos;
+        }
+        return seen_vertices.at(v);
     };
 
     /**

include/extractor/area/index_priority_queue.hpp

@@ -50,7 +50,7 @@ template <typename comp_t> class IndexPriorityQueue
      *             {@code v}. To construct a priority queue that has the lowest item on
      *             top the function should behave like {@code std::less}.
      */
-    IndexPriorityQueue(size_t size, comp_t comp) : size{size}, comp{comp}
+    IndexPriorityQueue(size_t size, comp_t comp) : vector_size{size}, comp{comp}
     {
         pq.resize(size + 1);
         qp.resize(size + 1);
@@ -171,10 +171,10 @@ template <typename comp_t> class IndexPriorityQueue
     /**
      * @brief Return the number of items in the queue.
      */
-    size_t num_items() { return n; }
+    size_t size() { return n; }
 
   private:
-    const size_t size;
+    const size_t vector_size;
     comp_t comp;
 
     /**

include/extractor/extraction_relation.hpp

@@ -3,201 +3,141 @@
 
 #include "util/exception.hpp"
 
+#include <boost/assert.hpp>
+#include <boost/flyweight.hpp>
+#include <boost/flyweight/no_tracking.hpp>
+
+#include <osmium/osm/area.hpp>
+#include <osmium/osm/item_type.hpp>
+#include <osmium/osm/node.hpp>
+#include <osmium/osm/object.hpp>
 #include <osmium/osm/relation.hpp>
+#include <osmium/osm/types.hpp>
+#include <osmium/osm/way.hpp>
 
-#include <boost/assert.hpp>
+#include <oneapi/tbb/concurrent_map.h>
+#include <oneapi/tbb/mutex.h>
 
+#include <osmium/storage/item_stash.hpp>
 #include <string>
-#include <unordered_map>
 #include <vector>
 
 namespace osrm::extractor
 {
 
-struct ExtractionRelation
+// It contains data of all parsed relations for each node/way element
+class ExtractionRelationContainer
 {
-    class OsmIDTyped
-    {
-      public:
-        OsmIDTyped(osmium::object_id_type _id, osmium::item_type _type) : id(_id), type(_type) {}
-
-        std::uint64_t GetID() const { return std::uint64_t(id); }
-        osmium::item_type GetType() const { return type; }
-
-        std::uint64_t Hash() const { return id ^ (static_cast<std::uint64_t>(type) << 56); }
-
-      private:
-        osmium::object_id_type id;
-        osmium::item_type type;
-    };
-
-    using AttributesList = std::vector<std::pair<std::string, std::string>>;
-    using MembersRolesList = std::vector<std::pair<std::uint64_t, std::string>>;
-
-    explicit ExtractionRelation(const OsmIDTyped &_id) : id(_id) {}
-    ExtractionRelation() : id{-1, osmium::item_type::relation} {};
-
-    void Clear()
-    {
-        attributes.clear();
-        members_role.clear();
-    }
+  public:
+    using rel_id_type = osmium::object_id_type;
+    using handle_type = osmium::ItemStash::handle_type;
+    using mutex_type = tbb::mutex;
 
-    const char *GetAttr(const std::string &attr) const
-    {
-        auto it = std::lower_bound(
-            attributes.begin(), attributes.end(), std::make_pair(attr, std::string()));
+    using RelationIDList = std::vector<rel_id_type>;
+    using id_and_type = std::pair<osmium::object_id_type, osmium::item_type>;
 
-        if (it != attributes.end() && (*it).first == attr)
-            return (*it).second.c_str();
+    using RelationHandleMap = tbb::concurrent_map<rel_id_type, handle_type>;
+    using MemberRelationMap = tbb::concurrent_map<id_and_type, RelationIDList>;
 
-        return nullptr;
-    }
+    ExtractionRelationContainer() = default;
+    ExtractionRelationContainer(ExtractionRelationContainer &&) = delete;
+    ExtractionRelationContainer(const ExtractionRelationContainer &) = delete;
 
-    void Prepare()
+    void AddRelation(const osmium::Relation &rel)
     {
-        std::sort(attributes.begin(), attributes.end());
-        std::sort(members_role.begin(), members_role.end());
-    }
+        BOOST_ASSERT(handles.find(rel.id()) == handles.end());
+        for (auto const &m : rel.members())
+        {
+            AddRelationMember(rel.id(), m.ref(), m.type());
+        }
 
-    void AddMember(const OsmIDTyped &member_id, const char *role)
-    {
-        members_role.emplace_back(std::make_pair(member_id.Hash(), std::string(role)));
+        handle_type handle;
+        {
+            mutex_type::scoped_lock lock(mutex);
+            handle = stash.add_item(rel);
+        }
+        handles.emplace(rel.id(), handle);
     }
 
-    const char *GetRole(const OsmIDTyped &member_id) const
+    void
+    AddRelationMember(rel_id_type relation_id, osmium::object_id_type id, osmium::item_type type)
     {
-        const auto hash = member_id.Hash();
-        auto it = std::lower_bound(
-            members_role.begin(), members_role.end(), std::make_pair(hash, std::string()));
-
-        if (it != members_role.end() && (*it).first == hash)
-            return (*it).second.c_str();
-
-        return nullptr;
+        parents[id_and_type{id, type}].push_back(relation_id);
     }
 
-    OsmIDTyped id;
-    AttributesList attributes;
-    MembersRolesList members_role;
-};
-
-// It contains data of all parsed relations for each node/way element
-class ExtractionRelationContainer
-{
-  public:
-    using AttributesMap = ExtractionRelation::AttributesList;
-    using OsmIDTyped = ExtractionRelation::OsmIDTyped;
-    using RelationList = std::vector<AttributesMap>;
-    using RelationIDList = std::vector<ExtractionRelation::OsmIDTyped>;
-    using RelationRefMap = std::unordered_map<std::uint64_t, RelationIDList>;
-
-    ExtractionRelationContainer() = default;
-    ExtractionRelationContainer(ExtractionRelationContainer &&) = default;
-    ExtractionRelationContainer(const ExtractionRelationContainer &) = delete;
+    std::size_t GetRelationsNum() const { return handles.size(); }
 
-    void AddRelation(ExtractionRelation &&rel)
+    const RelationIDList &_GetRelationsFor(osmium::object_id_type id, osmium::item_type type) const
     {
-        rel.Prepare();
+        if (type == osmium::item_type::area)
+        {
+            type = (id & 1) ? osmium::item_type::relation : osmium::item_type::way;
+            id /= 2;
+        }
+        auto it = parents.find(id_and_type{id, type});
+        if (it != parents.end())
+            return it->second;
 
-        BOOST_ASSERT(relations_data.find(rel.id.GetID()) == relations_data.end());
-        relations_data.insert(std::make_pair(rel.id.GetID(), std::move(rel)));
+        return empty_rel_list;
     }
 
-    void AddRelationMember(const OsmIDTyped &relation_id, const OsmIDTyped &member_id)
+    const RelationIDList &GetRelationsFor(const osmium::OSMObject &o) const
     {
-        switch (member_id.GetType())
-        {
-        case osmium::item_type::node:
-            node_refs[member_id.GetID()].push_back(relation_id);
-            break;
-
-        case osmium::item_type::way:
-            way_refs[member_id.GetID()].push_back(relation_id);
-            break;
-
-        case osmium::item_type::relation:
-            rel_refs[member_id.GetID()].push_back(relation_id);
-            break;
-
-        default:
-            break;
-        };
+        return _GetRelationsFor(o.id(), o.type());
     }
 
-    void Merge(ExtractionRelationContainer &&other)
+    // Note: non-const because SOL somehow chokes on const.
+    osmium::Relation &GetRelation(rel_id_type rel_id) const
     {
-        for (auto it : other.relations_data)
-        {
-            const auto res = relations_data.insert(std::make_pair(it.first, std::move(it.second)));
-            BOOST_ASSERT(res.second);
-            (void)res; // prevent unused warning in release
-        }
+        auto it = handles.find(rel_id);
+        if (it == handles.end())
+            throw osrm::util::exception("Can't find relation data for " + std::to_string(rel_id));
 
-        auto MergeRefMap = [&](RelationRefMap &source, RelationRefMap &target)
-        {
-            for (auto it : source)
-            {
-                auto &v = target[it.first];
-                v.insert(v.end(), it.second.begin(), it.second.end());
-            }
-        };
-
-        MergeRefMap(other.way_refs, way_refs);
-        MergeRefMap(other.node_refs, node_refs);
-        MergeRefMap(other.rel_refs, rel_refs);
+        return stash.get<osmium::Relation>(it->second);
     }
 
-    std::size_t GetRelationsNum() const { return relations_data.size(); }
+    void reset() { iter = handles.begin(); }
 
-    const RelationIDList &GetRelations(const OsmIDTyped &member_id) const
+    osmium::memory::Buffer read()
     {
-        auto getFromMap = [this](std::uint64_t id,
-                                 const RelationRefMap &map) -> const RelationIDList &
-        {
-            auto it = map.find(id);
-            if (it != map.end())
-                return it->second;
-
-            return empty_rel_list;
-        };
-
-        switch (member_id.GetType())
+        osmium::memory::Buffer buffer(16 * 1024, osmium::memory::Buffer::auto_grow::yes);
+        auto end = handles.end();
+        if (iter == end)
+            return osmium::memory::Buffer();
+        while (iter != end && buffer.written() < 12 * 1024)
         {
-        case osmium::item_type::node:
-            return getFromMap(member_id.GetID(), node_refs);
-
-        case osmium::item_type::way:
-            return getFromMap(member_id.GetID(), way_refs);
-
-        case osmium::item_type::relation:
-            return getFromMap(member_id.GetID(), rel_refs);
-
-        default:
-            break;
+            buffer.add_item(stash.get_item(iter->second));
+            buffer.commit();
+            ++iter;
         }
-
-        return empty_rel_list;
-    }
-
-    const ExtractionRelation &GetRelationData(const ExtractionRelation::OsmIDTyped &rel_id) const
-    {
-        auto it = relations_data.find(rel_id.GetID());
-        if (it == relations_data.end())
-            throw osrm::util::exception("Can't find relation data for " +
-                                        std::to_string(rel_id.GetID()));
-
-        return it->second;
-    }
+        return buffer;
+    };
 
   private:
+    osmium::ItemStash stash;
     RelationIDList empty_rel_list;
-    std::unordered_map<std::uint64_t, ExtractionRelation> relations_data;
+    RelationHandleMap handles;
+    RelationHandleMap::const_iterator iter;
+    MemberRelationMap parents;
+
+    mutex_type mutex;
+};
+
+/**
+ * @brief A copiable RelationMember
+ */
+class RelationMember
+{
+    osmium::object_id_type m_ref;
+    osmium::item_type m_type;
+    std::string m_role;
 
-    // each map contains list of relation id's, that has keyed id as a member
-    RelationRefMap way_refs;
-    RelationRefMap node_refs;
-    RelationRefMap rel_refs;
+  public:
+    RelationMember(const osmium::RelationMember &o)
+        : m_ref{o.ref()}, m_type{o.type()}, m_role(o.role()){};
+    osmium::object_id_type ref() const noexcept { return m_ref; }
+    osmium::item_type type() const noexcept { return m_type; }
+    const std::string &role() const noexcept { return m_role; }
 };
 
 } // namespace osrm::extractor