Merge pull request #1168 from joto/dependency-manager

New dependency manager

Author: lonvia

src/CMakeLists.txt

@@ -1,5 +1,6 @@
 
 set(osm2pgsql_lib_SOURCES
+  dependency-manager.cpp
   db-copy.cpp
   expire-tiles.cpp
   gazetteer-style.cpp
@@ -31,6 +32,7 @@ set(osm2pgsql_lib_SOURCES
   tagtransform.cpp
   util.cpp
   wildcmp.cpp
+  dependency-manager.hpp
   db-copy.hpp
   expire-tiles.hpp
   gazetteer-style.hpp

src/dependency-manager.cpp

@@ -0,0 +1,60 @@
+
+#include "dependency-manager.hpp"
+
+void full_dependency_manager_t::node_changed(osmid_t id)
+{
+    for (auto const way_id : m_object_store->get_ways_by_node(id)) {
+        way_changed(way_id);
+        m_ways_pending_tracker.mark(way_id);
+    }
+
+    for (auto const rel_id : m_object_store->get_rels_by_node(id)) {
+        m_rels_pending_tracker.mark(rel_id);
+    }
+}
+
+void full_dependency_manager_t::way_changed(osmid_t id)
+{
+    if (m_ways_pending_tracker.is_marked(id)) {
+        return;
+    }
+
+    for (auto const rel_id : m_object_store->get_rels_by_way(id)) {
+        m_rels_pending_tracker.mark(rel_id);
+    }
+}
+
+void full_dependency_manager_t::relation_changed(osmid_t id)
+{
+    for (auto const rel_id : m_object_store->get_rels_by_rel(id)) {
+        m_rels_pending_tracker.mark(rel_id);
+    }
+}
+
+void full_dependency_manager_t::relation_deleted(osmid_t id)
+{
+    for (auto const rel_id : m_object_store->get_ways_by_rel(id)) {
+        m_ways_pending_tracker.mark(rel_id);
+    }
+}
+
+bool full_dependency_manager_t::has_pending() const noexcept
+{
+    return !m_ways_pending_tracker.empty() || !m_rels_pending_tracker.empty();
+}
+
+void full_dependency_manager_t::process_pending(pending_processor &proc)
+{
+    osmid_t id;
+    while (id_tracker::is_valid(id = m_ways_pending_tracker.pop_mark())) {
+        proc.enqueue_way(id);
+    }
+
+    proc.process_ways();
+
+    while (id_tracker::is_valid(id = m_rels_pending_tracker.pop_mark())) {
+        proc.enqueue_relation(id);
+    }
+
+    proc.process_relations();
+}

src/dependency-manager.hpp

@@ -0,0 +1,163 @@
+#ifndef OSM2PGSQL_DEPENDENCY_MANAGER_HPP
+#define OSM2PGSQL_DEPENDENCY_MANAGER_HPP
+
+#include "id-tracker.hpp"
+#include "middle.hpp"
+#include "osmtypes.hpp"
+
+#include <cassert>
+#include <memory>
+
+struct pending_processor
+{
+    virtual ~pending_processor() = default;
+
+    virtual void enqueue_way(osmid_t id) = 0;
+    virtual void enqueue_relation(osmid_t id) = 0;
+
+    virtual void process_ways() = 0;
+    virtual void process_relations() = 0;
+};
+
+/**
+ * The job of the dependency manager is to keep track of the dependencies
+ * between OSM objects, that is nodes in ways and members of relations.
+ *
+ * Whenever an OSM object changes, this class is notified and can remember
+ * the ids for later use.
+ *
+ * This base class doesn't actually do the dependency management but is a
+ * dummy for cases where no dependency management is needed. See the
+ * full_dependency_manager_t class for the real dependency manager.
+ */
+class dependency_manager_t
+{
+public:
+    virtual ~dependency_manager_t() = default;
+
+    /**
+     * Mark a node as changed to trigger the propagation of this change to
+     * ways and relations.
+     *
+     * This has to be called *after* the object was stored in the object store.
+     */
+    virtual void node_changed(osmid_t) {}
+
+    /**
+     * Mark a way as changed to trigger the propagation of this change to
+     * relations.
+     *
+     * This has to be called *after* the object was stored in the object store.
+     */
+    virtual void way_changed(osmid_t) {}
+
+    /**
+     * Mark a relation as changed to trigger the propagation of this change to
+     * other relations.
+     *
+     * This has to be called *after* the object was stored in the object store.
+     */
+    virtual void relation_changed(osmid_t) {}
+
+    /**
+     * Mark a relation as deleted to trigger the propagation of this change to
+     * the way members.
+     */
+    virtual void relation_deleted(osmid_t) {}
+
+    /// Are there pending objects that need to be processed?
+    virtual bool has_pending() const noexcept { return false; }
+
+    /**
+     * Process all pending objects.
+     *
+     * \param pf Processor that we should feed the objects to and that
+     *        will handle the actual processing.
+     *
+     * \post !has_pending()
+     */
+    virtual void process_pending(pending_processor &) {}
+};
+
+/**
+ * The job of the dependency manager is to keep track of the dependencies
+ * between OSM objects, that is nodes in ways and members of relations.
+ *
+ * Whenever an OSM object changes, this class is notified and remembers
+ * the ids for later use. Later on the class can be told to process the
+ * ids it has remembered.
+ */
+class full_dependency_manager_t : public dependency_manager_t
+{
+public:
+    /**
+     * Constructor.
+     *
+     * \param object_store Pointer to the middle that keeps the actual
+     *        database of objects and their relationsships.
+     *
+     * \pre object_store != nullptr
+     */
+    explicit full_dependency_manager_t(std::shared_ptr<middle_t> object_store)
+    : m_object_store(std::move(object_store))
+    {
+        assert(m_object_store != nullptr);
+    }
+
+    void node_changed(osmid_t id) override;
+    void way_changed(osmid_t id) override;
+    void relation_changed(osmid_t id) override;
+    void relation_deleted(osmid_t id) override;
+
+    bool has_pending() const noexcept override;
+
+    void process_pending(pending_processor &proc) override;
+
+    /**
+     * Get access to the pending way ids. This is for debugging only.
+     *
+     * Note that the list of pending way ids will be empty after calling
+     * this.
+     *
+     * \tparam TOutputIterator Some output iterator type, for instance
+     *         created with std::back_inserter(some vector).
+     * \param it output iterator to which all ids should be written. *it
+     *        must be of type osmid_t.
+     */
+    template <typename TOutputIterator>
+    void get_pending_way_ids(TOutputIterator &&it)
+    {
+        osmid_t id;
+        while (id_tracker::is_valid(id = m_ways_pending_tracker.pop_mark())) {
+            *it++ = id;
+        }
+    }
+
+    /**
+     * Get access to the pending relation ids. This is for debugging only.
+     *
+     * Note that the list of pending relation ids will be empty after calling
+     * this.
+     *
+     * \tparam TOutputIterator Some output iterator type, for instance
+     *         created with std::back_inserter(some vector).
+     * \param it output iterator to which all ids should be written. *it
+     *        must be of type osmid_t.
+     */
+    template <typename TOutputIterator>
+    void get_pending_relation_ids(TOutputIterator &&it)
+    {
+        osmid_t id;
+        while (id_tracker::is_valid(id = m_rels_pending_tracker.pop_mark())) {
+            *it++ = id;
+        }
+    }
+
+private:
+    std::shared_ptr<middle_t> m_object_store;
+
+    id_tracker m_ways_pending_tracker;
+    id_tracker m_rels_pending_tracker;
+};
+
+#endif // OSM2PGSQL_DEPENDENCY_MANAGER_HPP

src/middle-pgsql.cpp

@@ -19,8 +19,6 @@
 #include <osmium/memory/buffer.hpp>
 #include <osmium/osm/types_from_string.hpp>
 
-#include <libpq-fe.h>
-
 #include "format.hpp"
 #include "middle-pgsql.hpp"
 #include "node-persistent-cache.hpp"
@@ -308,25 +306,40 @@ void middle_pgsql_t::node_delete(osmid_t osm_id)
     }
 }
 
-void middle_pgsql_t::node_changed(osmid_t osm_id)
+idlist_t middle_pgsql_t::get_ids(const char* stmt, osmid_t osm_id)
 {
-    assert(m_append);
-    if (!m_mark_pending) {
-        return;
-    }
+    idlist_t ids;
 
-    // Find all ways referencing this node and mark them as pending.
-    auto const res_ways =
-        m_db_connection.exec_prepared("mark_ways_by_node", osm_id);
-    for_each_id(res_ways, [this](osmid_t id) {
-        way_changed(id);
-        m_ways_pending_tracker->mark(id);
-    });
+    auto const res = m_db_connection.exec_prepared(stmt, osm_id);
+    ids.reserve(res.num_tuples());
+    for_each_id(res, [&ids](osmid_t id) { ids.push_back(id); });
+
+    return ids;
+}
+
+idlist_t middle_pgsql_t::get_ways_by_node(osmid_t osm_id)
+{
+    return get_ids("mark_ways_by_node", osm_id);
+}
+
+idlist_t middle_pgsql_t::get_rels_by_node(osmid_t osm_id)
+{
+    return get_ids("mark_rels_by_node", osm_id);
+}
+
+idlist_t middle_pgsql_t::get_rels_by_way(osmid_t osm_id)
+{
+    return get_ids("mark_rels_by_way", osm_id);
+}
+
+idlist_t middle_pgsql_t::get_rels_by_rel(osmid_t osm_id)
+{
+    return get_ids("mark_rels", osm_id);
+}
 
-    // Find all relations referencing this node and mark them as pending.
-    auto const res_rels = m_db_connection.exec_prepared("mark_rels_by_node", osm_id);
-    for_each_id(res_rels,
-                [this](osmid_t id) { m_rels_pending_tracker->mark(id); });
+idlist_t middle_pgsql_t::get_ways_by_rel(osmid_t osm_id)
+{
+    return get_ids("mark_ways_by_rel", osm_id);
 }
 
 void middle_pgsql_t::way_set(osmium::Way const &way)
@@ -431,31 +444,6 @@ void middle_pgsql_t::way_delete(osmid_t osm_id)
     m_db_copy.delete_object(osm_id);
 }
 
-void middle_pgsql_t::iterate_ways(pending_processor &pf)
-{
-    // enqueue the jobs
-    osmid_t id;
-    while (id_tracker::is_valid(id = m_ways_pending_tracker->pop_mark())) {
-        pf.enqueue_ways(id);
-    }
-
-    //let the threads work on them
-    pf.process_ways();
-}
-
-void middle_pgsql_t::way_changed(osmid_t osm_id)
-{
-    assert(m_append);
-
-    if (m_ways_pending_tracker->is_marked(osm_id)) {
-        return;
-    }
-
-    // Keep track of all relations having this way as member.
-    auto const res = m_db_connection.exec_prepared("mark_rels_by_way", osm_id);
-    for_each_id(res, [this](osmid_t id) { m_rels_pending_tracker->mark(id); });
-}
-
 void middle_pgsql_t::relation_set(osmium::Relation const &rel)
 {
     // Sort relation members by their type.
@@ -526,34 +514,10 @@ void middle_pgsql_t::relation_delete(osmid_t osm_id)
 {
     assert(m_append);
 
-    // Keep track of all member ways.
-    auto const res = m_db_connection.exec_prepared("mark_ways_by_rel", osm_id);
-    for_each_id(res, [this](osmid_t id) { m_ways_pending_tracker->mark(id); });
-
     m_db_copy.new_line(m_tables[REL_TABLE].m_copy_target);
     m_db_copy.delete_object(osm_id);
 }
 
-void middle_pgsql_t::iterate_relations(pending_processor &pf)
-{
-    // enqueue the jobs
-    osmid_t id;
-    while (id_tracker::is_valid(id = m_rels_pending_tracker->pop_mark())) {
-        pf.enqueue_relations(id);
-    }
-
-    //let the threads work on them
-    pf.process_relations();
-}
-
-void middle_pgsql_t::relation_changed(osmid_t osm_id)
-{
-    assert(m_append);
-
-    auto const res = m_db_connection.exec_prepared("mark_rels", osm_id);
-    for_each_id(res, [this](osmid_t id) { m_rels_pending_tracker->mark(id); });
-}
-
 idlist_t middle_query_pgsql_t::relations_using_way(osmid_t way_id) const
 {
     auto const result = m_sql_conn.exec_prepared("rels_using_way", way_id);
@@ -579,9 +543,6 @@ middle_query_pgsql_t::middle_query_pgsql_t(
 
 void middle_pgsql_t::start()
 {
-    m_ways_pending_tracker.reset(new id_tracker{});
-    m_rels_pending_tracker.reset(new id_tracker{});
-
     // Gazetter doesn't use mark-pending processing and consequently
     // needs no way-node index.
     // TODO Currently, set here to keep the impact on the code small.
@@ -782,8 +743,3 @@ middle_pgsql_t::get_query_instance()
 
     return std::shared_ptr<middle_query_t>(mid.release());
 }
-
-bool middle_pgsql_t::has_pending() const
-{
-    return !m_ways_pending_tracker->empty() || !m_rels_pending_tracker->empty();
-}