Refactor: Use idlist_t instead of osmium::index::IdSetSmall<osmid_t>

They are both basically the same class. We need a few functions from
IdSetSmall though.

Author: joto

src/CMakeLists.txt

@@ -19,6 +19,7 @@ target_sources(osm2pgsql_lib PRIVATE
   geom-from-osm.cpp
   geom-functions.cpp
   geom-pole-of-inaccessibility.cpp
+  idlist.cpp
   input.cpp
   logging.cpp
   middle.cpp

src/dependency-manager.cpp

@@ -15,17 +15,17 @@
 
 void full_dependency_manager_t::node_changed(osmid_t id)
 {
-    m_changed_nodes.set(id);
+    m_changed_nodes.push_back(id);
 }
 
 void full_dependency_manager_t::way_changed(osmid_t id)
 {
-    m_changed_ways.set(id);
+    m_changed_ways.push_back(id);
 }
 
 void full_dependency_manager_t::relation_changed(osmid_t id)
 {
-    m_changed_relations.set(id);
+    m_changed_relations.push_back(id);
 }
 
 void full_dependency_manager_t::after_nodes()
@@ -39,15 +39,13 @@ void full_dependency_manager_t::after_nodes()
     m_changed_nodes.clear();
 }
 
-static osmium::index::IdSetSmall<osmid_t>
-set_diff(osmium::index::IdSetSmall<osmid_t> const &set,
-         osmium::index::IdSetSmall<osmid_t> const &to_be_removed)
+static idlist_t set_diff(idlist_t const &set, idlist_t const &to_be_removed)
 {
-    osmium::index::IdSetSmall<osmid_t> new_set;
+    idlist_t new_set;
 
     for (auto const id : set) {
         if (!to_be_removed.get_binary_search(id)) {
-            new_set.set(id);
+            new_set.push_back(id);
         }
     }
 
@@ -92,7 +90,7 @@ void full_dependency_manager_t::after_relations()
 }
 
 void full_dependency_manager_t::mark_parent_relations_as_pending(
-    osmium::index::IdSetSmall<osmid_t> const &way_ids)
+    idlist_t const &way_ids)
 {
     assert(m_rels_pending_tracker.empty());
     m_object_store->get_way_parents(way_ids, &m_rels_pending_tracker);
@@ -103,8 +101,7 @@ bool full_dependency_manager_t::has_pending() const noexcept
     return !m_ways_pending_tracker.empty() || !m_rels_pending_tracker.empty();
 }
 
-idlist_t
-full_dependency_manager_t::get_ids(osmium::index::IdSetSmall<osmid_t> *tracker)
+idlist_t full_dependency_manager_t::get_ids(idlist_t *tracker)
 {
     tracker->sort_unique();
 

src/dependency-manager.hpp

@@ -13,8 +13,6 @@
 #include "idlist.hpp"
 #include "osmtypes.hpp"
 
-#include <osmium/index/id_set.hpp>
-
 #include <cassert>
 #include <memory>
 #include <utility>
@@ -59,8 +57,7 @@ class dependency_manager_t
     virtual void after_ways() {}
     virtual void after_relations() {}
 
-    virtual void mark_parent_relations_as_pending(
-        osmium::index::IdSetSmall<osmid_t> const & /*way_ids*/)
+    virtual void mark_parent_relations_as_pending(idlist_t const & /*way_ids*/)
     {
     }
 
@@ -112,8 +109,7 @@ class full_dependency_manager_t : public dependency_manager_t
     void after_ways() override;
     void after_relations() override;
 
-    void mark_parent_relations_as_pending(
-        osmium::index::IdSetSmall<osmid_t> const &ids) override;
+    void mark_parent_relations_as_pending(idlist_t const &ids) override;
 
     bool has_pending() const noexcept override;
 
@@ -128,7 +124,7 @@ class full_dependency_manager_t : public dependency_manager_t
     }
 
 private:
-    static idlist_t get_ids(osmium::index::IdSetSmall<osmid_t> *tracker);
+    static idlist_t get_ids(idlist_t *tracker);
 
     std::shared_ptr<middle_t> m_object_store;
 
@@ -140,7 +136,7 @@ class full_dependency_manager_t : public dependency_manager_t
      * the change file, too, and so we don't have to find out which ones they
      * are.
      */
-    osmium::index::IdSetSmall<osmid_t> m_changed_nodes;
+    idlist_t m_changed_nodes;
 
     /**
      * In append mode all new and changed ways will be added to this. After
@@ -149,17 +145,17 @@ class full_dependency_manager_t : public dependency_manager_t
      * relations that referenced deleted ways must be in the change file, too,
      * and so we don't have to find out which ones they are.
      */
-    osmium::index::IdSetSmall<osmid_t> m_changed_ways;
+    idlist_t m_changed_ways;
 
     /**
      * In append mode all new and changed relations will be added to this.
      * This is then used to remove already processed relations from the
      * pending list.
      */
-    osmium::index::IdSetSmall<osmid_t> m_changed_relations;
+    idlist_t m_changed_relations;
 
-    osmium::index::IdSetSmall<osmid_t> m_ways_pending_tracker;
-    osmium::index::IdSetSmall<osmid_t> m_rels_pending_tracker;
+    idlist_t m_ways_pending_tracker;
+    idlist_t m_rels_pending_tracker;
 };
 
 #endif // OSM2PGSQL_DEPENDENCY_MANAGER_HPP

src/idlist.cpp

@@ -0,0 +1,33 @@
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2023 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+#include "idlist.hpp"
+
+#include <algorithm>
+#include <iterator>
+#include <utility>
+
+void idlist_t::sort_unique()
+{
+    std::sort(m_list.begin(), m_list.end());
+    const auto last = std::unique(m_list.begin(), m_list.end());
+    m_list.erase(last, m_list.end());
+}
+
+void idlist_t::merge_sorted(const idlist_t &other)
+{
+    std::vector<osmid_t> new_list;
+
+    new_list.reserve(m_list.size() + other.m_list.size());
+    std::set_union(m_list.cbegin(), m_list.cend(), other.m_list.cbegin(),
+                   other.m_list.cend(), std::back_inserter(new_list));
+
+    using std::swap;
+    swap(new_list, m_list);
+}

src/idlist.hpp

@@ -38,8 +38,17 @@ class idlist_t
 
     auto end() const noexcept { return m_list.end(); }
 
+    auto cbegin() const noexcept { return m_list.cbegin(); }
+
+    auto cend() const noexcept { return m_list.cend(); }
+
     osmid_t operator[](std::size_t n) const noexcept { return m_list[n]; }
 
+    bool get_binary_search(osmid_t id) const noexcept
+    {
+        return std::binary_search(m_list.cbegin(), m_list.cend(), id);
+    }
+
     void clear() { m_list.clear(); }
 
     void push_back(osmid_t id) { m_list.push_back(id); }
@@ -64,6 +73,10 @@ class idlist_t
         return !(lhs == rhs);
     }
 
+    void sort_unique();
+
+    void merge_sorted(const idlist_t &other);
+
 private:
     std::vector<osmid_t> m_list;
 

src/middle-pgsql.cpp

@@ -56,8 +56,7 @@ static bool check_bucket_index(pg_conn_t const *db_connection,
 }
 
 static void send_id_list(pg_conn_t const &db_connection,
-                         std::string const &table,
-                         osmium::index::IdSetSmall<osmid_t> const &ids)
+                         std::string const &table, idlist_t const &ids)
 {
     std::string data;
     for (auto const id : ids) {
@@ -71,13 +70,12 @@ static void send_id_list(pg_conn_t const &db_connection,
 }
 
 static void load_id_list(pg_conn_t const &db_connection,
-                         std::string const &table,
-                         osmium::index::IdSetSmall<osmid_t> *ids)
+                         std::string const &table, idlist_t *ids)
 {
     auto const res = db_connection.exec(
         fmt::format("SELECT DISTINCT id FROM {} ORDER BY id", table));
     for (int n = 0; n < res.num_tuples(); ++n) {
-        ids->set(osmium::string_to_object_id(res.get_value(n, 0)));
+        ids->push_back(osmium::string_to_object_id(res.get_value(n, 0)));
     }
 }
 
@@ -827,10 +825,9 @@ void middle_pgsql_t::node_delete(osmid_t osm_id)
     }
 }
 
-void middle_pgsql_t::get_node_parents(
-    osmium::index::IdSetSmall<osmid_t> const &changed_nodes,
-    osmium::index::IdSetSmall<osmid_t> *parent_ways,
-    osmium::index::IdSetSmall<osmid_t> *parent_relations) const
+void middle_pgsql_t::get_node_parents(idlist_t const &changed_nodes,
+                                      idlist_t *parent_ways,
+                                      idlist_t *parent_relations) const
 {
     util::timer_t timer;
 
@@ -923,9 +920,8 @@ INSERT INTO osm2pgsql_changed_relations
               parent_ways->size(), parent_relations->size());
 }
 
-void middle_pgsql_t::get_way_parents(
-    osmium::index::IdSetSmall<osmid_t> const &changed_ways,
-    osmium::index::IdSetSmall<osmid_t> *parent_relations) const
+void middle_pgsql_t::get_way_parents(idlist_t const &changed_ways,
+                                     idlist_t *parent_relations) const
 {
     util::timer_t timer;
 

src/middle-pgsql.hpp

@@ -24,6 +24,7 @@
 #include <osmium/index/nwr_array.hpp>
 
 #include "db-copy-mgr.hpp"
+#include "idlist.hpp"
 #include "middle.hpp"
 #include "pgsql.hpp"
 
@@ -117,14 +118,11 @@ struct middle_pgsql_t : public middle_t
     void after_ways() override;
     void after_relations() override;
 
-    void get_node_parents(
-        osmium::index::IdSetSmall<osmid_t> const &changed_nodes,
-        osmium::index::IdSetSmall<osmid_t> *parent_ways,
-        osmium::index::IdSetSmall<osmid_t> *parent_relations) const override;
+    void get_node_parents(idlist_t const &changed_nodes, idlist_t *parent_ways,
+                          idlist_t *parent_relations) const override;
 
-    void get_way_parents(
-        osmium::index::IdSetSmall<osmid_t> const &changed_ways,
-        osmium::index::IdSetSmall<osmid_t> *parent_relations) const override;
+    void get_way_parents(idlist_t const &changed_ways,
+                         idlist_t *parent_relations) const override;
 
     class table_desc
     {

src/middle.hpp

@@ -10,12 +10,12 @@
  * For a full list of authors see the git log.
  */
 
-#include <osmium/index/id_set.hpp>
 #include <osmium/memory/buffer.hpp>
 #include <osmium/osm/entity_bits.hpp>
 
 #include <memory>
 
+#include "idlist.hpp"
 #include "osmtypes.hpp"
 #include "thread-pool.hpp"
 
@@ -149,16 +149,14 @@ class middle_t
 #endif
     }
 
-    virtual void get_node_parents(
-        osmium::index::IdSetSmall<osmid_t> const & /*changed_nodes*/,
-        osmium::index::IdSetSmall<osmid_t> * /*parent_ways*/,
-        osmium::index::IdSetSmall<osmid_t> * /*parent_relations*/) const
+    virtual void get_node_parents(idlist_t const & /*changed_nodes*/,
+                                  idlist_t * /*parent_ways*/,
+                                  idlist_t * /*parent_relations*/) const
     {
     }
 
-    virtual void get_way_parents(
-        osmium::index::IdSetSmall<osmid_t> const & /*changed_ways*/,
-        osmium::index::IdSetSmall<osmid_t> * /*parent_relations*/) const
+    virtual void get_way_parents(idlist_t const & /*changed_ways*/,
+                                 idlist_t * /*parent_relations*/) const
     {
     }
 

src/output-flex.cpp

@@ -998,7 +998,7 @@ void output_flex_t::select_relation_members()
                     "integer way ids."};
             }
 
-            m_stage2_way_ids->set(id);
+            m_stage2_way_ids->push_back(id);
         });
 
     lua_pop(lua_state(), 2); // return value (a table), ways field (a table)
@@ -1486,7 +1486,7 @@ void output_flex_t::init_lua(std::string const &filename)
     lua_remove(lua_state(), 1); // global "osm2pgsql"
 }
 
-idset_t const &output_flex_t::get_marked_way_ids()
+idlist_t const &output_flex_t::get_marked_way_ids()
 {
     if (m_stage2_way_ids->empty()) {
         log_info("Skipping stage 1c (no marked ways).");

src/output-flex.hpp

@@ -16,9 +16,9 @@
 #include "flex-table-column.hpp"
 #include "flex-table.hpp"
 #include "geom.hpp"
+#include "idlist.hpp"
 #include "output.hpp"
 
-#include <osmium/index/id_set.hpp>
 #include <osmium/osm/item_type.hpp>
 
 #include <lua.hpp>
@@ -35,8 +35,6 @@ class geom_transform_t;
 class thread_pool_t;
 struct options_t;
 
-using idset_t = osmium::index::IdSetSmall<osmid_t>;
-
 /**
  * When C++ code is called from the Lua code we sometimes need to know
  * in what context this happens. These are the possible contexts.
@@ -128,7 +126,7 @@ class output_flex_t : public output_t
 
     void wait() override;
 
-    idset_t const &get_marked_way_ids() override;
+    idlist_t const &get_marked_way_ids() override;
     void reprocess_marked() override;
 
     void pending_way(osmid_t id) override;
@@ -295,7 +293,7 @@ class output_flex_t : public output_t
 
     // This is shared between all clones of the output and must only be
     // accessed while protected using the lua_mutex.
-    std::shared_ptr<idset_t> m_stage2_way_ids = std::make_shared<idset_t>();
+    std::shared_ptr<idlist_t> m_stage2_way_ids = std::make_shared<idlist_t>();
 
     std::shared_ptr<db_copy_thread_t> m_copy_thread;