New ram middle that's much more memory-efficient

Replaces the somewhat dated middle_ram_t by a completely new
implementation for importing small to medium sized files into a
non-updateable database. It works completely in memory, no data is
written to disk.

The following traits of OSM objects can be stored. All are optional:
- Node locations for building geometries of ways.
- Way node ids for building geometries of relations based on ways.
- Tags and attributes for nodes, ways, and/or relations for full
  2-stage-processing support.
- Attributes for untagged nodes.

Author: joto

src/middle-ram.cpp

@@ -7,66 +7,197 @@
  * For a full list of authors see the git log.
  */
 
-/* Implements the mid-layer processing for osm2pgsql
- * using several arrays in RAM. This is fastest if you
- * have sufficient RAM+Swap.
- *
- * This layer stores data read in from the planet.osm file
- * and is then read by the backend processing code to
- * emit the final geometry-enabled output formats
-*/
+#include "logging.hpp"
+#include "middle-ram.hpp"
+#include "options.hpp"
+
+#include <osmium/builder/osm_object_builder.hpp>
+#include <osmium/util/delta.hpp>
+#include <osmium/util/string.hpp>
+
+// Workaround: This must be included before buffer_string.hpp due to a missing
+// include in the upstream code. https://github.com/mapbox/protozero/pull/104
+#include <protozero/config.hpp>
+
+#include <protozero/buffer_string.hpp>
+#include <protozero/varint.hpp>
 
+#include <algorithm>
 #include <cassert>
+#include <limits>
 #include <memory>
 
-#include <osmium/builder/attr.hpp>
+middle_ram_t::middle_ram_t(options_t const *options)
+{
+    assert(options);
 
-#include "middle-ram.hpp"
-#include "node-ram-cache.hpp"
-#include "options.hpp"
+    if (options->extra_attributes) {
+        m_store_options.untagged_nodes = true;
+    }
+}
 
-/* Object storage now uses 2 levels of storage arrays.
- *
- * - Low level storage of 2^16 (~65k) objects in an indexed array
- *   These are allocated dynamically when we need to first store data with
- *   an ID in this block
- *
- * - Fixed array of 2^(32 - 16) = 65k pointers to the dynamically allocated arrays.
- *
- * This allows memory usage to be efficient and scale dynamically without needing to
- * hard code maximum IDs. We now support an ID  range of -2^31 to +2^31.
- * The negative IDs often occur in non-uploaded JOSM data or other data import scripts.
- *
- */
+void middle_ram_t::set_requirements(output_requirements const &requirements)
+{
+    if (requirements.full_nodes) {
+        m_store_options.nodes = true;
+    }
+
+    if (requirements.full_ways) {
+        m_store_options.ways = true;
+        m_store_options.way_nodes = false;
+    }
+
+    if (requirements.full_relations) {
+        m_store_options.relations = true;
+    }
+
+    log_debug("Middle 'ram' options:");
+    log_debug("  locations: {}", m_store_options.locations);
+    log_debug("  way_nodes: {}", m_store_options.way_nodes);
+    log_debug("  nodes: {}", m_store_options.nodes);
+    log_debug("  untagged_nodes: {}", m_store_options.untagged_nodes);
+    log_debug("  ways: {}", m_store_options.ways);
+    log_debug("  relations: {}", m_store_options.relations);
+}
+
+void middle_ram_t::stop(thread_pool_t &)
+{
+    auto const mbyte = 1024 * 1024;
+
+    log_debug("Middle 'ram': Node locations: size={} bytes={}M",
+              m_node_locations.size(), m_node_locations.used_memory() / mbyte);
+
+    log_debug("Middle 'ram': Way nodes data: size={} capacity={} bytes={}M",
+              m_way_nodes_data.size(), m_way_nodes_data.capacity(),
+              m_way_nodes_data.capacity() / mbyte);
+
+    log_debug("Middle 'ram': Way nodes index: size={} capacity={} bytes={}M",
+              m_way_nodes_index.size(), m_way_nodes_index.capacity(),
+              m_way_nodes_index.used_memory() / mbyte);
+
+    log_debug("Middle 'ram': Object data: size={} capacity={} bytes={}M",
+              m_object_buffer.committed(), m_object_buffer.capacity(),
+              m_object_buffer.capacity() / mbyte);
+
+    std::size_t index_size = 0;
+    std::size_t index_capacity = 0;
+    std::size_t index_mem = 0;
+    for (auto const &index : m_object_index) {
+        index_size += index.size();
+        index_capacity += index.capacity();
+        index_mem += index.used_memory();
+    }
+    log_debug("Middle 'ram': Object indexes: size={} capacity={} bytes={}M",
+              index_size, index_capacity, index_mem / mbyte);
+
+    log_debug("Middle 'ram': Memory used overall: {}MBytes",
+              (m_node_locations.used_memory() + m_way_nodes_data.capacity() +
+               m_way_nodes_index.used_memory() + m_object_buffer.capacity() +
+               index_mem) /
+                  mbyte);
+
+    m_node_locations.clear();
+
+    m_way_nodes_index.clear();
+    m_way_nodes_data.clear();
+    m_way_nodes_data.shrink_to_fit();
+
+    m_object_buffer = osmium::memory::Buffer{};
+
+    for (auto &index : m_object_index) {
+        index.clear();
+    }
+}
+
+void middle_ram_t::store_object(osmium::OSMObject const &object)
+{
+    auto const offset = m_object_buffer.committed();
+    m_object_buffer.add_item(object);
+    m_object_buffer.commit();
+    m_object_index(object.type()).add(object.id(), offset);
+}
+
+bool middle_ram_t::get_object(osmium::item_type type, osmid_t id,
+                              osmium::memory::Buffer *buffer) const
+{
+    assert(buffer);
+
+    auto const offset = m_object_index(type).get(id);
+    if (offset == ordered_index_t::not_found_value()) {
+        return false;
+    }
+    buffer->add_item(m_object_buffer.get<osmium::memory::Item>(offset));
+    buffer->commit();
+    return true;
+}
+
+static void add_delta_encoded_way_node_list(std::string *data,
+                                            osmium::WayNodeList const &wnl)
+{
+    assert(data);
+
+    // Add number of nodes in list
+    protozero::add_varint_to_buffer(data, wnl.size());
+
+    // Add delta encoded node ids
+    osmium::DeltaEncode<osmid_t> delta;
+    for (auto const &nr : wnl) {
+        protozero::add_varint_to_buffer(data, delta.update(nr.ref()));
+    }
+}
 
 void middle_ram_t::node(osmium::Node const &node)
 {
     assert(node.visible());
-    m_cache->set(node.id(), node.location());
+
+    if (m_store_options.locations) {
+        m_node_locations.set(node.positive_id(), node.location());
+    }
+
+    if (m_store_options.nodes &&
+        (!node.tags().empty() || m_store_options.untagged_nodes)) {
+        store_object(node);
+    }
 }
 
 void middle_ram_t::way(osmium::Way const &way)
 {
     assert(way.visible());
-    m_ways.set(way.id(), new ramWay{way, m_extra_attributes});
+
+    if (m_store_options.way_nodes) {
+        auto const offset = m_way_nodes_data.size();
+        add_delta_encoded_way_node_list(&m_way_nodes_data, way.nodes());
+        m_way_nodes_index.add(way.id(), offset);
+    }
+
+    if (m_store_options.ways) {
+        store_object(way);
+    }
 }
 
-void middle_ram_t::relation(osmium::Relation const &rel)
+void middle_ram_t::relation(osmium::Relation const &relation)
 {
-    assert(rel.visible());
-    m_rels.set(rel.id(), new ramRel{rel, m_extra_attributes});
+    assert(relation.visible());
+
+    if (m_store_options.relations) {
+        store_object(relation);
+    }
 }
 
-size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
+void middle_ram_t::after_nodes() { m_node_locations.freeze(); }
+
+std::size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
 {
     assert(nodes);
-    size_t count = 0;
 
-    for (auto &n : *nodes) {
-        auto loc = m_cache->get(n.ref());
-        n.set_location(loc);
-        if (loc.valid()) {
-            ++count;
+    std::size_t count = 0;
+
+    if (m_store_options.locations) {
+        for (auto &nr : *nodes) {
+            nr.set_location(m_node_locations.get(nr.positive_ref()));
+            if (nr.location().valid()) {
+                ++count;
+            }
         }
     }
 
@@ -77,33 +208,89 @@ bool middle_ram_t::way_get(osmid_t id, osmium::memory::Buffer *buffer) const
 {
     assert(buffer);
 
-    auto const *ele = m_ways.get(id);
-
-    if (!ele) {
-        return false;
+    if (m_store_options.ways) {
+        return get_object(osmium::item_type::way, id, buffer);
     }
+    return false;
+}
 
-    // NOLINTNEXTLINE(google-build-using-namespace)
-    using namespace osmium::builder::attr;
-    osmium::builder::add_way(*buffer, _id(id), _tags(ele->tags),
-                             _nodes(ele->ndids));
+static void
+get_delta_encoded_way_nodes_list(std::string const &data, std::size_t offset,
+                                 osmium::builder::WayBuilder *builder)
+{
+    assert(builder);
 
-    return true;
+    char const *begin = data.data() + offset;
+    char const *const end = data.data() + data.size();
+
+    auto count = protozero::decode_varint(&begin, end);
+
+    osmium::DeltaDecode<osmid_t> delta;
+    osmium::builder::WayNodeListBuilder wnl_builder{*builder};
+    while (count > 0) {
+        auto const val = protozero::decode_varint(&begin, end);
+        wnl_builder.add_node_ref(delta.update(val));
+        --count;
+    }
 }
 
-size_t middle_ram_t::rel_way_members_get(osmium::Relation const &rel,
-                                         rolelist_t *roles,
-                                         osmium::memory::Buffer *buffer) const
+std::size_t
+middle_ram_t::rel_way_members_get(osmium::Relation const &rel,
+                                  rolelist_t *roles,
+                                  osmium::memory::Buffer *buffer) const
 {
     assert(buffer);
 
-    size_t count = 0;
-    for (auto const &m : rel.members()) {
-        if (m.type() == osmium::item_type::way && way_get(m.ref(), buffer)) {
-            if (roles) {
-                roles->emplace_back(m.role());
+    std::size_t count = 0;
+
+    for (auto const &member : rel.members()) {
+        switch (member.type()) {
+        case osmium::item_type::node:
+            if (m_store_options.nodes) {
+                auto const offset =
+                    m_object_index(osmium::item_type::node).get(member.ref());
+                if (offset != ordered_index_t::not_found_value()) {
+                    buffer->add_item(m_object_buffer.get<osmium::Node>(offset));
+                    buffer->commit();
+                    ++count;
+                }
+            }
+            break;
+        case osmium::item_type::way:
+            if (m_store_options.ways) {
+                auto const offset =
+                    m_object_index(osmium::item_type::way).get(member.ref());
+                if (offset != ordered_index_t::not_found_value()) {
+                    buffer->add_item(m_object_buffer.get<osmium::Way>(offset));
+                    buffer->commit();
+                    if (roles) {
+                        roles->emplace_back(member.role());
+                    }
+                    ++count;
+                }
+            } else if (m_store_options.way_nodes) {
+                auto const offset = m_way_nodes_index.get(member.ref());
+                if (offset != ordered_index_t::not_found_value()) {
+                    osmium::builder::WayBuilder builder{*buffer};
+                    builder.set_id(member.ref());
+                    get_delta_encoded_way_nodes_list(m_way_nodes_data, offset,
+                                                     &builder);
+                }
+                buffer->commit();
+                ++count;
+            }
+            break;
+        default: // osmium::item_type::relation
+            if (m_store_options.relations) {
+                auto const offset = m_object_index(osmium::item_type::relation)
+                                        .get(member.ref());
+                if (offset != ordered_index_t::not_found_value()) {
+                    buffer->add_item(
+                        m_object_buffer.get<osmium::Relation>(offset));
+                    buffer->commit();
+                    ++count;
+                }
             }
-            ++count;
         }
     }
 
@@ -115,33 +302,12 @@ bool middle_ram_t::relation_get(osmid_t id,
 {
     assert(buffer);
 
-    auto const *ele = m_rels.get(id);
-
-    if (!ele) {
-        return false;
+    if (m_store_options.relations) {
+        return get_object(osmium::item_type::relation, id, buffer);
     }
-
-    // NOLINTNEXTLINE(google-build-using-namespace)
-    using namespace osmium::builder::attr;
-    osmium::builder::add_relation(*buffer, _id(id),
-                                  _members(ele->members.for_builder()),
-                                  _tags(ele->tags));
-
-    return true;
+    return false;
 }
 
-void middle_ram_t::stop(thread_pool_t &)
-{
-    m_cache.reset();
-    m_ways.clear();
-    m_rels.clear();
-}
-
-middle_ram_t::middle_ram_t(options_t const *options)
-: m_cache(new node_ram_cache{options->alloc_chunkwise, options->cache}),
-  m_extra_attributes(options->extra_attributes)
-{}
-
 std::shared_ptr<middle_query_t> middle_ram_t::get_query_instance()
 {
     return shared_from_this();