Merge pull request #2221 from joto/flat-node-file-non-slim

Make --flat-nodes also work in non-slim mode

Author: lonvia

man/osm2pgsql.md

@@ -155,16 +155,17 @@ mandatory for short options too.
     timestamp and version.
 
 \--flat-nodes=FILENAME
-:   The flat-nodes mode is a separate method to store slim mode node information on disk.
-    Instead of storing this information in the main PostgreSQL database, this mode creates
-    its own separate custom database to store the information. As this custom database
-    has application level knowledge about the data to store and is not general purpose,
-    it can store the data much more efficiently. Storing the node information for the full
-    planet requires more than 300GB in PostgreSQL, the same data is stored in "only" 50GB using
+:   Use a file on disk to store node locations instead of storing them in
+    memory (in non-slim mode) or in the database (in slim mode). This is much
+    more efficient than storing the data in the database.
+    Storing the node information for the full
+    planet requires more than 500GB in PostgreSQL, the same data is stored in "only" 90GB using
     the flat-nodes mode. This can also increase the speed of applying diff files. This option
     activates the flat-nodes mode and specifies the location of the database file. It is a
     single large file. This mode is only recommended for full planet imports
-    as it doesn't work well with small imports. The default is disabled.
+    as it doesn't work well with small imports. The default is disabled. The
+    file will stay on disk after import, use \--drop to remove it (but you
+    can't do updates then).
 
 \--middle-schema=SCHEMA
 :   Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in the

src/command-line-parser.cpp

@@ -114,9 +114,9 @@ void parse_expire_tiles_param(char const *arg, uint32_t *expire_tiles_zoom_min,
 void check_options_non_slim(CLI::App const &app)
 {
     std::vector<std::string> const slim_options = {
-        "--flat-nodes",           "--middle-schema",
-        "--middle-with-nodes",    "--middle-way-node-index-id-shift",
-        "--tablespace-slim-data", "--tablespace-slim-index"};
+        "--middle-schema", "--middle-with-nodes",
+        "--middle-way-node-index-id-shift", "--tablespace-slim-data",
+        "--tablespace-slim-index"};
 
     for (auto const &opt : slim_options) {
         if (app.count(opt) > 0) {
@@ -190,10 +190,6 @@ void check_options(options_t *options)
         throw std::runtime_error{"--append can only be used with slim mode!"};
     }
 
-    if (options->droptemp && !options->slim) {
-        throw std::runtime_error{"--drop only makes sense with --slim."};
-    }
-
     if (options->cache < 0) {
         options->cache = 0;
         log_warn("RAM cache cannot be negative. Using 0 instead.");
@@ -494,7 +490,7 @@ options_t parse_command_line(int argc, char *argv[])
 
     // --drop
     app.add_flag("--drop", options.droptemp)
-        ->description("Drop middle tables after import (needs --slim).")
+        ->description("Drop middle tables and flat node file after import.")
         ->group("Middle options");
 
     // --extra-attributes
@@ -505,8 +501,8 @@ options_t parse_command_line(int argc, char *argv[])
 
     // --flat-nodes
     app.add_option("-F,--flat-nodes", options.flat_node_file)
-        ->description("File for storing node locations (needs --slim,"
-                      " default: store in database).")
+        ->description(
+            "File for storing node locations (default: store in database).")
         ->type_name("FILE")
         ->group("Middle options");
 

src/middle-ram.cpp

@@ -7,8 +7,10 @@
  * For a full list of authors see the git log.
  */
 
-#include "logging.hpp"
 #include "middle-ram.hpp"
+
+#include "logging.hpp"
+#include "node-persistent-cache.hpp"
 #include "options.hpp"
 #include "output-requirements.hpp"
 
@@ -75,6 +77,11 @@ middle_ram_t::middle_ram_t(std::shared_ptr<thread_pool_t> thread_pool,
     if (options->extra_attributes) {
         m_store_options.untagged_nodes = true;
     }
+
+    if (!options->flat_node_file.empty()) {
+        m_persistent_cache = std::make_shared<node_persistent_cache>(
+            options->flat_node_file, options->droptemp);
+    }
 }
 
 void middle_ram_t::set_requirements(output_requirements const &requirements)
@@ -94,6 +101,7 @@ void middle_ram_t::set_requirements(output_requirements const &requirements)
 
     log_debug("Middle 'ram' options:");
     log_debug("  locations: {}", m_store_options.locations);
+    log_debug("  locations_on_disk: {}", !!m_persistent_cache);
     log_debug("  way_nodes: {}", m_store_options.way_nodes);
     log_debug("  nodes: {}", m_store_options.nodes);
     log_debug("  untagged_nodes: {}", m_store_options.untagged_nodes);
@@ -107,8 +115,15 @@ void middle_ram_t::stop()
 
     auto const mbyte = 1024 * 1024;
 
-    log_debug("Middle 'ram': Node locations: size={} bytes={}M",
-              m_node_locations.size(), m_node_locations.used_memory() / mbyte);
+    if (m_persistent_cache) {
+        log_debug("Middle 'ram': Node locations on disk: size={} bytes={}M",
+                  m_persistent_cache->size(),
+                  m_persistent_cache->used_memory() / mbyte);
+    } else {
+        log_debug("Middle 'ram': Node locations in memory: size={} bytes={}M",
+                  m_node_locations.size(),
+                  m_node_locations.used_memory() / mbyte);
+    }
 
     log_debug("Middle 'ram': Way nodes data: size={} capacity={} bytes={}M",
               m_way_nodes_data.size(), m_way_nodes_data.capacity(),
@@ -180,7 +195,11 @@ void middle_ram_t::node(osmium::Node const &node)
     assert(node.visible());
 
     if (m_store_options.locations) {
-        m_node_locations.set(node.id(), node.location());
+        if (m_persistent_cache) {
+            m_persistent_cache->set(node.id(), node.location());
+        } else {
+            m_node_locations.set(node.id(), node.location());
+        }
     }
 
     if (m_store_options.nodes &&
@@ -222,7 +241,9 @@ void middle_ram_t::after_nodes()
     m_middle_state = middle_state::way;
 #endif
 
-    m_node_locations.log_stats();
+    if (!m_persistent_cache) {
+        m_node_locations.log_stats();
+    }
 }
 
 osmium::Location middle_ram_t::get_node_location(osmid_t id) const
@@ -237,10 +258,19 @@ std::size_t middle_ram_t::nodes_get_list(osmium::WayNodeList *nodes) const
     std::size_t count = 0;
 
     if (m_store_options.locations) {
-        for (auto &nr : *nodes) {
-            nr.set_location(m_node_locations.get(nr.ref()));
-            if (nr.location().valid()) {
-                ++count;
+        if (m_persistent_cache) {
+            for (auto &nr : *nodes) {
+                nr.set_location(m_persistent_cache->get(nr.ref()));
+                if (nr.location().valid()) {
+                    ++count;
+                }
+            }
+        } else {
+            for (auto &nr : *nodes) {
+                nr.set_location(m_node_locations.get(nr.ref()));
+                if (nr.location().valid()) {
+                    ++count;
+                }
             }
         }
     }

src/middle-ram.hpp

@@ -24,6 +24,7 @@
 #include <string>
 #include <utility>
 
+class node_persistent_cache;
 class thread_pool_t;
 
 /**
@@ -124,6 +125,10 @@ class middle_ram_t : public middle_t, public middle_query_t
 
     /// Options for this middle.
     middle_ram_options m_store_options;
+
+    /// File cache
+    std::shared_ptr<node_persistent_cache> m_persistent_cache;
+
 }; // class middle_ram_t
 
 #endif // OSM2PGSQL_MIDDLE_RAM_HPP

src/node-persistent-cache.hpp

@@ -33,6 +33,12 @@ class node_persistent_cache
     void set(osmid_t id, osmium::Location location);
     osmium::Location get(osmid_t id) const noexcept;
 
+    /// The number of locations stored.
+    std::size_t size() const { return m_index->size(); }
+
+    /// Return the approximate number of bytes used for internal storage.
+    std::size_t used_memory() const { return m_index->used_memory(); }
+
 private:
     using index_t =
         osmium::index::map::DenseFileArray<osmium::unsigned_object_id_type,

tests/bdd/regression/multipolygon.feature

@@ -137,6 +137,22 @@ Feature: Import and update of multipolygon areas
             | -k     | the default |
 
 
+     Scenario: Import with flat-node option
+        When running osm2pgsql pgsql with parameters
+            | -F | flat.store | --drop |
+
+        Then table planet_osm_polygon contains
+            | osm_id | landuse     | name      | round(ST_Area(way)) |
+            | -1     | residential | Name_rel  | 12895               |
+            | 4      | farmland    | Name_way3 | 3144                |
+            | -8     | residential | Name_rel2 | 12894               |
+            | 5      | farmland    | Name_way4 | 3144                |
+            | -14    | residential | Name_way5 | 12894               |
+            | -11    | residential | Name_rel6 | 11529               |
+            | -3     | residential | Name_rel11| 9286                |
+            | 83     | farmland    | NULL      | 24859               |
+
+
      Scenario: Import and update with flat-node option
         When running osm2pgsql pgsql with parameters
             | --slim | -F | flat.store |

tests/test-options-parse.cpp

@@ -48,8 +48,6 @@ TEST_CASE("Incompatible arguments", "[NoDB]")
 {
     bad_opt({"-a", "-c", "--slim"}, "options can not be used at the same time");
 
-    bad_opt({"--drop"}, "drop only makes sense with");
-
     bad_opt({"-j", "-k"}, "--hstore excludes --hstore-all");
 
     bad_opt({"-a"}, "--append can only be used with slim mode");