Split parse-osmium into input-handler and progress-display

Rename classes and put them in separate files. parse_osmium_t gets the
new name input_handler_t and parse_stats_t gets the new name
progress_display_t.

Author: joto

src/CMakeLists.txt

@@ -7,6 +7,7 @@ set(osm2pgsql_lib_SOURCES
   geometry-processor.cpp
   geom-transform.cpp
   id-tracker.cpp
+  input-handler.cpp
   middle-pgsql.cpp
   middle-ram.cpp
   node-persistent-cache.cpp
@@ -19,11 +20,11 @@ set(osm2pgsql_lib_SOURCES
   output-null.cpp
   output-pgsql.cpp
   output.cpp
-  parse-osmium.cpp
   pgsql.cpp
   processor-line.cpp
   processor-point.cpp
   processor-polygon.cpp
+  progress-display.cpp
   reprojection.cpp
   sprompt.cpp
   table.cpp
@@ -39,6 +40,7 @@ set(osm2pgsql_lib_SOURCES
   geometry-processor.hpp
   geom-transform.hpp
   id-tracker.hpp
+  input-handler.hpp
   middle-pgsql.hpp
   middle-ram.hpp
   middle.hpp
@@ -53,11 +55,11 @@ set(osm2pgsql_lib_SOURCES
   output-null.hpp
   output-pgsql.hpp
   output.hpp
-  parse-osmium.hpp
   pgsql.hpp
   processor-line.hpp
   processor-point.hpp
   processor-polygon.hpp
+  progress-display.hpp
   reprojection.hpp
   sprompt.hpp
   table.hpp

src/input-handler.cpp

@@ -0,0 +1,98 @@
+#include <osmium/osm.hpp>
+
+#include "format.hpp"
+#include "input-handler.hpp"
+#include "osmdata.hpp"
+
+input_handler_t::input_handler_t(osmium::Box const &bbox, bool append,
+                                 osmdata_t const *osmdata)
+: m_data(osmdata), m_bbox(bbox), m_append(append)
+{}
+
+void input_handler_t::node(osmium::Node const &node)
+{
+    if (m_type != osmium::item_type::node) {
+        m_type = osmium::item_type::node;
+        m_data->flush();
+    }
+
+    if (node.deleted()) {
+        if (!m_append) {
+            throw std::runtime_error{"Input file contains deleted objects but "
+                                     "you are not in append mode."};
+        }
+        m_data->node_delete(node.id());
+    } else {
+        // if the node is not valid, then node.location.lat/lon() can throw.
+        // we probably ought to treat invalid locations as if they were
+        // deleted and ignore them.
+        if (!node.location().valid()) {
+            fmt::print(
+                stderr,
+                "WARNING: Node {} (version {}) has an invalid location and has "
+                "been ignored. This is not expected to happen with recent "
+                "planet files, so please check that your input is correct.\n",
+                node.id(), node.version());
+
+            return;
+        }
+
+        if (!m_bbox.valid() || m_bbox.contains(node.location())) {
+            if (m_append) {
+                m_data->node_modify(node);
+            } else {
+                m_data->node_add(node);
+            }
+            m_progress.add_node(node.id());
+        }
+    }
+}
+
+void input_handler_t::way(osmium::Way &way)
+{
+    if (m_type != osmium::item_type::way) {
+        m_type = osmium::item_type::way;
+        m_data->flush();
+    }
+
+    if (way.deleted()) {
+        if (!m_append) {
+            throw std::runtime_error{"Input file contains deleted objects but "
+                                     "you are not in append mode."};
+        }
+        m_data->way_delete(way.id());
+    } else {
+        if (m_append) {
+            m_data->way_modify(&way);
+        } else {
+            m_data->way_add(&way);
+        }
+    }
+    m_progress.add_way(way.id());
+}
+
+void input_handler_t::relation(osmium::Relation const &rel)
+{
+    if (m_type != osmium::item_type::relation) {
+        m_type = osmium::item_type::relation;
+        m_data->flush();
+    }
+
+    if (rel.deleted()) {
+        if (!m_append) {
+            throw std::runtime_error{"Input file contains deleted objects but "
+                                     "you are not in append mode."};
+        }
+        m_data->relation_delete(rel.id());
+    } else {
+        if (rel.members().size() > 32767) {
+            return;
+        }
+        if (m_append) {
+            m_data->relation_modify(rel);
+        } else {
+            m_data->relation_add(rel);
+        }
+    }
+    m_progress.add_rel(rel.id());
+}

src/input-handler.hpp

@@ -0,0 +1,55 @@
+#ifndef OSM2PGSQL_INPUT_HANDLER_HPP
+#define OSM2PGSQL_INPUT_HANDLER_HPP
+
+/**
+ * \file
+ *
+ * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql).
+ *
+ * It contains the input_handler_t class.
+ */
+
+#include <osmium/fwd.hpp>
+#include <osmium/handler.hpp>
+#include <osmium/osm/box.hpp>
+
+#include "progress-display.hpp"
+
+class osmdata_t;
+
+/**
+ * When an OSM file is read, this handler is called for each node, way, and
+ * relation. Depending on the processing mode (create or append), the type
+ * of object and whether an object is added or deleted, the right functions
+ * of the osmdata_t class are called.
+ */
+class input_handler_t : public osmium::handler::Handler
+{
+public:
+    input_handler_t(osmium::Box const &bbox, bool append,
+                    osmdata_t const *osmdata);
+
+    void node(osmium::Node const &node);
+    void way(osmium::Way &way);
+    void relation(osmium::Relation const &rel);
+
+    progress_display_t const &progress() const noexcept { return m_progress; }
+
+private:
+    osmdata_t const *m_data;
+
+    // Bounding box for node import (or invalid Box if everything should be
+    // imported).
+    osmium::Box m_bbox;
+
+    // The progress meter will be updated as we go.
+    progress_display_t m_progress;
+
+    // Current type being parsed.
+    osmium::item_type m_type = osmium::item_type::node;
+
+    // Are we running in append mode?
+    bool m_append;
+};
+
+#endif // OSM2PGSQL_INPUT_HANDLER_HPP

src/osm2pgsql.cpp

@@ -29,6 +29,7 @@
 #include "options.hpp"
 #include "osmdata.hpp"
 #include "output.hpp"
+#include "progress-display.hpp"
 #include "reprojection.hpp"
 #include "util.hpp"
 #include "version.hpp"
@@ -84,7 +85,7 @@ int main(int argc, char *argv[])
 
         // Processing: In this phase the input file(s) are read and parsed,
         // populating some of the tables.
-        parse_stats_t stats;
+        progress_display_t progress;
         for (auto const &filename : options.input_files) {
             fmt::print(stderr, "\nReading in file: {}\n", filename);
             util::timer_t timer_parse;
@@ -99,13 +100,13 @@ int main(int argc, char *argv[])
                     "Unknown file format '{}'."_format(options.input_format)};
             }
 
-            stats.update(osmdata.process_file(file, options.bbox));
+            progress.update(osmdata.process_file(file, options.bbox));
 
-            stats.print_status(std::time(nullptr));
+            progress.print_status(std::time(nullptr));
             fmt::print(stderr, "  parse time: {}s\n", timer_parse.stop());
         }
 
-        stats.print_summary();
+        progress.print_summary();
 
         // Process pending ways and relations. Cluster database tables and
         // create indexes.

src/osmdata.cpp

@@ -14,6 +14,7 @@
 
 #include "db-copy.hpp"
 #include "format.hpp"
+#include "input-handler.hpp"
 #include "middle.hpp"
 #include "options.hpp"
 #include "osmdata.hpp"
@@ -369,8 +370,8 @@ class multithreaded_processor
 
 } // anonymous namespace
 
-parse_stats_t osmdata_t::process_file(osmium::io::File const &file,
-                                      osmium::Box const &bbox) const
+progress_display_t osmdata_t::process_file(osmium::io::File const &file,
+                                           osmium::Box const &bbox) const
 {
     if (!m_append && file.has_multiple_object_versions()) {
         throw std::runtime_error{
@@ -380,12 +381,12 @@ parse_stats_t osmdata_t::process_file(osmium::io::File const &file,
     fmt::print(stderr, "Using {} parser.\n",
                osmium::io::as_string(file.format()));
 
-    parse_osmium_t handler{bbox, m_append, this};
+    input_handler_t handler{bbox, m_append, this};
     osmium::io::Reader reader{file};
     osmium::apply(reader, handler);
     reader.close();
 
-    return handler.stats();
+    return handler.progress();
 }
 
 void osmdata_t::process_stage1b() const

src/osmdata.hpp

@@ -1,6 +1,14 @@
 #ifndef OSM2PGSQL_OSMDATA_HPP
 #define OSM2PGSQL_OSMDATA_HPP
 
+/**
+ * \file
+ *
+ * This file is part of osm2pgsql (https://github.com/openstreetmap/osm2pgsql).
+ *
+ * It contains the osmdata_t class.
+ */
+
 #include <memory>
 #include <vector>
 
@@ -9,13 +17,18 @@
 
 #include "dependency-manager.hpp"
 #include "osmtypes.hpp"
-#include "parse-osmium.hpp"
+#include "progress-display.hpp"
 
 class options_t;
 class output_t;
 struct middle_t;
 struct slim_middle_t;
 
+/**
+ * This class guides the processing of the OSM data through its multiple
+ * stages. It calls upon the major compontents of osm2pgsql, the dependency
+ * manager, the middle, and the outputs to do their work.
+ */
 class osmdata_t
 {
 public:
@@ -31,8 +44,8 @@ class osmdata_t
      * Process the specified OSM file (stage 1a). This is called once for
      * each input file.
      */
-    parse_stats_t process_file(osmium::io::File const &file,
-                               osmium::Box const &bbox) const;
+    progress_display_t process_file(osmium::io::File const &file,
+                                    osmium::Box const &bbox) const;
 
     /**
      * Rest of the processing (stages 1b, 2, and 3). This is called once