Merge pull request #1205 from joto/move-file-read-into-osmdata

Move file reading functionality from parse_osmium_t to osmdata_t

Author: lonvia

src/osm2pgsql.cpp

@@ -29,7 +29,6 @@
 #include "options.hpp"
 #include "osmdata.hpp"
 #include "output.hpp"
-#include "parse-osmium.hpp"
 #include "reprojection.hpp"
 #include "util.hpp"
 #include "version.hpp"
@@ -90,10 +89,17 @@ int main(int argc, char *argv[])
             fmt::print(stderr, "\nReading in file: {}\n", filename);
             util::timer_t timer_parse;
 
-            parse_osmium_t parser{options.bbox, options.append, &osmdata};
-            parser.stream_file(filename, options.input_format);
-
-            stats.update(parser.stats());
+            osmium::io::File file{filename, options.input_format};
+            if (file.format() == osmium::io::file_format::unknown) {
+                if (options.input_format.empty()) {
+                    throw std::runtime_error{
+                        "Cannot detect file format. Try using -r."};
+                }
+                throw std::runtime_error{
+                    "Unknown file format '{}'."_format(options.input_format)};
+            }
+
+            stats.update(osmdata.process_file(file, options.bbox));
 
             stats.print_status(std::time(nullptr));
             fmt::print(stderr, "  parse time: {}s\n", timer_parse.stop());

src/osmdata.cpp

@@ -8,7 +8,9 @@
 #include <utility>
 #include <vector>
 
+#include <osmium/io/any_input.hpp>
 #include <osmium/thread/pool.hpp>
+#include <osmium/visitor.hpp>
 
 #include "db-copy.hpp"
 #include "format.hpp"
@@ -367,6 +369,25 @@ class multithreaded_processor
 
 } // anonymous namespace
 
+parse_stats_t osmdata_t::process_file(osmium::io::File const &file,
+                                      osmium::Box const &bbox) const
+{
+    if (!m_append && file.has_multiple_object_versions()) {
+        throw std::runtime_error{
+            "Reading an OSM change file only works in append mode."};
+    }
+
+    fmt::print(stderr, "Using {} parser.\n",
+               osmium::io::as_string(file.format()));
+
+    parse_osmium_t handler{bbox, m_append, this};
+    osmium::io::Reader reader{file};
+    osmium::apply(reader, handler);
+    reader.close();
+
+    return handler.stats();
+}
+
 void osmdata_t::process_stage1b() const
 {
     if (m_dependency_manager->has_pending()) {

src/osmdata.hpp

@@ -4,8 +4,12 @@
 #include <memory>
 #include <vector>
 
+#include <osmium/fwd.hpp>
+#include <osmium/io/file.hpp>
+
 #include "dependency-manager.hpp"
 #include "osmtypes.hpp"
+#include "parse-osmium.hpp"
 
 class options_t;
 class output_t;
@@ -22,6 +26,18 @@ class osmdata_t
 
     void start() const;
     void flush() const;
+
+    /**
+     * Process the specified OSM file (stage 1a). This is called once for
+     * each input file.
+     */
+    parse_stats_t process_file(osmium::io::File const &file,
+                               osmium::Box const &bbox) const;
+
+    /**
+     * Rest of the processing (stages 1b, 2, and 3). This is called once
+     * after process_file() was called for each input file.
+     */
     void stop() const;
 
     void node_add(osmium::Node const &node) const;
@@ -37,6 +53,7 @@ class osmdata_t
     void relation_delete(osmid_t id) const;
 
 private:
+
     /**
      * Run stage 1b processing: Process dependent objects.
      * In append mode we need to process dependent objects that were marked

src/parse-osmium.cpp

@@ -23,12 +23,8 @@
 #include "parse-osmium.hpp"
 #include "format.hpp"
 #include "osmdata.hpp"
-#include "reprojection.hpp"
 
-#include <osmium/handler.hpp>
-#include <osmium/io/any_input.hpp>
 #include <osmium/osm.hpp>
-#include <osmium/visitor.hpp>
 
 void parse_stats_t::update(parse_stats_t const &other)
 {
@@ -73,36 +69,10 @@ void parse_stats_t::possibly_print_status()
 }
 
 parse_osmium_t::parse_osmium_t(osmium::Box const &bbox, bool append,
-                               osmdata_t *osmdata)
+                               osmdata_t const *osmdata)
 : m_data(osmdata), m_bbox(bbox), m_append(append)
 {}
 
-void parse_osmium_t::stream_file(std::string const &filename,
-                                 std::string const &format)
-{
-    osmium::io::File infile{filename, format};
-
-    if (!m_append && infile.has_multiple_object_versions()) {
-        throw std::runtime_error{
-            "Reading an OSM change file only works in append mode."};
-    }
-
-    if (infile.format() == osmium::io::file_format::unknown) {
-        throw std::runtime_error{
-            format.empty()
-                ? std::string{"Cannot detect file format. Try using -r."}
-                : "Unknown file format '{}'."_format(format)};
-    }
-
-    fmt::print(stderr, "Using {} parser.\n",
-               osmium::io::as_string(infile.format()));
-
-    m_type = osmium::item_type::node;
-    osmium::io::Reader reader{infile};
-    osmium::apply(reader, *this);
-    reader.close();
-}
-
 void parse_osmium_t::node(osmium::Node const &node)
 {
     if (m_type != osmium::item_type::node) {

src/parse-osmium.hpp

@@ -130,9 +130,8 @@ class parse_stats_t
 class parse_osmium_t : public osmium::handler::Handler
 {
 public:
-    parse_osmium_t(osmium::Box const &bbox, bool append, osmdata_t *osmdata);
-
-    void stream_file(std::string const &filename, std::string const &fmt);
+    parse_osmium_t(osmium::Box const &bbox, bool append,
+                   osmdata_t const *osmdata);
 
     void node(osmium::Node const &node);
     void way(osmium::Way &way);
@@ -141,15 +140,15 @@ class parse_osmium_t : public osmium::handler::Handler
     parse_stats_t const &stats() const noexcept { return m_stats; }
 
 private:
-    osmdata_t *m_data;
+    osmdata_t const *m_data;
 
     // Bounding box for node import or invalid Box if everything is imported
     osmium::Box m_bbox;
 
     parse_stats_t m_stats;
 
     // Current type being parsed.
-    osmium::item_type m_type = osmium::item_type::undefined;
+    osmium::item_type m_type = osmium::item_type::node;
 
     // Are we running in append mode?
     bool m_append;

tests/common-import.hpp

@@ -14,7 +14,6 @@
 #include "osmdata.hpp"
 #include "output-multi.hpp"
 #include "output.hpp"
-#include "parse-osmium.hpp"
 #include "taginfo-impl.hpp"
 
 #include "common-pg.hpp"
@@ -30,31 +29,19 @@ inline void parse_file(options_t const &options,
     osmdata_t osmdata{std::move(dependency_manager), mid, outs, options};
 
     osmdata.start();
-    parse_osmium_t parser{options.bbox, options.append, &osmdata};
 
-    std::string filep{TESTDATA_DIR};
-    filep += filename ? filename : options.input_files[0];
-
-    parser.stream_file(filep, "");
+    std::string filepath{TESTDATA_DIR};
+    if (filename) {
+        filepath += filename;
+    } else {
+        filepath += options.input_files[0];
+    }
+    osmium::io::File file{filepath};
+    osmdata.process_file(file, options.bbox);
 
     osmdata.stop();
 }
 
-class test_parse_t : public parse_osmium_t
-{
-public:
-    using parse_osmium_t::parse_osmium_t;
-
-    void stream_buffer(char const *buf, std::string const &fmt)
-    {
-        osmium::io::File infile{buf, std::strlen(buf), fmt};
-
-        osmium::io::Reader reader{infile};
-        osmium::apply(reader, *this);
-        reader.close();
-    }
-};
-
 namespace db {
 
 /**
@@ -65,7 +52,7 @@ class import_t
 {
 public:
     void run_import(options_t options, char const *data,
-                    std::string const &fmt = "opl")
+                    std::string const &format = "opl")
     {
         options.database_options = m_db.db_options();
 
@@ -96,9 +83,8 @@ class import_t
 
         osmdata.start();
 
-        test_parse_t parser(options.bbox, options.append, &osmdata);
-
-        parser.stream_buffer(data, fmt);
+        osmium::io::File file{data, std::strlen(data), format};
+        osmdata.process_file(file, options.bbox);
 
         osmdata.stop();
     }