Better progress output when processing input files

Author: joto

src/input.cpp

@@ -212,10 +212,10 @@ static void apply(osmium::OSMObject &object, osmdata_t &osmdata,
     if (last_type != object.type()) {
         if (last_type == osmium::item_type::node) {
             osmdata.after_nodes();
-            progress.after_nodes();
+            progress.start_way_counter();
         } else if (last_type == osmium::item_type::way) {
             osmdata.after_ways();
-            progress.after_ways();
+            progress.start_relation_counter();
         }
         last_type = object.type();
     }
@@ -242,15 +242,16 @@ void process_file(osmium::io::File const &file, osmdata_t &osmdata,
     }
 
     osmdata.after_relations();
-    progress.after_relations();
+    progress.print_summary();
 
     reader.close();
 }
 
 void process_files(std::vector<osmium::io::File> const &files,
-                   osmdata_t &osmdata, progress_display_t &progress,
-                   bool append)
+                   osmdata_t &osmdata, bool append, bool show_progress)
 {
+    progress_display_t progress{show_progress};
+
     if (files.size() == 1) {
         process_file(files.front(), osmdata, progress, append);
         return;
@@ -288,7 +289,7 @@ void process_files(std::vector<osmium::io::File> const &files,
     }
 
     osmdata.after_relations();
-    progress.after_relations();
+    progress.print_summary();
 
     for (auto &data_source : data_sources) {
         data_source.close();

src/input.hpp

@@ -47,7 +47,6 @@ prepare_input_files(std::vector<std::string> const &input_files,
  * Process the specified OSM files (stage 1a).
  */
 void process_files(std::vector<osmium::io::File> const &files,
-                   osmdata_t &osmdata, progress_display_t &progress,
-                   bool append);
+                   osmdata_t &osmdata, bool append, bool show_progress);
 
 #endif // OSM2PGSQL_INPUT_HPP

src/osm2pgsql.cpp

@@ -32,7 +32,6 @@
 #include "options.hpp"
 #include "osmdata.hpp"
 #include "output.hpp"
-#include "progress-display.hpp"
 #include "reprojection.hpp"
 #include "util.hpp"
 #include "version.hpp"
@@ -83,15 +82,7 @@ int main(int argc, char *argv[])
 
         // Processing: In this phase the input file(s) are read and parsed,
         // populating some of the tables.
-        util::timer_t timer_parse;
-
-        progress_display_t progress{get_logger().show_progress()};
-        process_files(files, osmdata, progress, options.append);
-
-        fmt::print(stderr, "  parse time: {}\n",
-                   util::human_readable_duration(timer_parse.stop()));
-
-        progress.print_summary();
+        process_files(files, osmdata, options.append, get_logger().show_progress());
 
         // Process pending ways and relations. Cluster database tables and
         // create indexes.

src/progress-display.cpp

@@ -1,18 +1,7 @@
 #include "format.hpp"
 #include "logging.hpp"
 #include "progress-display.hpp"
-
-void progress_display_t::print_summary() const
-{
-    std::time_t const now = std::time(nullptr);
-
-    log_info("Node stats: total({}), max({}) in {}s", m_node.count, m_node.max,
-             nodes_time(now));
-    log_info("Way stats: total({}), max({}) in {}s", m_way.count, m_way.max,
-             ways_time(now));
-    log_info("Relation stats: total({}), max({}) in {}s", m_rel.count,
-             m_rel.max, rels_time(now));
-}
+#include "util.hpp"
 
 static double count_per_second(osmid_t count, uint64_t elapsed) noexcept
 {
@@ -27,6 +16,40 @@ static double count_per_second(osmid_t count, uint64_t elapsed) noexcept
     return static_cast<double>(count) / elapsed;
 }
 
+static std::string cps_display(osmid_t count, uint64_t elapsed) noexcept
+{
+    double const cps = count_per_second(count, elapsed);
+
+    if (cps >= 1000.0) {
+        return "{:.0f}k/s"_format(cps / 1000);
+    }
+    return "{:.0f}/s"_format(cps);
+}
+
+void progress_display_t::print_summary() const
+{
+    std::time_t const now = std::time(nullptr);
+
+    if (m_enabled) {
+        fmt::print(stderr, "\r{:78s}\r", "");
+    }
+
+    log_info("Reading input files done in {}.",
+             util::human_readable_duration(overall_time(now)));
+
+    auto const nt = nodes_time(now);
+    log_info("  Processed {} nodes in {} - {}", m_node.count,
+             util::human_readable_duration(nt), cps_display(m_node.count, nt));
+
+    auto const wt = ways_time(now);
+    log_info("  Processed {} ways in {} - {}", m_way.count,
+             util::human_readable_duration(wt), cps_display(m_way.count, wt));
+
+    auto const rt = rels_time(now);
+    log_info("  Processed {} relations in {} - {}", m_rel.count,
+             util::human_readable_duration(rt), cps_display(m_rel.count, rt));
+}
+
 void progress_display_t::print_status(std::time_t now) const
 {
     if (m_enabled) {
@@ -50,3 +73,33 @@ void progress_display_t::possibly_print_status()
         print_status(now);
     }
 }
+
+uint64_t progress_display_t::nodes_time(std::time_t now) const noexcept
+{
+    if (m_node.count == 0) {
+        return 0;
+    }
+    return (m_way.start > 0 ? m_way.start : now) - m_node.start;
+}
+
+uint64_t progress_display_t::ways_time(std::time_t now) const noexcept
+{
+    if (m_way.count == 0) {
+        return 0;
+    }
+    return (m_rel.start > 0 ? m_rel.start : now) - m_way.start;
+}
+
+uint64_t progress_display_t::rels_time(std::time_t now) const noexcept
+{
+    if (m_rel.count == 0) {
+        return 0;
+    }
+    return now - m_rel.start;
+}
+
+uint64_t progress_display_t::overall_time(std::time_t now) const noexcept
+{
+    return now - m_node.start;
+}
+

src/progress-display.hpp

@@ -14,6 +14,7 @@
 
 #include <osmium/handler.hpp>
 
+#include "format.hpp"
 #include "osmtypes.hpp"
 
 /**
@@ -25,16 +26,9 @@ class progress_display_t : public osmium::handler::Handler
     struct Counter
     {
         std::size_t count = 0;
-        osmid_t max = 0;
         std::time_t start = 0;
 
         osmid_t count_k() const noexcept { return count / 1000; }
-
-        std::size_t add(osmid_t id) noexcept
-        {
-            max = id;
-            return ++count;
-        }
     };
 
 public:
@@ -45,60 +39,40 @@ class progress_display_t : public osmium::handler::Handler
 
     void node(osmium::Node const &node)
     {
-        if (m_node.add(node.id()) % 10000 == 0) {
+        if (++m_node.count % 10000 == 0) {
             possibly_print_status();
         }
     }
 
-    void after_nodes() { m_way.start = std::time(nullptr); }
-
     void way(osmium::Way const &way)
     {
-        if (m_way.add(way.id()) % 1000 == 0) {
+        if (++m_way.count % 1000 == 0) {
             possibly_print_status();
         }
     }
 
-    void after_ways() { m_rel.start = std::time(nullptr); }
 
     void relation(osmium::Relation const &relation)
     {
-        if (m_rel.add(relation.id()) % 10 == 0) {
+        if (++m_rel.count % 10 == 0) {
             possibly_print_status();
         }
     }
 
-    void after_relations() const { print_status(std::time(nullptr)); }
+    void start_way_counter() { m_way.start = std::time(nullptr); }
+
+    void start_relation_counter() { m_rel.start = std::time(nullptr); }
 
     void print_summary() const;
 
 private:
-    void possibly_print_status();
     void print_status(std::time_t now) const;
+    void possibly_print_status();
 
-    uint64_t nodes_time(std::time_t now) const noexcept
-    {
-        if (m_node.count == 0) {
-            return 0;
-        }
-        return (m_way.start > 0 ? m_way.start : now) - m_node.start;
-    }
-
-    uint64_t ways_time(std::time_t now) const noexcept
-    {
-        if (m_way.count == 0) {
-            return 0;
-        }
-        return (m_rel.start > 0 ? m_rel.start : now) - m_way.start;
-    }
-
-    uint64_t rels_time(std::time_t now) const noexcept
-    {
-        if (m_rel.count == 0) {
-            return 0;
-        }
-        return now - m_rel.start;
-    }
+    uint64_t nodes_time(std::time_t now) const noexcept;
+    uint64_t ways_time(std::time_t now) const noexcept;
+    uint64_t rels_time(std::time_t now) const noexcept;
+    uint64_t overall_time(std::time_t now) const noexcept;
 
     Counter m_node{};
     Counter m_way{};

tests/common-import.hpp

@@ -16,7 +16,6 @@
 #include "osmdata.hpp"
 #include "output-multi.hpp"
 #include "output.hpp"
-#include "progress-display.hpp"
 #include "taginfo-impl.hpp"
 
 #include "common-pg.hpp"
@@ -46,8 +45,7 @@ inline void parse_file(options_t const &options,
         filepath += options.input_files[0];
     }
     osmium::io::File file{filepath};
-    progress_display_t progress;
-    process_files({file}, osmdata, progress, options.append);
+    process_files({file}, osmdata, options.append, false);
 
     if (do_stop) {
         osmdata.stop();
@@ -158,8 +156,7 @@ class import_t
         for (auto const &data : input_data) {
             files.emplace_back(data.data(), data.size(), format);
         }
-        progress_display_t progress;
-        process_files(files, osmdata, progress, options.append);
+        process_files(files, osmdata, options.append, false);
 
         osmdata.stop();
     }